Selenium: Scroll till end of the page

Question

Selenium:

I am new to WebDriverJS. I have tried this approach in Java.

Long repaeted = 0l, scrollHeight = 0l, returnHeight = 0l;
while(true){
    if (repaeted == 0) {
        returnHeight = (Long) jse.executeScript("var scroll =document.documentElement.scrollHeight;window.scrollTo(0, scroll); return scroll;");
         System.out.println("Height : "+scrollHeight +"\t Chnage : "+returnHeight+ "\t Repeated : "+repaeted);
         scrollHeight = returnHeight;
     }else {
         returnHeight = (Long) jse.executeScript("var scroll =  document.documentElement.scrollHeight;window.scrollTo(0, scroll); return scroll;");
         System.out.println("Height : "+scrollHeight +"\t Chnage : "+returnHeight+ "\t Repeated : "+repaeted);
         if (scrollHeight.intValue() == returnHeight.intValue()) {
             System.out.println("Break.."+ returnHeight);
             break;
         } else { scrollHeight = returnHeight; }
     }
            repaeted++;
 } 

but I am facing problem in webdriverjs while iterating the loop.

var webdriver = require('..'),
    By = webdriver.By,
    until = webdriver.until;
// make sure chromedriver can be found on your system PATH
var driver = new webdriver.Builder()
    .forBrowser('chrome')
    .withCapabilities(webdriver.Capabilities.chrome())
    .build();


driver.get('https://in.yahoo.com/').then(function(){
        var window = new webdriver.WebDriver.Window(driver);
        window.maximize();
        driver.manage().timeouts().implicitlyWait(1000 * 3);
    })
    .then(function(){
        console.log('Entered');
        var check = 0, count = 0
        for(var i = 0; i< 50; i++){
        //driver.sleep(1000 * 2);
driver.executeScript('var dynamicscroll = document.documentElement.scrollHeight;window.scrollTo(0, dynamicscroll);return dynamicscroll;').then(function(height){
        console.log('Check : '+check+'  Height : '+height +'  Repeated : '+(count++));
        if(check === 0 || check !== height){console.log('continue'); check = height; }
        else { console.log('break'); i = 100; }
            });
        }
        })
    .then(null, function(err) {
      console.error("An error was thrown! By Promise..." + err);
    });

driver.quit();

In my code I have hardcoded for loop to iterate until 50 times and I want to quit/break the loop when the scroll height is reached to end. In this approach, I want to remove hardcode like java-code because I don't know how many times to iterate for other applications whose scroll is kept on increasing dynamically. For example, Facebook application, Yahoo News...

Alicea

Answer 1

Scrolling to the bottom of a dynamic page can be challenging depending on how it is implemented by the page.

First you'll have to find the container with the scrollbar since it can be different from the one linked to window.scrollTo.

Then scroll the container by increasing scrollTop until the scrollHeight becomes steady with no pending requests. To check if there are pending requests, either evalute jQuery.active if the page has JQuery or hook XMLHttpRequest to monitor the calls on send.

Here is an example using on a generic function to scroll to the bottom of the page a number of times or until the end:

var webdriver = require('selenium-webdriver');

var driver = new webdriver.Builder().forBrowser('chrome').build();

driver.get('https://groups.google.com/forum/#!search/webdriverjs');

 // scroll to the bottom 3 times
driver.executeAsyncScript(scrollBottom, 3)
  .then(n => console.log(`scrolled ${n} time(s)`));

 // scroll to the bottom until the end
driver.executeAsyncScript(scrollBottom)
  .then(n => console.log(`scrolled ${n} time(s)`));

function scrollBottom(){
  var count = arguments[arguments.length - 2] || 0x7fffffff;
  var callback = arguments[arguments.length - 1];

  /* get the scrollable container */
  var elm = document.elementFromPoint(window.innerWidth - 25, window.innerHeight / 2);
  for ( ;elm && (++elm.scrollTop, !elm.scrollTop); elm=elm.parentElement);
  elm = elm || document.documentElement;

  /* hook XMLHttpRequest to monitor Ajax requests */
  if (!('idle' in XMLHttpRequest)) (function(){
    var n = 0, t = Date.now(), send = XMLHttpRequest.prototype.send;
    var dispose = function(){ --n; t = Date.now(); };
    var loadend = function(){ setTimeout(dispose, 1) };
    XMLHttpRequest.idle = function() { return n > 0 ? 0 : Date.now() - t; };
    XMLHttpRequest.prototype.send = function(){
      ++n;
      this.addEventListener('loadend', loadend);
      send.apply(this, arguments);
    };
  })();

  /* scroll until steady scrollHeight or count of scroll and no pending request */
  var i = 0, scrollHeight = -1, scrollTop = -1;
  (function scroll(){
    if ((scrollHeight === elm.scrollHeight || i === count) && XMLHttpRequest.idle() > 60)
      return callback(i);
    scrollTop = elm.scrollTop;
    scrollHeight = elm.scrollHeight;
    if (i < count)
      i += (elm.scrollTop = 0x7fffffff, scrollTop !== elm.scrollTop);
    setTimeout(scroll, 100);
  })();
}

Or by scrolling until the height no longer increases during a specific time (5 seconds here) :

function scrollBottom(){
  var count = arguments[arguments.length - 2] || 0x7fffffff;
  var callback = arguments[arguments.length - 1];
  var timeout = 5000;  /* 5 seconds timeout */
  var i = 0;

  /* get the scrollable container */
  var elm = document.elementFromPoint(window.innerWidth - 25, window.innerHeight / 2);
  for ( ;elm && (++elm.scrollTop, !elm.scrollTop); elm=elm.parentElement);
  elm = elm || document.documentElement;

  /* scroll while the height is increasing or until timeout */
  (function scroll(){
    var endtime = Date.now() + timeout;
    var height = elm.scrollHeight;
    elm.scrollTop = 0x7fffffff;  /* scroll */

    setTimeout(function check(){
      if (Date.now() > endtime)            /* returns if waited more than 5 sec */
        callback(i);
      else if (elm.scrollHeight == height) /* wait again if same height  */
        setTimeout(check, 60);
      else if (++i === count)              /* returns if scrolled the expected count */
        callback(i);
      else                                 /* scroll again */
        setTimeout(scroll, 60);
    }, 250);
  })();
}

Psychopath

Can you check with this URL:https://in.yahoo.com/. and my question is i just want to break the loop when it reaches to end using webdriverjs. what URL you have checked contains div scroll. – Alicea

@Yash, it works with in.yahoo.com. Your question is how to scroll down to bottom of a web-page when the scroll height dynamically increases, which is exactly what this example does with webdriverjs. I added another example which is scrolling until the height no longer increases during a specific time. – Psychopath

Can you check that you code is throwing error ScriptTimeoutError: asynchronous script timeout: result was not received in 0 seconds every time while running. – Alicea

which one? It could be that you need need to set the timeout for the script: driver.manage().timeouts().setScriptTimeout(30000) – Psychopath

Thanks for immediate replay, now it is working fine. ):- – Alicea

iam facing page loading problem with this URL:https://in.news.yahoo.com/national, Can we solve it. – Alicea

Yahoo news is full of garbage and poorly implemented. Already crashed chrome twice by scrolling the page manually. I would stay away from it. – Psychopath

yes Yahoo news is not good, So i have checked with this URL:http://scrollmagic.io/examples/advanced/infinite_scrolling.html working good. Thanks once again. – Alicea

Answer 2

From experience, the quickest way to scroll to the end of a page is to look for the footer element and movetoit, usually #footer or .footer or just footer selector will do it. E.g.:

footer = driver.findElement({id: "footer"});
driver.executeScript("arguments[0].scrollIntoView(false);", footer);

In the case of 'endless' streams like Facebook, Twitter, etc. They may block you when you reach a limit so it's okay to combine max iterations with window.scrollTo(0, 300); recursively and wait for some seconds after each scroll.

Epitaph

Answer 3

Pure JavaScript:

In JavaScript we can use setTimeout() function. which will call the specified function recursively after the time delay you specified.

I have tested the google groups application, whose div tag vertical scroll dynamically increases. To load the content I used the time delay of 5000. you can test this code in browser's console use this URL: https://groups.google.com/forum/#!search/webdrierjs.

var i = 0, height = 0, check = 0, t = null;
flow();

function run(arg){
var objDiv = document.querySelector('div.IVILX2C-b-F');
objDiv.scrollTop = objDiv.scrollHeight;
return objDiv.scrollHeight;
}

function flow() {
i++;
    switch(i){ 
        case 0:     height  = run(i);
                    sleep(5000);
                break;
        case -1:    run(i);
                    clearTimeout(t); //stops flow
                break;
        default:    check = run(i);
                    console.log('Return Height : '+check +'  Count : '+i);
                    if(check === height){ i = -2;
                    console.log('Break message : '+i);
                    }else {
                    console.log('Changed...');
                    height = check;
                    }
                sleep(5000);
                break;
    }
}

function sleep(delay) { t=setTimeout("flow()",delay);} //starts flow control again after time specified.
//function sleep(delay) {  var start = new Date().getTime();     while (new Date().getTime() < start + delay);  flow(); } // stops execution and then continues.

but even I cannot run this script using WebDriver/WebDriverJS because it is not going to call recursive function on time delay.

Alicea

This code will throw an error when used in strict compliance mode (t is global). – Ithaman

@ A Red Herring thanks ):- now i changed t as local variable var t = null; and tested!. now the 'i' is increamenting properly. – Alicea

Answer 4

I am aware that this is kind of an old topic, yet I've still came across it while I had similar problem. Given sollutions didn't quite suite me. I wrote my own function - without using any "arguments[...]" as argument of driver.executeScript(...). ...I just don't get that "arguments[...]" to be honest... x)

Below I'm presenting my solution. (I believe its shorter and cleaner. And uses async/await syntax instead of ".then(s)")

// scroller.service.ts

import { WebDriver } from 'selenium-webdriver';

export async function scrollTillEnd(driver: WebDriver): Promise<void> {

    const scrollDownTillEnd = async () => {
        let counter = 0
        let heightBefore = 0
        let heightAfter = 0
        let shouldContinue = true

        const scrollDown = () => window.scrollBy(0, document.body.scrollHeight || document.documentElement.scrollHeight)
        const scrollAndCalc = async () => {
            heightBefore = document.body.scrollHeight || document.documentElement.scrollHeight
            scrollDown()
            await new Promise((res) => setTimeout(() => res(null), 2000)) // sleep in vanillaJS
            heightAfter = document.body.scrollHeight || document.documentElement.scrollHeight
            shouldContinue = heightAfter != heightBefore
            counter++
            console.log({ shouldContinue, heightBefore, heightAfter, counter })
        }

        while (shouldContinue) {
            await scrollAndCalc()
        }
    }

    await driver.executeScript(scrollDownTillEnd)
}

(executed on chrome 97 )

// example usage:

class App{
//    ...

    private async initDriver() {
        this.driver = await new DriverInitiator().getDriver()
        await this.driver.get(this.url)
    }

    private initPagesModels() {
        this.cookiesWelcome = new CookiesWelcomePage(this.driver)
    }

    async runExample() {
        await this.initDriver()
        this.initPagesModels()
        await this.driver.sleep(1000)

        await this.cookiesWelcome.acceptCookies()
        await scrollTillEnd(this.driver) // this is the part where i call my scrolling function :)

        console.log("selenium script ended.")

        await this.driver.sleep(5000)
        await this.driver.quit();

    }
}

Becka