Nightmare.js screenshot buffer length 0

Question

I'm running a nightmare.js script where where I'm trying to take a screenshot of multiple elements on a page.

The first element is captured just fine, but every other element that is below the fold is captured with a zero length. I am struggling to debug this issue. Any help would be incredibly appreciated.

Basically this script walks through a page and selects all the elements on the page that match a selector. Then, using async it collects the responses and returns a buffer of objects. The issue is that the elements below the fold do not get screenshotted (buffer length ends up at zero). I tried to wait() and scroll to the element, but I have not had any success as of yet.

import * as Nightmare from 'nightmare'
import * as vo from 'vo'
import * as async from 'async'
import * as fs from 'fs'

const urls:String[] = [
  'https://yahoo.com/'
]


Nightmare.action('snap', function(selector:String, done:Function) {
  const self = this;

  this.evaluate_now(function (selector) {
    return Array.from(document.querySelectorAll(selector))
    .map((ele:Element) => {
      if (ele) {
        const rect = ele.getBoundingClientRect()
        const r:Function = Math.round
        return {
          x: r(rect.left),
          y: r(rect.top),
          width: r(rect.width),
          height: r(rect.height)
        }
      }
    })
  }, function(err, clips) {
    if (err) return done(err)
    if (!clips) return done(new Error(`Selector not found`))
    let snaps = []
    const snap = (clip, cb) => {
      self
        .scrollTo(clip.y - clip.height, clip.x)
        .screenshot(clip, cb)
        .run()
    }
    async.mapSeries(clips.reverse(), snap, (err, res) => {
      done(err, res)
    })
  }, selector)
})

const scrape = (url) => {
  const nightmare = Nightmare({
    show: true
  });
  nightmare
    .goto(url)
    .snap('.navbar')
    .end()
    .then((buffers:Buffer[]) => {
      buffers.forEach((data, index) => {
        fs.writeFileSync(`images/navbar-${index}.png`, data)
      })
    })
}

urls.forEach(scrape)

Sontag

I'm trying to reproduce this issue but I figured out that there's no such element ".navbar" on the yahoo.com page. Could you please clarify that? – Turnery

Evgeny, Certainly. Try it on a page with bootstrap, i.e.: getbootstrap.com – Sontag

Were you able to replicate it, @EvgenySorokin? – Sontag

@Sontag Being little selfish for bounty :). Does my answer help? – Pinwork

@Pinwork wasn't intending on being selfish. I picked your answer as it fixes the specific example. Still having trouble when I pick multiple sites, but your answer certainly helps – Sontag

@Sontag Ah! selfish was for me, I was eagerly expecting for first bounty :). And yes you are right, as I ran under different use case it did Not give me good results. Screenshot was off the mark in many cases. To make it generic/robust across would be tough. I tried little more towards it. Hope that helps! – Pinwork

If you are running docker double check SHM the default of 64mb was to low – Abigail

Answer 1

Trying it from different flow, gave better results: The difference in approach is: first scroll to element and then take its bounds and then proceed for screenshot.

const Nightmare = require('nightmare');
const fs = require('fs');
const nightmare = Nightmare({
  show: true,
  openDevTools: false,
  gotoTimeout: 45000
});

nightmare.goto('https://www.google.co.in/?#safe=off&q=nightmare')
  .wait(1000)
  .evaluate(getElements, 'div.g')
  .then(() => {
    console.log("Calling screenshots: ");
    getAllScreenshots(0);
  })
  .catch(function(err) {
    console.log(err);
  });

function getAllScreenshots(index) {
  console.log("Called with index: ", index)
  nightmare.evaluate(function(index) {
      const r = Math.round;
      if(index >= window.__nightmare.output.length) {
        return false;
      }
      var element = window.__nightmare.output[index];
      console.log(index, element.innerHTML);
      element.scrollIntoView(false);
      var bound = element.getBoundingClientRect();
      return {
        x: r(bound.left)-10,
        y: r(bound.top)-10,
        width: r(element.clientWidth)+40,
        height: r(element.clientHeight)+10
      }
    }, index)
    .then(function(bound) {
      if(!bound) {
        return;
      }
      console.log("Taking screenshot: ", bound);
      nightmare.wait(500).screenshot(__dirname + '/images/navbar' + index + '.png', bound)
        .then(function() {
          console.log("Calling Next of: ", index);
          getAllScreenshots(index + 1);
        }).catch(function(err) {
          console.log(err);
        })
    })
    .catch(function(err) {
      console.log(err);
    });
}

function getElements(selector) {
  var elements = document.querySelectorAll(selector);
  window.__nightmare.output = elements;
  console.log(elements.length);
}

Pinwork

This is great. Thank you! – Sontag

Answer 2

Actually, screenshot() function takes coordinates from the visible screen.
Example, if (x,y) of any element is (10, 1000) and your window size is (800,600) then you can scroll (900:element.y, 0) and then take screenshot at (element.y-scroll.y=100, element.x)

I finally got the code working:

const Nightmare = require('nightmare');
const fs = require('fs');
const nightmare = Nightmare({
  show: true,
  openDevTools: true,
});

nightmare.goto('https://in.news.yahoo.com/')
  .wait(1000)
  .evaluate(getBounds, '.Cf')
  .then(function(rects) {
    console.log(rects);

    function getScreenshot(rects, index) {
      if (index == rects.length) return;
      nightmare.scrollTo(rects[index].y, 0)
        .screenshot(__dirname + '/images/navbar' + index + '.png', {
          //60 is height of the top element which remains
          x: rects[index].x-10,
          y: 60,
          width: rects[index].width+30,
          height: rects[index].height +60
        })
        .then(function() {
          console.log("Calling next. " + index);
          getScreenshot(rects, index + 1);
        }).catch(function(err) {
          console.log(err);
        })
    };

    getScreenshot(rects, 0);
  })
  .catch(function(err) {
    console.log(err);
  });

function getBounds(selector) {
  var elements = document.querySelectorAll(selector);
  if (elements && elements.length > 0) {
    var arr = [];
    const r = Math.round;
    for (var ii = 0; ii < elements.length; ii++) {
      var rect = elements[ii].getBoundingClientRect();
      arr.push({
        x: r(rect.left),
        y: r(rect.top),
        width: r(rect.width),
        height: r(rect.height)
      })
    }
    console.log("Elements found: ", arr.length);
    return arr;
  }
  return null;
}

Pinwork