Async/Await with Request-Promise returns Undefined

Question

I have two files; server.js and scrape.js, below are the code snippets as they currently stand.

server.js:

const scrape = require("./scrape");

async function start() {
    const response = await scrape.start();
    console.log(response);
}

start();

and scrape.js:

const cheerio = require("cheerio");
const request = require("request-promise");

go = async () => {

const options = {
  uri: "http://www.somewebsite.com/something",
  transform: function(body) {
    return cheerio.load(body);
  }
};

request(options)
  .then($ => {
    let scrapeTitleArray = [];
    $(".some-class-in-html").each(function(i, obj) {
      const data = $(this)
        .text()
        .trim();
      scrapeTitleArray.push(data);
    });
    return scrapeTitleArray;
  })
  .catch(err => {
    console.log(err);
  });
};

module.exports = {
  start: go
};

So when I spin up server.js, I return undefined to the console.log(response), when I actually want to return the array i've been pushing to, can you see where I'm going wrong?

Iminourea

Shouldn't "go" return a promise? – Brann

@Brann If i'm not mistaken, declaring it as an async function automatically returns a promise? – Iminourea

.each() uses a "callback". You need to wrap that in a promise and resolve it's result. Otherwise the code just skips straight to returning the empty array. You need to return the request result of course, but if you don't respect the callback then you're still going to have problems. – Susumu

go is an implicitly global variable. Don't do that. – Lookeron

Why do you define go as an async function but then nowhere use await? – Lookeron

Answer 1

You need to return something from your async function (a return inside a then does not return from the main function). Either a promise or something you await-ed.

Also, make sure to declare your go variable to avoid leaking it into the global space.

const go = async () => {

  const options = {
    uri: "http://www.somewebsite.com/something",
    transform: function(body) {
      return cheerio.load(body);
    }
  };

  return request(options)
    .then($ => {
      let scrapeTitleArray = [];
      $(".some-class-in-html").each(function(i, obj) {
        const data = $(this)
          .text()
          .trim();
        scrapeTitleArray.push(data);
      });
      return scrapeTitleArray;
    })
    .catch(err => {
      console.log(err);
    });
};

Since you are using an async function, you might want to take advantage of the await syntax also.

const go = async () => {

  const options = {
    uri: "http://www.somewebsite.com/something",
    transform: function(body) {
      return cheerio.load(body);
    }
  };

  try {
    const $ = await request(options);
    $(".some-class-in-html").each(function(i, obj) {
      const data = $(this)
        .text()
        .trim();
      scrapeTitleArray.push(data);
    });
    return scrapeTitleArray;
  }
  catch (err) {
    console.log(err);
  }
};

Lavation

.each() has a callback does it not? – Susumu

@NeilLunn Yes .each uses a synchronous callback (so it's ok to use here). – Strangury

So it is – Susumu

@NeilLunn (of course map should be preferred over each+push) – Lookeron

Answer 2

I believe your go function isn't returning any value.

You're calling request(options).then(...), but what follows from that promise is never returned by go. I recommend you add a return statement:

go = async () => {

  const options = {
    uri: "http://www.somewebsite.com/something",
    transform: function(body) {
      return cheerio.load(body);
    }
  };

  // The only difference is that it says "return" here:
  return request(options)
    .then($ => {
      let scrapeTitleArray = [];
      $(".some-class-in-html").each(function(i, obj) {
        const data = $(this)
          .text()
          .trim();
        scrapeTitleArray.push(data);
      });
      return scrapeTitleArray;
    })
    .catch(err => {
      console.log(err);
    });
};

Hypothesis