How to gather the result of Web APIs on nodeJS with 'request' and 'async'

Question

I'm writing a test code which gathers the result from Web APIs on node.js by using 'request.js' and 'async.js'.

Here's my sample code;

var request = require('request');
var async = require('async');

var addresses = [
    "Tokyo",
    "Moscow",
    "Bagdhad",
    "Mountain View",
    "New York",
];

function accessUrl( address, callback ) {  
    options ={
        headers: {'user-agent': 'Mozilla/5.0'},
        url: 'http://api.openweathermap.org/data/2.5/weather?q=' + address,
        json: true
    };

    request.get(options, function(err, response, body_json) {
        if( !err && response.statusCode === 200 ){
            return callback(address, body_json);
        }
        else{
            console.log('error');
        }
    });
}

function showContent( address, body_json ) {
    console.log( "%s : %s, %s (C)", address, body_json['weather'][0]['main'],
                 Math.round(body_json['main']['temp']-273.15));
    return [ address, body_json['weather'][0]['main'],
                 Math.round(body_json['main']['temp']-273.15)];
}

var result = [];
async.map( addresses, function( item, callback ) {
    result.push (accessUrl( item, showContent ));
    }, function(err) {
    });

console.log(result);

However, the result is;

~$ node asyncsample1.js
[ undefined, undefined, undefined, undefined, undefined ]
Tokyo : Clear, 23 (C)
Mountain View : Mist, 10 (C)
Bagdhad :  Clear, 10 (C)
New York : Clear, 22 (C)
Moscow : Clouds, 4 (C)

console.log() in the callback function showContent() shows the proper result but the gathered results are all 'undefined'.

How can I get the results in var result[]?

Jugum

Answer 1

Well, let me introduce you to async's concat function. Here is your modified program.

var request = require('request');
var async = require('async');

var addresses = [
    "Tokyo",
    "Moscow",
    "Bagdhad",
    "Mountain View",
    "New York",
];

function accessUrl( address, callback ) {  
    options ={
        headers: {'user-agent': 'Mozilla/5.0'},
        url: 'http://api.openweathermap.org/data/2.5/weather?q=' + address,
        json: true
    };

    request.get(options, function(err, response, body_json) {
        if( !err && response.statusCode === 200 ){
            return callback(null, [[ address, body_json['weather'][0]['main'],
                 Math.round(body_json['main']['temp']-273.15)]]);
        }
        else{
            return callback(err);
        }
    });
}

async.concat (addresses, accessUrl, function(err, result) {
    if (err) {
        console.error(err);
    } else {
        console.log(result);
    }
});

Output

[ [ 'Mountain View', 'Haze', 17 ],
  [ 'Bagdhad', 'Clear', 18 ],
  [ 'New York', 'Clear', 26 ],
  [ 'Tokyo', 'Clouds', 22 ],
  [ 'Moscow', 'Clouds', 3 ] ]

You don't have to synchronize the results yourself. concat does that for you.

If you are interested to know how async's concat works, try this

Giacopo

Answer 2

You are pushing undefined to results here, because accessUrl is asynchronous. It returns undefined. You don't have any result until the accessUrl callback is executed - but by that point, even though it returns a result, undefined has already been pushed and the result goes nowhere.

// wrong
var result = [];
async.map( addresses, function( item, callback ) {
    result.push (accessUrl( item, showContent ));
    }, function(err) {
    });

Any logic that depends on the result of an asynchronous function must go in the callback of that function. Also, async handles the output array for you, you just pass (err, result to the iterator.

//corrected
async.map( addresses
, function( item, callback ){
  accessUrl( item, function(address, body_json){
    result = showContent(address, body_json);
    // tell async.map there was no error and this is the mapped result
    callback(null, result); 
  });
}
, function(err, results) {
  console.log(results)
});

If I were you I would make a couple tweaks. First off handle the case where there is a request error. Second, I like to use done as the name for the callback within async, because I commonly find that I use async inside another asynchronous function, and I usually have an existing variable called callback already that is the callback of the parent function. So here is how I would do it:

//improved
function iterator(item, done){
  accessUrl( item, function(err, address, body_json){
  // you will have to tweak accessUrl to callback with three parameters instead of two

    if(err){ return done(err) }; 
    // the `return` is necessary, otherwise latter lines will still execute

    result = showContent(address, body_json);

    // tell async.map there was no error and this is the mapped result
    done(null, result); 
  });
};

async.map( addresses
, iterator
, function(err, results) {
  if(err){ console.log(err) };
  console.log(results)'
});

An alternative way, where you create a results array and add the results one by one. this is closer to your original idea, but since async.map automatically builds the results array, it didn't make sense to use this strategy with async.map.

//alternative
function printWeather(callback){
  var results = [];
  async.forEachSeries(addresses
  , function(item, done){
    accessUrl( item, function(err, address, body_json){
      if(err){ return done(err) }; 
      result = showContent(address, body_json);
      results.push(result);
      done(null); 
    });
  }
  , function(err) {
    if(err){ console.log(err) };
    console.log(results);
    //callback(err, results);
  });
};

Wahhabi

And let me confirm, does 'result' have to be handled only in async.map 'function(err, results){}'? – Jugum

The async.map function takes the addresses array and executes the iterator function on each item. Each time the iterator function is called, it calls back (err, result) where result is the mapped output of that one address. The completion callback of async.map takes (err, results) parameters, where results is the mapped outputs, in the same order that the addresses array was. – Wahhabi

see my latest edit for an alternative that manually adds results one by one – Wahhabi