NodeJS request multiple api endpoints

Question

Ok so I am trying to make two or more requests to API endpoints using the request module. I am rendering a HTML file and passing the returned JSON to a handlebars template using the below code:

res.render('list.html', {
  title: 'List',
  data: returnedJSON
}

I can then iterate over this JSON within my handlebars template fairly easily.

The problem I have, is that I am now needing to use multiple data sources where a category list will be built from a categories JSON response and a Staff list from a staff JSON response. I would like a simple solution where I can do this, but expand it to use any number of data sources.

Below is a full code snippet of what I have with one data source:

request({
    url: 'https://api.com/categories',
    headers: {
        'Bearer': 'sampleapitoken'
    }
}, function(error, response, body) {
    if(error || response.statusCode !== 200) {
        // handle error
    } else {
        var json = JSON.parse(body);
        res.render('list.html', {
            title: 'Listing',
            data: json
        });
    }
});

This works great for one endpoint, but as mentioned before I now need to use multiple requests and have multiple data sources for example:

request({
    url: ['https://api.com/categories','https://api.com/staff'],
    headers: {
        'Bearer': 'sampleapitoken'
    }
}, function(error, response, body1, body2) {
    if(error || response.statusCode !== 200) {
        // handle error
    } else {
        var json1 = JSON.parse(body1);
        var json2 = JSON.parse(body2);
        res.render('list.html', {
            title: 'Listing',
            staff: json1,
            categories: json2
        });
    }
});

I appreciate the above doesn't work like that, but I hope this can help convey what I am trying to achieve.

Thanks in advance :)

Sumatra

Couldn't you create new request objects for each URL and pool them at the end ? – Subclass

That's a good point ;) – Sumatra

Answer 1

You can use the async library to map your request objects and pass them to an actual request and return all results in one callback.

var async = require("async");
var request = require("request");

// create request objects
var requests = [{
  url: 'https://api.com/categories',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}, {
  url: 'https://api.com/staff',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}];

async.map(requests, function(obj, callback) {
  // iterator function
  request(obj, function(error, response, body) {
    if (!error && response.statusCode == 200) {
      // transform data here or pass it on
      var body = JSON.parse(body);
      callback(null, body);
    } else {
      callback(error || response.statusCode);
    }
  });
}, function(err, results) {
  // all requests have been made
  if (err) {
    // handle your error
  } else {
    console.log(results);
    for (var i = 0; i < results.length; i++) {
      // request body is results[i]
    }
  }
});

---

However a simpler way would to leverage promises, this can be done with bluebird and promisifying the request lib, or use the already promisified request lib request-promise. You'll still want to include a promise/A+ lib to map the results asynchronously.

var Promise = require("bluebird");
var request = require('request-promise');

// create request objects
var requests = [{
  url: 'https://api.com/categories',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}, {
  url: 'https://api.com/staff',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}];

Promise.map(requests, function(obj) {
  return request(obj).then(function(body) {
    return JSON.parse(body);
  });
}).then(function(results) {
  console.log(results);
  for (var i = 0; i < results.length; i++) {
    // access the result's body via results[i]
  }
}, function(err) {
  // handle all your errors here
});

---

It's important to note that all latest versions of node and browsers support Promises out of the box and this can be implemented without external libraries.

Hatten

This is fantastic, exactly what I was looking for. However I am getting the following TypeError: request(...).spread is not a function. – Sumatra

ahh that's an artifact. I'll remove that from the example. It should be then, not spread – Hatten

Awesome answer that just helped me get over a big hurdle in my program :) , props! – Lanettelaney

@Hatten do you know why you can't get to Promise via request-promise? I thought request-promise exposed the bluebird Promise object, but rp.promise().map() doesn't seem to work for me. github.com/request/… – Interracial

Hmm. @SebastianThomas Promise.map is a class function so invoking it like you are in your example. I've never leveraged the promise instance from Bluebird directly, always required my own, or more recently used node's built-in class – Hatten

Brilliant answer . the only thing I added on top of this was the .catch . Thanks to @SebastianThomas for including the reference to advanced use cases – Skite

how about Promise.all(requests.map(request)).then(results => results.map(JSON.parse)) – Bondon

Answer 2

Seems like promises could help.

The easiest would probably be to create a new request method that returns a promise (or promisifying with Bluebird etc), then wait for all promises to finish, and handle the data

function doReq(url, what) {
    return new Promise(function(resolve, reject) {
        request({
            url: url,
            headers: {
                'Bearer': 'sampleapitoken'
            }
        }, function(error, response) {
            if(error || response.statusCode !== 200) {
                reject(error);
            } else {
                var data = {};
                (Array.isArray(what) ? what : [what]).forEach(function(item, index) {
                    data[item] = JSON.parse(arguments[index + 2]);
                });
                resolve( data );
            }
        });
    });
}

Promise.all([
    doReq('https://api.com/categories', 'data'), 
    doReq(['https://api.com/categories','https://api.com/staff'], ['staff', 'categories'])
]).then(function() {
    var obj = {title : 'Listing'};
    [].slice.call(arguments).forEach(function(arg) {
        Object.keys(arg).forEach(function(key) {
            obj[key] = arg[key];
        });
    });
    res.render('list.html', obj);
});

Staircase