Using Promises with fs.readFile in a loop

Question

I'm trying to understand why the below promise setups don't work.

(Note: I already solved this issue with async.map. But I would like to learn why my attempts below didn't work.)

The correct behavior should be: bFunc should run as many time as necessary to fs read all the image files (bFunc below runs twice) and then cFunc console prints "End".

Thanks!

Attempt 1: It runs and stops at cFunc().

var fs = require('fs');

bFunc(0)
.then(function(){ cFunc() }) //cFunc() doesn't run

function bFunc(i){
    return new Promise(function(resolve,reject){

        var imgPath = __dirname + "/image1" + i + ".png";

        fs.readFile(imgPath, function(err, imagebuffer){

            if (err) throw err;
            console.log(i)

            if (i<1) {
                i++;
                return bFunc(i);
            } else {
                resolve();
            };

        });

    })
}

function cFunc(){
    console.log("End");
}

Attempt 2: In this case, I used a for-loop but it executes out of order. Console prints: End, bFunc done, bFunc done

var fs = require('fs');

bFunc()
        .then(function(){ cFunc() })

function bFunc(){
    return new Promise(function(resolve,reject){

        function read(filepath) {
            fs.readFile(filepath, function(err, imagebuffer){
                if (err) throw err;
                console.log("bFunc done")
            });
        }

        for (var i=0; i<2; i++){
            var imgPath = __dirname + "/image1" + i + ".png";
            read(imgPath);
        };

        resolve()
    });
}


function cFunc(){
    console.log("End");
}

Thanks for the help in advance!

Oller

In attempt 1, where is you error handler for bFunc? If an error is thrown, you will never know about it with your current code. – Lagomorph

What is the goal of this code? Please describe the problem you are trying to solve in words rather than just showing us code that doesn't do what you want. There are lots of things wrong with your code so I'd rather start from understand the problem to be solved than trying to rework all the things wrong with your code without knowing the actual end goal. – Armoire

@Lagomorph it doesn't throw an error at bFunc! But I guess I should put a catch at the end of the chain. – Oller

@Armoire I stripped out all the details of the code and just left the structure to simplify it. The problem is the same but if it helps, I was using Microsoft's Project Oxford for facial detection and then in each ".then" I augment the photo with glasses, hats, etc. I can share the source code with you. Thanks. – Oller

Well, think about it from our perspective. You give us two blocks of code that don't work and ask us how to fix them. But, you never really describe what they're supposed to do (what the correct behavior is). Are we supposed to guess from the malfunctioning code what the proper behavior is? Please describe the desired and proper behavior you want out of either code block. – Armoire

@Armoire I see your point and I have edited the post: The correct behavior should be: bFunc should run as many time as necessary to fs read all the image files (bFunc runs twice in this case) and then cFunc console prints "End". – Oller

Are you trying to call fs.readFile() multiple times to read files in parallel? Take a look at codendeavor.com/… – Martynne

Answer 1

So, anytime you have multiple async operations to coordinate in some way, I immediately want to go to promises. And, the best way to use promises to coordinate a number of async operations is to make each async operation return a promise. The lowest level async operation you show is fs.readFile(). Since I use the Bluebird promise library, it has a function for "promisifying" a whole module's worth of async functions.

var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));

This will create new parallel methods on the fs object with an "Async" suffix that return promises instead of use straight callbacks. So, there will be an fs.readFileAsync() that returns a promise. You can read more about Bluebird's promisification here.

So, now you can make a function that gets an image fairly simply and returns a promise whose value is the data from the image:

 function getImage(index) {
     var imgPath = __dirname + "/image1" + index + ".png";
     return fs.readFileAsync(imgPath);
 }

Then, in your code, it looks like you want to make bFunc() be a function that reads three of these images and calls cFunc() when they are done. You can do that like this:

var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));

 function getImage(index) {
     var imgPath = __dirname + "/image1" + index + ".png";
     return fs.readFileAsync(imgPath);
 }

 function getAllImages() {
    var promises = [];
    // load all images in parallel
    for (var i = 0; i <= 2; i++) {
        promises.push(getImage(i));
    }
    // return promise that is resolved when all images are done loading
    return Promise.all(promises);
 }

 getAllImages().then(function(imageArray) {
    // you have an array of image data in imageArray
 }, function(err) {
    // an error occurred
 });

If you did not want to use Bluebird, you could manually make a promise version of fs.readFile() like this:

// make promise version of fs.readFile()
fs.readFileAsync = function(filename) {
    return new Promise(function(resolve, reject) {
        fs.readFile(filename, function(err, data){
            if (err) 
                reject(err); 
            else 
                resolve(data);
        });
    });
};

Or, in modern versions of node.js, you can use util.promisify() to make a promisified version of a function that follows the node.js async calling convention:

const util = require('util');
fs.readFileAsync = util.promisify(fs.readFile);

Though, you will quickly find that once you start using promises, you want to use them for all async operations so you'll be "promisifying" lots of things and having a library or at least a generic function that will do that for you will save lots of time.

---

In even newer versions of node.js (version 10.0+), you can use the built-in version of the fs library that supports promises:

const fsp = require('fs').promises;

fsp.readFile("someFile").then(data => {
    console.log(data);
});

Armoire

please note that you now have to use Promise.promisifyAll instead of Promise.promisify – Intemperance

Note that node 8 comes with it's own promisify function: const {promisify} = require("util"); const fs = require("fs"); const readFile = promisify(fs.readFile); – Tanker

@Tanker - Using util.promisify() as an option in newer versions of node.js is already in my answer. – Armoire

Added info about the built-in fs promise support in node.js v10+. – Armoire

would be nice if you moved the util part to the top of the answer, would that be more standard 'now' ? – Otherworld

@MzA - The standard way now would be the last code block using fs.promises.readFile. I'll look at reorganizing the answer. – Armoire

@Armoire any reason why let jsondata = await fsPromises.readFile(jsonFileName) seems to have to wait a lot longer to complete (move on to the following statement) than let jsondata = fs.readFileSync(jsonFileName)?? I mean a LOT longer. – Dvinsk

@Dvinsk - How much longer and what size file? Interested in time to execute for both versions. Also, what version of node.js? And, is your server doing anything else when using .readFile() that would delay when you can process the resolved promise? Can you reproduce in a small test app that does nothing else? – Armoire

~10ms for fs.readFileSync vs ~15000ms for fsPromises.readFile!! File size is 16MB. The server is doing other stuff (dockerised running lots of stuff concurrently). Node 12.18.4. Must be something related to caching vs not caching? – Dvinsk

If the server is doing other stuff and there's that much of a time difference, then fsPromises.readFile() probably finishes, but your server is busy so it doesn't get a chance to process the completion until other stuff is done. This could be more about event loop processing than anything else. I'd suggest measuring in a simple test app that does nothing else and put fs.readFileSync() first so it is done before you start fsPromises.readFile(). For further discussion, please post all pertinent details in a new question. – Armoire

I made a more contained test... wasn't able to replicate such a HUGE difference, but still a very significant difference... posted question here: stackoverflow.com/questions/63971379 – Dvinsk

Great approach, I would just suggest to rename the variable of var Promise = require('bluebird'); to something else, otherwise, it will hide the existing Promise object from JS with Bluebird's Promise – Poona

@Poona - That was Bluebird's original purpose (to supply a Promise implementation when one didn't exist and is how their doc shows using it), thus why it was done that way. You will notice that this is a 2016 question (over 6 years old). But, now if you want to just use specific functions from Bluebird, you would want to import their library under a different name so as not to conflict with the built-in Promise. – Armoire

@Poona - Note that the answer also refers to the more modern const fsp = require('fs').promises; so you can just use the built-in promisified versions and there really isn't much of a use for Bluebird.promisifyAll() any more (at least not for the fs module). – Armoire

Answer 2

Node v10 has fs Promises API

const fsPromises = require('fs').promises

const func = async filenames => {

  for(let fn of filenames) {
    let data = await fsPromises.readFile(fn)
  }

}

func(['file1','file2'])
  .then(res => console.log('all read', res))
  .catch(console.log)

https://nodejs.org/api/fs.html#fs_fs_promises_api

Or if you want to read more files simultaneously:

const func = filenames => {
  return Promise.all(
    filenames.map(f => fsPromises.readFile(f))
  )
}

func(['./a','./b'])
  .then(res => console.log('all read', res))
  .catch(console.log)

Trotskyite

Note: as of 2019-06-28, this feature has been stable for sometime. – Placeeda

great info! Though your example makes little sense since the files are read in sync, right? Every promise is waited to be resolved before the next one is triggered. – Rahmann

@Rahmann This way it's in a loop ;) But here's another example with parallel execution – Trotskyite

import { readFile, writeFile } from 'fs/promises'; more modern – An

Answer 3

Your code should look more like this:

// promisify fs.readFile()
fs.readFileAsync = function (filename) {
    return new Promise((resolve, reject) => {
        fs.readFile(filename, (err, buffer) => {
            if (err) reject(err); else resolve(buffer);
        });
    });
};

const IMG_PATH = "foo";

// utility function
function getImageByIdAsync(i) {
    return fs.readFileAsync(IMG_PATH + "/image1" + i + ".png");
}

Usage with a single image:

getImageByIdAsync(0).then(imgBuffer => {
    console.log(imgBuffer);
}).catch(err => {
    console.error(err);
});

Usage with multiple images:

var images = [1,2,3,4].map(getImageByIdAsync);

Promise.all(images).then(imgBuffers => {
    // all images have loaded
}).catch(err => {
    console.error(err);
});

To promisify a function means to take an asynchronous function with callback semantics and derive from it a new function with promise semantics.

It can be done manually, like shown above, or – preferably – automatically. Among others, the Bluebird promise library has a helper for that, see http://bluebirdjs.com/docs/api/promisification.html

Stepdaughter

i just curious why you put the function fs.readFile() in try catch? Callback can handle the error, right? – Phenice

nice! thank you. ` return function () { return new Promise(function(resolve, reject) { try { fs.readFile ` that worked for me with gulp4 – Steinke

Answer 4

If you're using .mjs ECMAScript Modules import syntax then here's code that reads a file, based on this GitHub gist comment reply:

import { promises as fs } from 'fs';
let json = await fs.readFile('./package.json', 'utf-8');
console.log(json);

Here's code that does multiple files, based on this answer:

import { promises as fs } from 'fs';

const sequentially = async filenames => {
    for(let fn of filenames) {
        let json = await fs.readFile(fn, 'utf-8');
        console.log(json);
    }
}

const parallel = filenames => {
    return Promise.all(
      filenames.map(fn => fs.readFile(fn, 'utf-8'))
    )
  }

const fns = ['package.json', 'package-lock.json'];
await sequentially(fns);
await parallel(fns);
console.log('all read');

Vladamar

Answer 5

more modern

import { readFile, writeFile } from 'fs/promises';

An

Answer 6

I once wrote a library for using promises in a loop. It's called for-async. It's pretty small but getting all the details right is hard, so you can have a look at this for inspiration maybe.

This mainly takes care of running the promise-returning functions in sequence, waiting for the promise to resolve before proceeding to the next item and of handling errors. This differs from Promise.map because that function runs the promise-returning functions in parallel, which may or may not be what you want.

// example: looping over promise-returning functions with forAsync
forAsync(
  ['a.txt', 'b.txt', 'c.txt'], 
  readFile
)

// worker function that returns a promise
function readFile(name, idx) {
  return new Promise(function(resolve, reject) {
    setTimeout(function(){
      console.info('read file ' + idx + ': ' + name)
      resolve()
    }, 1000)
  })
}

// forAsync
function forAsync(arr, work) {
  function loop(arr, i) {
    return new Promise(function (resolve, reject) {
      if (i >= arr.length) {
        resolve();
      } else try {
        Promise.resolve(work(arr[i], i)).then(function () {
          return resolve(loop(arr, i + 1));
        }).catch(reject);
      } catch (error) {
        reject(error);
      }
    });
  }
  return loop(arr, 0);
}

Crafty

Answer 7

you can also use this module: 'fs-readfile-promise'

var readFile = require('fs-readfile-promise');
readFile(__dirname + '/file1.txt','utf-8').then(function (data){
    console.log("file's name:", data)
    return readFile(__dirname +'/'+data, 'utf-8')
}).then(function (data1){
    console.log('Content data:', data1)
}).catch( function (err){
    console.log(err)
})

Parton