Handling multipart/form-data POST with Express in Cloud Functions

Question

I've been trying to handle POSTs (multipart/form-data) with a Firebase function and Express but it just doesn't work. Tried this in local server and it works just fine. Everything's the same except it's not contained in a Firebase function.

Besides screwing up the request object it seems it also screws up the way busboy works.

I've tried different solutions presented here but they just don't work. As one user mentions, the callbacks passed to busboy (to be called when a 'field' is found or when it finishes going through the data) are never called and the function just hangs.

Any ideas?

Here's my function's code for reference:

const functions = require('firebase-functions');
const express = require('express');
const getRawBody = require('raw-body');
const contentType = require('content-type')
const Busboy = require('busboy');

const app = express();

const logging = (req, res, next) => {
  console.log(`> request body: ${req.body}`);
  next();
}

const toRawBody = (req, res, next) => {
  const options = {
      length: req.headers['content-length'],
      limit: '1mb',
      encoding: contentType.parse(req).parameters.charset
  };
  getRawBody(req, options)
      .then(rawBody => {
          req.rawBody = rawBody
          next();
      })
      .catch(error => {
          return next(error);
      });
};

const handlePostWithBusboy = (req, res) => {
  const busboy = new Busboy({ headers: req.headers });
  const formData = {};

  busboy.on('field', (fieldname, value) => {
      formData[fieldname] = value;
  });

  busboy.on('finish', () => {
      console.log(`> form data: ${JSON.stringify(formData)}`);
      res.status(200).send(formData);
  });

  busboy.end(req.rawBody);
}

app.post('/', logging, toRawBody, handlePostWithBusboy);

const exchange = functions.https.onRequest((req, res) => {
  if (!req.path) {
    req.url = `/${req.url}`
  }
  return app(req, res)
})
module.exports = {
  exchange
}

Canberra

Answer 1

Please read the documentation for handling multipart form uploads.

... if you want your Cloud Function to process multipart/form-data, you can use the rawBody property of the request.

Because of the way Cloud Functions pre-processes some requests, you can expect that some Express middleware will not work, and that's what you're running into.

Cuspidor

Doug, check out the answer I added. This is a frustrating bug I lost a lot of time. – Marillin

Updated link to example: cloud.google.com/functions/docs/samples/… – Cliffcliffes

Answer 2

The documentation Doug referred to in his answer is good. An important caveat though is that rawBody does not work in the emulator. The workaround is to do:

if (req.rawBody) {
    busboy.end(req.rawBody);
}
else {
    req.pipe(busboy);
}

As described in this issue: https://github.com/GoogleCloudPlatform/cloud-functions-emulator/issues/161#issuecomment-376563784

Marillin

FYI this is now fixed – Halflight

Answer 3

I've combined the previous two answers into a easy-to-use async function.

const Busboy = require('busboy');
const os = require('os');
const path = require('path');
const fs = require('fs');

module.exports = function extractMultipartFormData(req) {
  return new Promise((resolve, reject) => {
    if (req.method != 'POST') {
      return reject(405);
    } else {
      const busboy = new Busboy({ headers: req.headers });
      const tmpdir = os.tmpdir();
      const fields = {};
      const fileWrites = [];
      const uploads = {};

      busboy.on('field', (fieldname, val) => (fields[fieldname] = val));

      busboy.on('file', (fieldname, file, filename) => {
        const filepath = path.join(tmpdir, filename);
        const writeStream = fs.createWriteStream(filepath);

        uploads[fieldname] = filepath;

        file.pipe(writeStream);

        const promise = new Promise((resolve, reject) => {
          file.on('end', () => {
            writeStream.end();
          });
          writeStream.on('finish', resolve);
          writeStream.on('error', reject);
        });

        fileWrites.push(promise);
      });

      busboy.on('finish', async () => {
        const result = { fields, uploads: {} };

        await Promise.all(fileWrites);

        for (const file in uploads) {
          const filename = uploads[file];

          result.uploads[file] = fs.readFileSync(filename);
          fs.unlinkSync(filename);
        }

        resolve(result);
      });

      busboy.on('error', reject);

      if (req.rawBody) {
        busboy.end(req.rawBody);
      } else {
        req.pipe(busboy);
      }
    }
  });
};

Careless