Streaming file to S3 "Error: stream ended unexpectedly"

Question

Update:

I believe that this could be caused by the fact that I'm using the body parser provided by express. Could this be messing with the stream that multiparty is trying to parse?

I'm basing my solution on this answer.

What I'm trying to do: Stream a file from a client browser straight to S3 with my NodeJS server acting as a proxy (for security purposes). I don't want the file to touch the server's filesystem to avoid that bottleneck.

I'm getting the following error:

events.js:72
        throw er; // Unhandled 'error' event
              ^
Error: stream ended unexpectedly
    at Form.<anonymous> (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/index.js:619:24)
    at Form.EventEmitter.emit (events.js:117:20)
    at finishMaybe (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:443:14)
    at endWritable (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:452:3)
    at Form.Writable.end (/Users/kentcdodds/Developer/bucketstreams/node_modules/multiparty/node_modules/readable-stream/lib/_stream_writable.js:419:5)
    at onend (_stream_readable.js:457:10)
    at process._tickCallback (node.js:415:13)

I've looked at the code and can't quite seem to understand what's causing the issue. I'm using angular-file-upload because I'm using angular on the front end. Here's what the request looks like:

Request URL:http://local.bucketstreams.com:3000/upload/image
Request Headers CAUTION: Provisional headers are shown.
Accept:application/json, text/plain, */*
Cache-Control:no-cache
Content-Type:multipart/form-data; boundary=----WebKitFormBoundaryNKuH2H9IUB7kvmea
Origin:http://local.bucketstreams.com:3000
Pragma:no-cache
Referer:http://local.bucketstreams.com:3000/getting-started
User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36
Request Payload
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="type"

profile
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="user"

pie
------WebKitFormBoundaryNKuH2H9IUB7kvmea
Content-Disposition: form-data; name="file0"; filename="Screen Shot 2014-02-18 at 10.54.06 PM.png"
Content-Type: image/png


------WebKitFormBoundaryNKuH2H9IUB7kvmea--

And here's what my code looks like:

var ErrorController = require('../controller/ErrorController');
var AuthenticationController = require('../controller/AuthenticationController');
var logger = require('winston');

var http = require('http');
var util = require('util');
var multiparty = require('multiparty');
var knox = require('knox');
var Batch = require('batch');

var s3Client = knox.createClient({
  secure: false,
  key: process.env.S3_KEY,
  secret: process.env.S3_SECRET,
  bucket: process.env.S3_BUCKET_IMAGES
});

var Writable = require('readable-stream').Writable;
util.inherits(ByteCounter, Writable);
function ByteCounter(options) {
  Writable.call(this, options);
  this.bytes = 0;
}

ByteCounter.prototype._write = function(chunk, encoding, cb) {
  this.bytes += chunk.length;
  cb();
};

var supportedTypes = {
  profile: true,
  post: true
};

module.exports = function(app) {
  app.post('/upload/image', AuthenticationController.checkAuthenticated, function(req, res) {
    var type = req.body.type;
    var userId = req.user._id;
    if (!supportedTypes[type]) {
      return ErrorController.sendErrorJson(res, 401, 'Unsupported image upload type: ' + type);
    }

    var headers = {
      'x-amz-acl': 'public-read'
    };
    var form = new multiparty.Form();
    var batch = new Batch();
    batch.push(function(cb) {
      form.on('field', function(name, value) {
        if (name === 'path') {
          var destPath = value;
          if (destPath[0] !== '/') destPath = '/' + destPath;
          cb(null, destPath);
        }
      });
    });

    batch.push(function(cb) {
      form.on('part', function(part) {
        if (! part.filename) return;
        cb(null, part);
      });
    });

    batch.end(function(err, results) {
      if (err) throw err;
      form.removeListener('close', onEnd);
      var destPath = '/' + userId + results[0];
      var part = results[1];

      var counter = new ByteCounter();
      part.pipe(counter); // need this until knox upgrades to streams2
      headers['Content-Length'] = part.byteCount;
      s3Client.putStream(part, destPath, headers, function(err, s3Response) {
        if (err) throw err;
        res.statusCode = s3Response.statusCode;
        s3Response.pipe(res);
        console.log('https://s3.amazonaws.com/' + process.env.S3_BUCKET_IMAGES + destPath);
      });
      part.on('end', function() {
        console.log('part end');
        console.log('size', counter.bytes);
      });
    });

    form.on('close', function(error) {
      logger.error(error);
      return ErrorController.sendErrorJson(res, 500, 'There was a problem uploading the file.');
    });
    form.parse(req);
  });
};

It looks like the part that is blowing up is multiparty and I've looked into that code a little bit to no avail. I'm not certain if I'm making the request incorrectly or if there's something wrong with my server code. I don't think it has anything to do with my S3 bucket, but I suppose it could be that as well.

Anyway, any tips are welcome.

Incinerate

Why are pipe'ing the part to the counter instead of sending directly to S3? – Tman

@CharlieBrown that came from the example I'm using. I've tried reading through this code and I don't understand that part of it. Do you have a suggestion on a better way to do it? – Incinerate

In the example I posted, that code doesnt exist. If you pipe the part to the counter, its going to just go to the counter, not to the S3 endpoint. Just remove the counter code and pass part.byteCount directly to putStream. – Tman

@CharlieBrown you're right. Sorry I didn't look into it. See the comment on your answer and my edit. Thanks. – Incinerate

Answer 1

The crux to the solution is to not use the (apparently deprecated) bodyParser(). I'm not certain what it does, but it screws up the form's parts that multiparty uses. So instead, if you have the same problem I had, instead of using bodyParser(), use the things you need explicitely (for example):

app.use(express.urlencoded());
app.use(express.json());

And then for your multipart stuff, just use multiparty to parse the body yourself. The author of multiparty gives more info on the subject.

Incinerate

Answer 2

There is an example for S3 on the node-multiparty source.

https://github.com/andrewrk/node-multiparty/tree/master/examples

var http = require('http'),
    util = require('util'),
    multiparty = require('../'),
    knox = require('knox'),
    Batch = require('batch'),
    PORT = process.env.PORT || 27372

var s3Client = knox.createClient({
    secure: false,
    key: process.env.S3_KEY,
    secret: process.env.S3_SECRET,
    bucket: process.env.S3_BUCKET,
});

var server = http.createServer(function(req, res) {
    if (req.url === '/') {
        res.writeHead(200, {
            'content-type': 'text/html'
        });
        res.end(
            '<form action="/upload" enctype="multipart/form-data" method="post">' +
            '<input type="text" name="path"><br>' +
            '<input type="file" name="upload"><br>' +
            '<input type="submit" value="Upload">' +
            '</form>'
        );
    } else if (req.url === '/upload') {
        var headers = {
            'x-amz-acl': 'public-read',
        };
        var form = new multiparty.Form();
        var batch = new Batch();
        batch.push(function(cb) {
            form.on('field', function(name, value) {
                if (name === 'path') {
                    var destPath = value;
                    if (destPath[0] !== '/') destPath = '/' + destPath;
                    cb(null, destPath);
                }
            });
        });
        batch.push(function(cb) {
            form.on('part', function(part) {
                if (!part.filename) return;
                cb(null, part);
            });
        });
        batch.end(function(err, results) {
            if (err) throw err;
            form.removeListener('close', onEnd);
            var destPath = results[0],
                part = results[1];

            headers['Content-Length'] = part.byteCount;
            s3Client.putStream(part, destPath, headers, function(err, s3Response) {
                if (err) throw err;
                res.statusCode = s3Response.statusCode;
                s3Response.pipe(res);
                console.log("https://s3.amazonaws.com/" + process.env.S3_BUCKET + destPath);
            });
        });
        form.on('close', onEnd);
        form.parse(req);

    } else {
        res.writeHead(404, {
            'content-type': 'text/plain'
        });
        res.end('404');
    }

    function onEnd() {
        throw new Error("no uploaded file");
    }
});
server.listen(PORT, function() {
    console.info('listening on http://0.0.0.0:' + PORT + '/');
});

````

Tman

I was able to reproduce this successfully. Thanks a bunch. It looks like my issue is actually in the way I'm sending the file. Here's the difference in the requests: diffchecker.com/jdsi7yx8 – Incinerate