Get rawBody in express

Question

I am trying to retrieve something from the post, and need the rawBody property from the incoming request. How can I retrieve it?

I tried using express.bodyParser() and in my post handler, I was looking for req.rawBody, and it was undefined.

I even tried it with connect.bodyParser(), but I still have no luck with it. I am getting undefined for rawBody.

I was reading on the stackoverflow site saying that they had removed the rawBody functionality, but mentioned that it is a quick fix to add it to our own middleware file. I am a newbie, so I do not have a clue as to how to achieve this. Below is my code snippet.

/**
 * Module dependencies.
 */

var express = require('express')
  , connect = require('connect')
  , routes = require('./routes')
  , user = require('./routes/user')
  , http = require('http')
  , path = require('path');

var app = express();

// all environments
app.set('port', process.env.PORT || 3000);
app.set('views', __dirname + '/views');
app.set('view engine', 'jade');
app.use(express.favicon());
app.use(express.logger('dev'));
//app.use(express.bodyParser());
app.use(connect.bodyParser());
app.use(express.methodOverride());
app.use(app.router);
app.use(express.static(path.join(__dirname, 'public')));

// development only
if ('development' == app.get('env')) {
  app.use(express.errorHandler());
}

app.get('/', routes.index);
app.get('/users', user.list);


    
/**custom stuff**/

app.post('/upload',function(req, res){
        console.log(req.header('Content-Type'));
        console.log(req.header('Host'));
        console.log(req.header('User-Agent'));

        console.log(req.rawBody);
        console.log(req.body);
        res.send("<h1> Hello the response is "+req.body.username);
});

/** end**/

http.createServer(app).listen(app.get('port'), function(){
  console.log('Express server listening on port ' + app.get('port'));
});

Any help with this is much appreciated.

Seymour

Answer 1

You can use your own middle ware to do this:

app.use(function(req, res, next){
   var data = "";
   req.on('data', function(chunk){ data += chunk})
   req.on('end', function(){
      req.rawBody = data;
      next();
   })
})

// Your route registration:
app.get('/', function(){// whatever...})

app.post('/test', function(req, res){
    console.log(req.rawBody);
    res.send("your request raw body is:"+req.rawBody);
})

Febrile

Rikky, please don't mind me asking, but where do I put this in?? and how do I access it? I did a little reading on the middleware, but I did not understand. Also how do I integrate this with the bodyParser? – Seymour

You are welcome, you can replace this to bodyParser middle ware. Well, middle ware is just a process you need to do before or after a request was handled. Here, you want to bind a rawBody data that you processed into the request before the route run, then put it before any route registration. Now from your route, just use the req.rawBody, btw I'll update my answer to make it clearly :D – Febrile

oh! more update: you don't have to remove the bodyParser, just leave it there, you may need it for other purposes. So no integrate needed – Febrile

@Febrile both bodyParser and your middleware will read the incoming data, and since that's a stream only one of the two will get to read it. The other will never receive the end event and the request will hang. – Inobservance

Yes, you're right. I'm thinking of someway to get rid of it, but just a dirty hack :( – Febrile

@Inobservance which version of express did that comment relate to? I'm using this solution with v4.17.1 and both bodyParser and my middleware seem to be receiving both events, separately. I.e. it works fine here, in all edge cases I could think of testing. Did something maybe change in the Express implementation, since? (notably, registering multiple mock 'end' handler with a debug printf, for example, clearly triggers all of them.) – Latrena

I discovered the problem with the above solution: next was called from within end. Using promises and immediately yielding to the next middleware solves this problem. See my answer below. – Latrena

Answer 2

I'm back again :D. After read the connect.bodyParser I've found something: The bodyParser only parse the data which mime type is one of: application/json, application/x-www-form-urlencoded and multipart/form-data. So I think this is another approach, it's not generally elegant but acceptable: When you try to send raw data to server, change the mime type to something different. As your question it's a string, so I choose text/plain as example:

// if the request's mime type is text/plain, read it as raw data
var myRawParser = function(req, res, next){
    req.rawData = '';
    if(req.header('content-type') == 'text/plain'){
        req.on('data', function(chunk){
            req.rawData += chunk;
        })
        req.on('end', function(){
            next();
        })
    } else {
        next();
    }
}

// ... 
app.use(myRawParser);
app.use(express.bodyParser());
// ...

// Here is my test route:
app.post('/test', function(req, res){
    console.log('Mime type is:'+req.header('content-type'));
    console.log('Raw data is:'+req.rawData);
    console.log('Body via bodyParser is:');
    console.dir(req.body);
    res.send('Hello!');
})

I've tested it via curl:

$ curl -d 'test=hello' 127.0.0.1:3000/test

// console result:
Mime type is:application/x-www-form-urlencoded
Raw data is: 
Body via bodyParser is:
{ test: 'hello' }

and:

$ curl -d 'test=hello' -H  'Content-Type:text/plain' 127.0.0.1:3000/test
// console result:
Mime type is:text/plain
Raw data is:test=hello
Body via bodyParser is: 
{}

It's not actually integrate your middle ware to bodyParser, just make them work together.

Febrile

Answer 3

Building on @Rikky's solution, the way to avoid a race condition in the data event handlers is to continue calling the middleware chain immediately after setting the handlers. Don't wait for req.on('end') to call next(), because that next() call is what allows the json body parser to register its own data and end handlers; if you wait until end is triggered on the request, they'll miss all the relevant events. Instead, use promises:

const process = require('process');

const bodyParser = require('body-parser');
const express = require('express');

function main() {
  const app = express();

  app.use((req, res, next) => {
    req.rawBody = new Promise(resolve => {
      buf = '';
      req.on('data', x => buf += x);
      req.on('end', () => {
        resolve(buf);
      });
    });
    next();
  });

  app.use(bodyParser.json());

  app.use('/', async (req, res) => {
    console.log('raw body:', await req.rawBody);
    console.log('json parsed:', req.body);
    res.send('bye\n');
  });

  app.listen('3000', 'localhost', (e) => {
    if (e) {
      console.error(e);
      process.exit(1);
    }
    console.log('Listening on localhost:3000');
  });
}

main();

Latrena

Thank you so much! I couldn't make it to work at at all now everything works just fine! I don't know if its good idea but if you dont want to get promise in your rawBody just do app.use('/', async (req, res, next) => { req.rawBody = await req.rawBody; next(); }); after bodyParser middleware – Cyano

Answer 4

The best solution I found is here: https://mcmap.net/q/100370/-expressjs-raw-body

Because using

req.on('data', function(chunk){ data += chunk})

somehow shifted the bits of the files I send in multiform-data requests, so they were not valid anymore.

Boito