multer-s3-transform file upload is not working

Question

I'm using multer-s3-transform to upload file to s3, but before upload I want to resize file, and save in two different size. Following is my code, no error occur, but file is not uploading.

let multer = require("multer-s3-transform");
const aws = require('aws-sdk');
const sharp = require('sharp');

aws.config.update({
  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
  accessKeyId: process.env.AWS_ACCESS_KEY_ID
);

const s3 = new aws.S3();
var upload = multer({
  storage: multerS3({
    s3: s3,
    bucket: process.env.AWS_BUCKET_NAME,
    shouldTransform: function (req, file, cb) {
      console.log("hereeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee");
      let originalname = file.originalname;
      file.stream.once('data', async function (firstChunk) {
        let magic = firstChunk.toString('hex', 0, 4);
        let extName = path.extname(originalname).toLowerCase();
        // let filename = req.file.fieldname + '-' + Date.now() + path.extname(req.file.originalname);
        if (checkMagicNumbers(magic, extName, req)) {
          console.log("Valid File***************************");
          cb(null, true);
        }
      });
    },
    transforms: [{
      id: 'original',
      key: async function (req, file, cb) {
        let basePath = await getDynamicFilePath(req, false);
        console.log(basePath, "Path ^^^^^^^^^^^^^^^^^^^^^^^");
        let filePath = basePath;
        let filename;
        if (req.params.container === "resumes") {
          void function (req, file, callback) {
            filename = Date.now().toString() + '-' + file.originalname;
            filePath += "/cropped_" + filename;
          }()
        } else {
          filename = Date.now().toString();
          filePath += "/cropped_" + filename;
        }
        console.log(filePath, "path -------------------");
        cb(null, filePath);
      },
      transform: function (req, file, cb) {
        cb(null, sharp().resize(330, 512))
      }
    }
    {
      id: 'thumbnail',
      key: async function (req, file, cb) {
        let basePath = await getDynamicFilePath(req, false);
        console.log(basePath, "Path ^^^^^^^^^^^^^^^^^^^^^^^");
        let filePath = basePath;
        let filename;
        if (req.params.container === "resumes") {
          void function (req, file, callback) {
            filename = Date.now().toString() + '-' + file.originalname;
            filePath += "/normal" + filename;
          }()
        } else {
          filename = Date.now().toString();
          filePath += "/normal_" + filename;
        }
        console.log(filePath, "path -------------------");
        cb(null, filename);
      },
      transform: function (req, file, cb) {
        cb(null, sharp().resize(1200, 628));
      }
    }
    ]
  })
}).single("file");
upload(req, res, function (err, filePath) {
  console.log(err, "Error");
  console.log(filePath, "*************");
});

Everything is working if I use from multer-s3, except image resizing, but with multer-s3-transform no error occur, and file is not uploading, and never goes to this lines:

console.log(err, "Error"); console.log(filePath, "*************");

I have found the problem, its in following code block where I want to check file magic number.

file.stream.once('data', async function (firstChunk) {
    let magic = firstChunk.toString('hex', 0, 4);
    let extName = path.extname(originalname).toLowerCase();
    // let filename = req.file.fieldname + '-' + Date.now() + path.extname(req.file.originalname);
    if (checkMagicNumbers(magic, extName, req)) {
      console.log("Valid File***************************");
      cb(null, true);
    }
  });

If I comment this, every thing is working. It seems when taking magic number of file, file is changed to stream, and upload not work, a way to change stream back to file may work.

Compass

how did you fix it? Please share your answer, I need to do same – Sorcerer

Answer 1

For me changing to the following worked.

let uploadToCloud = (req, res) => {
let bufferUpload = multer({
  storage: multer.memoryStorage(),
  limits: {
    fileSize: 10485760
  }
}).single('file');
bufferUpload(req, res, async function (err) {
  try {
    let buffer = new Buffer(req.file.buffer);
    let magic = buffer.toString('hex', 0, 4);
    let extName = path.extname(req.file.originalname).toLowerCase();
    let filename;
    if (checkMagicNumbers(magic, extName, req)) {
       // Your codes
    }
  } catch (error) {
    res.status(400).json(error);
    res.end();
    return;
  }
 });
};

Compass

Answer 2

Why don't you calculate and check for magic numbers by converting file buffer to hex string instead of converting the file to stream and accessing the same in data event handler and then doing the conversion to hex?

shouldTransform: function(req, file, cb) {
      const originalname = file.originalname;
      const firstChunk = file.buffer;
      const magic = firstChunk.toString("hex", 0, 4);
      const extName = path.extname(originalname).toLowerCase();
      if (checkMagicNumbers(magic, extName, req)) {
        cb(null, true);
      }
    }

I hope this helps. Please let me know if you are facing any difficulties.

Subtonic

Thanks, but this line const firstChunk = file.buffer; return undefined – Compass

Could you check 2 things for me, see what the value for file is and try req.file.buffer instead of file.buffer? – Subtonic

file is { fieldname: 'file', originalname: 'pexels-photo-1004665.jpeg', encoding: '7bit', mimetype: 'image/jpeg' } and req.file.buffer gives error TypeError: Cannot read property 'buffer' of undefined – Compass

@Compass I faced this exact thing. Did you manage to get this to work? – Gerhart

@Gerhart Buffer only comes for storage type MemoryStorage – Cotta