NodeJS How do I Download a file to disk from an aws s3 bucket?
Asked Answered
O

9

88

My goal:

Display a dialog box prompting the user to save a file being downloaded from aws.

My problem:

I am currently using awssum-amazon-s3 to create a download stream. However I've only managed to save the file to my server or stream it to the command line... As you can see from my code my last attempt was to try and manually set the content disposition headers which failed. I cannot use res.download() as the headers have already been set?

How can I achieve my goal?

My code for node:

app.post('/dls/:dlKey', function(req, res, next){
        // download the file via aws s3 here
        var dlKey = req.param('dlKey');

        Dl.findOne({key:dlKey}, function(err, dl){
            if (err) return next(err);
            var files = dl.dlFile;

            var options = {
                BucketName    : 'xxxx',
                ObjectName    : files,
            };

            s3.GetObject(options, { stream : true }, function(err, data) {
                // stream this file to stdout
                fmt.sep();
                data.Headers['Content-Disposition'] = 'attachment';
                console.log(data.Headers);
                data.Stream.pipe(fs.createWriteStream('test.pdf'));
                data.Stream.on('end', function() {
                    console.log('File Downloaded!');
                });
            });
        });

        res.end('Successful Download Post!');
    });

My code for angular:

$scope.dlComplete = function (dl) {
        $scope.procDownload = true;
        $http({
            method: 'POST',
            url: '/dls/' + dl.dlKey
        }).success(function(data/*, status, headers, config*/) {
            console.log(data);
            $location.path('/#!/success');
        }).error(function(/*data, status, headers, config*/) {
            console.log('File download failed!');
        });
    };

The purpose of this code it to let users use a generated key to download a file once.

Outlander answered 3/3, 2014 at 10:1 Comment(1)
Unfortunately, you can't download a file to the user's disk via AJAX requests (see here and there for instance). What you can do instead is make the user send a POST FORM with the dlKey data.Launalaunce
L
99

This is the entire code using streaming on the latest version of aws-sdk

var express = require('express');
var app = express();
var fs = require('fs');

app.get('/', function(req, res, next){
    res.send('You did not say the magic word');
});


app.get('/s3Proxy', function(req, res, next){
    // download the file via aws s3 here
    var fileKey = req.query['fileKey'];

    console.log('Trying to download file', fileKey);
    var AWS = require('aws-sdk');
    AWS.config.update(
      {
        accessKeyId: "....",
        secretAccessKey: "...",
        region: 'ap-southeast-1'
      }
    );
    var s3 = new AWS.S3();
    var options = {
        Bucket    : '/bucket-url',
        Key    : fileKey,
    };

    res.attachment(fileKey);
    var fileStream = s3.getObject(options).createReadStream();
    fileStream.pipe(res);
});

var server = app.listen(3000, function () {
    var host = server.address().address;
    var port = server.address().port;
    console.log('S3 Proxy app listening at http://%s:%s', host, port);
});
Lossa answered 26/11, 2015 at 11:55 Comment(5)
I had no idea .createReadStream() was a thing. Thank you for this example!Exergue
how can I download that from the client? The stream works, and I can see on the client's console. But how can I open the "download dialog" by client side, to let the user actually get the file? I guess something with the Blob, but I can't get howDoublebank
var blob = new Blob([data], { type: 'image/jpg' }); where data is the response caught by angular side. It downloads the file, the size is correct, but file is corrupted and impossible to openDoublebank
@Doublebank - Were you able to get the file downloaded from angular?Ecumenicist
.createReadStream() is no longer available.Taeniacide
S
27

Simply create a ReadStream from S3, and WriteStream for the location to which you wish to download.

const AWS = require('aws-sdk');
const path = require('path');
const fs = require('fs');

AWS.config.loadFromPath(path.resolve(__dirname, 'config.json'));
AWS.config.update({
  accessKeyId: AWS.config.credentials.accessKeyId,
  secretAccessKey: AWS.config.credentials.secretAccessKey,
  region: AWS.config.region
});

const s3 = new AWS.S3();
const params = {
  Bucket: '<your-bucket>', 
  Key: '<path-to-your-file>'
};
const readStream = s3.getObject(params).createReadStream();
const writeStream = fs.createWriteStream(path.join(__dirname, 's3data.txt'));
readStream.pipe(writeStream);
Southwestwards answered 22/5, 2020 at 19:52 Comment(0)
W
24

This code worked for me with the most recent library:

var s3 = new AWS.S3();
var s3Params = {
    Bucket: 'your bucket',
    Key: 'path/to/the/file.ext'
};
s3.getObject(s3Params, function(err, res) {
    if (err === null) {
       res.attachment('file.ext'); // or whatever your logic needs
       res.send(data.Body);
    } else {
       res.status(500).send(err);
    }
});
Weisbrodt answered 24/6, 2015 at 20:39 Comment(3)
"aws-sdk": "^2.7.20",Fluctuant
data.Body is a Buffer which supports toString(). The object returned by the getObject did not have the .createReadStream() method shown in the example from 2015. There data object in the call back did not have the data.Stream attribute shown in the example from 2014.Fluctuant
data is undefined. I guess, data.Body is nothing but result.Body or response.Body. If that is the case, Please update the answer...Kazak
B
13

Using aws SDK v3

npm install @aws-sdk/client-s3

download code

import { GetObjectCommand } from "@aws-sdk/client-s3";
/**
 * download a file from AWS and send to your rest client
 */
app.get('/download', function(req, res, next){
    var fileKey = req.query['fileKey'];

    var bucketParams = {
        Bucket: 'my-bucket-name',
        Key: fileKey,
    };

    res.attachment(fileKey);
    var fileStream = await s3Client.send(new GetObjectCommand(bucketParams));
    // for TS you can add: if (fileStream.Body instanceof Readable)
    fileStream.Body.pipe(res)
});
Bagging answered 3/11, 2021 at 21:19 Comment(0)
L
7

You've already figured what's most important to solve your issue: you can pipe the file stream coming from S3 to any writable stream, be it a filestream… or the response stream that will be sent to the client!

s3.GetObject(options, { stream : true }, function(err, data) {
    res.attachment('test.pdf');
    data.Stream.pipe(res);
});

Note the use of res.attachment that will set the correct headers. You can also check out this answer regarding streams and S3.

Launalaunce answered 3/3, 2014 at 10:46 Comment(8)
Thanks for the quick post! The code you suggested compiles file, and looks as-if it should work however it still isn't prompting me with a download dialog on safari or chrome! I can't figure out why since it should have an the attachment header nowOutlander
Isn't this behavior (prompting the user for the download location) related to the browser's configuration?Launalaunce
Yes but even with the correct config I'm not getting a prompt. It's not even downloading(server or to local). I'll update my question with my angular code too because maybe thats the problem.Outlander
Can you try this simple gist and check if the file is correctly downloaded using your browser? If yes, this means that your Angular code is the issueLaunalaunce
I ran it but got: "Error: ENOENT, open 'test.pdf'"Outlander
Woops, just create a sample test.pdf file, or change the filename in your code so that it points to an existing fileLaunalaunce
Worked like a charm! Must be my angular code then! -Thanks for the all the help :DOutlander
GetObject is not a functionRhone
T
5

For this I use React frontend and node js backend. Frontend I use Axios. I used this click the button download file.

==== Node js backend code (AWS S3) ======

//inside GET method I called this function

    public download = (req: Request, res: Response) => {
    const keyName = req.query.keyName as string;
    if (!keyName) {
        throw new Error('key is undefined');
    }
    const downloadParams: AWS.S3.GetObjectRequest = {
        Bucket: this.BUCKET_NAME,
        Key: keyName
    };

    this.s3.getObject(downloadParams, (error, data) => {
        if (error) {
            return error;
        }
        res.send(data.Body);
        res.end();
    });
};

====== React js frontend code ========

//this function handle download button onClick

  const downloadHandler = async (keyName: string) => {
  const response = await axiosInstance.get( //here use axios interceptors
    `papers/paper/download?keyName=${keyName}`,{
      responseType:'blob', //very very important dont miss (if not downloaded file unsupported to view)
    }
  );
  const url = window.URL.createObjectURL(new Blob([response.data]));
  const link = document.createElement("a");
  link.href = url;
  link.setAttribute("download", "file.pdf"); //change "file.pdf" according to saved name you want, give extension according to filetype
  document.body.appendChild(link);
  link.click();
  link.remove();
};

------ OR (if you are using normal axios and not axios interceptors) -----

axios({
   url: 'http://localhost:5000/static/example.pdf',
   method: 'GET',
   responseType: 'blob', // very very important
}).then((response) => {
   const url = window.URL.createObjectURL(new Blob([response.data]));
   const link = document.createElement('a');
   link.href = url;
   link.setAttribute('download', 'file.pdf');
   document.body.appendChild(link);
   link.click();
});

For more refer below articles 1. article 1 2. article 2

Tetroxide answered 17/10, 2021 at 19:23 Comment(1)
"responseType: 'blob', // very very important" This line saved my day! Thanks manSandal
K
1

Using express, based on Jushua's answer and https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_GetObject_section.html

  public downloadFeedFile = (req: IFeedUrlRequest, res: Response) => {
    const downloadParams: GetObjectCommandInput = parseS3Url(req.s3FileUrl.replace(/\s/g, ''));
    logger.info("requesting S3 file  " + JSON.stringify(downloadParams));
    const run = async () => {
      try {
        const fileStream = await this.s3Client.send(new GetObjectCommand(downloadParams));
        if (fileStream.Body instanceof Readable){
          fileStream.Body.once('error', err => {
            console.error("Error downloading s3 file")
            console.error(err);
          });

          fileStream.Body.pipe(res);

        }
      } catch (err) {
        logger.error("Error", err);
      }
    };

  run();
  
  };
Kirtley answered 2/11, 2022 at 17:7 Comment(0)
D
0

Using @aws-sdk/client-s3's GetObjectCommand:

// The following example retrieves an object for an S3 bucket.
const input = {
  "Bucket": "examplebucket",
  "Key": "HappyFace.jpg"
};

const command = new GetObjectCommand(input);
const response = await client.send(command);

/* response ==
{
  "AcceptRanges": "bytes",
  "ContentLength": "3191",
  "ContentType": "image/jpeg",
  "ETag": "\"6805f2cfc46c0f04559748bb039d69ae\"",
  "LastModified": "Thu, 15 Dec 2016 01:19:41 GMT",
  "Metadata": {},
  "TagCount": 2,
  "VersionId": "null"
}
*/
// example id: to-retrieve-an-object-1481827837012

To stream the body to a file, in Typescript:

import { Readable } from 'stream';

if (!res.Body) {
  throw new Error("No body");
}

const writeStream = fs.createWriteStream(localPath);
const readStream = res.Body as Readable;
readStream.pipe(writeStream);
Demonolatry answered 29/3, 2023 at 8:16 Comment(0)
L
0
MIGHT HELP ANYONE

app.post("/api/posts",upload.single("demo"),async(req,res)=>{
        console.log('req.body',req.body);
        console.log('req.file',req.file);
        console.log('req.file.asliData',req.file.buffer);
        console.log('req.content-type',req.file.mimetype);
        // req.file.buffer sis the actual image and we need to send it to s3
    
        if (!req.file) {
          return res.status(400).send('No file uploaded');
        }
    
        //resize image only make sure image only
        // const buffer = sharp(req.file.buffer).resize({height:1920,width:1080,fit:"contain"}).toBuffer()
    
    
        const params = {
          Bucket:"usman.test.bucket",
          Key:req.file.originalname, //file name what it should be keep it unique
          Body:req.file.buffer,  // what we are passing should be buffer
          ContentType:req.file.mimetype  //type of file gfood practice
        }
        try {
          //Upload to bucket
          const command  = new PutObjectCommand(params)
          const result = await s3.send(command)
          res.status(200).send(`File uploaded to S3 ${result.ETag}`)
        } catch (error) {
          console.error('Error uploading file to S3:', error);
          res.status(500).send('File upload to S3 failed');
        }
    })
    
    app.get("/api/posts/:fileName", async (req, res) => {
      const fileName = req.params.fileName;
      const bucketName = "usman.test.bucket";
    
      try {
        const params = {
          Bucket: bucketName,
          Key: fileName
        };
    
        const command = new GetObjectCommand(params);
        const {Body,ContentType} = await s3.send(command);
    
        if (Body) {
          const contentType = ContentType;
          res.set("Content-Type", contentType);
          // In a regular HTTP response, the Content-Disposition response header is a header indicating if the content is expected to be displayed inline in the browser, that is, as a Web page or as part of a Web page, or as an attachment, that is downloaded and saved locally.
          // When the value is set to attachment, it tells the browser to treat the response as a downloadable file attachment.
          res.set("Content-Disposition", `attachment; filename="${fileName}"`);
    
          // Pipe the S3 response stream directly to the HTTP response
        
    
        // Body from the GetObjectCommand is a readable stream thats we use pipe (pipe read from codeevolutyiuon)
          Body.pipe(res);
    
          console.log(res);
        } else {
          res.status(404).send("File not found");
        }
      } catch (error) {
        console.error("Error retrieving file from S3:", error);
        res.status(500).send("Error retrieving file from S3");
      }
    });
Loci answered 27/4, 2024 at 12:50 Comment(0)

© 2022 - 2025 — McMap. All rights reserved.