OneDrive API Node.js - Can´t use :/createUploadSession Content-Range Error

Question

My problem was that I couldn´t upload files bigger than 4MB so I used the createuploadsession according to createuploadsession

I successfully get the uploadUrl value from the createuploadsession response. Now I try to make a PUT request with this code

var file = 'C:\\files\\box.zip'

fs.readFile(file, function read(e, f) {
    request.put({
        url: 'https://api.onedrive.com/rup/545d583xxxxxxxxxxxxxxxxxxxxxxxxx',
        headers: {
            'Content-Type': mime.lookup(file),
            'Content-Length': f.length,
            'Content-Range': 'bytes ' + f.length
        }
    }, function(er, re, bo) {
        console.log('#324324', bo);
    }); 
}); 

But I will get as response "Invalid Content-Range header value" also if I would try

'Content-Range': 'bytes 0-' + f.length
//or
'Content-Range': 'bytes 0-' + f.length + '/' + f.length

I will get the same response.
Also I don´t want to chunk my file I just want to upload my file complete in 1run. Does anybody have sample code for upload a file to the uploadUrl from the createuploadsession response. Also do I really need to get first this uploadurl before i can upload files bigger than 4mb or is there an alternative way?

Polypeptide

Answer 1

How about following sample script?

The flow of this script is as follows.

1. Retrieve access token from refresh token.
2. Create sesssion.
3. Upload file by every chunk. Current chunk size is max which is 60 * 1024 * 1024 bytes. You can change freely.

The detail information is https://dev.onedrive.com/items/upload_large_files.htm.

Sample script :

var fs = require('fs');
var request = require('request');
var async = require('async');

var client_id = "#####";
var redirect_uri = "#####";
var client_secret = "#####";
var refresh_token = "#####";
var file = "./sample.zip"; // Filename you want to upload.
var onedrive_folder = 'SampleFolder'; // Folder on OneDrive
var onedrive_filename = file; // If you want to change the filename on OneDrive, please set this.

function resUpload(){
    request.post({
        url: 'https://login.microsoftonline.com/common/oauth2/v2.0/token',
        form: {
            client_id: client_id,
            redirect_uri: redirect_uri,
            client_secret: client_secret,
            grant_type: "refresh_token",
            refresh_token: refresh_token,
        },
    }, function(error, response, body) { // Here, it creates the session.
        request.post({
            url: 'https://graph.microsoft.com/v1.0/drive/root:/' + onedrive_folder + '/' + onedrive_filename + ':/createUploadSession',
            headers: {
                'Authorization': "Bearer " + JSON.parse(body).access_token,
                'Content-Type': "application/json",
            },
            body: '{"item": {"@microsoft.graph.conflictBehavior": "rename", "name": "' + onedrive_filename + '"}}',
        }, function(er, re, bo) {
            uploadFile(JSON.parse(bo).uploadUrl);
        });
    });
}

function uploadFile(uploadUrl) { // Here, it uploads the file by every chunk.
    async.eachSeries(getparams(), function(st, callback){
        setTimeout(function() {
            fs.readFile(file, function read(e, f) {
                request.put({
                    url: uploadUrl,
                    headers: {
                        'Content-Length': st.clen,
                        'Content-Range': st.cr,
                    },
                    body: f.slice(st.bstart, st.bend + 1),
                }, function(er, re, bo) {
                    console.log(bo);
                });
            });
            callback();
        }, st.stime);
    });
}

function getparams(){
    var allsize = fs.statSync(file).size;
    var sep = allsize < (60 * 1024 * 1024) ? allsize : (60 * 1024 * 1024) - 1;
    var ar = [];
    for (var i = 0; i < allsize; i += sep) {
        var bstart = i;
        var bend = i + sep - 1 < allsize ? i + sep - 1 : allsize - 1;
        var cr = 'bytes ' + bstart + '-' + bend + '/' + allsize;
        var clen = bend != allsize - 1 ? sep : allsize - i;
        var stime = allsize < (60 * 1024 * 1024) ? 5000 : 10000;
        ar.push({
            bstart : bstart,
            bend : bend,
            cr : cr,
            clen : clen,
            stime: stime,
        });
    }
    return ar;
}

resUpload();

In my environment, this works fine. I could upload a 100 MB file to OneDrive using this script. If this doesn't work at your environment, feel free to tell me.

Kohlrabi

thank you again! worked like a charm! This getparams function is to high experience for me hahaha – Polypeptide

@Polypeptide Welcome. Thank you, too. getparams() creates data for using the chunk of the file. By this, separated data can be sent using URL created at the session. – Kohlrabi

I need to read more about chunk files I never worked with this but seems very important for API´s. I have one last question with GoogleDrive I can track the upload progress using this kind of code - I added it to your code pastebin.com/rNTns87A But I only get "OneDrive - We uploaded 5mb from 5mb". I had the same result for other API´s that didn´t support the upload progress. Do you also have an idea for tracking the upload progress? I can also make a new question if you prefer :) thank you – Polypeptide

@Polypeptide If you want to see the upload progress, you can see the progress by changing the chunk size sep in getparams(). Of course, you can modify the display of value. The current chunk size is 60 MB. So if the file size is less than 60 MB, it cannot be seen. – Kohlrabi

I set it down to 1MB, but I don´t know where I had to place the req.req.connection.bytesWritten. If I would use it at var req = request.put I would always get the full size. If my file is 5MB the console log would be "uploaded 5MB from 5MB". Sorry I´am new to this API stuff :( – Polypeptide

@Polypeptide If you set the chunk size to 1 MB, sep becomes allsize < 1048576 ? allsize : 1048576; – Kohlrabi

Thank you know I understand what you mean hehe and btw you can see the upload progress when you make a simple GET request to the uploadUrl. Thank you again for your great help! – Polypeptide

Let us continue this discussion in chat. – Polypeptide

Answer 2

This is the ES6 version of Tanaike's solution.

const fs        = require('fs')
const promisify = require('promisify')
const readFile  = promisify(fs.readFile)


const uploader = async function(messageId) {
  // const client = <setup your microsoft-client-here>

  const address = '/path/to/file_name.jpg'
  const name    = 'file_name.jpg'

  const stats = fs.statSync(address)
  const size  = stats['size']

  const uploadSession = { AttachmentItem: { attachmentType: 'file', name, size } }

  let location = ''

  function getparams() {
    const chSize = 10
    const mega   = 1024 * 1024

    const sep = size < (chSize * mega) ? size : (chSize * mega) - 1
    const arr = []

    for (let i = 0; i < size; i += sep) {
      const bstart = i
      const bend   = ((i + sep - 1) < size) ? (i + sep - 1) : (size - 1)
      const cr     = 'bytes ' + bstart + '-' + bend + '/' + size
      const clen   = (bend != (size - 1)) ? sep : (size - i)
      const stime  = size < (chSize * mega) ? 5000 : 10000

      arr.push({ bstart, bend, cr, clen, stime })
    }

    return arr
  }

  async function uploadFile(url) {
    const params = getparams()

    for await (const record of params) {      
      const file = await readFile(address)

      const result = await request({
        url,
        method: 'PUT',
        headers: {
          'Content-Length': record.clen,
          'Content-Range': record.cr,
        },
        body: file.slice(record.bstart, record.bend + 1),
        resolveWithFullResponse: true
      })

      location = (result.headers && result.headers.location) ? result.headers.location : null
      // await new Promise(r => setTimeout(r, record.stime)) // If you need to add delay
    }
  }

  const result = await client.api(`/me/messages/${messageId}/attachments/createUploadSession`).version('beta').post(uploadSession)

  try {
    await uploadFile(result.uploadUrl)
  } catch (ex) {
    console.log('ex.error:', ex.error)
    console.log('ex.statusCode:', ex.statusCode)
    await request.delete(result.uploadUrl)
  }

  return location
}

Circumvent

Answer 3

const fs = require('fs');
const axios = require('axios');

const client_id = 'face200d-b297-4a5a-8207-1cc297581a57'; // Your client ID
const client_secret = 'vOg8Q~wrSvUCwBbm6SfAs3QW4BM.VyTWHJQPKcfl'; // Your client secret
const refresh_token = '0.AX0ApesmMDcfP0GovwRmt8Axyw0gzvqXslpKggccwpdYGle1AF4'; // Your refresh token
const redirect_uri = 'http://localhost:3000/'; // Your redirect URI

const accessTokenFilePath = './access_token.json';

async function refreshAccessToken() {
    try {
        const refreshResponse = await axios.post('https://login.microsoftonline.com/common/oauth2/v2.0/token', null, {
            params: {
                client_id: client_id,
                client_secret: client_secret,
                refresh_token: refresh_token,
                redirect_uri: redirect_uri,
                grant_type: 'refresh_token'
            }
        });

        const newAccessToken = refreshResponse.data.access_token;
        fs.writeFileSync(accessTokenFilePath, JSON.stringify({ access_token: newAccessToken }));
        return newAccessToken;
    } catch (error) {
        console.error('Error refreshing access token:', error.message);
        throw error;
    }
}

async function uploadFileToOneDrive(filePath, accessToken) {
    try {
        const fileStats = fs.statSync(filePath);
        const fileSize = fileStats.size;

        // Create upload session
        const url = 'https://graph.microsoft.com/v1.0/me/drive/root:/envato';
        const fileName = filePath.split('/').pop();
        const uploadSessionResponse = await axios.post(`${url}/${fileName}:/createUploadSession`, null, {
            headers: {
                Authorization: `Bearer ${accessToken}`
            }
        });

        const uploadUrl = uploadSessionResponse.data.uploadUrl;

        // Upload file in chunks
        const chunkSize = 5 * 1024 * 1024; // 5MB chunk size
        let offset = 0;

        while (offset < fileSize) {
            const chunk = fs.readFileSync(filePath, {
                encoding: null,
                start: offset,
                end: offset + chunkSize - 1
            });

            const range = `bytes ${offset}-${offset + chunk.length - 1}/${fileSize}`;
            console.log(`Uploading range ${range}...`);

            await axios.put(uploadUrl, chunk, {
                headers: {
                    Authorization: `Bearer ${accessToken}`,
                    'Content-Length': chunk.length,
                    'Content-Range': range
                }
            });

            offset += chunk.length;
        }

        console.log('File uploaded successfully');
        return true;
    } catch (error) {
        console.error('Error uploading file to OneDrive:', error.message);
        return false;
    }
}

(async () => {
    try {
        const accessTokenData = fs.readFileSync(accessTokenFilePath, 'utf8');
        const { access_token: accessToken } = JSON.parse(accessTokenData);

        const success = await uploadFileToOneDrive('./sun-europe-trees-woods-yellow-above-natural-2023-02-27-20-26-28-utc.mp4', accessToken);
        if (!success) {
            const newAccessToken = await refreshAccessToken();
            await uploadFileToOneDrive('./sun-europe-trees-woods-yellow-above-natural-2023-02-27-20-26-28-utc.mp4', newAccessToken);
        }
    } catch (error) {
        console.error('Error:', error.message);
    }
})();

This code only uploading files less than 150 mb how can I upload larger files

Lamellate