Decode a Uint8Array into a JSON

Question

I am fetching data from an API in order to show sales and finance reports, but I receive a type gzip file which I managed to convert into a Uint8Array. I'd like to somehow parse-decode this into a JSON file that I can use to access data and create charts in my frontend with. I was trying with different libraries (pako and cborg seemed to be the ones with the closest use cases), but I ultimately get an error Error: CBOR decode error: unexpected character at position 0

This is the code as I have it so far:

let req = https.request(options, function (res) {
      console.log("Header: " + JSON.stringify(res.headers));
      res.setEncoding("utf8");
      res.on("data", function (body) {
        const deflatedBody = pako.deflate(body);
        console.log("DEFLATED DATA -----> ", typeof deflatedBody, deflatedBody);
        console.log(decode(deflatedBody));
      });
      res.on("error", function (error) {
        console.log("connection could not be made " + error.message);
      });
    });
    req.end();
  };

I hope anyone has stumbled upon this already and has some idea. Thanks a lot!

Suboceanic

JSON has a strict set of allowed characters/content. json.org/json-en.html please may you make sure that you're converting it correctly? – Misogyny

Answer 1

Please visit this answer https://mcmap.net/q/239365/-how-do-i-ungzip-decompress-a-nodejs-request-39-s-module-gzip-response-body to retrieve GZIP data from the response.

Assuming, You have already retrieved full data as UInt8Array.

You just need the UInt8Array as String

const jsonString = Buffer.from(dataAsU8Array).toString('utf8')

const parsedData = JSON.parse(jsonString)

console.log(parsedData)

Edit

Here is what worked for me

const {request} = require("https")
const zlib = require("zlib")


const parseGzip = (gzipBuffer) => new Promise((resolve, reject) =>{
    zlib.gunzip(gzipBuffer, (err, buffer) => {
        if (err) {
            reject(err)
            return
        }
        resolve(buffer)
    })
})

const fetchJson = (url) => new Promise((resolve, reject) => {
    const r = request(url)
    r.on("response", (response) => {
        if (response.statusCode !== 200) {
            reject(new Error(`${response.statusCode} ${response.statusMessage}`))
            return
        }

        const responseBufferChunks = []

        response.on("data", (data) => {
            console.log(data.length);
            responseBufferChunks.push(data)
        })
        response.on("end", async () => {
            const responseBuffer = Buffer.concat(responseBufferChunks)
            const unzippedBuffer = await parseGzip(responseBuffer)
            resolve(JSON.parse(unzippedBuffer.toString()))
        })
    })
    r.end()
})

fetchJson("https://wiki.mozilla.org/images/f/ff/Example.json.gz")
    .then((result) => {
        console.log(result)
    })
    .catch((e) => {
        console.log(e)
    })

Loxodromic

And in case JSON.prase returns some error, and the response string is too long, you may debug the JSON responseString here jsonformatter.curiousconcept.com – Loxodromic

Added full code for fetching and parsing gzip with required error handling in the Edit – Loxodromic

Thanks a lot! I still couldn't parse the decoded string I got, so I am trying another approach with python directly after downloading the file in .csv and it's quite a lot simpler for my use case. – Suboceanic

This solution doesn't work for me. Buffer.from(...) gives the error that Buffer is not defined. I guess Buffer is a nodejs thing that's not supported by the browser. And if I just run toString() on the Uint8Array, it generates a sequence of small numbers separated by commas. – Lefthander

The gzipped part helped identify the encoding issues I was having with the "toString" method. Thanks! – Boigie

Answer 2

Thank you, I actually just tried this approach and I get the following error:

SyntaxError: JSON Parse error: Unexpected identifier "x"

But I managed to print the data in text format using the below function:

getFinancialReports = (options, callback) => {
    // buffer to store the streamed decompression
    var buffer = [];

    https
      .get(options, function (res) {
        // pipe the response into the gunzip to decompress
        var gunzip = zlib.createGunzip();
        res.pipe(gunzip);

        gunzip
          .on("data", function (data) {
            // decompression chunk ready, add it to the buffer
            buffer.push(data.toString());
          })
          .on("end", function () {
            // response and decompression complete, join the buffer and return
            callback(null, buffer.join(""));
          })
          .on("error", function (e) {
            callback(e);
          });
      })
      .on("error", function (e) {
        callback(e);
      });
  };

Now I would need to pass this into a JSON object.

Suboceanic