Retrieving binary file content using Javascript, base64 encode it and reverse-decode it using Python

Question

I'm trying to download a binary file using XMLHttpRequest (using a recent Webkit) and base64-encode its contents using this simple function:

function getBinary(file){
    var xhr = new XMLHttpRequest();  
    xhr.open("GET", file, false);  
    xhr.overrideMimeType("text/plain; charset=x-user-defined");  
    xhr.send(null);
    return xhr.responseText;
}

function base64encode(binary) {
    return btoa(unescape(encodeURIComponent(binary)));
}

var binary = getBinary('http://some.tld/sample.pdf');
var base64encoded = base64encode(binary);

As a side note, everything above is standard Javascript stuff, including btoa() and encodeURIComponent(): https://developer.mozilla.org/en/DOM/window.btoa

This works pretty smoothly, and I can even decode the base64 contents using Javascript:

function base64decode(base64) {
    return decodeURIComponent(escape(atob(base64)));
}

var decodedBinary = base64decode(base64encoded);
decodedBinary === binary // true

Now, I want to decode the base64-encoded contents using Python which consume some JSON string to get the base64encoded string value. Naively this is what I do:

import urllib
import base64
# ... retrieving of base64 encoded string through JSON
base64 = "77+9UE5HDQ……………oaCgA="
source_contents = urllib.unquote(base64.b64decode(base64))
destination_file = open(destination, 'wb')
destination_file.write(source_contents)
destination_file.close()

But the resulting file is invalid, looks like the operation's messaed up with UTF-8, encoding or something which is still unclear to me.

If I try to decode UTF-8 contents before putting them in the destination file, an error is raised:

import urllib
import base64
# ... retrieving of base64 encoded string through JSON
base64 = "77+9UE5HDQ……………oaCgA="
source_contents = urllib.unquote(base64.b64decode(base64)).decode('utf-8')
destination_file = open(destination, 'wb')
destination_file.write(source_contents)
destination_file.close()

$ python test.py
// ...
UnicodeEncodeError: 'ascii' codec can't encode character u'\ufffd' in position 0: ordinal not in range(128)

As a side note, here's a screenshot of two textual representations of a same file; on left: the original; on right: the one created from the base64-decoded string: http://cl.ly/0U3G34110z3c132O2e2x

Is there a known trick to circumvent these problems with encoding when attempting to recreating the file? How would you achieve this yourself?

Any help or hint much appreciated :)

Polyandrist

As a side note, I tried to play with the codecs module for writing the destination file using the 'utf-8' codec with no luck as well, but I might have messed up something somewhere. – Polyandrist

That is strange, as \ufffd is very special: fileformat.info/info/unicode/char/fffd/index.htm – Winifredwinikka

@rocksportrocker> That would suppose the base64encode() function I'm using is unable to convert some characters… The strange thing is that the reverse operation works perfectly in javascript… – Polyandrist

Did you try to dump the first bytes values at different steps. Looks like one or your library is being too smart and not doing the convertion at byte level. I would suggest creating a simple UTF-8 text file with >255 code points and manually analyze bytes values at every step. You should stop where the error is. – Physiognomy

I'm not using any library unfortunately… JS stuff (btoa(), encodeURIComponent() and unescape()) are standard. Same by the Python part, nothing else than stdlib stuff used… I'll investigate with the strange Bytes values put this looks to be a real pain :( – Polyandrist

Answer 1

So I'm answering to myself — and sorry for that — but I think it might be useful for someone as lost as I was ;)

So you have to use ArrayBuffer and set the responseType property of your XMLHttpRequest object instance to arraybuffer for retrieving a native array of Bytes, which can be converted to base64 using the following convenient function (found there, author may be blessed here):

function base64ArrayBuffer(arrayBuffer) {
  var base64    = ''
  var encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'

  var bytes         = new Uint8Array(arrayBuffer)
  var byteLength    = bytes.byteLength
  var byteRemainder = byteLength % 3
  var mainLength    = byteLength - byteRemainder

  var a, b, c, d
  var chunk

  // Main loop deals with bytes in chunks of 3
  for (var i = 0; i < mainLength; i = i + 3) {
    // Combine the three bytes into a single integer
    chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2]

    // Use bitmasks to extract 6-bit segments from the triplet
    a = (chunk & 16515072) >> 18 // 16515072 = (2^6 - 1) << 18
    b = (chunk & 258048)   >> 12 // 258048   = (2^6 - 1) << 12
    c = (chunk & 4032)     >>  6 // 4032     = (2^6 - 1) << 6
    d = chunk & 63               // 63       = 2^6 - 1

    // Convert the raw binary segments to the appropriate ASCII encoding
    base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d]
  }

  // Deal with the remaining bytes and padding
  if (byteRemainder == 1) {
    chunk = bytes[mainLength]

    a = (chunk & 252) >> 2 // 252 = (2^6 - 1) << 2

    // Set the 4 least significant bits to zero
    b = (chunk & 3)   << 4 // 3   = 2^2 - 1

    base64 += encodings[a] + encodings[b] + '=='
  } else if (byteRemainder == 2) {
    chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1]

    a = (chunk & 64512) >> 10 // 64512 = (2^6 - 1) << 10
    b = (chunk & 1008)  >>  4 // 1008  = (2^6 - 1) << 4

    // Set the 2 least significant bits to zero
    c = (chunk & 15)    <<  2 // 15    = 2^4 - 1

    base64 += encodings[a] + encodings[b] + encodings[c] + '='
  }

  return base64
}

So here's a working code:

var xhr = new XMLHttpRequest();
xhr.open('GET', 'http://some.tld/favicon.png', false);
xhr.responseType = 'arraybuffer';
xhr.onload = function(e) {
    console.log(base64ArrayBuffer(e.currentTarget.response));
};
xhr.send();

This will log a valid base64 encoded string representing the binary file contents.

Edit: For older browsers not having access to ArrayBuffer and having btoa() failing on encoding characters, here's another way to get a base64 encoded version of any binary:

function getBinary(file){
    var xhr = new XMLHttpRequest();
    xhr.open("GET", file, false);
    xhr.overrideMimeType("text/plain; charset=x-user-defined");
    xhr.send(null);
    return xhr.responseText;
}

function base64Encode(str) {
    var CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
    var out = "", i = 0, len = str.length, c1, c2, c3;
    while (i < len) {
        c1 = str.charCodeAt(i++) & 0xff;
        if (i == len) {
            out += CHARS.charAt(c1 >> 2);
            out += CHARS.charAt((c1 & 0x3) << 4);
            out += "==";
            break;
        }
        c2 = str.charCodeAt(i++);
        if (i == len) {
            out += CHARS.charAt(c1 >> 2);
            out += CHARS.charAt(((c1 & 0x3)<< 4) | ((c2 & 0xF0) >> 4));
            out += CHARS.charAt((c2 & 0xF) << 2);
            out += "=";
            break;
        }
        c3 = str.charCodeAt(i++);
        out += CHARS.charAt(c1 >> 2);
        out += CHARS.charAt(((c1 & 0x3) << 4) | ((c2 & 0xF0) >> 4));
        out += CHARS.charAt(((c2 & 0xF) << 2) | ((c3 & 0xC0) >> 6));
        out += CHARS.charAt(c3 & 0x3F);
    }
    return out;
}

console.log(base64Encode(getBinary('http://www.google.fr/images/srpr/logo3w.png')));

Hope this helps others as it did for me.

Polyandrist

Second solution ("For older browsers") works in phantomjs (webkit). Thanks a lot :) – Hedonism

Thank you, thank you, thank you, thank you for your edit regarding older browsers! I can assure you, the excitement grows with the time looking for an answer and then finding it! *hugs* – Tweeze

IE doesn't seem to have overrideMimeType at all, and responseType is IE10+. Any solution for IE9? – Marvelmarvella

how would you go about decoding the base64 into a binary then? It seems that atob produce the same failures as btoa... – Elapse

This was very helpful for embedding font files via base64! – Shropshire

That first function will throw an error, since it's running synchronously, more here: #9855627 – Frohne