Develop Reference

Migrate FileReader ReadAsBinaryString() to ReadAsArrayBuffer() or ReadAsText()

I realize that the new Mozilla Firefox return allocation size overflow (on FileReader.ReadAsBinaryString()) when the file bigger than 200MB (something like that).
Here's some of my code on test for client web browser:
function upload(fileInputId, fileIndex)
{
var file = document.getElementById(fileInputId).files[fileIndex];
var blob;
var reader = new FileReader();
reader.readAsBinaryString(file);
reader.onloadend = function(evt)
{
xhr = new XMLHttpRequest();
xhr.open("POST", "upload.php", true);
XMLHttpRequest.prototype.mySendAsBinary = function(text){
var data = new ArrayBuffer(text.length);
var ui8a = new Uint8Array(data, 0);
for (var i = 0; i < text.length; i++){
ui8a[i] = (text.charCodeAt(i) & 0xff);
}
if(typeof window.Blob == "function")
{
blob = new Blob([data]);
}else{
var bb = new (window.MozBlobBuilder || window.WebKitBlobBuilder || window.BlobBuilder)();
bb.append(data);
blob = bb.getBlob();
}
this.send(blob);
}
var eventSource = xhr.upload || xhr;
eventSource.addEventListener("progress", function(e) {
var position = e.position || e.loaded;
var total = e.totalSize || e.total;
var percentage = Math.round((position/total)*100);
});
xhr.onreadystatechange = function()
{
if(xhr.readyState == 4)
{
if(xhr.status == 200)
{
console.log("Done");
}else{
console.log("Fail");
}
}
};
xhr.mySendAsBinary(evt.target.result);
};
}
So I tried change it to FileReader.ReadAsArrayBuffer(), the error has not shown up but the data are not the same (as it's not read as binary string).
Did anyone has any solution to solve this problem? Is there any way that we can upload bigger file from JS to Web Server in raw/string other than FileReader implementation?
I read on Mozilla JS Documentation that said:
This feature is non-standard and is not on a standards track. Do not
use it on production sites facing the Web: it will not work for every
user. There may also be large incompatibilities between
implementations and the behavior may change in the future. - Mozilla
If not ReadAsBinaryString, the how to implement ReadAsArrayBuffer or ReadAsText

To send Files to a web-server, you simply don't need js. HTML alone is well able to do this with the <form> element.
Now if you want to go through js, for e.g catch the different ProgressEvents, then you can send directly your File, no need to read it whatsoever on your side.
To do this, you've got two (or three) solutions.
If your server is able to handle PUT requests, you can simply xhr.send(file);.
Otherwise, you'd have to go through a FormData.
// if you really want to go the XHR way
document.forms[0].onsubmit = function handleSubmit(evt) {
if(!window.FormData) { // old browser use the <form>
return;
}
// now we handle the submit through js
evt.preventDefault();
var fD = new FormData(this);
var xhr = new XMLHttpRequest();
xhr.onprogress = function handleProgress(evt){};
xhr.onload = function handleLoad(evt){};
xhr.onerror = function handleError(evt){};
xhr.open(this.method, this.action);
// xhr.send(fD); // won't work in StackSnippet
log(fD, this.method, this.action); // so we just log its content
};
function log(formData, method, action) {
console.log('would have sent');
for(let [key, val] of formData.entries())
console.log(key, val);
console.log('through', method);
console.log('to', action);
}
<!-- this in itself is enough -->
<form method="POST" action="your_server.page">
<input type="file" name="file_upload">
<input type="submit">
</form>
Now, you sent a comment saying that you can't upload Files bigger than 1GB to your server.
This limitation is only due to your server's config, so the best if you want to accept such big files is to configure it correctly.
But if you really want to send your File by chunks, even then don't get off of the Blob interface.
Indeed Blobs have a slice() method, so use it.
document.forms[0].onsubmit = function handleSubmit(evt) {
evt.preventDefault();
var file = this.elements[0].files[0];
var processed = 0;
if(file) {
// var MAX_CHUNK_SIZE = Math.min(file.size, server_max_size);
// for demo we just split in 10 chunks
var MAX_CHUNK_SIZE = file.size > 10 ? (file.size / 10) | 0 : 1;
loadChunk(0);
}
function loadChunk(start) {
var fD = new FormData();
var sliced = file.slice(start, start+MAX_CHUNK_SIZE);
processed += sliced.size; // only for demo
fD.append('file_upload', sliced, file.name);
fD.append('starting_index', start);
if(start + MAX_CHUNK_SIZE >= file.size) {
fD.append('last_chunk', true);
}
var xhr = new XMLHttpRequest();
xhr.open('POST', 'your_server.page');
xhr.onload = function onchunkposted(evt) {
if(start + MAX_CHUNK_SIZE >= file.size) {
console.log('All done. Original file size: %s, total of chunks sizes %s', file.size, processed);
return;
}
loadChunk(start + MAX_CHUNK_SIZE);
};
// xhr.send(fD);
log(fD);
setTimeout(xhr.onload, 200); // fake XHR onload
}
};
function log(formData, method, action) {
console.log('would have sent');
for(let [key, val] of formData.entries())
console.log(key, val);
}
<form method="POST" action="your_server.page">
<input type="file" name="file_upload">
<input type="submit">
</form>
But you absolutely don't need to go through a FileReader for this operation.
Actually the only case where it could make sense to use a FileReader here would be for some Android browsers that don't support passing Blob into a FormData, even though they don't give a single clue about it.
So in this case, you'd have to set up your server to let you know the request was empty, and then only read the File as a dataURI that you would send in-place of the original File.

after a long week of research and sleepless nights, you can't upload binary strings without breaking it, also base64 doesn't work for all files, only images, the journey from the client-side to the server breaks the bytes being sent

Kaiido statement is correct
To send Files to a web-server, you simply don't need js
But that doesn't answer my question. Using the Simple XMLHttpRequest() can upload the file and track those progress as well. But still, it's not it. The direct upload, either from the <form> or using XMLHttpRequest() will need to increase your upload limit in php setting. This method is not convenience for me. How if the client upload file as 4GB? So I need to increase to 4GB. Then next time, client upload file as 6GB, then I have to increase to 6GB.
Using the slice() method is make sense for bigger file as we can send it part by part to server. But this time I am not using it yet.
Here's some of my test the worked as I want. I hope some expert could correct me if I am wrong.
My Upload.js
function upload(fileInputId, fileIndex)
{
var file = document.getElementById(fileInputId).files[fileIndex];
var blob;
var reader = new FileReader();
reader.readAsArrayBuffer(file);
reader.onloadend = function(evt)
{
xhr = new XMLHttpRequest();
xhr.open("POST", "upload.php?name=" + base64_encode(file.name), true);
XMLHttpRequest.prototype.mySendAsBinary = function(text){
var ui8a = new Uint8Array(new Int8Array(text));
if(typeof window.Blob == "function")
{
blob = new Blob([ui8a]);
}else{
var bb = new (window.MozBlobBuilder || window.WebKitBlobBuilder || window.BlobBuilder)();
bb.append(ui8a);
blob = bb.getBlob();
}
this.send(blob);
}
var eventSource = xhr.upload || xhr;
eventSource.addEventListener("progress", function(e) {
var position = e.position || e.loaded;
var total = e.totalSize || e.total;
var percentage = Math.round((position/total)*100);
console.log(percentage);
});
xhr.onreadystatechange = function()
{
if(xhr.readyState == 4)
{
if(xhr.status == 200)
{
console.log("Done");
}else{
console.log("Fail");
}
}
};
xhr.mySendAsBinary(evt.target.result);
};
}
Below is how the PHP server listen to the ArrayBuffer from JS
if(isset($_GET["name"])){
$name = base64_decode($_GET["name"]);
$loc = $name;
$inputHandler = fopen('php://input', "r");
$fileHandler = fopen($loc, "w+");
while(true) {
//$buffer = fgets($inputHandler, 1024);
$buffer = fread($inputHandler, 1000000);
if (strlen($buffer) == 0) {
fclose($inputHandler);
fclose($fileHandler);
return true;
}
//$b = base64_encode($buffer);
fwrite($fileHandler, $buffer);
}
}
The above method works well. The FileReader read the file as ArrayBuffer the upload to server. For me, migrating from ReadAsBinaryString() to ReadAsArrayBuffer() is important and ReadAsArrayBuffer() has some better performance rather than ReadAsBinaryString()
Here's some reason, why some developer relies to FileReader API:
Streaming. Using this method, the file will be stream, so we can avoid setting the php multiple time.
Easy Encrypt. As the file is sending via ArrayBuffer, it is easy for developer to Encrypt the file while upload in progress.
This method also support upload any type of file. I ve done some test and I realize that ReadAsArrayBuffer() method are more faster than ReadAsBinaryString() and direct form upload. You may try it.
Security Notice
The above code is only under test code, to use it in production, you have to consider sending the data in GET or POST under HTTPS.

javascript createObjectURL corrupt file

im using the following javascript to create an object url, the only problem is when loading the url blob:http:///mysite.com/randomhash the file is corrupt.
var audioFile = null;
var xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
audioFile = new Blob([xhttp.response], {type: 'audio/mp3'});
}
};
xhttp.open("GET", "myMp3File.mp3", false);
xhttp.send();
var file = new File([audioFile], "myMp3File.mp3", {type: "audio/mp3", lastModified: Date.now()});
any ideas as to why this would create a blob url with a corrupt mp3 ?

Multiple problems here.
First, you are dealing with asynchronous code. You will need to use a callback in order to use the response of your XHR request.
Then, even if you did so, there are good chances that it will still not work.
This is because the response you get is plain text UTF16 and that some bytes will get mangled by encoding.
The real solution in your case is to directly request the response as a Blob.
You can do so with the XMLHttpRequest.responseType parameter.
var xhttp = new XMLHttpRequest();
xhttp.responseType = 'blob';
xhttp.onload = function(e) {
var blob = xhttp.response;
callback(blob);
};
xhttp.open...
And now in your callback you will be able to create a blobURI directly from this blob.
function callback(blob) {
var url = URL.createObjectURL(blob);
...
Also, if all you want is to display this file, there is no point in creating a File from it, the only web API it would make a difference if it was a File and not just a Blob is Formdata.append method, and even there, it's not that useful.

Fetching non-utf8 data with XMLHttpRequest

I want to fetch a document from the web with xmlHttpRequest. However the text in question isn't utf8 (in this case it's windows-1251 but in the generic case, I wouldn't know that for sure).
However, if I use responseType="text" it treats it as though the string is utf8, ignoring the charset in the content-type (resulting in a nasty mess).
If I used 'blob' (probably the nearest thing to what I want), could I then convert that to a DomString taking into account the encoding?

I actually found an API which does what I want, from here:
https://developers.google.com/web/updates/2014/08/Easier-ArrayBuffer-String-conversion-with-the-Encoding-API
Basically, use responseType="arraybuffer", pick the encoding from the returned headers, and use DataView and TextDecoder. It does exactly what is required.
const xhr = new XMLHttpRequest();
xhr.responseType = "arraybuffer";
xhr.onload = function() {
const contenttype = xhr.getResponseHeader("content-type");
const charset = contenttype.substring(contenttype.indexOf("charset=") + 8);
const dataView = new DataView(xhr.response);
const decoder = new TextDecoder(charset);
console.log(decoder.decode(dataView));
}
xhr.open("GET", "https://people.w3.org/mike/tests/windows-1251/test.txt");
xhr.send(null);
fetch("https://people.w3.org/mike/tests/windows-1251/test.txt")
.then(response => {
const contenttype = response.headers.get("content-type");
const charset = contenttype.substring(contenttype.indexOf("charset=") + 8);
response.arrayBuffer()
.then(ab => {
const dataView = new DataView(ab);
const decoder = new TextDecoder(charset);
console.log(decoder.decode(dataView));
})
})

If I used 'blob' (probably the nearest thing to what I want), could I then convert that to a DomString taking into account the encoding?
https://medium.com/programmers-developers/convert-blob-to-string-in-javascript-944c15ad7d52 outlines a general approach you can use. To apply that to the case of fetching a remote document:
Create a FileReader to read in the fetch response as a Blob
Use FileReader.readAsText() to get back text from that Blob in the right encoding
Like this:
const reader = new FileReader()
reader.addEventListener("loadend", function() {
console.log(reader.result)
})
fetch("https://people.w3.org/mike/tests/windows-1251/test.txt")
.then(response => response.blob())
.then(blob => reader.readAsText(blob, "windows-1251"))
Or if you instead really want to use XHR:
const reader = new FileReader()
reader.addEventListener("loadend", function() {
console.log(reader.result)
})
const xhr = new XMLHttpRequest()
xhr.responseType = "blob"
xhr.onload = function() {
reader.readAsText(xhr.response, "windows-1251")
}
xhr.open("GET", "https://people.w3.org/mike/tests/windows-1251/test.txt", true)
xhr.send(null)
However, if I use responseType="text" it treats it as though the string is utf8, ignoring the charset in the content-type
Yes. That’s what’s required by the Fetch spec (which for this is what the XHR spec relies on too):
Objects implementing the Body mixin also have an associated package data algorithm, given bytes, a type and a mimeType, switches on type, and runs the associated steps:
…
↪ text
            Return the result of running UTF-8 decode on bytes.

Chrome extension: how to pass ArrayBuffer or Blob from content script to the background without losing its type?

I have this content script that downloads some binary data using XHR, which is sent later to the background script:
var self = this;
var xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.responseType = 'arraybuffer';
xhr.onload = function(e) {
if (this.status == 200) {
self.data = {
data: xhr.response,
contentType: xhr.getResponseHeader('Content-Type')
};
}
};
xhr.send();
... later ...
sendResponse({data: self.data});
After receiving this data in background script, I'd like to form another XHR request that uploads this binary data to my server, so I do:
var formData = new FormData();
var bb = new WebKitBlobBuilder();
bb.append(data.data);
formData.append("data", bb.getBlob(data.contentType));
var req = new XMLHttpRequest();
req.open("POST", serverUrl);
req.send(formData);
The problem is that the file uploaded to the server contains just this string: "[object Object]". I guess this happens because ArrayBuffer type is lost somehow while transferring it from content process to the background? How can I solve that?

Messages passed between a Content Script and a background page are JSON-serialized.
If you want to transfer an ArrayBuffer object through a JSON-serialized channel, wrap the buffer in a view, before and after transferring.
I show an isolated example, so that the solution is generally applicable, and not just in your case. The example shows how to pass around ArrayBuffers and typed arrays, but the method can also be applied to File and Blob objects, by using the FileReader API.
// In your case: self.data = { data: new Uint8Array(xhr.response), ...
// Generic example:
var example = new ArrayBuffer(10);
var data = {
// Create a view
data: Array.apply(null, new Uint8Array(example)),
contentType: 'x-an-example'
};
// Transport over a JSON-serialized channel. In your case: sendResponse
var transportData = JSON.stringify(data);
//"{"data":[0,0,0,0,0,0,0,0,0,0],"contentType":"x-an-example"}"
// At the receivers end. In your case: chrome.extension.onRequest
var receivedData = JSON.parse(transportData);
// data.data is an Object, NOT an ArrayBuffer or Uint8Array
receivedData.data = new Uint8Array(receivedData.data).buffer;
// Now, receivedData is the expected ArrayBuffer object
This solution has been tested successfully in Chrome 18 and Firefox.
new Uint8Array(xhr.response) is used to create a view of the ArrayBuffer, so that the individual bytes can be read.
Array.apply(null, <Uint8Array>) is used to create a plain array, using the keys from the Uint8Array view. This step reduces the size of the serialized message. WARNING: This method only works for small amounts of data. When the size of the typed array exceeds 125836, a RangeError will be thrown. If you need to handle large pieces of data, use other methods to do the conversion between typed arrays and plain arrays.
At the receivers end, the original buffer can be obtained by creating a new Uint8Array, and reading the buffer attribute.
Implementation in your Google Chrome extension:
// Part of the Content script
self.data = {
data: Array.apply(null, new Uint8Array(xhr.response)),
contentType: xhr.getResponseHeader('Content-Type')
};
...
sendResponse({data: self.data});
// Part of the background page
chrome.runtime.onMessage.addListener(function(data, sender, callback) {
...
data.data = new Uint8Array(data.data).buffer;
Documentation
MDN: Typed Arrays
MDN: ArrayBuffer
MDN: Uint8Array
MDN: <Function> .apply
Google Chrome Extension docs: Messaging > Simple one-time requests
"This lets you send a one-time JSON-serializable message from a content script to extension, or vice versa, respectively"
SO bonus: Upload a File in a Google Chrome Extension - Using a Web worker to request, validate, process and submit binary data.

For Chromium Extensions manifest v3, URL.createObjectURL() approach doesn't work anymore because it is prohibited in the service workers.
The best (easiest) way to pass data from a service worker to a content script (and vice-versa), is to convert the blob into a base64 representation.
const fetchBlob = async url => {
const response = await fetch(url);
const blob = await response.blob();
const base64 = await convertBlobToBase64(blob);
return base64;
};
const convertBlobToBase64 = blob => new Promise(resolve => {
const reader = new FileReader();
reader.readAsDataURL(blob);
reader.onloadend = () => {
const base64data = reader.result;
resolve(base64data);
};
});
Then send the base64 to the content script.
Service worker:
chrome.tabs.sendMessage(sender.tab.id, { type: "LOADED_FILE", base64: base64 });
Content script:
chrome.runtime.onMessage.addListener(async (request, sender) => {
if (request.type == "LOADED_FILE" && sender.id == '<your_extension_id>') {
// do anything you want with the data from the service worker.
// e.g. convert it back to a blob
const response = await fetch(request.base64);
const blob = await response.blob();
}
});

Is it possible to save a File object in LocalStorage and then reload a File via FileReader when a user comes back to a page?

For example, say the user loads some very large images or media files in to your web app. When they return you want your app to show what they've previously loaded, but can't keep the actual file data in LocalStorage because the data is too large.

This is NOT possible with localStorage. Data stored in localStorage needs to be one of the primitive types that can be serializable. This does not include the File object.
For example, this will not work as you'd expect:
var el = document.createElement('input');
el.type='file';
el.onchange = function(e) {
localStorage.file = JSON.stringify(this.files[0]);
// LATER ON...
var reader = new FileReader();
reader.onload = function(e) {
var result = this.result; // never reaches here.
};
reader.readAsText(JSON.parse(localStorage.f));
};
document.body.appendChild(el);
The solution is to use a more powerful storage option like writing the file contents to the HTML5 Filesystem or stashing it in IndexedDB.

Technically you can if you just need to save small files in localStorage.
Just base64 that ish and since it's a string... it's localStorage-friendly.
I think localStorage has a ~5MB limit. base64 strings are pretty low file size so this is a feasible way to store small images. If you use this lazy man's way, the downside is you'll have to mind the 5MB limit. I think it could def be a solution depending on your needs.

Yes, this is possible. You can insert whatever information about the file you want into LocalStorage, provided you serialize it to one of the primitive types supported. You can also serialize the whole file into LocalStorage and retrieve that later if you want, but there are limitations on the size of the file depending on browser.
The following shows how to achieve this using two different approaches:
(function () {
// localStorage with image
var storageFiles = JSON.parse(localStorage.getItem("storageFiles")) || {},
elephant = document.getElementById("elephant"),
storageFilesDate = storageFiles.date,
date = new Date(),
todaysDate = (date.getMonth() + 1).toString() + date.getDate().toString();
// Compare date and create localStorage if it's not existing/too old
if (typeof storageFilesDate === "undefined" || storageFilesDate < todaysDate) {
// Take action when the image has loaded
elephant.addEventListener("load", function () {
var imgCanvas = document.createElement("canvas"),
imgContext = imgCanvas.getContext("2d");
// Make sure canvas is as big as the picture
imgCanvas.width = elephant.width;
imgCanvas.height = elephant.height;
// Draw image into canvas element
imgContext.drawImage(elephant, 0, 0, elephant.width, elephant.height);
// Save image as a data URL
storageFiles.elephant = imgCanvas.toDataURL("image/png");
// Set date for localStorage
storageFiles.date = todaysDate;
// Save as JSON in localStorage
try {
localStorage.setItem("storageFiles", JSON.stringify(storageFiles));
}
catch (e) {
console.log("Storage failed: " + e);
}
}, false);
// Set initial image src
elephant.setAttribute("src", "elephant.png");
}
else {
// Use image from localStorage
elephant.setAttribute("src", storageFiles.elephant);
}
// Getting a file through XMLHttpRequest as an arraybuffer and creating a Blob
var rhinoStorage = localStorage.getItem("rhino"),
rhino = document.getElementById("rhino");
if (rhinoStorage) {
// Reuse existing Data URL from localStorage
rhino.setAttribute("src", rhinoStorage);
}
else {
// Create XHR, BlobBuilder and FileReader objects
var xhr = new XMLHttpRequest(),
blob,
fileReader = new FileReader();
xhr.open("GET", "rhino.png", true);
// Set the responseType to arraybuffer. "blob" is an option too, rendering BlobBuilder unnecessary, but the support for "blob" is not widespread enough yet
xhr.responseType = "arraybuffer";
xhr.addEventListener("load", function () {
if (xhr.status === 200) {
// Create a blob from the response
blob = new Blob([xhr.response], {type: "image/png"});
// onload needed since Google Chrome doesn't support addEventListener for FileReader
fileReader.onload = function (evt) {
// Read out file contents as a Data URL
var result = evt.target.result;
// Set image src to Data URL
rhino.setAttribute("src", result);
// Store Data URL in localStorage
try {
localStorage.setItem("rhino", result);
}
catch (e) {
console.log("Storage failed: " + e);
}
};
// Load blob as Data URL
fileReader.readAsDataURL(blob);
}
}, false);
// Send XHR
xhr.send();
}
})();
Source