Cloning a File object to a web Worker - javascript

I am experimenting with web Workers to improve file upload performance. I am working on an example from this post about large file uploads. I have a (somewhat) more complete code sample that works in Chrome (36.0.1985.143) and Safari (7.0.3 (9537.75.14)) but not in Firefox (31.0). I don't have server code to share, but the client side code is enough to see whether the browser is pushing the slices. According to MDN, File and FileList are both clonable objects, so is this a bug in Firefox?
The original link came via this post on StackOverflow.
In Firefox, I hit an error:
(DataCloneError: The object could not be cloned.)
on this line:
worker.postMessage({
'files' : files
});
Code follows:
index.html
<html>
<head>
<script>
var worker = new Worker('fileupload.js');
worker.onmessage = function(e) {
alert(e.data);
}
worker.onerror = werror;
function werror(e) {
console.log('ERROR: Line ', e.lineno, ' in ', e.filename, ': ', e.message);
}
function handleFileSelect(evt) {
evt.stopPropagation();
evt.preventDefault();
var files;
if(evt.dataTransfer === undefined ){
files = document.getElementById('files').files;
}else{
files = evt.dataTransfer.files||evt.target.files;
}
// FileList object.
worker.postMessage({
'files' : files
});
//Sending File list to worker
// files is a FileList of File objects. List some properties.
var output = [];
for (var i = 0, f; f = files[i]; i++) {
output.push('<li><strong>', escape(f.name), '</strong> (', f.type || 'n/a', ') - ', f.size, ' bytes, last modified: ', f.lastModifiedDate ? f.lastModifiedDate.toLocaleDateString() : 'n/a', '</li>');
}
document.getElementById('list').innerHTML = '<ul>' + output.join('') + '</ul>';
}
function handleDragOver(evt) {
evt.stopPropagation();
evt.preventDefault();
evt.dataTransfer.dropEffect = 'copy';
// Explicitly show this is a copy.
}
function setcode(){
// Setup the dnd listeners.
var dropZone = document.getElementById('drop_zone');
dropZone.addEventListener('dragover', handleDragOver, false);
dropZone.addEventListener('drop', handleFileSelect, false);
document.getElementById('files').addEventListener('change', handleFileSelect, false);
}
</script>
</head>
<body >
<input type="file" id="files" name="files" multiple />
<div id="drop_zone" style="width:500px;height:50%;">
Drop files here
</div>
<div>
<input type>
</div>
<output id="list"></output>
<script>
setcode();
</script>
</body>
</html>
fileupload.js
var file = [], p = true;
function upload(blobOrFile) {
var xhr = new XMLHttpRequest();
xhr.open('POST', '/server', false);
xhr.onload = function(e) {
};
xhr.send(blobOrFile);
}
function process() {
for (var j = 0; j <file.length; j++) {
var blob = file[j];
const BYTES_PER_CHUNK = 1024 * 1024;
// 1MB chunk sizes.
const SIZE = blob.size;
var start = 0;
var end = BYTES_PER_CHUNK;
while (start < SIZE) {
if ('mozSlice' in blob) {
var chunk = blob.mozSlice(start, end);
} else if ('webkitSlice' in blob) {
var chunk = blob.webkitSlice(start, end);
}else{
var chunk = blob.slice(start, end);
}
upload(chunk);
start = end;
end = start + BYTES_PER_CHUNK;
}
p = ( j = file.length - 1) ? true : false;
self.postMessage(blob.name + " Uploaded Succesfully");
}
}
self.onmessage = function(e) {
for (var j = 0; j < e.data.files.length; j++)
file.push(e.data.files[j]);
if (p) {
process()
}
}

Related

_formdataPolyfill2.default is not a constructor is thrown when I try to use formdata-polyfill npm

I am using web workers for uploading file and I am sending the file in the form of formData object as got to know that the web worker doesn't have access to DOM I used formdata-polyfill in place of default FormData , it is throwing this error and I don't know how to use this polyill properly.
here is my code,
//trying to send the formdata-polyfill object to worker
require('formdata-polyfill');
let data = new FormData();
data.append('server-method', 'upload');
data.append('file', event.target.files[0]);
// let data = new FormData(event.target.files[0]);
if (this.state.headerActiveTabUid === '1')
this.props.dispatch(handleFileUpload({upload: 'assets', data}));
//worker.js
var file = [], p = true, url,token;
function upload(blobOrFile) {
var xhr = new XMLHttpRequest();
xhr.open('POST', url, true);//add url to upload
xhr.setRequestHeader('Authorization', token);
xhr.onload = function(e) {
};
xhr.send(blobOrFile);
}
function process() {
for (var j = 0; j <file.length; j++) {
var blob = file[j];
const BYTES_PER_CHUNK = 1024 * 1024;
// 1MB chunk sizes.
const SIZE = blob.size;
var start = 0;
var end = BYTES_PER_CHUNK;
while (start < SIZE) {
if ('mozSlice' in blob) {
var chunk = blob.mozSlice(start, end);
} else {
var chunk = blob.slice(start, end);
}
upload(chunk);
start = end;
end = start + BYTES_PER_CHUNK;
}
p = ( j === file.length - 1);
self.postMessage(blob.name + " Uploaded Succesfully");
}
}
self.addEventListener('message', function(e) {
url = e.data.url;
token = e.data.id;
file.push(e.data.files);
if (p) {
process();
}
});

Javascript base64 convertion

my aswer i believe is easy. I need create a code to get base64 content in FileReader function to send to database. I try to put the base64 string and put in form type hidden but i cant do. I upload the source code bellow:
JSFIDDLE
<script type="text/javascript">
function readMultipleFiles(evt) {
//Retrieve all the files from the FileList object
var files = evt.target.files;
if (files) {
for (var i=0, f; f=files[i]; i++) {
var r = new FileReader();
r.onload = (function(f) {
return function(e) {
var contents = e.target.result;
alert(
"name: " + f.name + "n"
+ "starts with: " + contents.substr(1, contents.indexOf("n"))
);
document.write(f.name);
};
})(f);
r.readAsText(f);
}
} else {
alert("Failed to load files");
}
}
document.getElementById('fileinput').addEventListener('change', readMultipleFiles, false);
</script>
Try using the function from MDN which should also support unicode:
function readMultipleFiles(evt) {
//Retrieve all the files from the FileList object
var files = evt.target.files;
if (files) {
for (var i=0, f; f=files[i]; i++) {
var r = new FileReader();
r.onload = (function(f) {
return function(e) {
var contents = e.target.result;
alert(
"name: " + f.name + "n"
+ "starts with: " + contents.substr(1, contents.indexOf("n"))
);
document.getElementById('b64').innerHTML = b64EncodeUnicode(contents);
};
})(f);
r.readAsText(f);
}
} else {
alert("Failed to load files");
}
}
document.getElementById('fileinput').addEventListener('change',
readMultipleFiles, false);
function b64EncodeUnicode(str) {
return btoa(encodeURIComponent(str).replace(/%([0-9A-F]{2})/g,
function(match, p1) {
return String.fromCharCode('0x' + p1);
}));
}
Notes:
I tested the above using an ascii file and https://www.base64encode.org/
Removed the document.write. Instead the result is displayed in a span with id="b64" (document.getElementById('b64').innerHTML = ...)
As is, will work only for 1 file. Requires minor editing or putting back the document.write to support multiple files
Fiddle: https://jsfiddle.net/g6g1a51p/4/

Read File byte for byte and parse to int

I have to read data from an file. This data was written by an server byte-wise into the file. The file has an fix structure, now I want to read the Information in it with JS.
I have found http://www.html5rocks.com/en/tutorials/file/dndfiles/ and copied it down to fiddle: http://jsfiddle.net/egLof4ph/
function readBlob(opt_startByte, opt_stopByte) {
var files = document.getElementById('files').files;
if (!files.length) {
alert('Please select a file!');
return;
}
var file = files[0];
var start = parseInt(opt_startByte) || 0;
var stop = parseInt(opt_stopByte) || file.size - 1;
var reader = new FileReader();
// If we use onloadend, we need to check the readyState.
reader.onloadend = function(evt) {
if (evt.target.readyState == FileReader.DONE) { // DONE == 2
document.getElementById('byte_content').textContent = evt.target.result;
document.getElementById('byte_range').textContent = ['Read bytes: ', start + 1, ' - ', stop + 1,
' of ', file.size, ' byte file'].join('');
}
};
var blob = file.slice(start, stop);
var a = reader.readAsBinaryString(blob);
}
document.querySelector('.readBytesButtons').addEventListener('click', function(evt) {
if (evt.target.tagName.toLowerCase() == 'button') {
var startByte = evt.target.getAttribute('data-startbyte');
var endByte = evt.target.getAttribute('data-endbyte');
readBlob(startByte, endByte);
}
}, false);
I knew that the first 7 Bytes are crap and can throw them away. The next 68Bytes belong together and every value is 4bytes big. After the 68Bytes again 68 usable bytes come (that 68bytes are "timeslots").
My Question:
When I am using that Code I get many signs (A, Q, &&&, special chars,..), but the data are in reality longs. How can I parse them into Numbers? According to the Filereader API readAsBinarsString() returns raw binary data. And how to correctly parse the whole File?
So, the original File looks like this:
<7B>Metadata</7B><4B>long value</4B>....17times for each timeslot <4B>long value</4B>....17times again.... and this util the end of the file.
When I am using the above Code I get output like: �&�&WK��
Furthermore I have found: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Typed_arrays (since FileReader provides an method which returns an ArrayBuffer), so I guess I should use readAsArrayBuffer(), but how to use it to get to my data?
You really need binary ?
Note that readAsBinaryString method is now deprecated as per the 12 July 2012 Working Draft from the W3C.
function readBlob(opt_startByte, opt_stopByte) {
var files = document.getElementById('files').files;
if (!files.length) {
alert('Please select a file!');
return;
}
var file = files[0];
var start = parseInt(opt_startByte) || 0;
var stop = parseInt(opt_stopByte) || file.size - 1;
var reader = new FileReader();
reader.onloadend = function (evt) {
if (evt.target.readyState == FileReader.DONE) { // DONE == 2
var a = new Uint8Array(evt.target.result)
var binary = ""
for (var i =0; i <= a.length; i++) {
binary += Number(a[i]).toString(2)
}
document.getElementById('byte_content').textContent = binary;
document.getElementById('byte_range').textContent = ['Read bytes: ', start + 1, ' - ', stop + 1,
' of ', file.size, ' byte file'].join('');
}
};;
var blob = file.slice(start, stop);
var a = reader.readAsArrayBuffer(blob)
}
document.querySelector('.readBytesButtons').addEventListener('click', function (evt) {
if (evt.target.tagName.toLowerCase() == 'button') {
var startByte = evt.target.getAttribute('data-startbyte');
var endByte = evt.target.getAttribute('data-endbyte');
readBlob(startByte, endByte);
}
}, false);

How do I loop through a file, byte by byte, in JavaScript?

I need some help getting my head around how the file is accessed in JavaScript to do some operations on it.
I would like to loop through a file byte by byte using JavaScript.
I can already select which file I would like to read. And I can read preset byte of the file.
I've found this nice example on how to read a slice of a file here:
http://www.html5rocks.com/en/tutorials/file/dndfiles/
Here is the snippet of code which I'm playing with:
<style>
#byte_content {
margin: 5px 0;
max-height: 100px;
overflow-y: auto;
overflow-x: hidden;
}
#byte_range { margin-top: 5px; }
</style>
<input type="file" id="files" name="file" /> Read bytes:
<span class="readBytesButtons">
<button data-startbyte="0" data-endbyte="4">1-5</button>
<button data-startbyte="5" data-endbyte="14">6-15</button>
<button data-startbyte="6" data-endbyte="7">7-8</button>
<button>entire file</button>
</span>
<div id="byte_range"></div>
<div id="byte_content"></div>
<script>
function readBlob(opt_startByte, opt_stopByte) {
var files = document.getElementById('files').files;
if (!files.length) {
alert('Please select a file!');
return;
}
var file = files[0];
var start = parseInt(opt_startByte) || 0;
var stop = parseInt(opt_stopByte) || file.size - 1;
var reader = new FileReader();
// If we use onloadend, we need to check the readyState.
reader.onloadend = function(evt) {
if (evt.target.readyState == FileReader.DONE) { // DONE == 2
document.getElementById('byte_content').textContent = evt.target.result;
document.getElementById('byte_range').textContent =
['Read bytes: ', start + 1, ' - ', stop + 1,
' of ', file.size, ' byte file'].join('');
}
};
var blob = file.slice(start, stop + 1);
reader.readAsBinaryString(blob);
}
document.querySelector('.readBytesButtons').addEventListener('click', function(evt) {
if (evt.target.tagName.toLowerCase() == 'button') {
var startByte = evt.target.getAttribute('data-startbyte');
var endByte = evt.target.getAttribute('data-endbyte');
readBlob(startByte, endByte);
}
}, false);
</script>
Now I would like to loop through the file, four bytes at a time, but cannot seem to figure out how to do that. The reader does not seem to allow me to read more than once.
Once I can read from the file more than once, I should be able to iterate through it quite easily with something like this:
while( placemark != fileSize-4 ){
output = file.slice(placemark, placemark + 4);
console.log(output);
placemark = placemark + 5;
}
Thanks in advance!
Here is a link to a jsFiddle and plnkr version
I'm not sure it is what you wanted but maybe it can help, and anyway I had fun.
I tried setting reader and file vars as global :
var reader = new FileReader(), step = 4, stop = step, start = 0, file;
document.getElementById('files').addEventListener('change', load, true);
function load() {
var files = document.getElementById('files').files;
file = files[0];
reader.onloadend = function(evt) {
if (evt.target.readyState == FileReader.DONE) {
var result = evt.target.result;
document.getElementById('byte_content').textContent += result;
document.getElementById('byte_range').textContent = ['Read bytes: ', start, ' - ', start+result.length,
' of ', file.size, ' byte file'
].join('');
}
}
}
function next() {
if (!file) {
alert('Please select a file!');
return;
}
var blob = file.slice(start, stop);
reader.readAsBinaryString(blob);
start+= step;
stop = start+step;
}
function loop() {
if (!file) {
alert('Please select a file!');
return;
}
if (start < file.size) {
next();
setTimeout(loop, 50);
}
}
<input type="file" id="files" name="file" />Read bytes:
<span class="readBytesButtons">
<button onclick="next()">next</button>
<button onclick="loop()">loop</button>
</span>
<div id="byte_range"></div>
<div id="byte_content"></div>
I'd read the blob as an ArrayBuffer and use a DataView to read through the data
function readBlob(opt_startByte, opt_stopByte) {
var files = document.getElementById('files').files;
if (!files.length) {
alert('Please select a file!');
return;
}
var file = files[0];
var start = parseInt(opt_startByte) || 0;
var stop = parseInt(opt_stopByte) || file.size - 1;
var reader = new FileReader();
reader.onload = function(evt) {
var placemark = 0, dv = new DataView(this.result), limit = dv.byteLength - 4, output;
while( placemark <= limit ){
output = dv.getUint32(placemark);
console.log(' 0x'+("00000000" + output.toString(16)).slice(-8));
placemark += 4;
}
};
var blob = file.slice(start, stop + 1);
reader.readAsArrayBuffer(blob);
}
<input type="file" id="files" onchange="readBlob(0, 100)">
In the onload handler of FileReader, convert the result to string (toString()), then read 4 chars at a time with the string's slice method.
var contents = null;
reader.onload = function(){
contents = reader.result.toString();
}
var startByte = 0;
// read 4 bytes at a time
var step = 4;
// actual reading (doesn't alter the contents object)
console.log(contents.slice(startByte, step))
// update the next startByte position
startByte += step;

Asynchronous execution in javascript any solution to control execution?

I need a solution to control code execution in javascript.I want code on next line should not be executed unless the code on current line is completely executed.
Is there any solution?
function handleFileSelect(evt) {
var files = evt.target.files;
for (var i = 0; i < files.length; i++) {
alert("for");
f = files[i];
fileExtension = f.name.split('.').pop();
if(fileExtension != 'kml' && fileExtension !='kmz' && fileExtension != 'csv'){
alert('Unsupported file type ' + f.type + '(' + fileExtension + ')');
return;
}
var fileReaderkmlcsv = new FileReader();
fileReaderkmlcsv.onloadend = loadend;
fileReaderkmlcsv.onerror = function(event) {
alert("ERROR: " + event.target.error.code);
};
fileReaderkmlcsv.readAsText(f);
} //- end for
} //handleFileSelect
function loadend(theFile) {
alert("loadend");
//code for processing my files
}
The issue is that loadend is running as soon as any one of the FileReaders has completed loading. You'll need to redesign the code to wait for all 3 of them to finish, something like:
function handleFileSelect(evt) {
var files = evt.target.files;
var fileReaders = [];
var loadCount = 0;
for (var i = 0; i < files.length; i++) {
f = files[i];
fileExtension = f.name.split('.').pop();
if(fileExtension != 'kml' && fileExtension !='kmz' && fileExtension != 'csv'){
alert('Unsupported file type ' + f.type + '(' + fileExtension + ')');
return;
}
function fileLoaded() {
loadCount++;
//Check if we've loaded all the files
if (loadCount == files.length) {
loadend(fileReaders);
}
}
var fileReaderkmlcsv = new FileReader();
fileReaderkmlcsv.onloadend = fileLoaded;
fileReaderkmlcsv.onerror = function(event) {
alert("ERROR: " + event.target.error.code);
};
fileReaderkmlcsv.readAsText(f);
fileReaders.push(fileReaderkmlcsv);
}
}
function loadend(files) {
//files now contains an array of completed FileReader objects
}
Note that I don't have direct experience of the FileReader object itself - if onloadend doesn't fire if an error occurs, you'll need to put similar logic in the onerror event as well to make sure that the loadCount variable still gets incremented/checked etc.

Categories

Resources