Develop Reference

SHA256 Hashing Large files in angular 6 using Filereader Issue

I have problem with SHA256 hashing. if the file size is more then 250 MB it is terminating browser and crashing.
below is the hashing code please do help us.
let fileReader = new FileReader();
fileReader.readAsArrayBuffer(fileToSend);
fileReader.onload = (e) => {
const hash = CrypTo.SHA256(this.arrayBufferToWordArray(fileReader.result)).toString();
this.hashCode=hash;
this.fileHistory.MediaHash = hash;
this.fileHistory.FileName = fileToSend.name;
//Insert to file history
this.fileHistoryService.postFiles(this.fileHistory).subscribe(
data => {
this.hashCode=data["MediaHash"];
this.alertService.success('HASHFILE.FileUploadSuccessMessage', true);
this.hideGenerateHashCodeButton = true;
},
error => {
this.alertService.error('COMMONERRORMESSAGE.SomethingWentWrongErrorMessage');
});
}
arrayBufferToWordArray(fileResult) {
var i8a = new Uint8Array(fileResult);
var byteArray = [];
for (var i = 0; i < i8a.length; i += 4) {
byteArray.push(i8a[i] << 24 | i8a[i + 1] << 16 | i8a[i + 2] << 8 | i8a[i + 3]);
}
return CrypTo.lib.WordArray.create(byteArray, i8a.length);
}

Below code I tested for all the big files, which fixed my solution.
var hashdata = CrypTo.algo.SHA256.create();
var file =**<FiletoHash>**;
if(file){
var reader = new FileReader();
var size = file.size;
var chunk_size = Math.pow(2, 22);
var chunks = [];
var offset = 0;
var bytes = 0;
reader.onloadend = (e) =>{
if (reader.readyState == FileReader.DONE){
//every chunk read updating hash
hashdata.update(this.arrayBufferToWordArray(reader.result));
let chunk:any = reader.result;
bytes += chunk.length;
chunks.push(chunk);
if((offset < size)){
offset += chunk_size;
var blob = file.slice(offset, offset + chunk_size);
reader.readAsArrayBuffer(blob);
} else {
//use below hash for result
//finaly generating hash
var hash = hashdata.finalize().toString();
//debugger;
};
}
};
var blob = file.slice(offset, offset + chunk_size);
reader.readAsArrayBuffer(blob);
}
}
}
arrayBufferToWordArray(fileResult) {
var i8a = new Uint8Array(fileResult);
return CrypTo.lib.WordArray.create(i8a, i8a.length);
}

You should definitely use streams or something like it to avoid loading all the file into your memory.
Specifically using CryptoJS, I have seen that it's possible to perform progressive Hashing.
var sha256 = CryptoJS.algo.SHA256.create();
sha256.update("Message Part 1");
sha256.update("Message Part 2");
sha256.update("Message Part 3");
​
var hash = sha256.finalize();
So, use FileReader to read parts of the file, then every time you read a part, you update the sha256 until there is nothing more to read.
See :
filereader api on big files

JQuery FileReader onload not firing

I have a multiple file uploading. When I upload the images and binding to the model as follows not firing the FileReader onload function. It skip and fire remain
Here is my code
imageSelect: function (e) {
var dataModel = bindViewModel.selected.attachments;
var reader = new FileReader();
reader.onload = function () {
var uploadImg = new Image();
uploadImg.onload = function () {
for (var i = 0; i < e.files.length; i++) {
if (e.files[i].size < 1048576) {
var attachmentName = e.files[i].name;
var attachment = { id: i, citationId: bindViewModel.selected.id, attachmentName: attachmentName, attachmentUrl: reader.result };
dataModel.push(attachment);
if (dataModel[0].attachmentName == "" && dataModel[0].attachmentUrl == "") {
dataModel.splice($.inArray(dataModel[0], dataModel), 1);
}
uploadImg.src = reader.result;
reader.readAsDataURL(e.files[i].rawFile);
}
else {
app.ShowNotifications("Error", 'The ' + e.files[i].name + ' size greater than 1MB. \r\n Maximum allowed file size is 1MB.', "error");
}
}
};
};
}
Any one can have to help me?

Filereader read file using correct encoding when read as readAsArrayBuffer

I am working on reading .csv /xlsx file uploaded using javaScript and get the result as array containing each row . I was able to read the file and get data using FileReader and SheetJs with following code.
// code for the new excel reader
$scope.do_file = function(files)
{
$scope.fileContent = [];
var X = XLSX;
var global_wb;
var f = files[0];
var reader = new FileReader();
reader.onload = function(e)
{
var data = e.target.result;console.log(data);
global_wb = X.read(data, {type: 'array'});
var output = "";
var result = {};
global_wb.SheetNames.forEach(function(sheetName) {
var roa = X.utils.sheet_to_json(global_wb.Sheets[sheetName], {header:1});
if(roa.length) result[sheetName] = roa;
});
$scope.fileContent = result["Sheet1"];
if(!result["Sheet1"])
{
$scope.fileContent = result["contacts"].filter(function(el) { return typeof el != "object" || Array.isArray(el) || Object.keys(el).length > 0; });
}
};
reader.readAsArrayBuffer(f);
};
For reading most of the files the code works , but when file containing Hebrew text with Windows-1255 encoding i get gibberish data.
Looking in for more options i tried to read the file as text using reader.readAsText and change the encoding as necessary , check the following code:
function is_Hebrew(data)
{
var position = data.search(/[\u0590-\u05FF]/);
return position >= 0;
}
$scope.do_file = function(files)
{
var fullResult = [];
var file =files[0];
var reader = new FileReader();
reader.onload = function(e){
var data = e.target.result;
if(!is_Hebrew(data.toString()))
{
reader.readAsText(file,'ISO-8859-8');
}
};
reader.readAsText(file);
reader.onloadend = function(){
var lines = reader.result.split('\r\n');
console.log(lines);
lines.forEach(element => {
var cell = element.split(',');
fullResult.push(cell);
});
console.log(reader);
};
};
but the above code is not suitable as it does not read the file as each row identifying each cell. if any one of cell contains string with coma separated value (for example if a cell contains a string value such as "25,28,29" ) the array output gives wrong data as it considers each values as each cell.
So i decided to stick with first method but i am not able to change the encoding .Is there a possible way to change encoding in the first code where i have used the readAsArrayBuffer to read the file data ?

After going through lot of possible solutions i found that answer to the above question was to combine the above two methods. The first method for reading the xlsx files and second method for reading csv files. Also i have used an additional javaScript library called papaparse in the second method to solve the problem of reading data in each cell
$scope.is_Hebrew = function($data){
var position = $data.search(/[\u0590-\u05FF]/);
return position >= 0;
}
// code for the new excel reader
$scope.do_file = function(files)
{
var config = {
delimiter: "", // auto-detect
newline: "", // auto-detect
quoteChar: '"',
escapeChar: '"',
header: false,
trimHeader: false,
dynamicTyping: false,
preview: 0,
encoding: "",
worker: false,
comments: false,
step: undefined,
complete: undefined,
error: undefined,
download: false,
skipEmptyLines: false,
chunk: undefined,
fastMode: undefined,
beforeFirstChunk: undefined,
withCredentials: undefined
};
$scope.fileContent = [];
var f = files[0];
var fileExtension = f.name.replace(/^.*\./, '');
if(fileExtension == 'xlsx')
{
var X = XLSX;
var global_wb;
var reader = new FileReader();
reader.onload = function(e)
{
var data = e.target.result;
global_wb = X.read(data, {type: 'array'});
var result = {};
global_wb.SheetNames.forEach(function(sheetName) {
var roa = X.utils.sheet_to_json(global_wb.Sheets[sheetName], {header:1});
if(roa.length) result[sheetName] = roa;
});
$scope.fileContent = result["Sheet1"];
if(!result["Sheet1"])
{
$scope.fileContent = result["contacts"].filter(function(el) { return typeof el != "object" || Array.isArray(el) || Object.keys(el).length > 0; });
}
};
reader.readAsArrayBuffer(f);
}
else if(fileExtension == 'csv')
{
var reader = new FileReader();
reader.onload = function(e)
{
var data = e.target.result;
console.log(f);
console.log($scope.is_Hebrew(data.toString()));
if(!$scope.is_Hebrew(data.toString()))
{
reader.readAsText(f,'ISO-8859-8');
}
};
reader.readAsText(f);
reader.onloadend = function(e){
var c = Papa.parse(reader.result,[ config])
console.log(c);
$scope.fileContent = c["data"].filter(function(el) { return typeof el != "object" || Array.isArray(el) || Object.keys(el).length > 0; });
};
}
else
{
alert("File Not supported!");
}
$scope.fileContent.push([]);
};

How do I loop through a file, byte by byte, in JavaScript?

I need some help getting my head around how the file is accessed in JavaScript to do some operations on it.
I would like to loop through a file byte by byte using JavaScript.
I can already select which file I would like to read. And I can read preset byte of the file.
I've found this nice example on how to read a slice of a file here:
http://www.html5rocks.com/en/tutorials/file/dndfiles/
Here is the snippet of code which I'm playing with:
<style>
#byte_content {
margin: 5px 0;
max-height: 100px;
overflow-y: auto;
overflow-x: hidden;
}
#byte_range { margin-top: 5px; }
</style>
<input type="file" id="files" name="file" /> Read bytes:
<span class="readBytesButtons">
<button data-startbyte="0" data-endbyte="4">1-5</button>
<button data-startbyte="5" data-endbyte="14">6-15</button>
<button data-startbyte="6" data-endbyte="7">7-8</button>
<button>entire file</button>
</span>
<div id="byte_range"></div>
<div id="byte_content"></div>
<script>
function readBlob(opt_startByte, opt_stopByte) {
var files = document.getElementById('files').files;
if (!files.length) {
alert('Please select a file!');
return;
}
var file = files[0];
var start = parseInt(opt_startByte) || 0;
var stop = parseInt(opt_stopByte) || file.size - 1;
var reader = new FileReader();
// If we use onloadend, we need to check the readyState.
reader.onloadend = function(evt) {
if (evt.target.readyState == FileReader.DONE) { // DONE == 2
document.getElementById('byte_content').textContent = evt.target.result;
document.getElementById('byte_range').textContent =
['Read bytes: ', start + 1, ' - ', stop + 1,
' of ', file.size, ' byte file'].join('');
}
};
var blob = file.slice(start, stop + 1);
reader.readAsBinaryString(blob);
}
document.querySelector('.readBytesButtons').addEventListener('click', function(evt) {
if (evt.target.tagName.toLowerCase() == 'button') {
var startByte = evt.target.getAttribute('data-startbyte');
var endByte = evt.target.getAttribute('data-endbyte');
readBlob(startByte, endByte);
}
}, false);
</script>
Now I would like to loop through the file, four bytes at a time, but cannot seem to figure out how to do that. The reader does not seem to allow me to read more than once.
Once I can read from the file more than once, I should be able to iterate through it quite easily with something like this:
while( placemark != fileSize-4 ){
output = file.slice(placemark, placemark + 4);
console.log(output);
placemark = placemark + 5;
}
Thanks in advance!
Here is a link to a jsFiddle and plnkr version

I'm not sure it is what you wanted but maybe it can help, and anyway I had fun.
I tried setting reader and file vars as global :
var reader = new FileReader(), step = 4, stop = step, start = 0, file;
document.getElementById('files').addEventListener('change', load, true);
function load() {
var files = document.getElementById('files').files;
file = files[0];
reader.onloadend = function(evt) {
if (evt.target.readyState == FileReader.DONE) {
var result = evt.target.result;
document.getElementById('byte_content').textContent += result;
document.getElementById('byte_range').textContent = ['Read bytes: ', start, ' - ', start+result.length,
' of ', file.size, ' byte file'
].join('');
}
}
}
function next() {
if (!file) {
alert('Please select a file!');
return;
}
var blob = file.slice(start, stop);
reader.readAsBinaryString(blob);
start+= step;
stop = start+step;
}
function loop() {
if (!file) {
alert('Please select a file!');
return;
}
if (start < file.size) {
next();
setTimeout(loop, 50);
}
}
<input type="file" id="files" name="file" />Read bytes:
<span class="readBytesButtons">
<button onclick="next()">next</button>
<button onclick="loop()">loop</button>
</span>
<div id="byte_range"></div>
<div id="byte_content"></div>

I'd read the blob as an ArrayBuffer and use a DataView to read through the data
function readBlob(opt_startByte, opt_stopByte) {
var files = document.getElementById('files').files;
if (!files.length) {
alert('Please select a file!');
return;
}
var file = files[0];
var start = parseInt(opt_startByte) || 0;
var stop = parseInt(opt_stopByte) || file.size - 1;
var reader = new FileReader();
reader.onload = function(evt) {
var placemark = 0, dv = new DataView(this.result), limit = dv.byteLength - 4, output;
while( placemark <= limit ){
output = dv.getUint32(placemark);
console.log(' 0x'+("00000000" + output.toString(16)).slice(-8));
placemark += 4;
}
};
var blob = file.slice(start, stop + 1);
reader.readAsArrayBuffer(blob);
}
<input type="file" id="files" onchange="readBlob(0, 100)">

In the onload handler of FileReader, convert the result to string (toString()), then read 4 chars at a time with the string's slice method.
var contents = null;
reader.onload = function(){
contents = reader.result.toString();
}
var startByte = 0;
// read 4 bytes at a time
var step = 4;
// actual reading (doesn't alter the contents object)
console.log(contents.slice(startByte, step))
// update the next startByte position
startByte += step;

Asynchronous execution in javascript any solution to control execution?

I need a solution to control code execution in javascript.I want code on next line should not be executed unless the code on current line is completely executed.
Is there any solution?
function handleFileSelect(evt) {
var files = evt.target.files;
for (var i = 0; i < files.length; i++) {
alert("for");
f = files[i];
fileExtension = f.name.split('.').pop();
if(fileExtension != 'kml' && fileExtension !='kmz' && fileExtension != 'csv'){
alert('Unsupported file type ' + f.type + '(' + fileExtension + ')');
return;
}
var fileReaderkmlcsv = new FileReader();
fileReaderkmlcsv.onloadend = loadend;
fileReaderkmlcsv.onerror = function(event) {
alert("ERROR: " + event.target.error.code);
};
fileReaderkmlcsv.readAsText(f);
} //- end for
} //handleFileSelect
function loadend(theFile) {
alert("loadend");
//code for processing my files
}

The issue is that loadend is running as soon as any one of the FileReaders has completed loading. You'll need to redesign the code to wait for all 3 of them to finish, something like:
function handleFileSelect(evt) {
var files = evt.target.files;
var fileReaders = [];
var loadCount = 0;
for (var i = 0; i < files.length; i++) {
f = files[i];
fileExtension = f.name.split('.').pop();
if(fileExtension != 'kml' && fileExtension !='kmz' && fileExtension != 'csv'){
alert('Unsupported file type ' + f.type + '(' + fileExtension + ')');
return;
}
function fileLoaded() {
loadCount++;
//Check if we've loaded all the files
if (loadCount == files.length) {
loadend(fileReaders);
}
}
var fileReaderkmlcsv = new FileReader();
fileReaderkmlcsv.onloadend = fileLoaded;
fileReaderkmlcsv.onerror = function(event) {
alert("ERROR: " + event.target.error.code);
};
fileReaderkmlcsv.readAsText(f);
fileReaders.push(fileReaderkmlcsv);
}
}
function loadend(files) {
//files now contains an array of completed FileReader objects
}
Note that I don't have direct experience of the FileReader object itself - if onloadend doesn't fire if an error occurs, you'll need to put similar logic in the onerror event as well to make sure that the loadCount variable still gets incremented/checked etc.