jquery fileDownload: download multiple files into chunks - javascript

I'm using https://github.com/johnculviner/jquery.fileDownload to download an array of files. It seems working:
for (var index = 0; index < files.length; ++index) {
var file = files[index];
$.fileDownload(file)
.done(function() {
alert("Done " + file);
}
)
.fail(function() {
alert("Fail " + file);
}
);
}
How can I download the files in chunks of 5 files at a time?
The files are downloading, but no alert is showing for the moment. So I think maybe I have a problem here.
When trying to download 100 files, for example, I want to download 5, then other 5 etc until done for all. Is this possible?

Download first file
var firstfile = files[0]; //first file id
var downloadresult = downloadFileEx(firstfile);
Then chain other files with jquery then and use array.slice to pick the chunk
var begin=1; // start from second file
var end=5;
do {
for (var index = begin; index < files.slice(begin, end).length; i++) {
var file = files[index];
(function (index) {
downloadresult = downloadresult.then(function() {
return downloadFileEx(array[index]);
});
}(index));
}
index++;
}
begin = begin +5;
end = end + 5;
}
while (end < files.length);
write the file download functiom
function downloadFileEx(file) {
return $.fileDownload(file)
.done(function() {
alert("Done " + file); // remove this 100 alerts will be blocked by browser, maintain an array of files
}
)
.fail(function() {
alert("Fail " + file);
}
);
}

Related

nodejs - replace a string everywhere in a large file

I have some huge files which are difficult to read in memory. I need to read each line and then replace double quotes if found and edit the same file. Right now, I am reading the file line by line, storing in an array and overwriting the same file. But, that's giving memory issue for big files. Any pointers ?
Here is my present implementation :
var allData = fs.readFileSync(fileName, { encoding: 'utf8' }).toString().split("\n");
var finalString = "";
for (i in allData) {
allData[i] = allData[i].replace(/"/g, '""');
finalString = finalString.concat(allData[i]);
finalString = finalString.concat("\n");
}
fs.writeFileSync(fileName, finalString);
Is there a way to edit by reading one line at a time and changing that in the file?
I have seen the similar question with scramjet, but that gives an error and is not compatible with all nodejs versions : node.js modify file data stream?
After going through a lot of answers, this worked for me which took care of the required synchronous and asynchronous behaviour, large file and keeping the name same.
function format_file(fileName) {
return new Promise((resolve, reject) => {
if (fs.existsSync(fileName)) {
var fields = fileName.split('/');
var tempFile = "";
var arrayLength = fields.length;
for (var i = 0; i < arrayLength - 1; i++) {
tempFile = tempFile + fields[i] + "/";
}
tempFile = tempFile + "tempFile" + fields[arrayLength - 1];
console.log("tempfile name is : " + tempFile + " actualFileName is :" + fileName);
var processStream = new ProcessStream();
fs.createReadStream(fileName, { bufferSize: 128 * 4096 })
.pipe(processStream)
.pipe(fs.createWriteStream(tempFile)).on('finish', function() { // finished
fs.renameSync(tempFile, fileName);
console.log('done encrypting');
resolve('done');
});
} else {
reject('path not found')
}
});
}

Adding Multiple Files into a Zip File not Working

I'm using JSZip to add selected files into a single zip file.
The issue is only the last selected file is being added to the zip (which is also corrupted), see code below:
var zip = new JSZip();
var items = '';
var count = 0;
var zipName = 'resources.zip';
$$('input[type=checkbox]').each(function(e){
if(e.checked){
if(e.id!='select-all'){
items = items + "'" + e.getAttribute('data-href') + "'|" + e.getAttribute('data-file') + ",";
}
}
});
if(items!=''){
items = items.slice(0,-1)
var tmp = items.split(',');
for(i=0;i<tmp.length;i++){
var item = tmp[i].split('|');
var url = item[0];
var filename = item[1];
JSZipUtils.getBinaryContent(url, function (err, data) {
if(err) {
throw err; // or handle the error
}
zip.file(filename, data, {binary:true});
count++;
if(count == tmp.length) {
zip.generateAsync({type:'blob'}).then(function(content) {
saveAs(content, zipName);
});
}
});
}
}
Can anyone see the issue with my code?
Fixed it. Issue was I wasn't putting the urls into an array correctly.
Corrected line:
items.push(e.getAttribute('data-href') + '|' + e.getAttribute('data-file'));

Multifile upload as chunks

I'm setting up multi file upload in client-side.Below are the steps, I need to done:
User will upload multiple files at once. Ex: 5 files at a time.
Each file size is vary and large files. Ex: File1.size = 800mb
Slice that large files into chunks. Ex: chunk_size = 50mb
Each chunk will be send to backend and will get response from it.
I'm able to do it for single file upload, like splicing large file chunks.But I'm unable to do it for multiple files uploaded at a time.
Below is the code I tried:
var that = this;
var files = that.files || [];
var url = that.config.url;
var fileKeyUpload = that.config.fileKeyUpload;
for(var i=0; i < files.length; i++) { //multiple files loop
var start = 0; //chunk start as 0
var bytes_per_chunk = 52428800; // 50 MB per chunk
var blob = files[i];
var size = blob.size;
var end = bytes_per_chunk;
var getStatus = function() {
var nextChunk = start + bytes_per_chunk;
var percentage = nextChunk > size ? 100 : Math.round((start / size) * 100); // calculating percentage
uploadFile(blob.slice(start, end), files[key], start, percentage).then(function() {
if(start < size && bytes_per_chunk < size){
start = end;
end = start + bytes_per_chunk;
start < size ? getStatus() : null;
}
});
}
return getStatus();
}
function uploadFile(blob, file, offset, completedPrecentile) {
return new Promise(function(resolve) {
var xhr = new XMLHttpRequest();
xhr.open("POST", url);
xhr.setRequestHeader("Accept", "*/*");
xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest");
xhr.onreadystatechange = function() {
if (xhr.readyState === 4 && xhr.status === 200) {
if(that.config.hasOwnProperty("onFileUploadComplete")){
that.config.onFileUploadComplete(xhr, true, completedPrecentile, file);
resolve();
}
}
};
xhr.send(blob);
});
};
This is what I tried, it works for single file, Any help on how to achieve multiple file upload with slicing large files into chunks.
Unable to post working demo since local api is used.

Firefox Addon. How to determine is the element of the array a file or a directory?

I am trying to get a list of files and directories in Firefox Add-ons.
var io_file = require("sdk/io/file");
var HomePath = require('sdk/system').pathFor("Home");
var list_files = io_file.list(HomePath);
How to determine is the element of the array a file or a directory?
Do console.log('debug:', list_files) and then press Ctrl + Shift + J and then click on the link next to list_files, then it opens in varaible viewer and you can explore it there.
like this, notice the "Browser Console" box and see the "Properties" section below, tha comes up when I clicked on "[object MouseEvent]"s you see in the logs above.
const fileIO = require("sdk/io/file");
let path = "/Users/Work/";
let list = fileIO.list(path);
for (i = 0; i < list.length; i++) {
let item = fileIO.join(path, list[i]);
if (fileIO.isFile(item)) {
console.log(item + " is a file");
}
else {
console.log(item + " is a directory");
}
}
or
const {Cu} = require("chrome");
const {TextEncoder, OS} = Cu.import("resource://gre/modules/osfile.jsm", {});
var iterator = new OS.File.DirectoryIterator('/home/user');
var promise = iterator.forEach(
function onEntry(entry) {
if (entry.isDir) { // is Directory
console.log(JSON.stringify(entry));
} else { // is File
console.log(JSON.stringify(entry));
}
}
);
promise.then(
function onSuccess() {
iterator.close();
return subdirs;
},
function onFailure(reason) {
iterator.close();
throw reason;
}
);

javaScript file loads in wrong order

I'm new to javaScript and am trying to load a CSV or TXT file into the browser.
When the file is selected an event handler displays the file name and details, once the user hits the load button the script should double check the file extension, load the file then carry out some further checks on the file.
My problem is that the file load function seems to always be called last meaning the other checks happen first.
The file is held here: http://bananamountain.net/project/20140703pm/file-loader2.html
Code pasted below:
</head>
<body>
<script>
// Check for the various File API support.
if (window.File && window.FileReader && window.FileList && window.Blob) {
// Great success! All the File APIs are supported.
} else {
alert('The File APIs are not fully supported in this browser.');
}
</script>
<h3>File Load test</h3>
<p>Use only test-data-csv.csv just now</p>
<input type="file" id="file" name="file" required="required" accept=".csv, .txt" />
<button onclick="handleFileLoad()">Load button</button>
<output id="list"></output>
<script>
// global variables
var content;
var fileName;
var splitString = ",";
var rows = new Array();
var headerRow = new Array();
var values = new Array();
function handleFileLoad() {
//var suitableFileType = checkFileType();
//document.write("<strong>Suitable file type: " + suitableFileType + "</strong><br />");
loadFile();
var suitableContent = checkFileContent();
document.write("<strong>Suitable file content: " + suitableContent + "</strong><br />");
}
function checkFileType() {
document.write("inside checkFileType<br/>");
// var testFile = fileName.split(".")[1].toUpperCase();
// document.write("file extension is '" + testFile+ "'<br />");
if ((fileName.split(".")[1].toUpperCase() === "CSV")) {
document.write('suitable file selected<br/>');
return (true);
} else if (fileName.split(".")[1].toUpperCase() === "TXT") {
document.write('suitable file selected<br/>');
return (true);
}else {
document.write('Invalid file format! \nPlease select a suitable .txt or .csv file<br/>');
return (false);
}
} // end of checkFileType - tested WORKING
function loadFile() {
// checkFileType();
var file = document.getElementById("file").files[0];
var reader = new FileReader();
var link_reg = /(http:\/\/|https:\/\/)/i;
reader.onload = function(file) {
// content = reader.result;
content = file.target.result;
rows = file.target.result.split(/[\r\n|\n]+/);
for (var i=0; i<rows.length; i++) {
document.write("row found at line " + i + " is " + rows[i] +".<br/>");
}
};
reader.readAsText(file);
/*
var suitableFileType = checkFileType();
document.write("<strong>Suitable file type: " + suitableFileType + "</strong><br />");
var suitableContent = checkFileContent();
document.write("<strong>Suitable file content: " + suitableContent + "</strong><br />");
var splitStringFound = getSplitString();
document.write("<strong>Split string found: " + splitStringFound + "</strong><br />");
document.write("<strong>Split String: " + splitString + "</strong><br/>");
var replacedHeaders = checkHeaderRow();
document.write("<strong>Header row complete<br />" + replacedHeaders +" headers replaced</strong><br/>");
document.write(content);
document.write(fileName);
document.write(splitString);
document.write(rows);
document.write( headerRow);
document.write(values);*/
return;
}
function checkFileContent() {
document.write("inside check file content<br/>");
// check for file content
// identifies blank lines and deletes them
// checking content of rows
for (var i=0;i<rows.length;i++) {
document.write ("Row " + i + " is " + rows[i]);
}
var filteredArr = rows.filter(function (val) {
return !(val === "" || typeof val == "undefined" || val === null || val === ",," || val === "\t\t");
});
// identifies empty file (e.g. all blank lines deleted)
if (filteredArr.length === 0) {
document.write("Empty file - no data found <br/>");
rows = filteredArr;
return false;
// check for row deletions
} else if (rows.length < filteredArr.length) {
rows = filteredArr;
document.write("blank rows deleted - " + (rows.length - filteredArr.length) + " rows remaining. <br/>");
return ("deletions");
} else {
document.write("No blank rows <br/>");
return true;
}
} // end of check file content - empty file tested, file with one line tested
function checkHeaderRow() {
// check for header row
// words in first non-empty row
var replaceCount = 0;
var checkArray = rows[0].split(splitString);
for (var i = 1; i < checkArray.length; i++) {
// start at array[1] as array[0] not likely to be a header value
// loop through inserting non numeric values into headerRow array
if (isNaN(checkArray[i])) {
headerRow[i - 1] = checkArray[i];
// need a flag to remove this from file once it has been done
replaceCount++;
} else {
headerRow[i - 1] = "Risk " + i;
}
}
// if non numeric values in array[1] delete rows[0]
// so the header row is not included with the data set
if (isNaN(checkArray[1])) {
rows[0] = null;
}
return (replaceCount);
} // end of checkHeaderRow works for all non-numeric, all numeric and mixed
function getSplitString() {
// call countCharacter to return count of comma and tab characters in first five lines
var tabCount = countCharacter("\t");
var commaCount = countCharacter(",");
// compare tabCount and commaCount values
if (tabCount === 0 && commaCount === 0) {
document.write("Cannot detect the value seperator,\n please ammend file to seperate values with tabs or commas");
return false;
}
else if (tabCount === commaCount) {
splitString = prompt("Cannot detect the value seperator,\n please input \"\\t\" for tabs or \",\" for commas");
if ((splitString === null) || (splitString != '\t') || (splitString != ',')) {
document.write("please check file and try again<br/>");
splitString = ',';
return false;
} // NOT WORKING
else {
return true;
}
} else if (tabCount>commaCount) {
splitString = "\t";
if (commaCount!=0) {
document.write("tab character selected as value seperator.<br/>");
// alert as this may not be the case
}
return true;
} else {
splitString=",";
if (tabCount!=0){
document.write("tab character selected as value seperator.<br/>");
// alert as this may not be the case
}
return true;
}
} // end of getSplitString - NOT FULLY WORKING
function splitRows() {
// what if rows is now empty? (e.g. header row only in file)
if (rows[0] != null) {
for (var i=0; i<rows.length;i++) {
values.push(rows[i].split(splitString));
}
return values;
} else {
return false;
}
} // end of splitRows fully working
function checkEmptyCells () {
for (var i=0; i<values.length; i++) {
for (var j=0; j<values[i].length; j++)
if (!((values[i][j] === "") || (typeof values[i][j] == "undefined") || (values[i][j] === null) || (values[i][j] === ",,") || (values[i][j] === "\t\t"))) {
// remove line values[i][j]
document.write("in here");
}
}
} // NOT FININSHED - STOPPED HERE
function countCharacter (character) {
// count the instances of a specified character over first 5 lines (or length of rows array)
// number of rows to loop through
var loopCount=0;
var characterCount=0;
if (rows.length < 5) {
loopCount = rows.length;
} else {
loopCount = 5;
}
for (var count=0; count < loopCount; count++) {
characterCount += rows[count].split(character).length-1;
}
return characterCount;
} // End of countCharacter - WORKING - TESTED
function handleFileSelect(evt) {
var files = evt.target.files; // FileList object
// files is a FileList of File objects. List some properties.
var output = [];
for (var i = 0, f; f = files[i]; i++) { // THIS IS NOT NEEDING TO BE IN A LOOP
output.push('<strong>', escape(f.name), '</strong> ', ' - ',
f.size, ' bytes, last modified: ',
f.lastModifiedDate ? f.lastModifiedDate.toLocaleDateString() : 'n/a',
'');
fileName = escape(f.name);
}
document.getElementById('list').innerHTML = '<div class="file-name">' + output.join('') + '</div>';
}
document.getElementById('file').addEventListener('change', handleFileSelect, false);
</script>
</body>
</html>
So first of all I think I asked the question wrong and it was for that reason that there were limited responses. I'm adding the answer so that if anyone has the same issue and comes across this that it can help them.
The problem was not in the html file loading but a file being loaded through javascript. The script carried out some checks on the file, loaded the file and then carried out further checks on the contents of the file.
This was all happening correctly however javascript does a thing called asyncronious loading where it calls the functions in turn but moves to the next function before the current function has finished doing what it is doing.
Imagine you go to a bar, order drinks, pay for your drinks and go to your table. Javascript would do this but without the normal pauses of waiting to get served, the useing of the drinks and getting change.
Essentially my code was going back to the table without drinks (or checking the contents of the file without it finishing loading).
To fix it I put a time out in, this probably isn't the best as the load speed will depend on the size of the file. However it works for just now and allows me to get on with other stuff.
A snippet of the working code is as follows:
{
function handleFileLoad() {
if (checkFileType()) {
values = [];
loadFile();
} else {
alert("Invalid file format! \nPlease select a suitable .txt or .csv file<br/>");
return;
}
//setTimeout(fileContentChecks(), 1000);
if (!setTimeout(fileContentChecks(), 1000)) {
return;
} else {
setData(); //PROBABLY PUT THESE IN A FUNCTION OR TWO
setComboLists(); //SO THESE CAN BE CALLED LATER TO UPDATE PAGE
UpdateAssetList();
UpdateXAxisList();
UpdateYAxisList();
UpdateTable();
}
}
function checkFileType() { // CHECK FILE NAME EXTENSION
if ((fileName.split(".")[1].toUpperCase() === "CSV")) {
return (true);
} else if (fileName.split(".")[1].toUpperCase() === "TXT") {
return (true);
} else {
return (false);
}
} // end of checkFileType - tested WORKING
function loadFile() { // LOADS FILE AND SPLITS INTO ROWS
var file = document.getElementById("file").files[0];
var reader = new FileReader();
var link_reg = /(http:\/\/|https:\/\/)/i;
reader.onload = function(file) {
content = file.target.result;
rows = file.target.result.split(/[\r\n|\n]+/);
};
reader.readAsText(file);
// NEEDS TIMEOUT HERE.....
return;
} // end of loadFile - TESTED WORKS WHEN STEPPING THROUGH - NEEDS TIMEOUT
function fileContentChecks() {
if (checkFileContent()) {
if (getSplitString()) {
checkHeaderRow();
} else {
alert("Seperator value not found"); // not sure if this is required?
return false;
}
} else {
alert("File contents not verified, please check file and try again.");
return false;
}

Categories

Resources