I have some huge files which are difficult to read in memory. I need to read each line and then replace double quotes if found and edit the same file. Right now, I am reading the file line by line, storing in an array and overwriting the same file. But, that's giving memory issue for big files. Any pointers ?
Here is my present implementation :
var allData = fs.readFileSync(fileName, { encoding: 'utf8' }).toString().split("\n");
var finalString = "";
for (i in allData) {
allData[i] = allData[i].replace(/"/g, '""');
finalString = finalString.concat(allData[i]);
finalString = finalString.concat("\n");
}
fs.writeFileSync(fileName, finalString);
Is there a way to edit by reading one line at a time and changing that in the file?
I have seen the similar question with scramjet, but that gives an error and is not compatible with all nodejs versions : node.js modify file data stream?
After going through a lot of answers, this worked for me which took care of the required synchronous and asynchronous behaviour, large file and keeping the name same.
function format_file(fileName) {
return new Promise((resolve, reject) => {
if (fs.existsSync(fileName)) {
var fields = fileName.split('/');
var tempFile = "";
var arrayLength = fields.length;
for (var i = 0; i < arrayLength - 1; i++) {
tempFile = tempFile + fields[i] + "/";
}
tempFile = tempFile + "tempFile" + fields[arrayLength - 1];
console.log("tempfile name is : " + tempFile + " actualFileName is :" + fileName);
var processStream = new ProcessStream();
fs.createReadStream(fileName, { bufferSize: 128 * 4096 })
.pipe(processStream)
.pipe(fs.createWriteStream(tempFile)).on('finish', function() { // finished
fs.renameSync(tempFile, fileName);
console.log('done encrypting');
resolve('done');
});
} else {
reject('path not found')
}
});
}
Related
I'm using JSZip to add selected files into a single zip file.
The issue is only the last selected file is being added to the zip (which is also corrupted), see code below:
var zip = new JSZip();
var items = '';
var count = 0;
var zipName = 'resources.zip';
$$('input[type=checkbox]').each(function(e){
if(e.checked){
if(e.id!='select-all'){
items = items + "'" + e.getAttribute('data-href') + "'|" + e.getAttribute('data-file') + ",";
}
}
});
if(items!=''){
items = items.slice(0,-1)
var tmp = items.split(',');
for(i=0;i<tmp.length;i++){
var item = tmp[i].split('|');
var url = item[0];
var filename = item[1];
JSZipUtils.getBinaryContent(url, function (err, data) {
if(err) {
throw err; // or handle the error
}
zip.file(filename, data, {binary:true});
count++;
if(count == tmp.length) {
zip.generateAsync({type:'blob'}).then(function(content) {
saveAs(content, zipName);
});
}
});
}
}
Can anyone see the issue with my code?
Fixed it. Issue was I wasn't putting the urls into an array correctly.
Corrected line:
items.push(e.getAttribute('data-href') + '|' + e.getAttribute('data-file'));
I am trying to build a file of json data from repeated calls to a restAPI. The final file to be written is the sum of the data received from all the calls. At present the file is being written with contents of the first call then overwritten by the contents of the first + second call (see console output below code).
As I have to make many calls, once the code is working, I would like to only write the file once the request has finished and the json string has been built. Does anyone now how I would go about doing this? Maybe with a callback(?), which I still don't have the hang of, once the requests have finished or the json string has finished being built.
"use strict";
const fs = require('fs');
const request = require('request');
var parse = require('csv-parse');
const path = "../path tocsv.csv";
const pathJSON = "../pathtoJSON.json";
var shapes = "https://url";
var options = {
url: '',
method: 'GET',
accept: "application/json",
json: true,
};
var csvData = [];
var jsonData = "[";
fs.createReadStream(path)
.pipe(parse({delimiter: ','}))
.on('data', function(data) {
csvData.push(data[1]);
})
.on('end',function() {
var start = Date.now();
var records = csvData.length //2212 objects
console.log(records);
var dataLength = 2 //set low at moment
for (var i = 0; i < dataLength; i += 1) {
var url = shapes + csvData[i];
options.url = url; //set url query
request(options, function(error, response, body) {
var time = Date.now() - start;
var s = JSON.stringify(body.response);
console.log( '\n' + (Buffer.byteLength(s)/1000).toFixed(2)+
" kilobytes downloaded in: " + (time/1000) + " sec");
console.log(i)
buildJSON(s);
});
}
function buildJSON(s) {
var newStr = s.substring(1, s .length-1);
jsonData += newStr + ',';
writeFile(jsonData);
}
function writeFile(jsonData) {
fs.writeFile(pathJSON, jsonData, function(err) {
if (err) {
return console.log(err);
} else {
console.log("file complete")
}
});
}
});
128.13 kilobytes downloaded in: 2.796 sec
2
file complete
256.21 kilobytes downloaded in: 3.167 sec
2
file complete
Perhaps writing to the file after all requests are complete will help. In the current code, the writeFile function is called each time a request is completed (which overwrites the file each time)
A quick way to fix this is to count requests (and failures) and write to file only after all the requests are complete.
"use strict";
const fs = require('fs');
const request = require('request');
var parse = require('csv-parse');
const path = "../path tocsv.csv";
const pathJSON = "../pathtoJSON.json";
var shapes = "https://url";
var options = {
url: '',
method: 'GET',
accept: "application/json",
json: true,
};
var csvData = [];
var jsonData = "[";
fs.createReadStream(path)
.pipe(parse({
delimiter: ','
}))
.on('data', function (data) {
csvData.push(data[1]);
})
.on('end', function () {
var start = Date.now();
var records = csvData.length //2212 objects
console.log(records);
var dataLength = 2 //set low at moment
var jsonsDownloaded = 0; // Counter to track complete JSON requests
var jsonsFailed = 0; // Counter to handle failed JSON requests
for (var i = 0; i < dataLength; i += 1) {
var url = shapes + csvData[i];
options.url = url; //set url query
request(options, function (error, response, body) {
if(error){
jsonsFailed++;
writeFile(jsonData);
return;
}
jsonsDownloaded++;
var time = Date.now() - start;
var s = JSON.stringify(body.response);
console.log('\n' + (Buffer.byteLength(s) / 1000).toFixed(2) +
" kilobytes downloaded in: " + (time / 1000) + " sec");
console.log(i)
buildJSON(s);
});
}
function buildJSON(s) {
var newStr = s.substring(1, s.length - 1);
jsonData += newStr + ',';
writeFile(jsonData);
}
function writeFile(jsonData) {
if(dataLength - (jsonsDownloaded + jsonsFailed) > 0){
return;
}
fs.writeFile(pathJSON, jsonData, function (err) {
if (err) {
return console.log(err);
} else {
console.log("file complete")
}
});
}
});
Note:
Requests being fired in quick succession like (2000 requests in a for loop) in my experience does not work well.. Try batching them. Also, doing it this way does not guarantee order (if that is important in your usecase)
An alternative would be to open your file in append mode. You can do this by passing an extra options object with flag set to your fs.writeFile call.
fs.writeFile(pathJSON, jsonData, {
flag: 'a'
}, function (err) {
if (err) {
return console.log(err);
}
});
References:
fs.writeFile Docs
File system flags
I'm trying to convert a .wav file to a byte array string. I need to do this on the back-end targeting the file is becoming an issue.
files.forEach(file => {
let index = files.indexOf(file)
let reader = new FileReader();
reader.readAsArrayBuffer(file);
console.log(reader.result);
reader.onload = function (event) {
let byteArray = new Uint8Array(reader.result);
let FileName = file.name;
let dataAsByteArrayString = byteArray.toString();
var listHtml = $list.html();
The above code uses npm's filereader which says to target a file. I'm having difficulty doing this since this is not a front end drag and drop of the file.
my file that is generated is called "response.wav" how would I convert this file using JavaScript and node extensions? Thanks!
I dont know if this will help but the last project I worked on we parsed a .wav file using the Node buffer API and wrote it using the Node file API.
If you have more questions about the code I can direct you to the person that worked on this file the most. I hope it helps somewhat
https://github.com/IntelliSound/intelliSound-Server/blob/development/lib/sound-data-parser.js
'use strict';
function ParsedWave(buffer) {
const RIFF_HEADER_OFFSET = 0;
const FILE_SIZE_OFFSET = 4;
const RIFF_FORMAT_OFFSET = 8;
const SUBCHUNK1_ID_OFFSET = 12;
const AUDIO_FORMAT_OFFSET = 20;
const NUMBER_OF_CHANNELS_OFFSET = 22;
const SAMPLE_RATE_OFFSET = 24;
const BITS_PER_SAMPLE_OFFSET = 34;
const SUBCHUNK2_ID_OFFSET = 36;
const SUBCHUNK2_SIZE_OFFSET = 40;
const DATA_OFFSET = 44;
this.buffer = buffer;
this.riff = buffer.slice(RIFF_HEADER_OFFSET, RIFF_HEADER_OFFSET + 4).toString('utf8');
this.fileSize = buffer.readUInt32LE(FILE_SIZE_OFFSET);
this.riffType = buffer.slice(RIFF_FORMAT_OFFSET, RIFF_FORMAT_OFFSET + 4).toString('utf8');
this.subChunk1Id = buffer.slice(SUBCHUNK1_ID_OFFSET, SUBCHUNK1_ID_OFFSET + 4).toString('utf8');
this.audioFormat = buffer.readUInt16LE(AUDIO_FORMAT_OFFSET);
this.numberOfChannels = buffer.readUInt16LE(NUMBER_OF_CHANNELS_OFFSET);
this.sampleRate = buffer.readUInt32LE(SAMPLE_RATE_OFFSET);
this.bitsPerSample = buffer.readUInt16LE(BITS_PER_SAMPLE_OFFSET);
this.subChunk2Id = buffer.slice(SUBCHUNK2_ID_OFFSET, SUBCHUNK2_ID_OFFSET + 4).toString('utf8');
this.subChunk2Size = buffer.readUInt32LE(SUBCHUNK2_SIZE_OFFSET);
this.data = buffer.slice(DATA_OFFSET, this.subChunk2Size + DATA_OFFSET);
}
// Andrew - The bufferMapper function is going to accept a parsed wave-file and output
// an array of values corresponding to the data subchunk in a format which can
// be accepted as input to the neural network.
const bufferMapper = parsedWave => {
const SIXTEEN_BIT_ZERO = 32768;
const SIXTEEN_BIT_MAX = 65535;
parsedWave.neuralArray = [];
for (let i = 0; i < parsedWave.data.length; i += 2) {
const sample = parsedWave.data.readInt16LE(i);
const unsignedSample = sample + SIXTEEN_BIT_ZERO;
const sigmoidSample = unsignedSample / SIXTEEN_BIT_MAX;
parsedWave.neuralArray.push(sigmoidSample);
}
return parsedWave;
};
module.exports = data => {
const parsedWaveFile = new ParsedWave(data);
if (parsedWaveFile.riff !== 'RIFF') {
throw new TypeError('incorrect file type, must be RIFF format');
}
if (parsedWaveFile.fileSize > 10000000) {
throw new TypeError('file too large, please limit file size to less than 10MB');
}
if (parsedWaveFile.riffType !== 'WAVE') {
throw new TypeError('file must be a WAVE');
}
if (parsedWaveFile.subChunk1Id !== 'fmt ') {
throw new TypeError('the first subchunk must be fmt');
}
if (parsedWaveFile.audioFormat !== 1) {
throw new TypeError('wave file must be uncompressed linear PCM');
}
if (parsedWaveFile.numberOfChannels > 2) {
throw new TypeError('wave file must have 2 or less channels');
}
if (parsedWaveFile.sampleRate > 48000) {
throw new TypeError('wave file must have sample rate of less than 48k');
}
if (parsedWaveFile.bitsPerSample !== 16) {
throw new TypeError(`file's bit depth must be 16`);
}
if (parsedWaveFile.subChunk2Id !== 'data') {
throw new TypeError('subchunk 2 must be data');
}
const neuralMappedWaveFile = bufferMapper(parsedWaveFile);
return neuralMappedWaveFile;
};
Using the included fs module, you can read in your wav file like so:
const fs = require('fs');
const path = './path/to/my.wav';
fs.readFile(path, (err, data) => {
// Data is a Buffer object
});
For documentation on working with a Node.JS Buffer see here. Now, if you were more interested in the file conversion portion, there are a couple of libraries out there. If you just need the conversion functionality, and not implementing yourself, node-fluent-ffmpeg may work for you. If you want to implement it yourself, this node-wav file may be a good reference (too much to paste here).
If you need to go from Buffer to ArrayBuffer, this SO shows some options.
I'm using https://github.com/johnculviner/jquery.fileDownload to download an array of files. It seems working:
for (var index = 0; index < files.length; ++index) {
var file = files[index];
$.fileDownload(file)
.done(function() {
alert("Done " + file);
}
)
.fail(function() {
alert("Fail " + file);
}
);
}
How can I download the files in chunks of 5 files at a time?
The files are downloading, but no alert is showing for the moment. So I think maybe I have a problem here.
When trying to download 100 files, for example, I want to download 5, then other 5 etc until done for all. Is this possible?
Download first file
var firstfile = files[0]; //first file id
var downloadresult = downloadFileEx(firstfile);
Then chain other files with jquery then and use array.slice to pick the chunk
var begin=1; // start from second file
var end=5;
do {
for (var index = begin; index < files.slice(begin, end).length; i++) {
var file = files[index];
(function (index) {
downloadresult = downloadresult.then(function() {
return downloadFileEx(array[index]);
});
}(index));
}
index++;
}
begin = begin +5;
end = end + 5;
}
while (end < files.length);
write the file download functiom
function downloadFileEx(file) {
return $.fileDownload(file)
.done(function() {
alert("Done " + file); // remove this 100 alerts will be blocked by browser, maintain an array of files
}
)
.fail(function() {
alert("Fail " + file);
}
);
}
I used phogap function for recording audio and uploading audio , file.
Audio File is recording and stored in music folder and I get the path and name of the file by the below function.
function captureSuccess(mediaFiles) {
var i, len;
for (i = 0, len = mediaFiles.length; i < len; i += 1) {
uploadFile(mediaFiles[i]);
}
alert('over');
}
function captureAudio() {
navigator.device.capture.captureAudio(captureSuccess, captureError, { limit: 1, duration: 10 });
}
function uploadFile(mediaFile) {
var ft = new FileTransfer(),
path = mediaFile.fullPath,
name = mediaFile.name;
temp = path; //Assigned path to GLOBAL VARIABLE
temp1 = name;
alert(temp);
}
I call the function to upload the created audio file.
function uplod() {
var options = new FileUploadOptions();
options.fileKey = "file";
options.fileName = name;
options.mimeType = "audio/mpeg";
options.chunkedMode = false;
var ft = new FileTransfer();
ft.upload(temp,
"http:/URL/upload.php",
function (result) {
alert("success" + result.bytesSent + result.responseCode);
console.log('Upload success: ' + result.responseCode);
console.log(result.bytesSent + ' bytes sent');
},
function (error) {
console.log('Error uploading file ' + recordingPath + ': ' + error.code);
},
options);
}
But on running, I get these error, got stucked up, is the error is with URL or PATH ? Unicode ? and path of file is like this
C:/Users/AB/Music/captureAudio (4).mp3
0x80070459 - JavaScript runtime error: No mapping for the Unicode character exists in the target multi-byte code page.
WinRT information: No mapping for the Unicode character exists in the target multi-byte code page.
I got the answer by using Ajax post and by using Formdata() to server there by getting in the form of arrays.