Problem Statement:
I am getting a text file where the byte array of a binary file is stored in comma-separated values in a single line.
for Eg: 82,19,176,106,0,0,0,4,247,35,186,20,87,143,18,120,44,76,100
The string is very long and everything is in a single line , i have no control on this because it depends on binary file size.
I have to read this byte array and convert it back to the original binary file.
Implemented Logic:
using Node.js and FS
var instream = fs.createReadStream('stream1.txt',{ highWaterMark: 1 * 1024 , encoding: 'utf8' });
instream.on("data", function(line) {
lineCount++;
var splitArray = line.split(',');
var uintArray = new Uint8Array(splitArray);
chunks.push(uintArray);
console.log(lineCount);
});
instream.on("end", function() {
var fullUint8Array = concatenate(chunks);
fs.writeFile("abc.prt", Buffer.from(fullUint8Array), function (err) {
if (err) {
console.log(err);
} else {
console.log("Done");
}
});
});
I am not able to get the original binary file. It is always getting corrupted.
If I am reading a file in single chunk and try the above solution it will work. But always this cannot be done because if try to convert a very big string array to uint8Array it gives memory error.
But when I read the string in chunks and do I am not able to get the binary file.
I am not able to get what I am doing wrong. Technology to be used Node.JS, javascript.
Updated The Question with samples
This is a sample stream. (stream1.txt)
This is the original binary file which is needed as output after reading stream1.txt.
Link to the files
Code for concatenate
//For joining uInt8Arrays
function concatenate(arrays) {
let totalLength = 0;
for (const arr of arrays) {
totalLength += arr.length;
}
const result = new Uint8Array(totalLength);
let offset = 0;
for (const arr of arrays) {
result.set(arr, offset);
offset += arr.length;
}
return result;
}
I am not able to get the original binary file. It is always getting
corrupted.
No, it's not corrupted. The string was splitted by comma, its unencoded values were put in Uint8Array and later on file is saved with that data.
This is more or less what's happening
let line = "82,19,176,106,0,0,0,4,247,35,186,20,87,143,18,120,44,76,100";
let result = line.split(',').map(pr => String.fromCharCode(Number(pr))).join('');
console.log(result);
// Solution 1
let encoded = line.split('').map(npr => npr.charCodeAt(0));
result = encoded.map(pr => String.fromCharCode(pr)).join('');
console.log(result);
// Solution 2
const encoder = new TextEncoder();
const decoder = new TextDecoder();
encoded = encoder.encode(line);
result = decoder.decode(encoded);
console.log(result);
If you apply code above it might look like this:
const fs = require('fs');
let lineCount = 0;
let chunks = [];
const encoder = new TextEncoder();
function concatenate(chunks) {
return chunks.reduce((acc, chunk) => {
return new Uint8Array([...acc, ...chunk]);
}, new Uint8Array([]));
}
var instream = fs.createReadStream('stream1.txt',{ highWaterMark: 1 * 1024 , encoding: 'utf8' });
instream.on("data", function(line) {
lineCount++;
var splitArray = line.split(',');
var uintArray = encoder.encode(line);
chunks.push(uintArray);
});
instream.on("end", function() {
var fullUint8Array = concatenate(chunks);
fs.writeFile("abc.prt", Buffer.from(fullUint8Array, 'utf-8'), function (err) {
if (err) {
console.log(err);
} else {
console.log("Done");
}
});
});
If I am reading a file in single chunk and try the above solution it
will work. But always this cannot be done because if try to convert a
very big string array to uint8Array it gives memory error.
You can reduce memory footprint by creating write stream and putting data immediately there.
Example
const fs = require('fs');
let lineCount = 0;
let chunks = [];
const encoder = new TextEncoder();
var outputStream = fs.createWriteStream("abc.prt");
var inputStream = fs.createReadStream('stream1.txt',{ highWaterMark: 1 * 1024 , encoding: 'utf8' });
outputStream.on("open", function() {
inputStream.on("data", function(line) {
lineCount++;
var splitArray = line.split(',');
var uintArray = encoder.encode(line);
outputStream.write(uintArray);
});
inputStream.on("end", function() {
outputStream.close();
})
})
If you are reading a file in chunks, you need to adjust your splitting logic to cope with that. Your code probably does produce the corrupt results because an input string like 82,19,176,106,0,0 could be read as 82,19,17+6,106,0,0 or 82,19+,176,106,+0,0.
Instead, you need to make sure that you always read whole byte values. If it is not followed by a comma or the eof, you cannot process it yet. I'd recommend to do this with a Transform stream (see also this article about the technique):
import { createReadStream, createWriteStream } from 'fs';
import { pipeline, Transform } from 'stream';
const parseCommaSeparatedBytes = new Transform({
transform(chunk, encoding, callback) {
const prefix = this.leftover || '';
const string = prefix + chunk.toString();
// TODO: validate inputs to be numeric and in the byte range
const splitArray = string.split(',');
if (splitArray.length)
this.leftover = splitArray.pop();
this.push(new Uint8Array(splitArray));
callback();
},
flush(callback) {
const last = this.leftover || '';
if (last.length)
this.push(new Uint8Array([last]));
callback();
},
});
const instream = createReadStream('stream1.txt', {
highWaterMark: 1024,
encoding: 'utf8'
});
const outstream = createWriteStream('abc.prt');
pipeline(instream, parseCommaSeparatedBytes, outstream, function (err) {
if (err) {
console.error(err);
} else {
console.log("Done");
}
});
How can I find and edit an attribute value in a JavaScript file using nodeJS.
for example: If i have a file called app.js containing following code.
let title = "hello world";
document.getElementById("arr").innerHTML = title;
how can I change the value of title let title = "hello world"; or any other attribute using nodeJS.
Instead of trying to parse/edit JS files why not use something like JSON which is easy to serialize?
Your gulp task can write whatever values are needed to a file:
const fs = require('fs')
fs.writeFileSync('./blah.json', JSON.stringify({ title: "whatever" }))
Then your code references the JSON file
const title = require('./blah.json').title
document.getElementById("arr").innerHTML = title;
I was able to edit a JavaScript file using regex. my code is below:
function updateFile(filename, replacements) {
return new Promise(function(resolve) {
fs.readFile(filename, 'utf-8', function(err, data) {
var regex, replaceStr;
if (err) {
throw (err);
} else {
regex = new RegExp("(\\" + 'let' + "\\s* ]*" + replacements[0].rule + "\\s*=\\s*)([^\\n;}]+)([\\s*;}])");
replaceStr = "$1" + replacements[0].replacer + "$3";
data = data.replace(regex, replaceStr);
}
fs.writeFile(filename, data, 'utf-8', function(err) {
if (err) {
throw (err);
} else {
resolve();
}
});
});
})
}
usage:
var file = src/app.js
var myNewValue = "Hello";
updateFile(file, [{
rule: 'newApp',
replacer: myNewValue
}], function (err) {
sails.log.info((err));
});
this will edit,
let newApp = lol; to let newApp = Hello;
I have a function that scans a directory and creates a JSON file with the audio files metadata. I want it to check if the file already exists and only overwrite if there is any diference between the file that was created from the last time the script was run and the data from the the second time it runs.
This is my code:
var fs = require('fs');
var nodeID3 = require('node-id3');
var path = require('path');
var tracksPath = './public/tracks/';
var dataPath = './public/data/';
fs.readdir(tracksPath,function(err,files){
if(err) {
throw err;
}
//Read the tracks metadata
var tracksMetadata = [];
files.forEach(function(trackName){
var trackFile = nodeID3.read(tracksPath + trackName);
//If the track returns metadata push it to the array
if (trackFile.title && trackFile.artist){
var metadata = {
"filename" : trackName,
"title" : trackFile.title,
"artist" : trackFile.artist
};
tracksMetadata.push(metadata);
}
//If no metadata is found ignore and log it to the console
else if (trackName.charAt(0) != "."){
var filename = {
"filename" : trackName
};
tracksMetadata.push(filename);
console.log(trackName + " doesn't have metadata. Ignoring.");
}
if(fs.existsSync(dataPath + "metadata.json")){
fs.readFile(dataPath + "metadata.json",'utf8', function (err, data){
if (err) throw err;
console.log(JSON.parse(JSON.stringify(data)));
console.log(JSON.parse(JSON.stringify(tracksMetadata)));
console.log(Boolean(JSON.parse(JSON.stringify(data)) == JSON.parse(JSON.stringify(tracksMetadata))));
});
}
});
fs.writeFile(path.join(dataPath, 'metadata.json'),
JSON.stringify(tracksMetadata),'utf8', function(err){
if(err){
throw err;
}
console.log("Tracks Metadata JSON created succesfully");
});
});
Right now I'm only writing to the console a Boolean value that checks wether the data from the file and the data generated by the function are equal and so far I get false.
What should I do?
I'm calling the function getKeywords from another function and got an Unrecheable code detected section and don't understand why. Any help?
var env = require('dotenv').config();
var request = require('request')
var getKeywords = function(){
request.get('URI', //URI IS CORRECT IN MY CODE
function(err, httpResponse, body){
if(err){ //UNREACHABLE CODE DETECTED
console.error("request.post Error:", err);
return false;
} //UNREACHABLE CODE DETECTED
else{
console.log('Im here');
return JSON.parse(httpResponse.body).keywords;
}
});
}
module.export = getKeywords;
Here is the calling code.
var getKeywords = require('./getKeywords.js');
var keywords = new getKeywords();
var env = require('dotenv').config();
var difflib = require('difflib');
var postMention = require('./postMention.js');
var detection = function(obj, i){
var keyword = keywords[i];
var mentionObject = {
//some json
//postMention(mentionObject);
}
}
module.exports = detection;
Some tools have the ability to analyze every call to your function. It may be possible that all the places in your code that call the function you never set err parameter to true.
I am having problem with asynchronous file read and write operations. only the last file is written to the server.
js:
function uploadassignment(req, res){
var path;
var multiparty = require("multiparty");
var form = new multiparty.Form();
console.log(req.query);
var filelength = req.query.filecount;
console.log(filelength);
form.parse(req, function(err, fields, files){
console.log(req.body);
for(i=0;i<filelength;i++){
var img = files.file[i];
console.log(img);
console.log('divide');
var fs = require('fs');
fs.readFile(img.path, function(err, data){
var originalfile = img.originalFilename.split('.');
console.log(originalfile);
var file_ext = originalfile[1];
path = "public/assignments/"+img.originalFilename;
console.log(path);
fs.writeFile(path, data, function(error){
if(error)console.log(error);
});
})
}
});
};
This is a common bug caused by using a loop variable without a closure. By the time the callback for the read operation is invoked, the loop has terminated and the index i points to the last element (and hence your img contains the last file). Create a function (a closure) that accepts the index as the parameter and call this function from the loop:
function blah(i) {
var img = files.file[i];
console.log(img);
console.log('divide');
var fs = require('fs');
fs.readFile(img.path, function(err, data){
var originalfile = img.originalFilename.split('.');
console.log(originalfile);
var file_ext = originalfile[1];
path = "public/assignments/"+img.originalFilename;
console.log(path);
fs.writeFile(path, data, function(error){
if(error)console.log(error);
});
})
}
for(i=0;i<filelength;i++) blah(i);
This isn't quite an answer, but it is too long for a comment.
What is not working? The file reading/writing bit of your code works fine:
var fs = require("fs")
img = {
path: "./test.txt",
originalFilename: "test.txt"
}
fs.readFile(img.path, function(err, data){
if(err)console.log(err);
var originalfile = img.originalFilename.split('.');
console.log(originalfile);
var file_ext = originalfile[1];
path = "public/assignments/"+img.originalFilename;
console.log(path);
fs.writeFile(path, data, function(error){
if(error)console.log(error);
});
})
With a directory structure like:
script.js
text.txt
public
assignments
I think your problem might be that you are assigning "fs" locally, then trying to call it from an async function. That might be why only the last one works (maybe.)
Try moving var fs = require('fs'); to the top of your code.