I'm trying to add an object to a very large JSON file in Node.js (but only if the id doesn't match an existing object). What I have so far:
example JSON file:
[
{
id:123,
text: "some text"
},
{
id:223,
text: "some other text"
}
]
app.js
var fs = require('fs');
var jf = require('jsonfile')
var util = require('util')
var file = 'example.json'
// Example new object
var newThing = {
id: 324,
text: 'more text'
}
// Read the file
jf.readFile(file, function(err, obj) {
// Loop through all the objects in the array
for (i=0;i < obj.length; i++) {
// Check each id against the newThing
if (obj[i].id !== newThing.id) {
found = false;
console.log('thing ' + obj[i].id + ' is different. keep going.');
}else if (obj[i].id == newThing.id){
found = true;
console.log('found it. stopping.');
break;
}
}
// if we can't find it, append it to the file
if(!found){
console.log('could not find it so adding it...');
fs.appendFile(file, ', ' + JSON.stringify(newTweet) + ']', function (err) {
if (err) throw err;
console.log('done!');
});
}
})
This is so close to what I want. The only problem is the trailing ] character at the end of the JSON file. Is there a way to delete it using the file system API or something? Or is there a much easier way to do exactly what I want?
The proper way to handle this is to parse the JSON file, modify the object, and output it again.
var obj = require('file.json');
obj.newThing = 'thing!';
fs.writeFile('file.json', JSON.stringify(obj), function (err) {
console.log(err);
});
For my project I ended up using this code.
function appendJsonToFile(file, entry, key, callback){
if(!_.isObject(entry)){
return callback('Type object expected for param entry', null);
}
fs.readFile(file, 'utf8', function(err, data){
if(err){
return callback(err, null);
}
var json;
try{
json = JSON.parse(data);
} catch(e){
return callback(e, null);
}
if(!_.isArray(json[key])){
return callback('Key "' + key + '" does not point to an array', null);
}
json[key].push(entry);
fs.writeFile(file, JSON.stringify(json), function (err) {
if(err){
return callback(err, null);
}
callback(null, file);
});
});
}
Related
Stackoverflow JS Genius's!
I have an issue with my current project, it's using node's HTTP createServer, using Formidable to parse the body data.
See code below. (http-listener.js)
var listenport = 7200;
const server = http.createServer((req, res) => {
// Set vars ready
var data = '';
var plateImg = '';
var overview1 = '';
var overview2 = '';
new formidable.IncomingForm().parse(req)
// I originally thought it was sent in files, but it isnt, it's fields.
.on('file', function(name, file) {
console.log('Got file:', name);
})
// This is the correct procedure for my issue.
.on('field', function(name, field) {
console.log('Got a field:', name);
if(name.toLowerCase() === "anpr.xml")
{
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(field, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Console log parsed json data.
console.log("Read: "+result.EventNotificationAlert.ANPR.licensePlate);
console.log(result);
data = result;
});
}
if(name.toLowerCase() === "licenseplatepicture.jpg")
{
plateImg = field
// This doesnt work?
// I need to store these fields as an image. ? Is this possible with it being sent as a field and not as a file upload.
// This is the only option I have as I can't control the client sending this data (It's a camera)
fs.writeFile(config.App.ImageDir+'/Plate.jpg', plateImg, function(err) {
if(err)console.log(err);
});
}
if(name.toLowerCase() === "detectionpicture.jpg")
{
if(overview1 == '')
{
overview1 = field;
}
else if(overview2 == '')
{
overview2 = field;
}
else
{
// do nothing else.
console.log("Couldn't send images to variable.");
}
}
})
.on('error', function(err) {
alert(err);
})
.on('end', function() {
// Once finished, send to ANPR data to function to handle data and insert to database. WORKS
// Call anpr function.
ANPR_ListenData(data, plateImg, overview1, overview2, function(result) {
if(result.Status > 0)
{
console.log("Accepted by: "+result.Example);
// reset var
data = '';
plateImg = '';
overview1 = '';
overview2 = '';
res.writeHead(200, {'content-type':'text/html'});
res.end();
}
});
});
});
server.listen(listenport, () => {
console.log('ANPR Server listening on port: ' + listenport);
});
Basically the images that are sent in the fields: licenseplatepicture.jpg etc I want to store them directly into my app image directory.
Unfortunately I have no control over how the chunks are sent to this server due to it being a network camera, I simply need to write a procedure.
The full request chunk is quite large so I will upload the file to OneDrive for you to glance at and understand the request.
Any help with this will be appreciated. I've tried everything I can possibly think of, but the file saves unreadable :(. I don't know where else to look or what else I can try, other than what I've already done & tried.
Request Txt File: https://1drv.ms/t/s!AqAIyFoqrBTO6hTwCimcHDHODqEi?e=pxJY00
Ryan.
I fixed this by using Busboy package instead of Formidable.
This is how my http listener looks like using Busboy.
var inspect = util.inspect;
var Busboy = require('busboy');
http.createServer(function(req, res) {
if (req.method === 'POST') {
//vars
var ref = Math.random().toString(36).substring(5) + Math.random().toString(36).substring(2, 15);;
var xml = '';
var parseXml = '';
var over1, over2 = '';
var i = 0;
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
if(filename.toLowerCase() === "licenseplatepicture.jpg")
{
var saveTo = config.App.ImageDir+"/"+ref+"_Plate.jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
if(filename.toLowerCase() === "detectionpicture.jpg")
{
i++;
var saveTo = config.App.ImageDir+"/"+ref+"_Front_"+i+".jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
file.on('data', function(data) {
if(filename.toLowerCase() === "anpr.xml")
{
xml += data;
}
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
});
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
// No fields according to busboy
});
busboy.on('finish', function() {
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(xml, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Set parsed var
parseXml = result;
});
var images = '';
if(i = 2)
{
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"${ref}_Front_2.jpg"}`;
} else {
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"null"}`;
}
// Once parsed, send on to ANPR listen function.
ANPR_ListenData(ref, parseXml, images, function(result) {
if(result.Status == 1)
{
console.log('Data transfered for: '+parseXml.EventNotificationAlert.ANPR.licensePlate);
console.log('Accepted Camera: '+result.Example);
res.writeHead(200, { Connection: 'close', Location: '/' });
res.end();
}
});
});
req.pipe(busboy);
}
}).listen(7200, function() {
console.log('Listening for requests');
});
Hope this helps someone else in the future. Certainly caused me a lot of a wasted time.
Busboy was the better package to use when I was reading into it more, it makes more sense for what I was attempting to achieve.
Ryan :).
All the best.
I am trying to send a soap request with an attachment. Everything works fine except that the attachment i send is always of zero bytes. The soap server accepts a Base64 encoded file and i had achieved to do it in Java using the code
OutputStream outputStream = new ByteArrayOutputStream()
outputStream.writeTo(fileOutputStream);
Base64.encode(outputStream.toByteArray())//argument passed to the function which sends this to the SOAP API
I want to replicate the same with node but i am unable to do so. Below is the function i am using to achieve this. I am reading some files from the client and trying to send it to the SOAP API. I have marked the place in the code responsible to read and append the data the rest is just for reference.
function createSoapEntryWithAtt(req,response){
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
let filesArr = []
for(objkeys in files){
filesArr.push(files[objkeys])
}
return Promise.all(filesArr.map(item => {
return new Promise((res,rej) => {
var oldpath = item.path;
var newpath = 'C:/user/' + item.name;
**var data = fs.readFileSync(oldpath).toString('base64');
let result = []
for (var i = 0; i < data.length; i += 2)// trying to create a 64bit byte array
result.push('0x' + data[i] + '' + data[i + 1])**
console.log(result)
if(data)
res({ [`${item.name}`]: result })
rej("Error occured")
})
})).then(data => {
let url = config.url
var credentials = {
AuthenticationInfo: {
userName: "user",
password: "passwd"
}
}
let args = {
Notes: "Testing From Node App",
}
let count = 0
for (index in data) {
if (count <= 3) {
**for(keys in data[index]){
//console.log(data[index][keys])
args[`Attachment${++count}_Name`] = keys
args[`Attachment${++count}_Data`] = data[index][keys]//Attaching the file read
}
}**
}
soap.createClient(url, function (err, client) {
client.addSoapHeader(credentials)
client.CreateWorkInfo(args, function (err, res) {
if (err) {
console.log("Error is ----->" + err)
} else {
console.log("Response is -----> " + res)
response.end();
}
})
})
})
});
}
Please ignore this question .... and thanks and sorry if anyone wasted time on this question. The error was a careless mistake from my side in the line args["Attachment${++count}_Name"] = keys
args["Attachment${++count}_Data"] = data[index][keys]. Here as i am incrementing the count in both lines there is a mismatch in the sense that Attachment name will be 1 and then in the second line Attachment data will be 02 and hence the name does not contain any data.
How can I find and edit an attribute value in a JavaScript file using nodeJS.
for example: If i have a file called app.js containing following code.
let title = "hello world";
document.getElementById("arr").innerHTML = title;
how can I change the value of title let title = "hello world"; or any other attribute using nodeJS.
Instead of trying to parse/edit JS files why not use something like JSON which is easy to serialize?
Your gulp task can write whatever values are needed to a file:
const fs = require('fs')
fs.writeFileSync('./blah.json', JSON.stringify({ title: "whatever" }))
Then your code references the JSON file
const title = require('./blah.json').title
document.getElementById("arr").innerHTML = title;
I was able to edit a JavaScript file using regex. my code is below:
function updateFile(filename, replacements) {
return new Promise(function(resolve) {
fs.readFile(filename, 'utf-8', function(err, data) {
var regex, replaceStr;
if (err) {
throw (err);
} else {
regex = new RegExp("(\\" + 'let' + "\\s* ]*" + replacements[0].rule + "\\s*=\\s*)([^\\n;}]+)([\\s*;}])");
replaceStr = "$1" + replacements[0].replacer + "$3";
data = data.replace(regex, replaceStr);
}
fs.writeFile(filename, data, 'utf-8', function(err) {
if (err) {
throw (err);
} else {
resolve();
}
});
});
})
}
usage:
var file = src/app.js
var myNewValue = "Hello";
updateFile(file, [{
rule: 'newApp',
replacer: myNewValue
}], function (err) {
sails.log.info((err));
});
this will edit,
let newApp = lol; to let newApp = Hello;
I have a function that scans a directory and creates a JSON file with the audio files metadata. I want it to check if the file already exists and only overwrite if there is any diference between the file that was created from the last time the script was run and the data from the the second time it runs.
This is my code:
var fs = require('fs');
var nodeID3 = require('node-id3');
var path = require('path');
var tracksPath = './public/tracks/';
var dataPath = './public/data/';
fs.readdir(tracksPath,function(err,files){
if(err) {
throw err;
}
//Read the tracks metadata
var tracksMetadata = [];
files.forEach(function(trackName){
var trackFile = nodeID3.read(tracksPath + trackName);
//If the track returns metadata push it to the array
if (trackFile.title && trackFile.artist){
var metadata = {
"filename" : trackName,
"title" : trackFile.title,
"artist" : trackFile.artist
};
tracksMetadata.push(metadata);
}
//If no metadata is found ignore and log it to the console
else if (trackName.charAt(0) != "."){
var filename = {
"filename" : trackName
};
tracksMetadata.push(filename);
console.log(trackName + " doesn't have metadata. Ignoring.");
}
if(fs.existsSync(dataPath + "metadata.json")){
fs.readFile(dataPath + "metadata.json",'utf8', function (err, data){
if (err) throw err;
console.log(JSON.parse(JSON.stringify(data)));
console.log(JSON.parse(JSON.stringify(tracksMetadata)));
console.log(Boolean(JSON.parse(JSON.stringify(data)) == JSON.parse(JSON.stringify(tracksMetadata))));
});
}
});
fs.writeFile(path.join(dataPath, 'metadata.json'),
JSON.stringify(tracksMetadata),'utf8', function(err){
if(err){
throw err;
}
console.log("Tracks Metadata JSON created succesfully");
});
});
Right now I'm only writing to the console a Boolean value that checks wether the data from the file and the data generated by the function are equal and so far I get false.
What should I do?
I have a root directory say "A" inside this directory i am having lots of directories say "1","2","3","4","5"........ and in all these subdirectories i have single file called cucumber.json. All i want to do is read the cucumber.json file and get the accumulated result. How can i achieve this using node js.
In the below screen shot my root directory is "cucumber" and inside that i have lot of sub directories. All these sub directories contains a single file named cucumber.json.
Are there any dedicated node package which can make my work easy.
Let me know if any further info is required.
Hi there please try the following (javascript):
// Require filesystem package for IO operations
var fs = require('fs');
// Put the path you are looking for here
var path = "d:\\nodef";
//Call the function defined below
recursiveloop(path, function(err,result){
/* begin processing of each result */
// For each file in the array
for(i=0;i<result.length;i++)
{
//Write the name of the file
console.log('Processing: ' + result[i]);
//Read the file
fs.readFile(result[i], 'utf8', function(err, data){
//If there is an error notify to the console
if(err) console.log('Error: ' + err);
//Parse the json object
var obj = JSON.parse(data);
//Print out contents
console.log('Name: ' + obj.name);
console.log('Position: ' + obj.position);
})
}
});
// Asynchronous function to read folders and files recursively
function recursiveloop(dir, done)
{
var results = [];
fs.readdir(dir, function(err, list){
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
recursiveloop(file, function(err, res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
}