I use request to read a website then parseString to convert and later save as json. The problem is saving. Looks like "for loop" keeps looping and when is saving the information uses the last info from var cta to assign to information. I have try asyn await but it didnt work. thanks in advance.
var fs = require('fs');
var request = require('request');
var parseString = require('xml2js').parseString;
var json = {} ;
var cta = [
{
"_id": 1,
"name": "PRWeb",
"feedtype": "Misc",
"feedaddress": "https://www.prweb.com/rss2/daily.xml",
"status": true
},
{
"_id": 2,
"name": "Business Wire",
"feedtype": "Industrie",
"feedaddress": "https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRVQ==",
"status": false
},
{
"_id": 3,
"name": "News Wire",
"feedtype": "Daily News",
"feedaddress": "https://www.newswire.com/newsroom/rss/custom/all-press-releases",
"status": true
}
];
for (var i = 0; i < cta.length; i++) {
function getDatos(url, callback) {
request(url, function(error, response, data){
callback(error, response, data);
});
}
if (cta[i].status === true) {
console.log("cta: " + cta[i].feedaddress);
agrabar = cta[i];
console.log("agrabar: " + agrabar.feedaddress);
getDatos(agrabar.feedaddress, function(error, response, data){
parseString(data, {explicitArray: false}, function (err, result) {
if (err) {
return console.log('unable to parse XML');
}
json = {rssfeeder: agrabar, feed: result.rss.channel.item};
console.log(json);
fs.appendFile ("output.json", JSON.stringify(json, null, 4), function(err) {
if (err) throw err;
console.log('complete');
});
});
});
}
}
console.log('DONE!!!')
You seem to be mixing asynchronous code with synchronous code. It'd be great if you read more about how (and in what order) callbacks are handled in Javascript.
Example : Your final line of code console.log('DONE!!!') should actually be printed at the last when everything has finished, but when you'll run it, you'll be surprised that it's actually the first line to be printed to console. This is because the function getDatos is an asynchronous function, meaning that it will execute at some later point in time. Your for loop executes synchronously meaning that getDatos will be called thrice in the correct order, but owing to function closure in JS and asynchronicity, getDatos will be called after your final console.log has been done.
Furthermore, it's a good practice to use async await and Promises in modern JS, since it makes reading the code much more easier. I have modified your code to do what you intend to do and here it is. Hope it helps!
var fs = require('fs');
var request = require('request');
var parseString = require('xml2js').parseString;
var cta = [
{
"_id": 1,
"name": "PRWeb",
"feedtype": "Misc",
"feedaddress": "https://www.prweb.com/rss2/daily.xml",
"status": true
},
{
"_id": 2,
"name": "Business Wire",
"feedtype": "Industrie",
"feedaddress": "https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtRVQ==",
"status": false
},
{
"_id": 3,
"name": "News Wire",
"feedtype": "Daily News",
"feedaddress": "https://www.newswire.com/newsroom/rss/custom/all-press-releases",
"status": true
}
];
function getDatos(cta_object) {
if (cta_object.status === false){
return new Promise((resolve, reject) => resolve(false));
}
else {
return new Promise((resolve, reject) => {
request(cta_object.feedaddress, (err, response, data) => {
if (err) reject(err);
else resolve(data);
})
})
.then((data) => {
return new Promise((resolve, reject) => {
parseString(data, {explicitArray: false}, (err, result) => {
if (err) {
console.error("Unable to parse XML!");
return reject(err);
}
else return resolve(result);
});
})
})
.then((result) => {
console.log(result);
return {
'rssfeeder': cta_object,
'feed': result.rss.channel.item
};
})
.then((json) => {
return new Promise((resolve, reject) => {
fs.appendFile ("output.json", JSON.stringify(json, null, 4), function(err) {
if (err) reject(err) ;
console.log("complete!");
resolve(true);
});
})
})
.catch((err) => {
throw(err);
})
}
};
Promise.all(cta.map((cta_obj) => getDatos(cta_obj)))
.then(() => console.log('DONE!!!'))
.catch((err) => {
throw err;
})
Related
I have a JSON file and I want to add a square bracket at the beginning and end of JSON.
eg.
Input
{
"name": "Ram",
"age": 25
},
{
"name": "Laxman",
"age": 24
}
Expected output:
[
{
"name": "Ram",
"age": 25
},
{
"name": "Laxman",
"age": 24
}
]
this is a sample response, I am having a large JSON data in a file.
The best option (in my opinion) would be to open a new reader;
Open a BufferedReader
Append [
Append JSON file
Append ]
From there you can use the BufferedReader or write it into a new file.
So, we need to consider 2 situations here:
The first one is when you're responsible for creating this input file. Then, assuming you already have those objects in an array and just need to save the array itself instead of the individual objects using a for.
const fs = require('fs')
const objs = [ { "name": "Ram","age": 25},{ "name": "Laxman","age": 24} ]
const jsonData = JSON.stringify(objs)
fs.writeFile("inputFile.json", jsonData, (err) => {
if (err) {
console.log(err);
}
});
The second situation if when you don't have control to modify the input file in its creation, and are just transforming the file previously saved. In this case you'll need to completely rewrite the file due to fs limitations for positional "inserts". To do so, read the previous file into a buffer, prepend it with the opening bracket "[" and append the closing one "]" at the end. As follows:
const fs = require('fs')
const filename = 'inputFile.json'
const fileBuffer = fs.readFileSync(filename)
const newBuffer = Buffer.concat([Buffer.from('['), fileBuffer, Buffer.from(']')])
fs.writeFileSync(filename, newBuffer)
I had the same problem and after hours of searching, I found nothing so I wrote a custom code that works like a charm. Hope it helps!:)
const fs = require('fs')
var response = {};
const fsOps = async (params) => {
try {
const path = "tmp/" + params.user + ".json";
const data = params.data;
const chunksNumber = params.chunksNumber;
var chunkID = params.chunkID;
//ON FIRST CHUNK ADD [
if (chunkID === 1) {
fs.appendFile(
path, ("["), 'utf-8', function (err) {
if (err) throw err;
}
);
if (chunksNumber !== 1)
fs.appendFile(
path, JSON.stringify(data, null, 2) + ',', 'utf-8', function (err) {
if (err) throw err;
}
);
}
//WRITE CHUNKS
if (chunkID !== 1 && chunkID < chunksNumber) {
fs.appendFile(
path, JSON.stringify(data, null, 2) + ',', 'utf-8', function (err) {
if (err) throw err;
}
);
}
//ON LAST CHUNK WRITE THE LAST CHUNK AND ADD ]
if (chunkID === chunksNumber) {
console.log("LAST CHUNK")
fs.appendFile(
path, JSON.stringify(data, null, 2), 'utf-8', function (err) {
if (err) throw err;
}
);
//APPEND ] on the end of file
fs.appendFile(
path, ("]"), 'utf-8', function (err) {
if (err) throw err;
}
);
//READ THE FILE
fs.readFile(path, (err, data) => {
if (err) {
console.error(err)
return;
} else {
response = data;
}
})
//DELETE FILE
fs.unlink(path, (err) => {
if (err) {
console.error(err)
return err
}
})
}
//Return object with all the part data
return JSON.parse(response);
} catch (err) {
//IN CASE OF ERROR DELETE FILE
fs.unlink(path, (err) => {
if (err) {
console.error(err)
return err
}
})
return err;
}
}
module.exports = fsOps;
I am calling below restAPI using node JS.
I am using below code.
var Request = require("request");
Request.get("http://localhost:8080/ords/hr/rest-v3/item/Sugar", (error, response, body) => {
if (error) {
return console.dir(error);
}
console.log(response.body);
});
It is giving below output which is correct.
{"items":[{"itemid":101,"itemname":"Sugar","itemcost":32.5}],"hasMore":false,"limit":0,"offset":0,"count":1,"links":[{"rel":"self","href":"http://localhost:8080/ords/hr/rest-v3/item/Sugar"},{"rel":"describedby","href":"http://localhost:8080/ords/hr/metadata-catalog/rest-v3/item/item"}]}
How can I access only itemcost, not entire body. I am new to node so not familiar with many things.
Can you please help me on that.
May be it can help you.
var Request = require("request");
Request.get("http://localhost:8080/ords/hr/rest-v3/item/Sugar", (error, response, body) => {
if (error) {
return console.dir(error);
}
let jsonData = JSON.parse(response.body);
let obj= new Object(jsonData);
obj.items.forEach(itemChild => {
let cost=itemChild.itemcost;
console.log(cost);
})
});
you can iterate the response.body.items and get the costs of all items in the items array like this,
var Request = require("request");
Request.get("http://localhost:8080/ords/hr/rest-v3/item/Sugar", (error, response, body) => {
if (error) {
return console.dir(error);
}
console.log(response.body);
const itemCosts = response.body.items.map(item => item.itemcost);
console.log(itemCosts);
});
Considering your input you can simply do this:
let input = {
"items": [
{
"itemid": 101,
"itemname": "Sugar",
"itemcost": 32.5
}
],
"hasMore": false,
"limit": 0,
"offset": 0,
"count": 1,
"links": [
{
"rel": "self",
"href": "http://localhost:8080/ords/hr/rest-v3/item/Sugar"
},
{
"rel": "describedby",
"href": "http://localhost:8080/ords/hr/metadata-catalog/rest-v3/item/item"
}
]
};
let json = JSON.parse(JSON.stringify(input));
json.items.map(item =>{
console.log(item.itemcost)
})
You can find running solution here
I'm trying to execute two asynchronous functions to DynamoDB.
I will need the return data from both before continuing with the following step which is sending an email that contains their data.
How do I tackle this issue?
I'm using the following code:
var productParams = {
TableName: productsTable,
FilterExpression: 'client = :this_client',
ExpressionAttributeValues: { ':this_client': "someclient" }
};
dynamoClient.scan(productParams, function (err, data) {
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
} else {
console.log("Query succeeded.");
data.Items.forEach(item => {
products.push(item)
});
}
});
var retailerParams = {
TableName: retailersTable,
FilterExpression: 'leadTime = :this_leadTime',
ExpressionAttributeValues: { ':this_leadTime': 42 }
};
dynamoClient.scan(retailerParams, function (err, data) {
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
} else {
console.log("Query succeeded.");
data.Items.forEach(item => {
retailers.push(item)
});
}
});
var email = {
"Source": "somemail#gmail.com",
"Template": "some_template",
"Destination": {
"ToAddresses": ["somemail#gmail.com"]
},
"TemplateData": `{somedata}`
}
await ses.sendTemplatedEmail(email).promise();
You can convert both DynamoDB calls into promises (by chaining scan calls with .promise() calls) and await them using Promise.all before sending the email:
var productParams = {
TableName: productsTable,
FilterExpression: 'client = :this_client',
ExpressionAttributeValues: { ':this_client': "someclient" }
};
const productsPromise = dynamoClient.scan(productParams).promise()
.then(data => {
data.Items.forEach(item => {
products.push(item)
});
})
.catch(err => {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
});
var retailerParams = {
TableName: retailersTable,
FilterExpression: 'leadTime = :this_leadTime',
ExpressionAttributeValues: { ':this_leadTime': 42 }
};
const retailersPromise = dynamoClient.scan(retailerParams).promise()
.then(data => {
console.log("Query succeeded.");
data.Items.forEach(item => {
retailers.push(item)
});
})
.catch(err => {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
});
await Promise.all([
productsPromise,
retailersPromise
]);
var email = {
"Source": "somemail#gmail.com",
"Template": "some_template",
"Destination": {
"ToAddresses": ["somemail#gmail.com"]
},
"TemplateData": `{somedata}`
}
await ses.sendTemplatedEmail(email).promise();
here is my code in which I try to update some documents of my database (MongoDB)
using a for loop. however I want to run the next code after the loop is completed,
as for example, I want to use some variables calculated inside the loop after it finished.
How can I do such using callback, promise, etc?
numPktUpdated = 0;
for (key in InvoicesPUT) {
Invoice.findOneAndUpdate({ InvoiceNumber: InvoicesPUT[key].InvoiceNumber }, InvoicesPUT[key]).exec()
.then((doc) => {
console.log("Update Succeeded")
numPktUpdated = numPktUpdated + 1;
})
.catch((err) => {
return resp.send(JSON.stringify({
"status": "error",
"message": "DB Error while Updating: Wrong Packet"
}));
console.log(err);
})
}
resp.send(numPktUpdated);
Here numPktUpdated = 0 is sent to client, although its real value after the loop is something else.
Thanks.
You should be able to do this with Promise.all():
Promise.all(InvoicesPUT.map(InvoicePUT => {
return Invoice.findOneAndUpdate({
InvoiceNumber: InvoicePUT.InvoiceNumber
}, InvoicePUT).exec()
.then(doc => {
console.log("Update Succeeded");
numPktUpdated += 1;
})
}))
.catch(err => {
return resp.send(JSON.stringify({
"status": "error",
"message": "DB Error while Updating: Wrong Packet"
}));
console.log(err);
})
.then(() => {
// What you want to do after the loop ...
})
Try to put code to the function, then run it with async
async function forLoopFunction(args){//here is a for loop
}
let ret = await forLoopFunction(args);
If you are using an older version of Node v7.6, which brought support for the async / await pattern, you could use a simple Promise:
function updateDocs() {
return new Promise(function(resolve, reject) {
numPktUpdated = 0;
for (key in InvoicesPUT) {
Invoice.findOneAndUpdate({ InvoiceNumber: InvoicesPUT[key].InvoiceNumber }, InvoicesPUT[key]).exec()
.then((doc) => {
console.log("Update Succeeded")
numPktUpdated = numPktUpdated + 1;
resolve(numPktUpdated);
})
.catch((err) => {
console.log(err);
reject(err);
return resp.send(JSON.stringify({
"status": "error",
"message": "DB Error while Updating: Wrong Packet"
}));
})
}
});
}
updateDocs().then(function(numPktUpdated) {
resp.send(numPktUpdated);
})
.catch(function(err) {
// handle error
});
Thanks, It worked with just a minor modification:
Promise.all(InvoicesPUT.map(invoice => {
return Invoice.findOneAndUpdate({InvoiceNumber: invoice.InvoiceNumber}, invoice).exec()
.then((doc) => {
console.log("Update Succeeded")
numPktUpdated = numPktUpdated + 1;
})
}))
.catch((err) => {
return resp.send(JSON.stringify({
"status": "error",
"message": "DB Error while Updating: Wrong Packet"
}));
console.log(err);
})
.then((resolve) => {
return resp.send(JSON.stringify({
"status": "succed",
"LastSavedGUID": 654323,
"SyncDate": 1,
"pagenumber": numPktUpdated
}));
})
I have a big issue with my function in sails.js (v12). I'm trying to get all userDetail using async (v2.3) for deep populating my user info:
UserController.js:
userDetail: function (req, res) {
var currentUserID = authToken.getUserIDFromToken(req);
async.auto({
//Find the User
user: function (cb) {
User
.findOne({ id: req.params.id })
.populate('userFollowing')
.populate('userFollower')
.populate('trips', { sort: 'createdAt DESC' })
.exec(function (err, foundedUser) {
if (err) {
return res.negotiate(err);
}
if (!foundedUser) {
return res.badRequest();
}
// console.log('foundedUser :', foundedUser);
cb(null, foundedUser);
});
},
//Find me
me: function (cb) {
User
.findOne({ id: currentUserID })
.populate('myLikedTrips')
.populate('userFollowing')
.exec(function (err, user) {
var likedTripIDs = _.pluck(user.myLikedTrips, 'id');
var followingUserIDs = _.pluck(user.userFollowing, 'id');
cb(null, { likedTripIDs, followingUserIDs });
});
},
populatedTrip: ['user', function (results, cb) {
Trip.find({ id: _.pluck(results.user.trips, 'id') })
.populate('comments')
.populate('likes')
.exec(function (err, tripsResults) {
if (err) {
return res.negotiate(err);
}
if (!tripsResults) {
return res.badRequest();
}
cb(null, _.indexBy(tripsResults, 'id'));
});
}],
isLiked: ['populatedTrip', 'me', 'user', function (results, cb) {
var me = results.me;
async.map(results.user.trips, function (trip, callback) {
trip = results.populatedTrip[trip.id];
if (_.contains(me.likedTripIDs, trip.id)) {
trip.hasLiked = true;
} else {
trip.hasLiked = false;
}
callback(null, trip);
}, function (err, isLikedTrip) {
if (err) {
return res.negotiate(err);
}
cb(null, isLikedTrip);
});
}]
},
function finish(err, data) {
if (err) {
console.log('err = ', err);
return res.serverError(err);
}
var userFinal = data.user;
//userFinal.trips = data.isLiked;
userFinal.trips = "test";
return res.json(userFinal);
}
);
},
I tried almost everthing to get this fix but nothing is working...
I am able to get my array of trips(data.isLiked) but I couldn't get my userFInal trips.
I try to set string value on the userFinal.trips:
JSON response
{
"trips": [], // <-- my pb is here !!
"userFollower": [
{
"user": "5777fce1eeef472a1d69bafb",
"follower": "57e44a8997974abc646b29ca",
"id": "57efa5cf605b94666aca0f11"
}
],
"userFollowing": [
{
"user": "57e44a8997974abc646b29ca",
"follower": "5777fce1eeef472a1d69bafb",
"id": "5882099b9c0c9543706d74f6"
}
],
"email": "test2#test.com",
"userName": "dany",
"isPrivate": false,
"bio": "Hello",
"id": "5777fce1eeef472a1d69bafb"
}
Question
How should I do to get my array of trips (isLiked) paste to my user trips array?
Why my results is not what I'm expecting to have?
Thank you for your answers.
Use .toJSON() before overwriting any association in model.
Otherwise default toJSON implementation overrides any changes made to model associated data.
var userFinal = data.user.toJSON(); // Use of toJSON
userFinal.trips = data.isLiked;
return res.json(userFinal);
On another note, use JS .map or _.map in place of async.map as there is not asynchronous operation in inside function. Otherwise you may face RangeError: Maximum call stack size exceeded issue.
Also, it might be better to return any response from final callback only. (Remove res.negotiate, res.badRequest from async.auto's first argument). It allows to make response method terminal