Develop Reference

How to make array wait for nested loops to complete before returning

I have the following nested loop function which loops through nested JSON objects. It gathers the data and adds each iteration to an array and each array will be a row of data in a MySql database.
Only the final full iterated data is being added to the MySql database though, so I believe either the data is being saved over previous data in the loop, or else the array variable is not waiting for all of the data? I'm not sure
function getCategories(jsonContent) {
var values = [];
try {
for(let i=0; i<jsonContent.length; i++) {
if (jsonContent[i].subcategories) {
var subcat = jsonContent[i].subcategories.length;
for (let n=0; n<subcat; n++) {
if(jsonContent[i].subcategories[n].facetValueData) {
var subsubcat = jsonContent[i].subcategories[n].facetValueData.length;
for (let p=0; p<subsubcat; p++) {
var products = jsonContent[i].subcategories[n].facetValueData[p].productData.length
for (let f=0; f<products; f++) {
var productDataFinal = jsonContent[i].subcategories[n].facetValueData[p].productData[f]
values.push([
0,
jsonContent[i].id, jsonContent[i].categoryName,
jsonContent[i].subcategories[n].name, jsonContent[i].subcategories[n].code,
jsonContent[i].subcategories[n].facetValueData[p].name, jsonContent[i].subcategories[n].facetValueData[p].code,
productDataFinal.productDisplayName, productDataFinal.code,
productDataFinal.url, productDataFinal.imageUrl,
productDataFinal.price.value, productDataFinal.originPrice.value,
productDataFinal.price.currencyIso, productDataFinal.inStock,
productDataFinal.newProduct,
])
}
}
} else {
console.log(`No facetvalue data found at ${jsonContent[i]}`)
}
}
} else {
console.log(`No subcats found at ${jsonContent[i]}`)
}
}
return values;
} catch (e) {
console.error(e)
}
}
The data is then added to the database with this:
con.connect(function(err) {
if (err) {console.log("Error adconnecting to db" + err)}
else {console.log("Connected!")}
let valueData = getCategories(jsonContent)
let sqlQuery = 'INSERT INTO data VALUES ?'
con.query(sqlQuery, [valueData], function (err, result) {
if (err) {console.log("Error adding to db" + err)}
else {console.log("Added to table"+result)}
});
});
if I add the following code just above the return values; line to write the values array to a seperate json file, all of the correct arrays of data are added to the file, so they must get lost along the way to the database somehow?
fs.writeFileSync("data.json", JSON.stringify(values, null, 2), (err) => {
if (err) {
console.log(err)
}
console.log(`\n ${parsedResults.length} Results exported successfully to ${outputFile}\n`)
})

How do I update an object from an outer function?

Struggling with this mongoose function. I'm trying to add an embedded object inside another embedded object, but it is adding the new bid object to the wrong place - the new listings variable I created for my iterator.
My for loops are trying to find the exact Listing to update so I think the action of assigning them is messing them up. For instance if the Listing was users[2].listings[10].bids , how do I get to that object so I can update it?
function create(req, res) {
db.Listing.findById(req.params.listingId, function(err, foundListing) {
// console.log(foundListing._id );
if (err) {
console.log('Error', err);
}
// res.json(foundListing);
// get array of all users
db.User.find({}, function(err, users) {
// for loop iterates through all users' listings
for (let i = 0; i < users.length; i++) {
let listings = users[i].listings
// for loop iterates through all listings ids
for (let j = 0; j < listings.length; j++) {
// finds match
// comparing _id with _id returning false. Not sure why, will return later
if (listings[j].topic === foundListing.topic && listings[j].created === foundListing.created) {
console.log("Found match: " + foundListing.topic);
// get current user id to add to bid object
db.User.findById(req.user, function(err, user) {
if (err) {
console.log(err);
}
var newBid = new db.Bid(req.body); // add data validation later
newBid.uid = user._id
// pushes new bid object into embedded listing
listings[j].bids.push(newBid);
listings[j].save(function(err, savedBid) {
console.log('newBid created: ', newBid);
console.log(listings[j]);
res.json(newBid);
});
});
}
}
}
})
if (err) {
console.log(err);
}
});
};
EDIT - Got this far, but now it doesn't seem like my array is saving.
function create(req, res) {
db.Listing.findById(req.params.listingId, function(err, foundListing) {
if (err) {
console.log('Error:', err);
}
db.User.findOne({ 'listings._id': req.params.listingId }, function(err, foundUser) {
// console.log(foundUser.listings);
if (err) {
console.log('Error: ', err);
}
for (let i = 0; i < foundUser.listings.length; i++) {
// console.log(foundUser.listings[i]._id);
if (foundUser.listings[i]._id == req.params.listingId) {
console.log( 'found it! - ' + foundUser.listings[i].topic);
var newBid = new db.Bid(req.body);
// console.log(newBid)
foundUser.listings[i].bids.push(newBid);
console.log(foundUser.listings[i].bids)
foundUser.listings[i].save(function(err, savedListing) {
// console.log(foundUser.listings[i])
if (err) {
console.log('Error: ', err);
return;
}
console.log('newBid created: ', newBid);
console.log('savedListing', savedListing);
res.json(newBid);
})
}
}
})
});
};

The code you have is pretty hard to grasp. I think you might be using the wrong tools to solve the problem. MongoDb and Mongoose seem to have pretty advanced querying, so that you can specify what exact user, listing etc. you're interested in.
I think it would be worth your time to look a bit at the documentation for Mongoose queries and maybe MongoDb queries as well. The . used for listings.id in db.User.findOne({ 'listings.id': req.params.listingId } is described here.
I can't really test that this code compiles or that it would work, because I don't know enough about your database models and so on, but something along these lines should be possible:
function create(req, res) {
db.Listing.findById(req.params.listingId, function(err, foundListing) {
// use the power of mongo db and mongoose.
// you can specify more exactly what you're looking for
// in this case we're interested in ONE user that
// has a listing with a specific id
// (if you still problems with comparing identical IDs, then that's a serious issue..
// maybe try logging req.params.listingId and check if it really exists
// in your database.)
db.User.findOne({ 'listings.id': req.params.listingId }, function(err, foundUser) {
var newBid = new db.Bid(req.body); // add data validation later
newBid.uid = foundUser._id
// Old code: foundListing.bids.push(newBid)
// Change according to first question/answer at http://mongoosejs.com/docs/faq.html
// which was linked in the thread you linked to in comment.
// The essence is that Mongoose doesn't get
// notified about changes to arrays by default
// To get around this you can update using "set"
// but then you need both a value and an index:
var oldBidsLength = foundListing.bids.length;
foundListing.bids.set(oldBidsLength, newBid);
foundListing.save(function(err, savedBid) {
// savedBid = saved LISTING?
console.log('newBid created: ', newBid);
console.log('savedBid', savedBid);
res.json(savedBid);
}
}
})
});
};
Things that may be off with this code example is things like if db.User.findOne({ 'listings.id': req.params.listingId } should be used with listings._id. I don't know enough about your models.

Asynchronous add to array in Nodejs

var veh = [];
app.get('/updateProf', isLoggedIn, function(req, res) {
for (var i = 0; i < req.user.local.vehicles.length; i++){
Vehicles.findById(req.user.local.vehicles[i], function(err, vehicle) {
veh.push(vehicle);
console.log("GET Json: " + veh);
});
}
console.log(veh);
res.json(veh);
veh.length = 0;
});
So I am doing a get request to obtain all my vehicles that a user owns and return its json, it works fine after a refresh, but when I go to the page it shows a empty array on the initial load, if I refresh the page, the array is populated. I think the issue is something to do with it being asynchronous but I'm having a hard time thinking this way and need some advice on how to tackle this.

Yes!!
You will have to wait for all of the callbacks to finish before returning the JSON.
A solution is to keep a count of how many callbacks have been executed and when all of them have been executed you can return the JSON.
var veh = [];
app.get('/updateProf', isLoggedIn, function(req, res) {
number_processed = 0;
total = req.user.local.vehicles.length;
for (var i = 0; i < req.user.local.vehicles.length; i++){
Vehicles.findById(req.user.local.vehicles[i], function(err, vehicle) {
if(!err){
veh.push(vehicle);
}
number_processed = number_processed + 1;
if(number_processed === total){
res.json(veh);
}
console.log("GET JSON: " + veh);
});
}
veh.length = 0;
});

If you are using a more recent version of Mongoose, then you can directly use Promises that are returned by Mongoose for each query.
For example, your query can be simplified to
Vehicles.find({ _id: {'$in': req.user.local.vehicles }})
.exec()
.then(function(vehicleArr) {
res.json(vehicleArr);
});
Note that I use the $in operator to directly translate your loop into an IN condition, which takes an array of what you want to compare (in this case, it's an array of IDs)
The then() function just executes on completion of the query.

Async is a utility library to deal with this.
var async = require('async');
app.get('/updateProf', isLoggedIn, function(req, res) {
async.map(req.user.local.vehicles, function(vehicle, cb){
Vehicles.findById(vehicle, function(err, vehicle) {
if (err) cb(err, null);
console.log('GET Json: ' + vehicle);
cb(null, vehicle);
});
}, function (err, results) {
console.log(results);
res.json(results);
});
});

Async : Get empty array after For inside the callback

I try to store the result of a DB query in an array but I always got an empty array. I don't understand very well how Async works but I think this code should be work because I store The variable before It finish
Note: following is an array also and I understand this problem is because Async behavior but I dont know what to do to solve it
code:
exports.getfollowingUser = function(req, res){
followedUser=[];
following = req.user.follow;
for (i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){
followedUser[i]= followUser;
});
}
console.log(followedUser) // result empty
res.render('following', {
followedUser: followedUser
});
};
EDIT :1 Follow schema
module.exports = mongoose.model('Friendship',{
follower: String,
followed: String
});
EDIT :2 User schema
module.exports = mongoose.model('User',{
email:{
type: String,
unique: true,
lowercase: true },
password:String,
profile:{
fullname: String,
gender: String,
role: {type: String, default: 'Autorizado'},
country: String },
});
Note: I'm trying to get the friends that the User (Login) is following and show them in a view.

In your for loop node just make request to your DB and continue processing. It doesn't wait for result from database. This is how async works. So after your for loop node didn't yet receive results from DB and followedUser is empty. To fix this without third-part libraries you can do:
exports.getfollowingUser = function(req, res){
findFollowedUser(req.user.follow, function(error, followedUser) {
console.log(followedUser);
res.render('following', {
followedUser: followedUser
});
});
function findFollowedUser(following, callback) {
var followedUser=[];
var waiting = following.length;
var wasError = false;
for (var i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){
if (wasError) {
return;
}
if (err) {
wasError = err;
return callback(err);
}
followedUser[i]= followUser;
waiting--;
if (!waiting) {
callback(null, followedUser);
}
});
}
}
};
Another way (and I think better) is to use some flow control libraries. For example:
q
bluebird
async
fibers
My personal preference: bluebird - extremely fast promise realization. Actually promises is upcoming standard of javascript. So I'd recommend you to take a closer look at it.
Also I'd recommend you to watch this video. This is very simplified explanation of javascript async model (how event loop works).

Look at the flow of execution:
exports.getfollowingUser = function(req, res){
followedUser=[]; // 1
following = req.user.follow;
for (i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){ // 2,3,4
followedUser[i]= followUser; // 7,8,9
});
}
console.log(followedUser) // result empty // 5
res.render('following', {followedUser: followedUser}); // 6
};
As you see, at the time of your console.log [5], followedUser[i]= followUser hasn't yet executed [7,8,9]. This is why you're getting the empty array.
Here's how you could fix it
exports.getfollowingUser = function(req, res){
followedUser=[]; // 1
following = req.user.follow;
var counter = 0;
var length = following.length;
for (i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){ // 2,3,4
followedUser[i]= followUser; // 7,8,9
counter++;
if(counter>=length) allDone(); // 10
});
}
console.log(followedUser) // result empty // 5
res.render('following', {followedUser: followedUser}); // 6
function allDone(){
console.log(followedUser) // result empty // 11
res.render('following', {followedUser: followedUser}); // 12
}
};
Here the same statements but inside the allDone()[11,12] will do what you're expecting them to do, as they execute after [7,8,9].
This is just to explain, there are better libraries to handle stuff like this:
Async: https://github.com/caolan/async#each
Promises: https://gist.github.com/neilk/8467412
Also helpful: https://blog.engineyard.com/2015/taming-asynchronous-javascript-with-async

node.js Wait for task to be finished

So I am writing this node.js program to import XML files into arrays of JSON objects. I got 2 files to import, teachers.xml and students.xml.
Teachers and student contains several thousands of info about teachers and students. I got that part pretty well covered with my code.
This is my javascript file for parsing the files :
var fs = require('fs');
var xmldom = require('xmldom');
var async = require('async');
// Parse `catalog` xml file and call `callback(catalog domxml root)`
function parse_catalog(catalog, callback) {
// Read xml file content
fs.readFile(catalog, function (err, data) {
if (err) {
throw 'Error reading XML catalog';
} else {
// Parse xml content and return dom root
var domRoot = new xmldom.DOMParser().parseFromString(data.toString());
// Call callback passing dom root
callback(domRoot)
}
});
}
I got two methods like this to convert the xml to json and its working perfectly (one for teachers and one for students)
// Convert teacher XML into json format in a array
function convert_teachers(domCatalog) {
var teachers = domCatalog.getElementsByTagName('teacher');
var teachers_arr= [];
for (var i = 0; i < teachers .length; i++) {
var teacher= teachers[i];
...
//Reading the xml
teachers_arr.push({
...
//Create the json
});
}
console.log("Teachers are now in JSON format ");
}
So in the end all I have to do is this :
parse_catalog('teachers.xml', convert_teachers);
When I do this :
parse_catalog('teachers.xml', convert_teachers);
parse_catalog('students.xml', convert_students);
One or the other will finish first depending on the number of elements to import, witch is normal I think.
What I want is to wait for both to be imported and then do some javascript manipulations and this is where I am stuck.
I tried to do this with async, but it doesnt not wait until the two files are finished to import.
async.parallel([
function(callback) {
parse_catalog('teachers.xml', convert_teachers);
callback();
},
function(callback) {
parse_catalog('students.xml', convert_students);
callback();
}
], function(err) {
if (err) return next(err);
console.log("Finished");
//Enventually some Javascript manipulations on the two arrays
});
Actually it outputs :
Finished
Teachers are now in JSON format
Students are now in JSON format
or depending of the file size
Finished
Students are now in JSON format
Teachers are now in JSON format
What I would like is more like :
Teachers are now in JSON format (or students)
Students are now in JSON format (or teachers)
Finished
I plan to load 2 more files, and the order they will be loaded doesn't matter to me.
Any leads ? Thanks!

You're executing callback() in your async.parallel() functions too soon because by that time fs.readFile() hasn't even started yet. Try something like this instead:
function parse_catalog(catalog, callback) {
// Read xml file content
fs.readFile(catalog, function(err, data) {
if (err)
return callback(err);
// Parse xml content and return dom root
var domRoot = new xmldom.DOMParser().parseFromString(data.toString());
// Call callback passing dom root
callback(null, domRoot);
});
}
// Convert teacher XML into json format in a array
function convert_teachers(domCatalog) {
var teachers = domCatalog.getElementsByTagName('teacher');
var teachers_arr = [];
for (var i = 0; i < teachers .length; i++) {
var teacher = teachers[i];
...
//Reading the xml
teachers_arr.push({
...
//Create the json
});
}
console.log('Teachers are now in JSON format');
return teachers_arr;
}
// and similarly for `convert_students`
async.parallel({
teachers: function(callback) {
parse_catalog('teachers.xml', function(err, domCatalog) {
if (err)
return callback(err);
var teachers = convert_teachers(domCatalog);
callback(null, teachers);
});
},
students: function(callback) {
parse_catalog('students.xml', function(err, domCatalog) {
if (err)
return callback(err);
var students = convert_students(domCatalog);
callback(null, students);
});
}
}, function(err, results) {
if (err) return next(err);
console.log('Finished');
// here you have `results.teachers` and `results.students`
console.dir(results);
});