Below is a snapshot of my code from routes/index.js
exports.index = function(req, res){
var results=new Array();
for(var i=0; i<1000;i++){
//do database query or time intensive task here based on i
// add each result to the results array
}
res.render('index', { title: 'Home !' , results:results });
};
If I run this code , due to the asynchronous nature of javascript the last line gets executed before the loop gets completely processed . Hence my webpage doesnt have the results . How do I structure this such a way that the page gets loaded once the queries are completed ?
Updated
Inside the loop I have database code ( Redis ) like the below -
client.hgetall("game:" +i, function(err, reply) {
results.push(reply.name);
});
Use the async library:
exports.index = function(req, res){
var results=new Array();
async.forEach(someArray, function(item, callback){
// call this callback when any asynchronous processing is done and
// this iteration of the loop can be considered complete
callback();
// function to run after loop has completed
}, function(err) {
if ( !err) res.render('index', { title: 'Home !' , results:results });
});
};
If one of the tasks inside the loop is asynchronous, you need to pass the asynchronous task a callback that calls callback(). If you don't have an array to use in forEach, just populate one with integers 1-1000.
Edit: Given your latest code, just put the async callback() right after responses.push(reply.name).
exports.index = function(req, res) {
var events = require("events");
var e = new events.EventEmitter();
e.expected = 1000;
e.finished = 0;
e.results = [];
e.on("finishedQuery", (function(err, r) {
this.finished += 1;
this.results.push(r && r.name);
if (this.finished === this.expected) {
res.render('index', { title: 'Home !' , results:this.results });
};
}).bind(e));
for (var i = 0; i < e.expected; i++) {
client.hgetall("game:" + i, function(err, reply) {
e.emit("finishedQuery", err, reply);
});
};
};
Of course, the above code doesn't handle [1 or more] errors. You'd need to add logic that only responds (1) on the first error or (2) if no errors occur.
Related
I've been puzzling over this loop for a few days and can't quite get to what I need.
I loop over my array of search results (mdb_results) and extract from each object a .name to use as a _search term in a Google CSE Image Search.
Th CSE returns another array (cse) which I want to append to the object from which I extracted the _search term (mdb_results[i]).
router.get('/json', function(req, res, next) {
var ImageSearch = require('node-google-image-search');
MongoClient.connect(url, function(err,db){
db.collection('mycatalog')
.find({$text: {$search:"FOO" }})
.toArray(function(err, mdb_results){
for (var i=0; i<mdb_results.length; i++){
var _search = mdb_results[i].name ;
ImageSearch(_search, function(cse){
// How to add cse.img to mdb_results[i].images ??
// mdb_results[i].images = cse;
// gives undefined
},0,2);
};
res.send(mdb_results);
});
});
});
My initial mdb_results looks like this.
[{"name":"FOO"},{"name":"FOOOO"}]
I'm trying to achieve something like this,
[{"name":"FOO",
"images":[{"img1":"http://thumbnail1"},{"img2":"http://thumbnail2"}]
},
{"name":"FOOOO",
"images":[{"img1":"http://thumbnaila"},{"img2":"http://thumbnailb"}]
}]
Can anyone show me how to achieve this?
Thanks
The issue is that you're expecting an asynchronous operation to work synchronously:
router.get('/json', function(req, res, next) {
var ImageSearch = require('node-google-image-search');
MongoClient.connect(url, function(err,db){
db.collection('mycatalog')
.find({$text: {$search:"FOO" }})
.toArray(function(err, mdb_results){
for (var i=0; i<mdb_results.length; i++){
var _search = mdb_results[i].name ;
// This search is asynchronous, it won't have returned by the time
// you return the result below.
ImageSearch(_search, function(cse){
// How to add cse.img to mdb_results[i].images ??
// mdb_results[i].images = cse;
// gives undefined
},0,2);
};
// At this point, ImageSearch has been called, but has not returned results.
res.send(mdb_results);
});
});
});
You're going to need to use promises or the async library.
Here's an example:
router.get('/json', function(req, res, next) {
var ImageSearch = require('node-google-image-search');
MongoClient.connect(url, function(err,db){
db.collection('mycatalog')
.find({$text: {$search:"FOO" }})
.toArray(function(err, mdb_results){
// async.forEach will iterate through an array and perform an asynchronous action.
// It waits for you to call callback() to indicate that you are done
// rather than waiting for it to execute synchronously.
async.forEach(mdb_results, function (result, callback) {
ImageSearch(result.name, function(cse){
// This code doesn't get executed until the network call returns results.
result.images = cse;
// This indicate that you are done with this iteration.
return callback();
},0,2);
},
// This gets call once callback() is called for each element in the array,
// so this only gets fired after ImageSearch returns you results.
function (err) {
res.send(mdb_results);
});
});
});
var veh = [];
app.get('/updateProf', isLoggedIn, function(req, res) {
for (var i = 0; i < req.user.local.vehicles.length; i++){
Vehicles.findById(req.user.local.vehicles[i], function(err, vehicle) {
veh.push(vehicle);
console.log("GET Json: " + veh);
});
}
console.log(veh);
res.json(veh);
veh.length = 0;
});
So I am doing a get request to obtain all my vehicles that a user owns and return its json, it works fine after a refresh, but when I go to the page it shows a empty array on the initial load, if I refresh the page, the array is populated. I think the issue is something to do with it being asynchronous but I'm having a hard time thinking this way and need some advice on how to tackle this.
Yes!!
You will have to wait for all of the callbacks to finish before returning the JSON.
A solution is to keep a count of how many callbacks have been executed and when all of them have been executed you can return the JSON.
var veh = [];
app.get('/updateProf', isLoggedIn, function(req, res) {
number_processed = 0;
total = req.user.local.vehicles.length;
for (var i = 0; i < req.user.local.vehicles.length; i++){
Vehicles.findById(req.user.local.vehicles[i], function(err, vehicle) {
if(!err){
veh.push(vehicle);
}
number_processed = number_processed + 1;
if(number_processed === total){
res.json(veh);
}
console.log("GET JSON: " + veh);
});
}
veh.length = 0;
});
If you are using a more recent version of Mongoose, then you can directly use Promises that are returned by Mongoose for each query.
For example, your query can be simplified to
Vehicles.find({ _id: {'$in': req.user.local.vehicles }})
.exec()
.then(function(vehicleArr) {
res.json(vehicleArr);
});
Note that I use the $in operator to directly translate your loop into an IN condition, which takes an array of what you want to compare (in this case, it's an array of IDs)
The then() function just executes on completion of the query.
Async is a utility library to deal with this.
var async = require('async');
app.get('/updateProf', isLoggedIn, function(req, res) {
async.map(req.user.local.vehicles, function(vehicle, cb){
Vehicles.findById(vehicle, function(err, vehicle) {
if (err) cb(err, null);
console.log('GET Json: ' + vehicle);
cb(null, vehicle);
});
}, function (err, results) {
console.log(results);
res.json(results);
});
});
I created a Node.js script that creates a large array of randomly generated test data and I want to write it to a Redis DB. I am using the redis client library and the async library. Initially, I tried executing a redisClient.hset(...) command within the for loop that generates my test data, but after some Googling, I learned the Redis method is asynchronous while the for loop is synchronous. After seeing some questions on StackOverflow, I can't get it to work the way I want.
I can write to Redis without a problem with a small array or larger, such as one with 100,000 items. However, it does not work well when I have an array of 5,000,000 items. I end up not having enough memory because the redis commands seem to be queueing up, but aren't executed until after async.each(...) is complete and the node process does not exit. How do I get the Redis client to actually execute the commands, as I call redisClient.hset(...)?
Here a fragment of the code I am working with.
var redis = require('redis');
var async = require('async');
var redisClient = redis.createClient(6379, '192.168.1.150');
var testData = generateTestData();
async.each(testData, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
});
callback();
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
You could call eachLimit to ensure you are not executing too many redisClient.hset calls at the same time.
To avoid overflowing the call stack you could do setTimeout(callback, 0); instead of calling the callback directly.
edit:
Forget what I said about setTimeout. All you need to do is call the callback at the right place. Like so:
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
callback();
});
You may still want to use eachLimit and try out which limit works best.
By the way - async.each is supposed to be used only on code that schedules the invocation of the callback in the javascript event queue (e.g. timer, network, etc) . Never use it on code that calls the callback immediately as was the case in your original code.
edit:
You can implement your own eachLimit function that instead of an array takes a generator as it's first argument. Then you write a generator function to create the test data. For that to work, node needs to be run with "node --harmony code.js".
function eachLimit(generator, limit, iterator, callback) {
var isError = false, j;
function startNextSetOfActions() {
var elems = [];
for(var i = 0; i < limit; i++) {
j = generator.next();
if(j.done) break;
elems.push(j.value);
}
var activeActions = elems.length;
if(activeActions === 0) {
callback(null);
}
elems.forEach(function(elem) {
iterator(elem, function(err) {
if(isError) return;
else if(err) {
callback(err);
isError = true;
return;
}
activeActions--;
if(activeActions === 0) startNextSetOfActions();
});
});
}
startNextSetOfActions();
}
function* testData() {
while(...) {
yield new Data(...);
}
}
eachLimit(testData(), 10, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
if(err) callback(err);
else {
console.log("Item was persisted. Result: " +reply);
callback();
}
});
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
I have a for loop in NodeJS. Inside the loop is a function that gets data from a database.
var player2.schedule = {
opponent: //string
datetime: //date object
}
var schedule = "";
for (i=0; i<player2.schedule.length; i++) {
User.findOne({ id: player2.schedule[i].opponent }.select("name").exec(function(err, opponent) {
schedule += opponent.name;
});
}
The loop adds to a variable schedule with the results from the database call each time the loop goes round.
Now my problem is if I have code after the for loop that relies on this schedule variable, it can't. Because the variable is updated in the callback function from the database call, any code after the for loop happens asynchronously, so the variable hasn't been updated in time.
How can I make sure the next batch of code waits for the for loop and callbacks to finish first?
Here is a simple example using async:
var async = require('async');
async.each(player2.schedule, function(item, cb) {
User.findOne({ id: item.opponent })
.select("name")
.exec(function(err, opponent) {
if (err)
return cb(err);
schedule += opponent.name;
cb();
});
}, function(err) {
if (err)
throw err;
console.log('All done');
});
You can use async library function whilst() to wait for all queries to finish:
var async = require('async');
var i = 0;
var len = player2.schedule.length;
async.whilst(
function () { return i < len; },
function (callback) {
User.findOne({ id:player2.schedule[i].opponent}.select("name").exec(function(err, opponent) {
schedule += opponent.name;
i++;
callback(null); // assume no error (null)
});
},
function (err) {
// this function is executed after all queries are done
// schedule now has everything from the loop
}
);
I try to store the result of a DB query in an array but I always got an empty array. I don't understand very well how Async works but I think this code should be work because I store The variable before It finish
Note: following is an array also and I understand this problem is because Async behavior but I dont know what to do to solve it
code:
exports.getfollowingUser = function(req, res){
followedUser=[];
following = req.user.follow;
for (i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){
followedUser[i]= followUser;
});
}
console.log(followedUser) // result empty
res.render('following', {
followedUser: followedUser
});
};
EDIT :1 Follow schema
module.exports = mongoose.model('Friendship',{
follower: String,
followed: String
});
EDIT :2 User schema
module.exports = mongoose.model('User',{
email:{
type: String,
unique: true,
lowercase: true },
password:String,
profile:{
fullname: String,
gender: String,
role: {type: String, default: 'Autorizado'},
country: String },
});
Note: I'm trying to get the friends that the User (Login) is following and show them in a view.
In your for loop node just make request to your DB and continue processing. It doesn't wait for result from database. This is how async works. So after your for loop node didn't yet receive results from DB and followedUser is empty. To fix this without third-part libraries you can do:
exports.getfollowingUser = function(req, res){
findFollowedUser(req.user.follow, function(error, followedUser) {
console.log(followedUser);
res.render('following', {
followedUser: followedUser
});
});
function findFollowedUser(following, callback) {
var followedUser=[];
var waiting = following.length;
var wasError = false;
for (var i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){
if (wasError) {
return;
}
if (err) {
wasError = err;
return callback(err);
}
followedUser[i]= followUser;
waiting--;
if (!waiting) {
callback(null, followedUser);
}
});
}
}
};
Another way (and I think better) is to use some flow control libraries. For example:
q
bluebird
async
fibers
My personal preference: bluebird - extremely fast promise realization. Actually promises is upcoming standard of javascript. So I'd recommend you to take a closer look at it.
Also I'd recommend you to watch this video. This is very simplified explanation of javascript async model (how event loop works).
Look at the flow of execution:
exports.getfollowingUser = function(req, res){
followedUser=[]; // 1
following = req.user.follow;
for (i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){ // 2,3,4
followedUser[i]= followUser; // 7,8,9
});
}
console.log(followedUser) // result empty // 5
res.render('following', {followedUser: followedUser}); // 6
};
As you see, at the time of your console.log [5], followedUser[i]= followUser hasn't yet executed [7,8,9]. This is why you're getting the empty array.
Here's how you could fix it
exports.getfollowingUser = function(req, res){
followedUser=[]; // 1
following = req.user.follow;
var counter = 0;
var length = following.length;
for (i = 0; i < following.length; i++) {
User.find(following[i].followed, function (err, followUser){ // 2,3,4
followedUser[i]= followUser; // 7,8,9
counter++;
if(counter>=length) allDone(); // 10
});
}
console.log(followedUser) // result empty // 5
res.render('following', {followedUser: followedUser}); // 6
function allDone(){
console.log(followedUser) // result empty // 11
res.render('following', {followedUser: followedUser}); // 12
}
};
Here the same statements but inside the allDone()[11,12] will do what you're expecting them to do, as they execute after [7,8,9].
This is just to explain, there are better libraries to handle stuff like this:
Async: https://github.com/caolan/async#each
Promises: https://gist.github.com/neilk/8467412
Also helpful: https://blog.engineyard.com/2015/taming-asynchronous-javascript-with-async