Develop Reference

Async for Array problems

I've been pointed towards using the async module, but I'm not quite sure how to use waterfall to solve my problem.
My original code had problems with asynchronicity.
var Image = require('./models/image');
var User = require('./models/user');
var query = Image.find({});
query.limit(10);
query.sort('-date')
query.exec(function (err, collected) {
if (err) return console.error(err);
var i = 0;
var authors = [];
while (i < 8) {
var search = User.find({'twitter.id' : collected[i].author});
search.exec(function (err, user){
if (err) return console.error(err);
var result = (user[0].twitter.username);
authors.push(result);
});
i = i + 1;
}
}
console.log(authors);
I want the authors array to hold all the found usernames. However when that last console.log() call returns '[]'

So, you want to wait for all of the searches to complete first. You should put all your async calls into an array, and then use an async library to chain them together (waterfall) or execute simultaneously (parallel). Parallel tends to execute "faster":
var searches = [];
while (i < 8) {
var search = User.find({'twitter.id' : collected[i].author});
searches.push(function(cb) {
search.exec(function (err, user){
if (err) cb(err, null);
else cb(null, user[0].twitter.username);
});
});
i++;
}
async.parallel(searches, function( err, authors ) {
if ( err ) return console.error( err );
console.log(authors);
// Authors only defined here.
// TODO: More code
});
// Authors not defined here.

Wait for functions in for loop to finish

I have a for loop in NodeJS. Inside the loop is a function that gets data from a database.
var player2.schedule = {
opponent: //string
datetime: //date object
}
var schedule = "";
for (i=0; i<player2.schedule.length; i++) {
User.findOne({ id: player2.schedule[i].opponent }.select("name").exec(function(err, opponent) {
schedule += opponent.name;
});
}
The loop adds to a variable schedule with the results from the database call each time the loop goes round.
Now my problem is if I have code after the for loop that relies on this schedule variable, it can't. Because the variable is updated in the callback function from the database call, any code after the for loop happens asynchronously, so the variable hasn't been updated in time.
How can I make sure the next batch of code waits for the for loop and callbacks to finish first?

Here is a simple example using async:
var async = require('async');
async.each(player2.schedule, function(item, cb) {
User.findOne({ id: item.opponent })
.select("name")
.exec(function(err, opponent) {
if (err)
return cb(err);
schedule += opponent.name;
cb();
});
}, function(err) {
if (err)
throw err;
console.log('All done');
});

You can use async library function whilst() to wait for all queries to finish:
var async = require('async');
var i = 0;
var len = player2.schedule.length;
async.whilst(
function () { return i < len; },
function (callback) {
User.findOne({ id:player2.schedule[i].opponent}.select("name").exec(function(err, opponent) {
schedule += opponent.name;
i++;
callback(null); // assume no error (null)
});
},
function (err) {
// this function is executed after all queries are done
// schedule now has everything from the loop
}
);

How to call a function after an asynchronous for loop of Object values finished executing

I want to call a function after an asynchronous for loop iterating through values of an Javascript object finishes executing. I have the following code
for (course in courses) {
var url = '...' + courses[course];
request(url, (function (course) {
return function (err, resp, body) {
$ = cheerio.load(body);
//Some code for which I use object values
};
})(course));
}

This can be done in vanilla JS, but I recommend the async module, which is the most popular library for handling async code in Node.js. For example, with async.each:
var async = require('async');
var courseIds = Object.keys(courses);
// Function for handling each course.
function perCourse(courseId, callback) {
var course = courses[courseId];
// do something with each course.
callback();
}
async.each(courseIds, perCourse, function (err) {
// Executed after each course has been processed.
});
If you want to use a result from each iteration, then async.map is similar, but passes an array of results to the second argument of the callback.
If you prefer vanilla JS, then this will work in place of async.each:
function each(list, func, callback) {
// Avoid emptying the original list.
var listCopy = list.slice(0);
// Consumes the list an element at a time from the left.
// If you are concerned with overhead in using the shift
// you can accomplish the same with an iterator.
function doOne(err) {
if (err) {
return callback(err);
}
if (listCopy.length === 0) {
return callback();
}
var thisElem = listCopy.shift();
func(thisElem, doOne);
}
doOne();
}
(taken from a gist I wrote a while back)
I strongly suggest that you use the async library however. Async is fiddly to write, and functions like async.auto are brilliant.

A possible simple JS solution would be to do something like this.
var courses = {
lorum: 'fee',
ipsum: 'fy',
selum: 'foe'
};
var keys = Object.keys(courses);
var waiting = keys.length;
function completedAll() {
console.log('completed all');
}
function callOnCourseComplete(course, func) {
console.log('completed', course);
waiting -= 1;
if (!waiting) {
func();
}
}
var delay = 10000;
keys.forEach(function(course) {
var url = '...' + courses[course];
console.log('request', url);
setTimeout((function(closureCourse) {
return function( /* err, resp, body */ ) {
// Some code for which I use object values
callOnCourseComplete(closureCourse, completedAll);
};
}(course)), (delay /= 2));
});
Update: Probably a better Javascript solution would be to use Promises
const courses = {
lorum: 'fee',
ipsum: 'fy',
selum: 'foe',
};
function completedAll() {
console.log('completed all');
}
function callOnCourseComplete(courseName) {
console.log('completed', courseName);
}
let delay = 10000;
const arrayOfPromises = Object.keys(courses).map(courseName => (
new Promise((resolve, reject) => {
const url = `...${courses[courseName]}`;
console.log('request', url);
setTimeout((err, resp, body) => {
if (err) {
reject(err);
}
// Some code for which I use object values
resolve(courseName);
}, (delay /= 2));
}))
.then(callOnCourseComplete));
Promise.all(arrayOfPromises)
.then(completedAll)
.catch(console.error);

Node.js - wait for multiple async calls

I'm trying to make multiple MongoDB queries before I render a Jade template, but I can't quite figure out how to wait until all the Mongo Queries are completed before rendering the template.
exports.init = function(req, res){
var NYLakes = {};
var NJLakes = {};
var filterNY = {"State" : "NY"};
db.collection('lakes').find(filterNY).toArray(function(err, result) {
if (err) throw err;
NYLakes = result;
});
var filterNJ = {"State" : "NJ"};
db.collection('lakes').find(filterNJ).toArray(function(err, result) {
if (err) throw err;
NJLakes = result;
});
res.render('explore/index', {
NYlakes: NYLakes,
NJlakes: NJLakes
});
};

I'm a big fan of underscore/lodash, so I usually use _.after, which creates a function that only executes after being called a certain number of times.
var finished = _.after(2, doRender);
asyncMethod1(data, function(err){
//...
finished();
});
asyncMethod2(data, function(err){
//...
finished();
})
function doRender(){
res.render(); // etc
}
Since javascript hoists the definition of functions defined with the function funcName() syntax, your code reads naturally: top-to-bottom.

Assuming you want to run the two operations in parallel rather than waiting for one to finish before starting the next, you'll need to track how many operations have completed in each callback.
In raw node.js javascript, one way to do this would be this:
exports.init = function(req, res){
var NYLakes = null;
var NJLakes = null;
var filterNY = {"State" : "NY"};
db.collection('lakes').find(filterNY).toArray(function(err, result) {
if (err) throw err;
NYLakes = result;
complete();
});
var filterNJ = {"State" : "NJ"};
db.collection('lakes').find(filterNJ).toArray(function(err, result) {
if (err) throw err;
NJLakes = result;
complete();
});
function complete() {
if (NYLakes !== null && NJLakes !== null) {
res.render('explore/index', {
NYlakes: NYLakes,
NJlakes: NJLakes
});
}
}
};
Basically what's happening here is that you check at the end of each operation if all of them have finished, and at that point you finish off the operation.
If you're doing a lot of these things, take a look at the async library as an example of a tool to make it easier to manage this sort of thing.

You can use async module:
var states = [{"State" : "NY"},{"State" : "NJ"}];
var findLakes = function(state,callback){
db.collection('lakes').find(state).toArray(callback);
}
async.map(states, findLakes , function(err, results){
// do something with array of results
});

Wait.for https://github.com/luciotato/waitfor
using Wait.for:
exports.init = function(req, res){
var NYLakes = {};
var NJLakes = {};
var coll = db.collection('lakes');
var filterNY = {"State" : "NY"};
var a = wait.forMethod(coll,'find',filterNY);
NYLakes = wait.forMethod(a,'toArray');
var filterNJ = {"State" : "NJ"};
var b = wait.forMethod(coll,'find',filterNJ);
NJLakes = wait.forMethod(b,'toArray');
res.render('explore/index',
{
NYlakes: NYLakes,
NJlakes: NJLakes
}
);
};
Requesting in parallel using wait.for parallel map:
exports.init = function(req, res){
var coll = db.collection('lakes');
//execute in parallel, wait for results
var result = wait.parallel.map(
[{coll:coll,filter:{"State" : "NY"}}
, {coll:coll,filter:{"State" : "NJ"}}]
, getData);
res.render('explore/index',
{
NYlakes: result[0],
NJlakes: result[1]
}
);
};
//map function
function getData(item,callback){
try{
var a = wait.forMethod(item.coll,'find',item.filter);
var b = wait.forMethod(a,'toArray');
callback (null, b);
} catch(err){
callback(err);
}
I'm not familiar with mongo, so you may have to adjust the calls.

This seems like the least lines of code using await:
var async = require("async"); //include async module
...
async function getData() { //make sure to use async function
var NYlakes = await db.collection('lakes').find(filterNY); //can append additional logic after the find()
var NJlakes = await db.collection('lakes').find(filterNJ);
res.json({"NYLakes": NYLakes, "NJLakes": NJLakes}); //render response
}
getData();
Side note: In this case await is serving as a Promise.all() be careful not to abuse the await function.

returning a variable form an async function

I have a module with a function which generates the value for a vaariable for a variable "stitcheBook". I can see and use this value using a callback.
However, I want to have this value available to me as a module property. How can i achieve this?
Note: I wish the output of the _BookStitcher.stitchAllStories function to go into the _BookStitcher.stitchedBook property.
module.exports = _BookStitcher = (function() {
var db = require('../modules/db');
var stitchedBook = {};
var stitchAllStories = function(callback) {
db.dbConnection.smembers("storyIdSet", function (err, reply) {
if (err) throw err;
else {
var storyList = reply;
console.log(storyList);
// start a separate multi command queue
multi = db.dbConnection.multi();
for (var i=0; i<storyList.length; i++) {
multi.hgetall('story/' + String(storyList[i]) + '/properties');
};
// drains multi queue and runs atomically
multi.exec(function (err, replies) {
stitchedBook = replies;
// console.log(stitchedBook);
callback(stitchedBook);
});
};
});
};
return {
stitchedBook : stitchedBook,
stitchAllStories: stitchAllStories
}
})();
EDIT: to add: I know that I can actually set the value from outside by doing something like this;
_BookStitcher.stitchAllStories(function (reply) {
console.log("Book has been stitched!\n\n")
console.log("the Book is;\n");
console.log(reply);
_BookStitcher.stitchedBook = reply;
console.log("-------------------------------------------------------------------------\n\n\n");
console.log(_BookStitcher.stitchedBook);
});
I was wondering if there was a way of doing it from inside the _BookStitcher module itself.

You could take advantage of how object references work in JavaScript, and assign it to a property:
module.exports = _BookStitcher = (function() {
var db = require('../modules/db');
// CHANGE HERE
var stitched = { book: null };
var stitchAllStories = function(callback) {
db.dbConnection.smembers("storyIdSet", function (err, reply) {
if (err) throw err;
else {
var storyList = reply;
console.log(storyList);
// start a separate multi command queue
multi = db.dbConnection.multi();
for (var i=0; i<storyList.length; i++) {
multi.hgetall('story/' + String(storyList[i]) + '/properties');
};
// drains multi queue and runs atomically
multi.exec(function (err, replies) {
// CHANGE HERE
stitched.book = replies;
// console.log(stitchedBook);
callback(replies);
});
};
});
};
return {
stitched : stitched,
stitchAllStories: stitchAllStories
};
}());
So instead of having it inside _BookStitcher.stitchedBook, you'd have it at _BookStitcher.stitched.book.
But that looks awful, and I'd never use it! You can't know when the value will be available, it's only safe to use it from the callback, when you're sure it's been set.