Develop Reference

Waiting for data from async nested Functions within JQuery $.each in Javascript

this is a follow up question to Asynchron Errorhandling inside $.each. As mentioned in the comments there, i want to handle data after the last async job from a $.each loop.
So for instance:
var errors = 0;
var started = 0;
var successful = 0;
$.each(..., function(){
started++;
connection.query('INSERT INTO tableName SET ?', post, function(err, result)
{
if (err) {
if (err.code === 'ER_DUP_ENTRY')
{ errors++; }
else
{ throw err; }
} else { successful++;}
if (started == successful + errors) {
// all done
console.log(errors + " errors occurred");
}
});
});
In this case everything logs out properly when the // all done comment is reached. But what if i want to use this data later on instead of just logging it out.
Is there a way to wait for this data outside of the $.each scope? Or do i always have to handle everything in the nested function?

You can use promises instead
var promises = [];
$.each(..., function() {
var promise = new Promise(function(resolve, reject) {;
connection.query('INSERT INTO tableName SET ?', post, function(err, result) {
if (err) {
resolve(err.code);
} else {
resolve(result);
}
});
});
promises.push(promise);
});
var result = Promise.all(promises);
And then when you want to use the data, you do
result.then(function(data) {
// use data array looking like ["result data", "result data", "ER_DUP_ENTRY" .. etc]
})

Nodejs mongoose Mass/Batch update from file

So i have a csv file containing my information, i need to do a mass add/update
exports.add_questions_from_file = function (file_path, surveyid, callback)
{
var U = [{}];
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
console.log(data);
d = data.split(/\r\n|\n/);
for (x=0;x <d.length;x++)
{
line = d[x].split(',');
if (line[0] == "") {return};
RQuestion.add_by_line (line,function (err, question)
{
U.push({id:question.id});
console.log(U);
});
}
}
});
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":U}},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
};
But even though im using callback functions the update seems to happen with the same object always, even the console.log here
U.push({id:question.id});
console.log(U);
returns the same object (even that all the other were created)
Im doing something wrong?

I see a few issues.
First for:
if (line[0] == "") {return};
Don't you mean to use a break or continue instead? Otherwise the entire function will quit if there is a blank line anywhere in the file. This is very important because Survey.update won't get called either.
Second: I assumed that RQuestion.add_by_line and Survey.update are doing something async like updating a database. Your code needs to be restructured to wait for those async items to complete before moving on to the next step. I'd recommend an npm package named async for that.
fs.readFile(file_path, 'utf8', function(err, data){
if (err){
console.log(err);
callback(err,null);
}else{
d = data.split(/\r\n|\n/);
async.map(d, function(line, callback) {
//this function is called for each line
add_by_line (line,function (err, question)
{
callback(err,{id:question.id});
});
}, function(err, results) {
//this function is called when all of the items are done
console.log("done with async");
console.dir(results);
Survey.update({_id:surveyid},{$push:{"SurveyQuestions":results},function (err,numAffected, rawResponse) {
console.log(rawResponse);
RET = {"module":"survey","operation": "add", "status":"OK"};
callback(RET);
});
});
}
});

node function result undefined

this function check_auth return always undefined ? cookie is ok result is ok, maybe the problem is the async call if yes how can fix it?
I don't wont connection if the function is not satisfied
function check_auth(req,res){
var cookie=check_cookie(req);
var check_result_ret;
if(cookie){
console.log("exist cookie: "+cookie);
var sql = 'SELECT session_id,username,id FROM session WHERE session_id = ' + connection.escape(cookie);
connection.query(sql, function(err, results) {
if (err) throw err;
if (results.length > 0 && results[0]['session_id'] ==cookie) {
users[results[0]['session_id']]={username:results[0]["username"],id:results[0]["id"]};
check_result_ret=1;
}else{
check_result_ret=0;
}
});
}else{
check_result_ret=0;
}
return check_result_ret;
}
server node part
switch (path) {
case '/':
var test=check_auth(request,response);
console.log("result",test);
if(test==1){
fs.readFile("main.html", 'utf-8', function (error, data) {
//make somthing
});
}else{
send404(response);
}
break; etc....

The problem is check_auth returns check_result_ret before the callback function(err, results) of connection.query is called .
I guess you need to read more on how asynchronous programming works .This might help http://www.sebastianseilund.com/nodejs-async-in-practice
maybe this helps,
check_auth(req,res){
...do stuff
connection.query(sql, function(err, results) {
if (err) throw err;
if (results.length > 0 && results[0]['session_id'] ==cookie) {
users[results[0]['session_id']]={username:results[0]["username"],id:results[0["id"]};
check_result_ret=1;
}
else{
check_result_ret=0;
}
var test = check_result_ret;
if(test==1){
fs.readFile("main.html", 'utf-8', function (error, data) {
//make somthing afer reading the file
anotherAsyncCall(input,options,function(output){
//do something with output
});
});
}
else{
send404(response);
}
})

Node.js - wait for multiple async calls

I'm trying to make multiple MongoDB queries before I render a Jade template, but I can't quite figure out how to wait until all the Mongo Queries are completed before rendering the template.
exports.init = function(req, res){
var NYLakes = {};
var NJLakes = {};
var filterNY = {"State" : "NY"};
db.collection('lakes').find(filterNY).toArray(function(err, result) {
if (err) throw err;
NYLakes = result;
});
var filterNJ = {"State" : "NJ"};
db.collection('lakes').find(filterNJ).toArray(function(err, result) {
if (err) throw err;
NJLakes = result;
});
res.render('explore/index', {
NYlakes: NYLakes,
NJlakes: NJLakes
});
};

I'm a big fan of underscore/lodash, so I usually use _.after, which creates a function that only executes after being called a certain number of times.
var finished = _.after(2, doRender);
asyncMethod1(data, function(err){
//...
finished();
});
asyncMethod2(data, function(err){
//...
finished();
})
function doRender(){
res.render(); // etc
}
Since javascript hoists the definition of functions defined with the function funcName() syntax, your code reads naturally: top-to-bottom.

Assuming you want to run the two operations in parallel rather than waiting for one to finish before starting the next, you'll need to track how many operations have completed in each callback.
In raw node.js javascript, one way to do this would be this:
exports.init = function(req, res){
var NYLakes = null;
var NJLakes = null;
var filterNY = {"State" : "NY"};
db.collection('lakes').find(filterNY).toArray(function(err, result) {
if (err) throw err;
NYLakes = result;
complete();
});
var filterNJ = {"State" : "NJ"};
db.collection('lakes').find(filterNJ).toArray(function(err, result) {
if (err) throw err;
NJLakes = result;
complete();
});
function complete() {
if (NYLakes !== null && NJLakes !== null) {
res.render('explore/index', {
NYlakes: NYLakes,
NJlakes: NJLakes
});
}
}
};
Basically what's happening here is that you check at the end of each operation if all of them have finished, and at that point you finish off the operation.
If you're doing a lot of these things, take a look at the async library as an example of a tool to make it easier to manage this sort of thing.

You can use async module:
var states = [{"State" : "NY"},{"State" : "NJ"}];
var findLakes = function(state,callback){
db.collection('lakes').find(state).toArray(callback);
}
async.map(states, findLakes , function(err, results){
// do something with array of results
});

Wait.for https://github.com/luciotato/waitfor
using Wait.for:
exports.init = function(req, res){
var NYLakes = {};
var NJLakes = {};
var coll = db.collection('lakes');
var filterNY = {"State" : "NY"};
var a = wait.forMethod(coll,'find',filterNY);
NYLakes = wait.forMethod(a,'toArray');
var filterNJ = {"State" : "NJ"};
var b = wait.forMethod(coll,'find',filterNJ);
NJLakes = wait.forMethod(b,'toArray');
res.render('explore/index',
{
NYlakes: NYLakes,
NJlakes: NJLakes
}
);
};
Requesting in parallel using wait.for parallel map:
exports.init = function(req, res){
var coll = db.collection('lakes');
//execute in parallel, wait for results
var result = wait.parallel.map(
[{coll:coll,filter:{"State" : "NY"}}
, {coll:coll,filter:{"State" : "NJ"}}]
, getData);
res.render('explore/index',
{
NYlakes: result[0],
NJlakes: result[1]
}
);
};
//map function
function getData(item,callback){
try{
var a = wait.forMethod(item.coll,'find',item.filter);
var b = wait.forMethod(a,'toArray');
callback (null, b);
} catch(err){
callback(err);
}
I'm not familiar with mongo, so you may have to adjust the calls.

This seems like the least lines of code using await:
var async = require("async"); //include async module
...
async function getData() { //make sure to use async function
var NYlakes = await db.collection('lakes').find(filterNY); //can append additional logic after the find()
var NJlakes = await db.collection('lakes').find(filterNJ);
res.json({"NYLakes": NYLakes, "NJLakes": NJLakes}); //render response
}
getData();
Side note: In this case await is serving as a Promise.all() be careful not to abuse the await function.

node.js mongodb - collection.find().toArray(callback) - callback doesn't get called

I am just starting out with mongodb, but I am running into a problem when trying to use .find() on a collection.
I've created a DataAccessObject which opens a specific databate and then lets your perform operations on it. Here is the code:
The constructor:
var DataAccessObject = function(db_name, host, port){
this.db = new Db(db_name, new Server(host, port, {auto_reconnect: true}, {}));
this.db.open(function(){});
}
A getCollection function:
DataAccessObject.prototype.getCollection = function(collection_name, callback) {
this.db.collection(collection_name, function(error, collection) {
if(error) callback(error);
else callback(null, collection);
});
};
A save function:
DataAccessObject.prototype.save = function(collection_name, data, callback){
this.getCollection(collection_name, function(error, collection){
if(error) callback(error);
else{
//in case it's just one article and not an array of articles
if(typeof (data.length) === 'undefined'){
data = [data];
}
//insert to collection
collection.insert(data, function(){
callback(null, data);
});
}
});
}
And what seems to be the problematic one - a findAll function:
DataAccessObject.prototype.findAll = function(collection_name, callback) {
this.getCollection(collection_name, function(error, collection) {
if(error) callback(error)
else {
collection.find().toArray(function(error, results){
if(error) callback(error);
else callback(null, results);
});
}
});
};
Whenever I try to dao.findAll(error, callback), the callback never gets called.
I've narrowed the problem down to the following part of the code:
collection.find().toArray(function(error, result){
//... whatever is in here never gets executed
});
I've looked at how other people do it. In fact, I'm following this tutorial very closely. No one else seems to have this problem with colelction.find().toArray(), and it doesn't come up in my searches.
Thanks,
Xaan.

You are not using the open callback so if you are trying to make the findall request right after creating the dao then it won't be ready.
If your code is like this, it will not work.
var dao = new DataAccessObject("my_dbase", "localhost", 27017);
dao.findAll("my_collection",function() {console.log(arguments);});
I tested it and it doesn't find records, and it also gives no error. I think it should give an error.
But if you change it so that you give a callback to the constructor, then it should work.
var DataAccessObject = function(db_name, host, port, callback){
this.db = new Db(db_name, new Server(host, port, {auto_reconnect: true}, {}));
this.db.open(callback);
}
And make your code like this.
var dao = new DataAccessObject("my_dbase", "localhost", 27017, function() {
dao.findAll("my_collection",function() {console.log(arguments);});
});