I'm trying to use the async waterfall method but when it gets to one of the functions, it hangs. I suspect it's because the save() operation is too slow for the execution context, but that's why I was starting to use async's waterfall, so I can wait for the value returned until it goes to the next function in the series (passing along the proper data with it which would be the counted in my case below).
// In my user controller:
async.waterfall([
function(callback) {
getSubmission(id, function(submission) {
if (submission) {
callback(null, submission);
}
});
},
function(submission, callback) {
var submissionId = submission._id;
getViews(submissionId, ip, function(count) {
if (count) {
callback(null, count, submissionId);
}
});
},
// Those top two functions work perfectly passing what
// I need to this one which is where I'm having trouble
function(views, submissionId, callback) {
// addView is called because it is actually
// inserting a row in the db, but never returns from the caller
addView(submissionId, ip, function(added) {
// this callback doesn't fire
if (added) {
callback(null, added);
}
});
},
function(added, callback) {
console.log(added);
}
]);
This is what addView() is (also within user controller which is where the previous async.waterfall code also is in) :
var addView = function(submissionId, ip, callback) {
Submission.addView({
submissionId : submissionId,
ip: ip
}, function(err, counted) {
if (err) {
throw err;
}
if (counted) {
callback(counted);
}
});
};
This is what it's calling (inside my Submission model file) when it calls Submission.addView():
exports.addView = function(obj, fn) {
var ip = obj.ip,
submissionId = obj.submissionId,
submissionView = new SubmissionView(obj);
// it gets to this point
submissionView.save({
ip : ip,
submission_id : submissionId
}, function(err, counted) {
fn(err, counted);
});
};
Whenever async "hangs", it's usually because a callback hasn't been called.
You need to make sure that you call the callback in all code paths. I would also recommend that you reserve the first parameter of any async callback to be an error, even if you don't use it as that is the pattern used throughout node.js. Some modules rely on this pattern. e.g. domains.
If you make the below change, then I would expect some error to pop up somewhere:
getSubmission(id, function(submission) {
if (submission) {
callback(null, submission);
}
});
should be something like this:
getSubmission(id, function(err, submission) {
if(err){
return callback(err);
}
if (!submission) {
return callback('no submission found');
}
callback(null, submission);
});
Related
I have an app.get that will return customer data and customer purchases. Inside this app.get I need run two mysql calls and build a an array to pass back.
How can I execute one query after another and process that data?
app.get('/customer', function (req,res) {
var response1 = [];
var response2 = [];
var processedData = [];
connection.query('QUERY HERE', function(err, rows, fields) {
if (!err){
response.push({rows});
} else {
res.status(400).send(err);
}
});
//for loop 'response' results and perform another query
for (var i = 0; i < response1.length; i++) {
var row = response1[i];
connection.query('QUERY HERE FOR row.customerid', function(err, rows, fields) {
if (!err){
processedData.push({'Customer Name:' : row.customername, 'purchases' : rows});
} else {
res.status(400).send(err);
}
});
}
//Send json back
res.setHeader('Content-Type', 'application/json');
res.status(200).send(JSON.stringify(processedData));
});
There is a very convenient module called async.js that provides a bunch of functions for doing complex async operations. Particularly,
async.waterfall() is great when you need to pass down results from one async operation/task to another.
async.mapSeries() is great when you need to create a new array with results from an array of async operation/tasks.
Let's use both.
If I understood your code correctly, the code would look something similar to
app.get('/customer', function (req, res) {
async.waterfall([
// each task is passed a callback 'cb' as last argument;
// you MUST call it at least and at most once within each task;
// if you pass an error into the callback as the first argument, it will stop the async function
function task1 (cb1) {
//connection.query('QUERY HERE', function(err, rows, fields) {
// if (err) return cb1(err); // stop waterfall() if an error occurred
// cb1(null, rows, fields); // pass results down to next task
//});
connection.query('QUERY HERE', cb1); // shorter version
},
function task2 (rows, fields, cb2) {
// iterate and run async operation over each element in array 'rows'
async.mapSeries(rows, function getPurchases (row, cb3) {
connection.query('QUERY HERE FOR row.customerid', function (err, purchases, fields) {
if (err) return cb3(err); // stop mapSeries() if an error occurred
cb3(null, { 'Customer Name': row.customername, 'purchases': purchases })
});
}, function (err, customers) {
// when mapSeries() is done iterating OR if an error occurred, it will come here
if (err) return cb2(err); // stop waterfall() if an error occurred
cb2(null, customers)
});
// }, cb2); // shorter version
}
], function (err, customers) {
// when waterfall() is done all its tasks OR if an error occurred, it will come here
// handle error and send response here
});
});
I'm currently writing a simple api where you post an array (length = 200) and since each element in the array needs to do 1-2 look up requests, I'm using the async library to control the flow of things. I'm using node 0.12.5 & Express.
router.post('/data', function(req, res, next) {
var cloudDB = db.cloudant.use('events');
var tempStorage = {"docs": []};
// This each loop is to make sure all events get iterated through before submitting response
async.each(req.body, function(singleEvent, loopCallback) {
// this should be async waterfall or something better to organize it
async.waterfall(
[
function(callback) { // get user data from db
db.getUserInfo(singleEvent.email, function (error, dbResponse) {
if(error) { // Houston, we have a problem
return callback(error);
}
return callback(null, dbResponse);
})
},
function(dbResponse, callback) { // decide what to do about results
if(!dbResponse) { // we were unable to get the user from DB
db.searchForUser(singleEvent.email, function (err, searchResponse) {
if(err)
return callback(err);
else
return callback(null, JSON.parse(searchResponse));
})
}
else {
return callback(null,JSON.parse(dbResponse));
}
},
function(userInfo, callback) { // combine data into proper logic
callback(null,combineEventAndUserData(singleEvent,userInfo));
}
],
function (err, result) {
// User event has been processed, so if there are no errors, lets add it to the queue
if(err) {
console.log(err);
}
else {
tempStorage.docs.push(result);
}
loopCallback(); // We're done with this singleEvent
}
)
}, function(err) { // function gets called when all singleEvents have been looped through
console.log("Finished each");
if(err) {
res.status(500).send(err);
}
else {
cloudDB.bulk(tempStorage, function(err, body) {
if(!err) {
res.status(200).send(body);
}
else {
res.status(500).send(err);
}
})
}
});
});
So, this code works! However... (sniff sniff), I seem to have created a memory leak. I have taken a look at both memwatch-next and heapdump, and all I've been able to tell was that 'arrays' keep growing when I look at the heap dump.
I don't know why, but I have a suspicion that this might have something to do with closures and how I'm storing the items generated from each of the waterfalls and perhaps the tempStorage.docs is not being released? Am I storing the tempStorage in the correct way? Or should I change how I do that?
i have the following routing function:
router.route('/api/teamUsersWithStat/:team_id')
.get(function (req, res) {
var user_stat = academy_team_user_stat.build();
user_stat.usersByTeam(req.params.team_id, function (result) {
if (result) {
async.each(result, function () {
var i = 0;
user_stat.findModulesTaken(res.user_id, res.team_id, function (modules) {
result[i].modules = modules;
i++;
});
}, res.json(result))
} else {
res.status(401).send("Team not found");
}
}, function (error) {
res.send("Team not found");
});
});
as you can see im using the async.each method to collect additional data to my existing array.
However the res.json(result) is called without it running the actual loop.
(i can tell this as in my javascript i am debugging the response).
So what am i doing wrong?
You're calling your res.json method straight away, you're also reinitializing i inside the loop so it's always 0.
Also, each requires a callback in order to procede to the next iteration.
The following is how I'd do it:
async.each(result, function (r, callback) {
user_stat.findModulesTaken(res.user_id, res.team_id, function (modules) {
result[result.indexOf(r)].modules = modules;
callback();
});
}, function(err) {
if(err)
return res.json(err);
res.json(result);
});
res.json(result) is called as a function, and therefore invoked immediately. To make sure res.json is invoked after the async.each(), you need to pass a function as callback:
async.each(result, function () {
...
}, function(err) {
if(!err) res.json(result);
));
Given the following prototype function:
Client.prototype.getLocalIp = function() {
var rtc = new window.RTCPeerConnection({iceServers: []});
rtc.createDataChannel('', {reliable: false});
var that = this;
rtc.onicecandidate = function(event) {
if (event.candidate) {
that.localIp = grep(event.candidate.candidate);
}
};
rtc.createOffer(function (offer) {
that.localIp = grep(offer.sdp);
rtc.setLocalDescription(offer);
}, function (error) {
console.warn('Fetching local IP failed', error);
});
var grep = function(sdpOrCandidate) {
// Does lots of string processing stuff and returns a string
}
console.log("Returning from function");
console.log(this.localIp);
}
How can I stop the function from returning until the grep function has finished doing its business and returned a value? Here's a JSFiddle demonstrating what I mean: http://jsfiddle.net/tjkxcL1j/
If you look in your browser console you should see that the getLocalIp() function is returning null first until the async stuff from rtc.onicecandidate and/or rtc.createOffer is finished.
Your function needs to accept a callback argument
Client.prototype.getLocalIp = function getLocalIp(done) {
// ...
rtc.createOffer(function (offer) {
that.localIp = grep(offer.sdp);
rtc.setLocalDescription(offer);
// call the callback here
done(null, that.localIp);
},
function (error) {
console.warn('Fetching local IP failed', error);
// call the callback with an error here
done(error);
});
};
Then you can use it like this
client.getLocalIp(function(err, ip){
if (err) return console.error(err.message);
console.log("client ip", ip);
});
However, as #zerkms mentions in a comment, this is only going to work if actually async operations are happening. Examples include accessing information over a network or accessing the disk.
I am trying to learn node and understand how callbacks are working. I am doing this by trying to use the async lib. I am trying to hit my db with 2 separate calls and use the async lib to let me know when my object is ready to build. Here is my aysnc code:
async.parallel({
one: function(callback) {
getUser(id, callback);
},
two: function(callback) {
getUserServices(id, callback);
}
}, function(err, results) {
if(err) {
console.log(err);
new Error();
}
res.json(result);
});
Here are what my functions look like where I am calling the db. There are both basically the same function:
var getUser = function(id, callback) {
var query = client.query('SELECT * FROM USERS WHERE USER_ID=$1', [id]);
query.on('row', function(row, result) {
result.addRow(row);
});
query.on('end', function(result) {
callback(result);
});
};
My code hits the db and returns the user, but when it goes back up to the async code the user is in the err object. What am I doing wrong? How do I properly set up callbacks?
As pointed by damphat, your code should be
//additionally, handle errors
query.on('error', function(err){
callback(err) // The first argument to callback should be an error object
})
query.on('end', function(result){
callback(null, result) //passing null error object
})