How can I execute multiple HTTP requests sequentially using async.js . I checked the async.js documentation but couldn't figure out, how to do that. I want to achieve the same thing as below code using async.js callback style.
var http = require('http');
var Q = require('q');
var URL="http://localhost:3000";
var getPromise=function(url) {
var deferred = Q.defer();
var req = http.get(url, function(response) {
if(response.statusCode < 200 || response.statusCode > 299){
deferred.reject(new Error('ErrorCode '+response.statusCode))
}
var result="";
response.on('data',function(chunk){result +=chunk;} )
response.on('end',function(){deferred.resolve(result);} )
});
req.on('error',function(err){
console.error('Error with the request:', err.message);
deferred.reject(err);
});
req.end();
return deferred.promise;
}
getPromise('http://localhost:3000/olympic/2016/ranking/4')
.then(function(data){
console.log("Response 1 "+data)
return getPromise(URL+'/iso/country/'+JSON.parse(data).Country);
})
.then(function(data){
console.log("Response 2 "+data)
return getPromise(URL+'/olympic/2016/medal/'+JSON.parse(data).iso);
})
.then(function(data){
console.log("Response 3 "+data)
})
.catch(function(err){
console.log(err)
});
I got it, I needed async.waterfall it takes an array of functions and execute them one by one. Also we can pass result from previous function execution to the next one
var async = require('async');
async.waterfall([
function task1(done) {
console.log('start!');
setTimeout(function(){
console.log("T1 Complete");
// <- set value to passed to step 2
done(null, 'Value from step 1');
},5000);
},
function task2(task1Result, done) {
console.log(task1Result);
setTimeout(function(){
console.log("T2 Complete");
// <- set value to passed to step 3
done(null, 'Value from step 2');
},1000);
},
function task3 (task2Result, done) {
console.log(task2Result);
setTimeout(function(){
console.log("T3 Complete");
// <- no value set for the next step.
done(null);
},100);
}
],
function (err) {
if (err) {
throw new Error(err);
} else {
console.log('No error happened in any steps, operation done!');
}
});
Looking over the code a bit and trying to understand it more, I believe that async.waterfall is the function you'll need. What this will do is run each function in order, passing its results to the next function in the sequence. Here's an example:
async.waterfall([
function(callback)
{
// Function 1: do request here...
callback(null, val); // replace null with a value if you want the waterfall to error and go straight to the end
},
function(val, callback) {
// Function 2: do your second request here
callback(null, val1, val2, val3); // you can pass any number of variables you like, just make sure the next function in the sequence expects them
},
function(val1, val2, val3, callback)
{
// Function 3: do your third request here
callback(null, result);
} // this can go on for as long as you like
], function(err, result)
{
// this will be called immediately if the first parameter in any of the callbacks is not null, or when all the functions have run
});
Related
I have an app.get that will return customer data and customer purchases. Inside this app.get I need run two mysql calls and build a an array to pass back.
How can I execute one query after another and process that data?
app.get('/customer', function (req,res) {
var response1 = [];
var response2 = [];
var processedData = [];
connection.query('QUERY HERE', function(err, rows, fields) {
if (!err){
response.push({rows});
} else {
res.status(400).send(err);
}
});
//for loop 'response' results and perform another query
for (var i = 0; i < response1.length; i++) {
var row = response1[i];
connection.query('QUERY HERE FOR row.customerid', function(err, rows, fields) {
if (!err){
processedData.push({'Customer Name:' : row.customername, 'purchases' : rows});
} else {
res.status(400).send(err);
}
});
}
//Send json back
res.setHeader('Content-Type', 'application/json');
res.status(200).send(JSON.stringify(processedData));
});
There is a very convenient module called async.js that provides a bunch of functions for doing complex async operations. Particularly,
async.waterfall() is great when you need to pass down results from one async operation/task to another.
async.mapSeries() is great when you need to create a new array with results from an array of async operation/tasks.
Let's use both.
If I understood your code correctly, the code would look something similar to
app.get('/customer', function (req, res) {
async.waterfall([
// each task is passed a callback 'cb' as last argument;
// you MUST call it at least and at most once within each task;
// if you pass an error into the callback as the first argument, it will stop the async function
function task1 (cb1) {
//connection.query('QUERY HERE', function(err, rows, fields) {
// if (err) return cb1(err); // stop waterfall() if an error occurred
// cb1(null, rows, fields); // pass results down to next task
//});
connection.query('QUERY HERE', cb1); // shorter version
},
function task2 (rows, fields, cb2) {
// iterate and run async operation over each element in array 'rows'
async.mapSeries(rows, function getPurchases (row, cb3) {
connection.query('QUERY HERE FOR row.customerid', function (err, purchases, fields) {
if (err) return cb3(err); // stop mapSeries() if an error occurred
cb3(null, { 'Customer Name': row.customername, 'purchases': purchases })
});
}, function (err, customers) {
// when mapSeries() is done iterating OR if an error occurred, it will come here
if (err) return cb2(err); // stop waterfall() if an error occurred
cb2(null, customers)
});
// }, cb2); // shorter version
}
], function (err, customers) {
// when waterfall() is done all its tasks OR if an error occurred, it will come here
// handle error and send response here
});
});
I do have a async parallel block to execute two queries into MongoDB.
on each step of function(callback) I have a valid return result, no any errors are firing. I did debug and all steps are working except that final callback not firing and I can't event reach with breakpoint to its condition.
Does anybody have idea what that may happen?
async.parallel([
// first Query
function(callback){
this._game.findOne({'_id': gameId}, function(err, result){
if(err){return callback(err);}
else if(result) {
callback(null, result);
}
});
},
// second Query
function(callback){
this._player.findOne({'_id': playerId}, function(err, result){
if(err){return callback(err);}
else if(result) {
callback(null, result);
}
});
}
],
// Final callback to send all 2 Results -> which is not working...
function(error, results){
if(results){
res.status(200);
res.send(results);
}
}
);`
You're not dealing with the possibility that Mongo doesn't find any results. In this case it will call the callback with err and result as null. When this happens the functions that you're running in parallel don't call their callbacks, so the final function never runs. What you do in this case is up to you but you need to call the callback function with something.
The reason that you're not getting any results is most likely that playerId and gameId are strings and _id is by default an ObjectId. Add a var ObjectId = require('mongodb').ObjectId;, then replace playerId with ObjectId(playerId) and you'll likely see results.
This is how i would debug it :
async.parallel([
// first Query
function(fq_callback){
this._game.findOne({'_id': gameId}, function(fq_err, fq_result){
if(fq_err){
console.log('err from _game.findOne');
return fq_callback(fq_err);
}
else if(fq_result) {
console.log('result from _game.findOne');
fq_callback(null, fq_result);
}
});
},
// second Query
function(sq_callback){
this._player.findOne({'_id': playerId}, function(sq_err, sq_result){
if(sq_err){
console.log('err from _player.findOne');
return sq_callback(sq_err);
}
else if(sq_result) {
console.log('result from _player.findOne');
sq_callback(null, sq_result);
}
});
}
],
// Final callback to send all 2 Results -> which is not working...
function(fn_error, fn_results){
console.log('in final callback');
if(fn_error) console.log('we have error');
if(fn_results){
console.log('we have results');
res.status(200);
res.send(results);
}else{
console.log('we have NO results');
}
}
);
I'm very new to JavaScript and callbacks, so my apologies if this is a stupid question. Based on the async docs for parallel, I came up with this example code that executed the expected way based on the docs:
async = require('async')
async.parallel([
function(callback){
setTimeout(function(){
callback(null, 'one');
}, 800);
},
function(callback){
setTimeout(function(){
callback(null, 'two');
}, 100);
}
],
function(err, results){
console.log(results)
})
Running this with node foo.js prints a results array ['one', 'two'] as the docs indicate it will. The thing I don't understand is how exactly this works: when you pass callback as a parameter to each function and then callback is called as callback(null, res), what exactly is being called her? I've not actually defined any function callback, nor have I passed any sort of operating callback function as a parameter, yet everything magically works fine. Am I totally missing the point here? Or is this actually the under-the-hood magic of async?
You are not the one passing callback to the task functions, the async module is. It's a special function that the module passes to your task functions that when called, checks if any more tasks are left.
Here is something similar to what is being done inside async:
function myParallel(tasks, finalcb) {
var tasksLeft = tasks.length,
results = [],
ignore = false;
function callback(err, result) {
if (ignore) return;
if (err) {
ignore = true;
finalcb && finalcb(err);
} else if (--tasksLeft === 0) {
ignore = true;
finalcb && finalcb(null, results);
} else
results.push(result);
}
tasks.forEach(function(taskfn) {
taskfn(callback);
});
}
myParallel([
function(callback) {
setTimeout(function() {
callback(null, 'one');
}, 800);
},
function(callback) {
setTimeout(function() {
callback(null, 'two');
}, 100);
}
], function(err, results){
console.log(results)
});
I am trying to learn node and understand how callbacks are working. I am doing this by trying to use the async lib. I am trying to hit my db with 2 separate calls and use the async lib to let me know when my object is ready to build. Here is my aysnc code:
async.parallel({
one: function(callback) {
getUser(id, callback);
},
two: function(callback) {
getUserServices(id, callback);
}
}, function(err, results) {
if(err) {
console.log(err);
new Error();
}
res.json(result);
});
Here are what my functions look like where I am calling the db. There are both basically the same function:
var getUser = function(id, callback) {
var query = client.query('SELECT * FROM USERS WHERE USER_ID=$1', [id]);
query.on('row', function(row, result) {
result.addRow(row);
});
query.on('end', function(result) {
callback(result);
});
};
My code hits the db and returns the user, but when it goes back up to the async code the user is in the err object. What am I doing wrong? How do I properly set up callbacks?
As pointed by damphat, your code should be
//additionally, handle errors
query.on('error', function(err){
callback(err) // The first argument to callback should be an error object
})
query.on('end', function(result){
callback(null, result) //passing null error object
})
var async = require('async');
function callbackhandler(err, results) {
console.log('It came back with this ' + results);
}
function takes5Seconds(callback) {
console.log('Starting 5 second task');
setTimeout( function() {
console.log('Just finshed 5 seconds');
callback(null, 'five');
}, 5000);
}
function takes2Seconds(callback) {
console.log('Starting 2 second task');
setTimeout( function() {
console.log('Just finshed 2 seconds');
callback(null, 'two');
}, 2000);
}
async.series([takes2Seconds(callbackhandler),
takes5Seconds(callbackhandler)], function(err, results){
console.log('Result of the whole run is ' + results);
})
The output looks like below :
Starting 2 second task
Starting 5 second task
Just finshed 2 seconds
It came back with this two
Just finshed 5 seconds
It came back with this five
I was expecting the takes2Second function to finish completely before the takes5Second starts.
Is that how it is supposed to work. Please let me know.
And the final function never runs. Thanks.
Not quite. You are executing the functions immediately (as soon as the array is evaluated), which is why they appear to start at the same time.
The callback passed to each of the functions to be executed is internal to the async library. You execute it once your function has completed, passing an error and/or a value. You don't need to define that function yourself.
The final function never runs in your case because the callback function that async needs you to invoke to move on to the next function in the series never actually gets executed (only your callbackHandler function gets executed).
Try this instead:
async.series([
takes2Seconds,
takes5seconds
], function (err, results) {
// Here, results is an array of the value from each function
console.log(results); // outputs: ['two', 'five']
});
James gave you a good overview of async.series. Note that you can setup anonymous functions in the series array and then call your actual functions with parameters
var async = require('async')
var param1 = 'foobar'
function withParams(param1, callback) {
console.log('withParams function called')
console.log(param1)
callback()
}
function withoutParams(callback) {
console.log('withoutParams function called')
callback()
}
async.series([
function(callback) {
withParams(param1, callback)
},
withoutParams
], function(err) {
console.log('all functions complete')
})
My preferred way to create the async series is using operational array as follow;
var async = require('async'),
operations = [];
operations.push(takes2Seconds);
operations.push(takes5seconds);
async.series(operations, function (err, results) {
// results[1]
// results[2]
});
function takes2Seconds(callback) {
callback(null, results);
}
function takes5seconds(callback) {
callback(null, results);
}
async.series
([
function (callback)
{
response=wsCall.post(user,url,method,response);
console.log("one");
callback();
}
,
function (callback)
{
console.log("two");
//console.log(response);
callback();
}
]
,
function(err)
{
console.log('Both a and b are saved now');
console.log(response);
});
In async.series,all the functions are executed in series and the consolidated outputs of each function is passed to the final callback. e.g
var async = require('async');
async.series([
function (callback) {
console.log('First Execute..');
callback(null, 'userPersonalData');
},
function (callback) {
console.log('Second Execute.. ');
callback(null, 'userDependentData');
}
],
function (err, result) {
console.log(result);
});
Output:
First Execute..
Second Execute..
['userPersonalData','userDependentData'] //result