I have something like this
var async = require('async');
var request = require('request');//request urls
var urls = ['url1','url2', 'url3' ];
function requestUrl( url, callback){
console.log(callback.toString());
request(url, function(err, resp, body){
callback(null, body)
})
}
function cb(err, results){ console.log(results); }
then I call it via
async.map(urls, requestUrl, cb);
My 'cb' never gets called. I am printing out the 'callback' using .toString() method function in the second parameter and it looks like this.
function (err, v) {
results[index] = v;
callback(err);
}
Why is it ignoring my callback and putting it in its own?
async.map(arr, iterator, [callback])
arr - An array to iterate over.
iterator(item, callback) -The iterator is passed a callback(err, transformed) which must be called once it has completed with an error (which can be null) and a transformed item.
callback(err, results) - Optional A callback which is called when all iterator functions have finished, or an error occurs. Results is an array of the transformed items from the arr.
Results is an array.
Take a look at this runnable.
var async = require('async');
var request = require('request');
var urls = ['http://www.google.fr', 'http://twitter.fr'];
function requester(url, done) {
console.log('request: %s', url);
request(url, function(err, r, body) {
console.log('request is done');
console.log('have error on it ?', err !== null);
if (err) return done(err);
var res = res;
var b = body;
done(null, {
response: r,
body: b
});
})
}
async.map(urls, requester, function(err, r) {
console.log('async callback');
console.log('have error on it ?', err !== null);
if (err){
console.log('on error', err);
return;
}
console.log('it\'s a success! Now take a look at result :)');
console.log('results[%s]', r.length);
});
If you call your own endpoint (like http://localhost:3000/your/endpoint) make sure this request don't crash your node app.
This was my fault. When you request data and that request fails async tends to fail silently along with it not returning your request. If nothing seems to be returning from your requests I would say print out any and all errors and check if any conditions not being met.
For instance I had an
if
else if
which but neither was executing leaving it hanging and causing erratic behavior that was hard to pinpoint. 3 hour frustrating lesson in clean programming.
Related
I am trying to call two functions and pass the output of the first function as a parameter into the second.
Function 1:
module.exports.getAllStatisticsByUserId = function(id, callback){
User.findById(id, (err, user) =>{
if(err)
throw err;
if(user)
callback(null, user.statistics);
});
}
Function 2:
module.exports.getGameByStatisticsId = function(id, callback){
Statistics.findById(id, (err, statistics) =>{
if(err)
throw err;
if(statistics)
callback(null, statistics.game);
});
};
I am trying to execute the second method by passing the output of the first method as a parameter but the asynchronous nature of javascript is messing it up. I have tried implementing promises to no avail.
Can anyone suggest some good javascript practices to deal with calling functions asynchronously when they need each other? Any help would be appreciated.
After fixing the issue I mentioned above, you can call them in sequence like this:
module.exports.getAllStatisticsByUserId = function(id, callback){
User.findById(id, (err, user) =>{
if(err) callback(err);
if(user) callback(null, user.statistics);
});
};
module.exports.getGameByStatisticsId = function(id, callback){
Statistics.findById(id, (err, statistics) =>{
if(err) callback(err);
if(statistics) callback(null, statistics.game);
});
};
someService.getAllStatisticsByUserId(id, (err, statistics) => {
if (err || !statistics) {
// handle error
return;
}
someService.getGameByStatisticsId(statistics.id, (err, game) => {
if (err || !game) {
// handle error
return;
}
// handle game
});
});
However, as noted in Mongoose documentation:
When a callback function is not passed, an instance of Query is returned, which provides a special query builder interface.
A Query has a .then() function, and thus can be used as a promise.
So you can simply rewrite the calls like this:
someService.getAllStatisticsByUserId(id).then(statistics =>
someService.getGameByStatisticsId(statistics.id)
).then(game => {
// handle game
}).catch(err => {
// handle error
});
or convert it into an async/await function:
async function getGameByUserId(id) {
try {
const statistics = await someService.getAllStatisticsByUserId(id);
const game = await someService.getGameByStatisticsId(statistics.id);
// handle game
} catch (error) {
// handle error
}
}
Note that an async function always returns a Promise, so you must await it or chain it with a .then() to ensure completion of the query and resolve the returned value, if any.
It looks like you should be able to write:
getAllStatisticsByUserId("me", (err, stats) => {
getGameByStatisticsId(stats.id, (err, game) => {
console.log(game);
});
});
Here's how it would look if these functions returned promises instead.
getAllStatisticsByUserId("me")
.then(stats => getGameByStatisticsId(stats.id))
.then(game => console.log(game))
Even better, if you're able to use a version of Node that supports async/await then you could write.
let stats = await getAllStatisticsByUserId("me");
let game = await getGameByStatisticsId(stats.id);
console.log(game);
This would mean slightly rewriting the original functions (unless User.findById and Statistics.findById already return promises).
module.exports.getAllStatisticsByUserId = function(id, callback){
return new Promise((resolve, reject) => {
User.findById(id, (err, user) =>{
if(err) return reject(err);
return resolve(user.statistics);
});
});
}
Hello try to use an mongodb distinct query with NodeJS (Async). In the GUI of Mongo this query works but in Node it returns the following error
TypeError: db.collection(...).distinct(...).toArray is not a function
The error returns on the following statement:
mongo.connect(uri, function (err, db) {
console.info('MONGODB START CHECK COLLECTIONS')
var tasks = [ // Load businessrules
function (callback) {
db.collection('businessrules').find({typeBusinessRule: 'SpiderGraphExeption'}).toArray(function (err, businessrules) {
if (err) return callback(err);
locals.businessrules = businessrules;
callback();
});
},
// Load stgOmniTracker
function (callback) {
db.collection('stgOmniTracker').find({}).toArray(function (err, tickets) {
if (err) return callback(err);
locals.tickets = tickets;
callback();
});
},
// HERE STARTS THE ERROR
function (callback) {
db.collection('stgOmniTracker').distinct("Responsible Group").toArray(function (err, group) {
if (err) return callback(err);
locals.group = group;
callback();
});
}
];
console.info('--------------- START ASYNC ------------------------')
async.parallel(tasks, function (err) {
if (err) return next(err);
var businessrules = locals.businessrules, tickets = locals.tickets, resultSet = {}, aggCountsPerDayCattegory = [], group = locals.group
db.close()
}
I hope you can help me to fix this. Manny Thanks
Erik
In the mongodb docs you can see that distinct returns null.
distinct(key[, query][, options], callback)
Arguments:
key (string) – key to run distinct against.
[query] (object) – option query to narrow the returned objects.
[options] (object) – additional options during update.
callback (function) – this will be called after executing this method.
The first parameter will contain the Error object if an error occured, or null otherwise.
While the second parameter will contain the results from distinct or null if an error occured.
**Returns: **
null
It takes a callback as the last argument. In this callback, the second argument contains the results you are looking for.
So your updated code would look like this:
function (callback) {
db.collection('stgOmniTracker')
.distinct("Responsible Group", function (err, group) {
if (err) return callback(err);
locals.group = group;
callback();
});
}
db.collection('stgOmniTracker').distinct("Responsible Group", function(err, result)
{
//your code here
});
I have an app.get that will return customer data and customer purchases. Inside this app.get I need run two mysql calls and build a an array to pass back.
How can I execute one query after another and process that data?
app.get('/customer', function (req,res) {
var response1 = [];
var response2 = [];
var processedData = [];
connection.query('QUERY HERE', function(err, rows, fields) {
if (!err){
response.push({rows});
} else {
res.status(400).send(err);
}
});
//for loop 'response' results and perform another query
for (var i = 0; i < response1.length; i++) {
var row = response1[i];
connection.query('QUERY HERE FOR row.customerid', function(err, rows, fields) {
if (!err){
processedData.push({'Customer Name:' : row.customername, 'purchases' : rows});
} else {
res.status(400).send(err);
}
});
}
//Send json back
res.setHeader('Content-Type', 'application/json');
res.status(200).send(JSON.stringify(processedData));
});
There is a very convenient module called async.js that provides a bunch of functions for doing complex async operations. Particularly,
async.waterfall() is great when you need to pass down results from one async operation/task to another.
async.mapSeries() is great when you need to create a new array with results from an array of async operation/tasks.
Let's use both.
If I understood your code correctly, the code would look something similar to
app.get('/customer', function (req, res) {
async.waterfall([
// each task is passed a callback 'cb' as last argument;
// you MUST call it at least and at most once within each task;
// if you pass an error into the callback as the first argument, it will stop the async function
function task1 (cb1) {
//connection.query('QUERY HERE', function(err, rows, fields) {
// if (err) return cb1(err); // stop waterfall() if an error occurred
// cb1(null, rows, fields); // pass results down to next task
//});
connection.query('QUERY HERE', cb1); // shorter version
},
function task2 (rows, fields, cb2) {
// iterate and run async operation over each element in array 'rows'
async.mapSeries(rows, function getPurchases (row, cb3) {
connection.query('QUERY HERE FOR row.customerid', function (err, purchases, fields) {
if (err) return cb3(err); // stop mapSeries() if an error occurred
cb3(null, { 'Customer Name': row.customername, 'purchases': purchases })
});
}, function (err, customers) {
// when mapSeries() is done iterating OR if an error occurred, it will come here
if (err) return cb2(err); // stop waterfall() if an error occurred
cb2(null, customers)
});
// }, cb2); // shorter version
}
], function (err, customers) {
// when waterfall() is done all its tasks OR if an error occurred, it will come here
// handle error and send response here
});
});
In my node.js project, I need to make iterative API request so I'm using async.each method.
The console complains that url is not defined and I understand this because I've only declared urls which is an array of url.
I'm just not sure how I can put together request() from request module inside async.each. To satisfy async.each() I've placed urls as the first argument but request() itself requires a query string argument which is url.
I'm also using _.extend from Underscore to merge two responses but I'm not sure where I have it currently is in the right place.
Can someone please point me in the right direction?
var urls = [];
Object.keys(result.items).forEach(function(item) {
urls.push("https://www.googleapis.com/youtube/v3/videos?part=contentDetails&id=" + result.items[item].contentDetails
.videoId + "&key=xxx");
})
async.each(
urls,
request(url, function(err, response, body) {
if (err) {
console.log(err);
return;
}
body = JSON.parse(body);
}),
function() {
var extended = _.extend(result, body);
getVideoDetailsCallback(null, extended)
});
It seems you're calling request with callbacks and all, and not just referencing it, which means you probably need an anonymous function call.
Also, if you want an array at the end, or whatever you're extending, you could just use async.map instead, and do something like
var urls = Object.keys(result.items).map(function(item) {
return "https://www.googleapis.com/youtube/v3/videos?part=contentDetails&id=" + result.items[item].contentDetails.videoId + "&key=xxx";
});
async.map(urls, function(url, callback) {
request(url, function(err, response, body) {
if (err) {
return callback(err);
}
callback(null, JSON.parse(body));
});
}, function(err, extended) {
if (err) {
// handle error
}
// extended is an array containing the parsed JSON
getVideoDetailsCallback(null, extended);
});
If you want to get the results from each iteration. I suggest using async.map. Map returns an array of results to the final callback.
Here's an example,
var async = require('async');
async.map([ 1, 2, 3, 4 ], function ( num, cb ) {
cb(null, num);
}, function (error, results) {
console.log(results);
});
// output: [ 1, 2, 3, 4 ]
As for your code snippet, your second argument is the wrong type. You're invoking request which doesn't return a function. The second argument is expected to be a function so what you need to do is wrap the request around function definition.
async.map(urls, function(url, callback) {
request(options, callback);
}, function (error, results) {
// do something with results
});
I am trying to learn node and understand how callbacks are working. I am doing this by trying to use the async lib. I am trying to hit my db with 2 separate calls and use the async lib to let me know when my object is ready to build. Here is my aysnc code:
async.parallel({
one: function(callback) {
getUser(id, callback);
},
two: function(callback) {
getUserServices(id, callback);
}
}, function(err, results) {
if(err) {
console.log(err);
new Error();
}
res.json(result);
});
Here are what my functions look like where I am calling the db. There are both basically the same function:
var getUser = function(id, callback) {
var query = client.query('SELECT * FROM USERS WHERE USER_ID=$1', [id]);
query.on('row', function(row, result) {
result.addRow(row);
});
query.on('end', function(result) {
callback(result);
});
};
My code hits the db and returns the user, but when it goes back up to the async code the user is in the err object. What am I doing wrong? How do I properly set up callbacks?
As pointed by damphat, your code should be
//additionally, handle errors
query.on('error', function(err){
callback(err) // The first argument to callback should be an error object
})
query.on('end', function(result){
callback(null, result) //passing null error object
})