In my node.js project, I need to make iterative API request so I'm using async.each method.
The console complains that url is not defined and I understand this because I've only declared urls which is an array of url.
I'm just not sure how I can put together request() from request module inside async.each. To satisfy async.each() I've placed urls as the first argument but request() itself requires a query string argument which is url.
I'm also using _.extend from Underscore to merge two responses but I'm not sure where I have it currently is in the right place.
Can someone please point me in the right direction?
var urls = [];
Object.keys(result.items).forEach(function(item) {
urls.push("https://www.googleapis.com/youtube/v3/videos?part=contentDetails&id=" + result.items[item].contentDetails
.videoId + "&key=xxx");
})
async.each(
urls,
request(url, function(err, response, body) {
if (err) {
console.log(err);
return;
}
body = JSON.parse(body);
}),
function() {
var extended = _.extend(result, body);
getVideoDetailsCallback(null, extended)
});
It seems you're calling request with callbacks and all, and not just referencing it, which means you probably need an anonymous function call.
Also, if you want an array at the end, or whatever you're extending, you could just use async.map instead, and do something like
var urls = Object.keys(result.items).map(function(item) {
return "https://www.googleapis.com/youtube/v3/videos?part=contentDetails&id=" + result.items[item].contentDetails.videoId + "&key=xxx";
});
async.map(urls, function(url, callback) {
request(url, function(err, response, body) {
if (err) {
return callback(err);
}
callback(null, JSON.parse(body));
});
}, function(err, extended) {
if (err) {
// handle error
}
// extended is an array containing the parsed JSON
getVideoDetailsCallback(null, extended);
});
If you want to get the results from each iteration. I suggest using async.map. Map returns an array of results to the final callback.
Here's an example,
var async = require('async');
async.map([ 1, 2, 3, 4 ], function ( num, cb ) {
cb(null, num);
}, function (error, results) {
console.log(results);
});
// output: [ 1, 2, 3, 4 ]
As for your code snippet, your second argument is the wrong type. You're invoking request which doesn't return a function. The second argument is expected to be a function so what you need to do is wrap the request around function definition.
async.map(urls, function(url, callback) {
request(options, callback);
}, function (error, results) {
// do something with results
});
Related
I'm trying to create a function that parses a json when it gets passed a URL. It parses it and if I try to log it inside the request it works, but I cannot return this value.
function JaySon(){
request({url: url,json: true}, function (error, response, body) {
return body;
})
}
var url ="http://api.geonames.org/findNearbyPlaceNameJSON?lat=51.9877644&lng=-1.47866&username=demo";
var x = JaySon(url);
console.log("json:"+x);
The console just logs "json:undefined", from my understanding it is because console.log gets ran faster than the parsing finishes. How could I go about fixing this issue?
This is due to the fact that you're not returning anything in your function :
async function JaySon(url) {
return new Promise((resolve, reject) => {
request({ url: url, json: true }, function(err, res, body) {
if (err) reject(err);
else resolve(body);
});
});
}
use it like :
Jayson('your_url').then(data => data)
or with async/await, but then it has to be wrap inside an async function
Another possible reason may also be due to the fact that your function doesn't take any arguments thus there's no request made to the URL you have passed when invoking the function.
Your code should be something like
function JaySon(url){
request({url: url,json: true}, function (error, response, body) {
return body;
})
}
Hello try to use an mongodb distinct query with NodeJS (Async). In the GUI of Mongo this query works but in Node it returns the following error
TypeError: db.collection(...).distinct(...).toArray is not a function
The error returns on the following statement:
mongo.connect(uri, function (err, db) {
console.info('MONGODB START CHECK COLLECTIONS')
var tasks = [ // Load businessrules
function (callback) {
db.collection('businessrules').find({typeBusinessRule: 'SpiderGraphExeption'}).toArray(function (err, businessrules) {
if (err) return callback(err);
locals.businessrules = businessrules;
callback();
});
},
// Load stgOmniTracker
function (callback) {
db.collection('stgOmniTracker').find({}).toArray(function (err, tickets) {
if (err) return callback(err);
locals.tickets = tickets;
callback();
});
},
// HERE STARTS THE ERROR
function (callback) {
db.collection('stgOmniTracker').distinct("Responsible Group").toArray(function (err, group) {
if (err) return callback(err);
locals.group = group;
callback();
});
}
];
console.info('--------------- START ASYNC ------------------------')
async.parallel(tasks, function (err) {
if (err) return next(err);
var businessrules = locals.businessrules, tickets = locals.tickets, resultSet = {}, aggCountsPerDayCattegory = [], group = locals.group
db.close()
}
I hope you can help me to fix this. Manny Thanks
Erik
In the mongodb docs you can see that distinct returns null.
distinct(key[, query][, options], callback)
Arguments:
key (string) – key to run distinct against.
[query] (object) – option query to narrow the returned objects.
[options] (object) – additional options during update.
callback (function) – this will be called after executing this method.
The first parameter will contain the Error object if an error occured, or null otherwise.
While the second parameter will contain the results from distinct or null if an error occured.
**Returns: **
null
It takes a callback as the last argument. In this callback, the second argument contains the results you are looking for.
So your updated code would look like this:
function (callback) {
db.collection('stgOmniTracker')
.distinct("Responsible Group", function (err, group) {
if (err) return callback(err);
locals.group = group;
callback();
});
}
db.collection('stgOmniTracker').distinct("Responsible Group", function(err, result)
{
//your code here
});
I understand why I get this error, however I have no idea how to fix this. This is what my first function looks like:
All I am doing is doing a get request for all of my urls which are stored in the database, and then for each URL response, I go through the body and save the values information by calling saveInformation(value) or saveInfoTwo(value)
function getUrlInfo(urls, callback) {
urls.map(function(url) {
request.get(urls, {
timeout: 1500
}, function(error, response, body) {
if (error) {console.log(error);
} else {
parseString(body, function(error, result) {
if (error) {console.log(error);
} else {
result.Values.forEach(function(listValues) {
listValues.forEach(function(value) {
saveInformation(value);
});
saveInfoTwo(value);
});
}
});
}
callback(null);
});
});
}
My async waterfall looks like this:
async.waterfall([
getUrls,
getUrlInfo
], function(err, result) {
mongoose.connection.close();
});
Where getUrls is just a method that looks for all the urls and adds them to the urls array, so that getUrlInfo can use it.
function getUrls(callback) {
var urls = [];
urlSchema.find(function(error, data) {
data.forEach(function(result) {
clusters.push(result.url);
});
callback(null, urls);
});
}
I use mongodb as my database.
This works great when there is only one url. As soon as I add another url, I get this error:
Error: Callback was already called.
Now, after trying to debug, it seems like everything works fine for the first url, and then when the second url finishes going through its foreach loops, the error is thrown.
How can I fix this?
I am pretty sure that the callback in my getUrlInfo() waits for all of the urls and foreach loops to finish, then continues? - which is not the case I guess.
Any help would be appreciated!
To summarise: How can I prevent the error appearing when I add another url?
Your map call the function for the first URL, when it finishes, the callback is called. When the second one is executed, when it finishes, it will call callback again, thus you are having this error.
The solution, wait for all urls to finish and then call the callback.
As #Bergi said on the comments you can use async map
function getUrlInfo(urls, callback) {
async.map(urls, function(url, cb) {
//do what you must
cb();
}, callback);
}
I have something like this
var async = require('async');
var request = require('request');//request urls
var urls = ['url1','url2', 'url3' ];
function requestUrl( url, callback){
console.log(callback.toString());
request(url, function(err, resp, body){
callback(null, body)
})
}
function cb(err, results){ console.log(results); }
then I call it via
async.map(urls, requestUrl, cb);
My 'cb' never gets called. I am printing out the 'callback' using .toString() method function in the second parameter and it looks like this.
function (err, v) {
results[index] = v;
callback(err);
}
Why is it ignoring my callback and putting it in its own?
async.map(arr, iterator, [callback])
arr - An array to iterate over.
iterator(item, callback) -The iterator is passed a callback(err, transformed) which must be called once it has completed with an error (which can be null) and a transformed item.
callback(err, results) - Optional A callback which is called when all iterator functions have finished, or an error occurs. Results is an array of the transformed items from the arr.
Results is an array.
Take a look at this runnable.
var async = require('async');
var request = require('request');
var urls = ['http://www.google.fr', 'http://twitter.fr'];
function requester(url, done) {
console.log('request: %s', url);
request(url, function(err, r, body) {
console.log('request is done');
console.log('have error on it ?', err !== null);
if (err) return done(err);
var res = res;
var b = body;
done(null, {
response: r,
body: b
});
})
}
async.map(urls, requester, function(err, r) {
console.log('async callback');
console.log('have error on it ?', err !== null);
if (err){
console.log('on error', err);
return;
}
console.log('it\'s a success! Now take a look at result :)');
console.log('results[%s]', r.length);
});
If you call your own endpoint (like http://localhost:3000/your/endpoint) make sure this request don't crash your node app.
This was my fault. When you request data and that request fails async tends to fail silently along with it not returning your request. If nothing seems to be returning from your requests I would say print out any and all errors and check if any conditions not being met.
For instance I had an
if
else if
which but neither was executing leaving it hanging and causing erratic behavior that was hard to pinpoint. 3 hour frustrating lesson in clean programming.
I'm trying to use the async waterfall method but when it gets to one of the functions, it hangs. I suspect it's because the save() operation is too slow for the execution context, but that's why I was starting to use async's waterfall, so I can wait for the value returned until it goes to the next function in the series (passing along the proper data with it which would be the counted in my case below).
// In my user controller:
async.waterfall([
function(callback) {
getSubmission(id, function(submission) {
if (submission) {
callback(null, submission);
}
});
},
function(submission, callback) {
var submissionId = submission._id;
getViews(submissionId, ip, function(count) {
if (count) {
callback(null, count, submissionId);
}
});
},
// Those top two functions work perfectly passing what
// I need to this one which is where I'm having trouble
function(views, submissionId, callback) {
// addView is called because it is actually
// inserting a row in the db, but never returns from the caller
addView(submissionId, ip, function(added) {
// this callback doesn't fire
if (added) {
callback(null, added);
}
});
},
function(added, callback) {
console.log(added);
}
]);
This is what addView() is (also within user controller which is where the previous async.waterfall code also is in) :
var addView = function(submissionId, ip, callback) {
Submission.addView({
submissionId : submissionId,
ip: ip
}, function(err, counted) {
if (err) {
throw err;
}
if (counted) {
callback(counted);
}
});
};
This is what it's calling (inside my Submission model file) when it calls Submission.addView():
exports.addView = function(obj, fn) {
var ip = obj.ip,
submissionId = obj.submissionId,
submissionView = new SubmissionView(obj);
// it gets to this point
submissionView.save({
ip : ip,
submission_id : submissionId
}, function(err, counted) {
fn(err, counted);
});
};
Whenever async "hangs", it's usually because a callback hasn't been called.
You need to make sure that you call the callback in all code paths. I would also recommend that you reserve the first parameter of any async callback to be an error, even if you don't use it as that is the pattern used throughout node.js. Some modules rely on this pattern. e.g. domains.
If you make the below change, then I would expect some error to pop up somewhere:
getSubmission(id, function(submission) {
if (submission) {
callback(null, submission);
}
});
should be something like this:
getSubmission(id, function(err, submission) {
if(err){
return callback(err);
}
if (!submission) {
return callback('no submission found');
}
callback(null, submission);
});