An API call returns the next 'page' of results. How do I recurse over that result callback elegantly?
Here is an example of where I need to do this:
var url = 'https://graph.facebook.com/me/?fields=posts&since=' + moment(postFromDate).format('YYYY-MM-DD') + '&access_token=' + User.accessToken;
request.get({
url: url,
json: true
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
_.each(body.posts.data, function (post) {
User.posts.push(post); //push some result
});
if (body.pagination.next) { // if set, this is the next URL to query
//?????????
}
} else {
console.log(error);
throw error;
}
});
I would suggest wrapping the call in a function and just keep calling it until necessary.
I would also add a callback to know when the process has finished.
function getFacebookData(url, callback) {
request.get({
url: url,
json: true
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
_.each(body.posts.data, function (post) {
User.posts.push(post); //push some result
});
if (body.pagination.next) { // if set, this is the next URL to query
getFacebookData(body.pagination.next, callback);
} else {
callback(); //Call when we are finished
}
} else {
console.log(error);
throw error;
}
});
}
var url = 'https://graph.facebook.com/me/?fields=posts&since=' +
moment(postFromDate).format('YYYY-MM-DD') + '&access_token=' + User.accessToken;
getFacebookData(url, function () {
console.log('We are done');
});
Related
I've read a couple of posts about this here (callbacks) but I still don't really fully understand how to solve my problem. So I was hoping that somebody here could help me with mine and I would get it better.
Simple put I want the ID I get from the first request to be used for the second request.
I'm new to JavaScript and NodeJS in general.
function idRequest(name) {
var options = {
...
};
function callback(error, response, body) {
if (response.statusCode == 200 && !error) {
const info = JSON.parse(body);
//console.log(info.accountId);
return info.accountId;
}
}
request(options, callback);
}
function requestById(accountId) {
var options = {
...
};
function callback(error, response, body) {
if (response.statusCode == 200 && !error) {
const info = JSON.parse(body);
console.log(info);
}
}
request(options, callback);
}
var id = idRequest('..');
requestById(id);
Try by returning a promise from the first function and inside it resolve the callback, so the once it is resolved , you can use it's then to trigger the second function
function idRequest(name) {
var options = {
...
};
function callback(error, response, body) {
if (response.statusCode == 200 && !error) {
const info = JSON.parse(body);
//console.log(info.accountId);
return info.accountId;
}
}
return new Promise(function(resolve, reject) {
resolve(request(options, callback))
})
}
function requestById(accountId) {
var options = {
...
};
function callback(error, response, body) {
if (response.statusCode == 200 && !error) {
const info = JSON.parse(body);
console.log(info);
}
}
request(options, callback);
}
var id = idRequest('..').then(function(data) {
requestById(data);
});
since callback is a async call, so var id will be undefined, when you call the requestById(id);
so either you can use the promise method, answered by #brk or you can call your requestById(id) function directly from the first callback.
I have duplicated error handling code in my Node.js code, how can I make it better to get rid of duplicated code. I specifically want to ask error handling about this callback way, not the Promise way.
var request = require('request');
var URL = 'http://localhost:3000';
var getRanking = function get_rank(error, response, body) {
if (error) {
handleError(error);
} else {
if (response.statusCode != 200) {
handleError(response);
} else {
console.log('Response 1 ' + body);
request(URL + '/iso/country/' + JSON.parse(body).Country, getISO);
}
}
}
var getISO = function get_iso(error, response, body) {
if (error) {
handleError(error);
} else {
if (response.statusCode != 200) {
handleError(response)
} else {
console.log("Response 2 "+body);
request(URL+'/olympic/2016/medal/'+JSON.parse(body).iso,getMedalCount);
}
}
}
var getMedalCount = function get_medal_count(error, response, body) {
if (error) {
handleError(error);
} else {
if (response.statusCode != 200) {
handleError(response);
} else {
console.log("Response 3 " + body);
}
}
}
function handleError(err) {
console.log('Error ' + JSON.stringify(err))
}
request(URL+'/olympic/2016/ranking/4', getRanking);
Create a funciton handleResponse and write the response handling duplicated code in that funciton.
Call that function with required parameters as given,
var request = require('request');
var URL = 'http://localhost:3000';
var getRanking = function get_rank(error, response, body) {
handleResponse(error, response, body, 'getISO');
}
var getISO = function get_iso(error, response, body) {
handleResponse(error, response, body, 'getMedalCount');
}
var getMedalCount = function get_medal_count(error, response, body) {
handleResponse(error, response, body null);
}
function handleResponse(error, response, body, url) {
if (error) {
handleError(error);
} else {
if (response.statusCode != 200) {
handleError(response);
} else {
if(url == 'getISO')
{
request(URL + '/iso/country/' + JSON.parse(body).Country, getISO);
}
else if(url == 'getMedalCount')
{
request(URL+'/olympic/2016/medal/'+JSON.parse(body).iso,getMedalCount);
}
}
}
}
function handleError(err) {
console.log('Error ' + JSON.stringify(err))
}
request(URL+'/olympic/2016/ranking/4', getRanking);
You can try following code:
var request = require('request');
var URL = 'http://localhost:3000';
var getRanking = function get_rank(error, response, body) {
if (error) {
throw error;
}
if (response.statusCode != 200) {
throw new Error(response);
}
console.log('Response 1 ' + body);
request(URL + '/iso/country/' + JSON.parse(body).Country, getISO);
}
var getISO = function get_iso(error, response, body) {
if (error) {
throw error;
}
if (response.statusCode != 200) {
throw new Error(response);
}
console.log("Response 2 "+body);
request(URL+'/olympic/2016/medal/'+JSON.parse(body).iso,getMedalCount);
}
}
var getMedalCount = function get_medal_count(error, response, body) {
if (error) {
throw error;
}
if (response.statusCode != 200) {
throw new Error(response);
return;
}
console.log("Response 3 " + body);
}
try {
request(URL+'/olympic/2016/ranking/4', getRanking);
} catch(ex) {
console.log(ex);
}
Ok, as far as noone suggested such optimization, I will.
Instead of such block:
if (error) {
handleError(error);
} else {
if (response.statusCode != 200) {
handleError(response);
} else {
console.log("Response 3 " + body);
}
}
You can do this way:
if (error || response.statusCode != 200) handlerError(error || response);
else console.log("Response 3 " + body);
You can use following code to handle response.
var request = require('request');
var URL = "http://localhost:3000";
var getRanking = function get_rank (body) {
console.log("Response 1 " + body);
request(URL + '/iso/country/' + JSON.parse(body).Country, handleResponse.bind(null, getISO));
}
var getISO = function get_iso (body) {
console.log("Response 2 " + body);
request(URL + '/olympic/2016/medal/' + JSON.parse(body).iso, handleResponse.bind(null, getMedalCount));
}
var getMedalCount = function get_medal_count (body) {
console.log("Response 3 " + body);
}
function handleResponse (callback, error, response, body) {
console.log(error, response, body, callback)
if (error || response.statusCode != 200) {
console.log('Error ' + JSON.stringify(error))
}
else {
callback(body);
}
}
request(URL + '/olympic/2016/ranking/4', handleResponse.bind(null, getRanking));
The simpliest way to shorten your code would probably be the following :
function handleError(err, res) {
if(err){
console.log('Error '+JSON.stringify(err));
return false;
} else if(res.statusCode != 200) {
console.log('Error '+JSON.stringify(res));
return false;
} else {
return true;
}
}
// and then, use it in your functions like that :
var getRanking = function get_rank(error,response,body) {
if (handleError(err, res)) { // if there is no error, do your stuff
console.log("Response 1 "+body);
request(URL+'/iso/country/'+JSON.parse(body).Country,getISO);
}
}
But I think that's not very appropriate to JS, because JS can be used as a functional programming language (which is better) and this approach looks more procedural (like C language).
So, in my opinion, the below solution would be proper :
function handleError(successFunc) {
return function (error, response, body) {
if (error) {
throw new Error(error);
} else if(response.statusCode != 200) {
throw new Error(response);
} else {
successFunc(response);
}
}
}
// then, just pass your successFunc to your error handler :
var getRanking = function (response) {
// do your success stuff.
}
try {
request(URL+'/olympic/2016/ranking/4', handleError(getRanking));
} catch (e) {
// catch your error here
}
If your request is a success, getRanking will be executed, if this is a failure, the error will be logged and thrown.
With this solution, you can pass any function to your error handler and it will be used if the request succeed, if the request fails, the error handler will throw an error and then, you will be able to catch it.
Hope it helps,
Best regards
I'm trying to return the body of a post request of another site in node.js, but the function below doesn't return anything
// returns "undefined"
mysqlVerify = (socialclub_id, session_id) => {
request({
url: 'http://myapi.site/VerifyUser',
method: 'post',
form: {
socialclub_id: socialclub_id,
session_id: session_id
}
}, (error, response, body) => {
if(error) {
return false // this isnt returning
} else {
console.log(response.statusCode, body)
return body == "1" // this isnt returning
}
})
}
The other site is receiving the post request, and I am also getting the right body back when I use console.log, but the return just doesn't work. What am I doing wrong?
You can't use return inside a callback to return a value when your function is called. You could pass mysqlVerify a callback (a function which is run once the result is determined) and call it once you get a response, like so:
mysqlVerify = (socialclub_id, session_id, callback) => {
request({
url: 'http://myapi.site/VerifyUser',
method: 'post',
form: {
socialclub_id: socialclub_id,
session_id: session_id
}
}, (error, response, body) => {
if(error) {
callback(false) // call the function if false
} else {
console.log(response.statusCode, body)
callback(body == "1") // call the function if condition met
}
});
}
The callback function can then do whatever you need with the result of mysqlVerify.
Basically I have a method fetching values from an API and works, but inside async.parallel the results listed are all undefined.
Here's the method:
function getNumberOfSharesFromFacebookApi(url ,callback) {
request(facebookApiUrl + url + '&format=json', function (error, response, body) {
let res = 0;
if (!error && response.statusCode == 200) {
try {
res = JSON.parse(body)[0]['total_count'];
} catch(err) { }
}
callback(res);
});
}
Here's the async call:
async.parallel ([
callback => {
getNumberOfSharesFromFacebookApi(urlsToTest[0], callback);
},
callback => {
getNumberOfSharesFromFacebookApi(urlsToTest[1], callback);
},
callback => {
getNumberOfSharesFromFacebookApi(urlsToTest[2], callback);
},
callback => {
getNumberOfSharesFromFacebookApi(urlsToTest[3], callback);
}
],
(err, results) => {
console.log(results);
});
on your request function, place your response as the 2nd argument of the callback. request uses node style callbacks wherein the 1st argument is an error.
function getNumberOfSharesFromFacebookApi(url ,callback) {
request(facebookApiUrl + url + '&format=json', function (error, response, body) {
if(error) return callback(error); //callback on error
let res = 0;
if (!error && response.statusCode == 200) {
try {
res = JSON.parse(body)[0]['total_count'];
} catch(err) { }
}
callback(null, res);
});
}
I want to make http requests to an API-s to collect for each user it's data and insert into mongodb.
The problem I am having is, it is doing all the requests at once, and seems it gets stuck somewhere and I don't know what is going on.
Al thou I am using async library and add the request() method inside each iteration, and I dont know if this is the right way, here is the code:
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.forEach(students, function iteratee(student, callback) {
if (student.worksnap.user != null) {
var options = {
url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
}
};
request(options, getTimeEntriesFromWorksnap);
}
callback(); // tell async that the iterator has completed
}, function (err) {
console.log('iterating done');
});
});
}
function getTimeEntriesFromWorksnap(error, response, body) {
console.log(response.statusCode);
if (!error && response.statusCode == 200) {
parser.parseString(body, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
_.forEach(timeEntries, function (timeEntry) {
_.forEach(timeEntry, function (item) {
saveTimeEntry(item);
});
});
});
}
}
function saveTimeEntry(item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
throw err;
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('item inserted...');
}
});
});
}
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
I am new to JavaScript, especially when dealing with async.
Any help?
Use Async.eachLimit() to make batched request to the api...Try this iterateThruAllStudents() function.
I already had same question before here
See tutorial of limiting here.
Though i am making the limit as 5 but you can do whatever you want(10,50 etc).
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
async.eachLimit(students,5,function iteratee(student, callback) {
if (student.worksnap.user != null) {
var options = {
url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
}
};
request(options,getTimeEntriesFromWorksnap(callback));
}
}, function (err) {
console.log(err);
console.log('iterating done');
});
});
}
function getTimeEntriesFromWorksnap(cb) {
return function(error, response, body){
console.log(response.statusCode);
if (!error && response.statusCode == 200) {
parser.parseString(body, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
async.each(timeEntries,function(timeEntry,cb1){
async.each(timeEntry,function(item,cb2){
saveTimeEntry(item,cb2);
},function(err){
if(err)
cb1(err);
else
cb1();
})
},function(err){
if(err)
cb(err);
else
cb();
});
//_.forEach(timeEntries, function (timeEntry) {
// _.forEach(timeEntry, function (item) {
// saveTimeEntry(item);
// });
//});
});
}
cb(null);
}
}
function saveTimeEntry(item,cb2) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
return cb2(err);
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
//return cb2(err);//Do it if you wanna throw an error.
} else {
console.log('item inserted...');
}
cb2();
});
});
}
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
In your example you missed iteratee param in the each method of async - iteratee(item, callback). Look at this example here.
You need to call callback each time inside your iteratee function to tell async continue doing its processing.
each(collection, iteratee, [callback])
collection - collection to iterate over.
iteratee(item, callback) - function to apply to each item in coll. The iteratee is passed a callback(err) which must be called once it has completed. If no error has occurred, the callback should be run without arguments or with an explicit null argument. The array index is not passed to the iteratee. If you need the index, use forEachOf.
callback(err) - Optional callback which is called when all iteratee functions have finished, or an error occurs.
If you need synchronous behavior, no probs! There is also eachSeries method with the same signature except every collection item will be iterated synchronously.
UPDATE:
Changes should be implemented:
Pass async callback:
request(options, getTimeEntriesFromWorksnap(callback));
Return necessary for request callback function:
function getTimeEntriesFromWorksnap(callback) {
return function(error, response, body) {
// ...
saveTimeEntry(item, callback);
// ...
}
}
Call callback only after record is saved in database:
function saveTimeEntry(item, callback) {
// ..
student.save(callback);
// ..
}
Refactor nested loops (not sure what timeEntries, timeEntry are, so use appropriate async method to iterate these data structures):
async.each(timeEntries, function (timeEntry, callback) {
async.each(timeEntry, function (item, callback) {
saveTimeEntry(item, callback);
}, callback);
}, callback);