I'm looking to execute a callback upon the full completion of a recursive function that can go on for an undetermined amount of time. I'm struggling with async issues and was hoping to get some help here. The code, using the request module, is as follows:
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, callback);
});
};
var recurse = function(startingPlace, callback) {
request.get({
url: 'bbb'
}, function(error, response, body) {
// store body somewhere outside these funtions
// make second request
request.get({
url: 'ccc'
}, function(error, response, body) {
var anArray = JSON.parse(body).stuff;
if (anArray) {
anArray.forEach(function(thing) {
request.get({
url: 'ddd'
}, function(error, response, body) {
var nextPlace = JSON.parse(body).place;
recurse(nextPlace);
});
})
}
});
});
callback();
};
start(function() {
// calls final function to print out results from storage that gets updated each recursive call
finalFunction();
});
It seems that once my code goes past the for loop in the nested requests, it continues out of the request and ends the initial function call while the recursive calls are still going on. I want it to not finish the highest-level iteration until all the nested recursive calls have completed (which I have no way of knowing how many there are).
Any help is GREATLY appreciated!
In your example you have no recursive calls. If I understand correctly you want to say that recurse(point, otherFunc); is the beginning of a recursive call.
Then just go back to the definition of the recursive call (which you have not shown in your post) and do this (add a third argument for a callback function to be called in the end of recursion; the caller will pass it as a parameter):
function recurse(startingPlace, otherFunc, callback_one) {
// code you may have ...
if (your_terminating_criterion === true) {
return callback_one(val); // where val is potentially some value you want to return (or a json object with results)
}
// more code you may have
}
Then in the original code that you posted, make this call instead (in the inner-most part):
recurse(startingPlace, otherFunc, function (results) {
// results is now a variable with the data returned at the end of recursion
console.log ("Recursion finished with results " + results);
callback(); // the callback that you wanted to call right from the beginning
});
Just spend some time and try to understand my explanation. When you understand, then you will know node. This is the node philosophy in one post. I hope it is clear. Your very first example should look like this:
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, otherFunc, function (results) {
console.log ("Recursion finished with results " + results);
callback();
});
});
};
Below is only additional information in case you are interested. Otherwise you are set with the above.
Typically in node.js though, people return an error value as well, so that the caller knows if the function that was called has finished successfully. There is no big mystery here. Instead of returning just results people make a call of the form
return callback_one(null, val);
Then in the other function you can have:
recurse(startingPlace, otherFunc, function (recError, results) {
if (recErr) {
// treat the error from recursion
return callback(); // important: use return, otherwise you will keep on executing whatever is there after the if part when the callback ends ;)
}
// No problems/errors
console.log ("Recursion finished with results " + results);
callback(); // writing down `return callback();` is not a bad habit when you want to stop execution there and actually call the callback()
});
Update with my suggestion
This is my suggestion for the recursive function, but before that, it looks like you need to define your own get:
function myGet (a, callback) {
request.get(a, function (error, response, body) {
var nextPlace = JSON.parse(body).place;
return callback(null, nextPlace); // null for no errors, and return the nextPlace to async
});
}
var recurse = function(startingPlace, callback2) {
request.get({
url: 'bbb'
}, function(error1, response1, body1) {
// store body somewhere outside these funtions
// make second request
request.get({
url: 'ccc'
}, function(error2, response2, body2) {
var anArray = JSON.parse(body2).stuff;
if (anArray) {
// The function that you want to call for each element of the array is `get`.
// So, prepare these calls, but you also need to pass different arguments
// and this is where `bind` comes into the picture and the link that I gave earlier.
var theParallelCalls = [];
for (var i = 0; i < anArray.length; i++) {
theParallelCalls.push(myGet.bind(null, {url: 'ddd'})); // Here, during the execution, parallel will pass its own callback as third argument of `myGet`; this is why we have callback and callback2 in the code
}
// Now perform the parallel calls:
async.parallel(theParallelCalls, function (error3, results) {
// All the parallel calls have returned
for (var i = 0; i < results.length; i++) {
var nextPlace = results[i];
recurse(nextPlace, callback2);
}
});
} else {
return callback2(null);
}
});
});
};
Note that I assume that the get request for 'bbb' is always followed by a get request for 'ccc'. In other words, you have not hidden a return point for the recursive calls where you have the comments.
Typically when you write a recursive function it will do something and then either call itself or return.
You need to define callback in the scope of the recursive function (i.e. recurse instead of start), and you need to call it at the point where you would normally return.
So, a hypothetical example would look something like:
get_all_pages(callback, page) {
page = page || 1;
request.get({
url: "http://example.com/getPage.php",
data: { page_number: 1 },
success: function (data) {
if (data.is_last_page) {
// We are at the end so we call the callback
callback(page);
} else {
// We are not at the end so we recurse
get_all_pages(callback, page + 1);
}
}
}
}
function show_page_count(data) {
alert(data);
}
get_all_pages(show_page_count);
I think you might find caolan/async useful. Look especially into async.waterfall. It will allow you to pass results from a callback from another and when done, do something with the results.
Example:
async.waterfall([
function(cb) {
request.get({
url: 'aaa.com'
}, function(err, res, body) {
if(err) {
return cb(err);
}
cb(null, JSON.parse(body).id);
});
},
function(id, cb) {
// do that otherFunc now
// ...
cb(); // remember to pass result here
}
], function (err, result) {
// do something with possible error and result now
});
If your recursive function is synchronous, just call the callback on the next line:
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, otherFunc);
// Call output function AFTER recursion has completed
callback();
});
};
Else you need to keep a reference to the callback in your recursive function.
Pass the callback as an argument to the function and call it whenever it is finished.
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, otherFunc, callback);
});
};
Build your code from this example:
var udpate = function (callback){
//Do stuff
callback(null);
}
function doUpdate() {
update(updateDone)
}
function updateDone(err) {
if (err)
throw err;
else
doUpdate()
}
doUpdate();
With ES6, 'es6-deferred' & 'q'. You could try as following,
var Q = require('q');
var Deferred = require('es6-deferred');
const process = (id) => {
var request = new Deferred();
const ids =//do something and get the data;
const subPromises = ids.map(id => process(id));
Q.all(subPromises).then(function () {
request.resolve();
})
.catch(error => {
console.log(error);
});
return request.promise
}
process("testId").then(() => {
console.log("done");
});
Related
I am creating project using javascript and nodejs. I am integrating callback in my function inside for loop with condition basis,but am unable to do this.my problem is callback is completed on first iteration of loop. here is my code:
function tagEndpointNames(callback) {
var data = userGenerateToken();
var sql = "SELECT * FROM topology_data WHERE topology_coordinates !='' and topology_uuid is not null"
var query = conn.query(sql, function(err, tagEndpointNames) {
for (var i = 0; i < tagEndpointNames.length; i++) {
var topologytagData = {
"topology_tag": tagEndpointNames[i].topology_uuid
}
var tpCooridinates = JSON.parse(tagEndpointNames[i].topology_coordinates);
for (var j = 0; j < tpCooridinates.stageObjects.length; j++) {
if (tpCooridinates.stageObjects.length) {
if (tpCooridinates.stageObjects[j].endPointId) {
if (isGuid(tpCooridinates.stageObjects[j].endPointId)) {
var endPointUUID = tpCooridinates.stageObjects[j].endPointId;
var _ro = require('request');
var url = url;
var _d = '';
_ro({
url: url,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + data['access_token']
},
json: topologytagData
}, function(_e, _r, _b) {
if (_r.statusCode == 200 && !_e) {
callback()
//return;
} else {
callback()
console.log("andarss")
return;
}
})
}
}
}
}
}
})
}
Here is the function call:
tagEndpointNames(function(){
console.log ('Server Closed during MIGRATION JOB 4');
server.close(function () {
process.exit(0);
});
})
When you are running asynchronous process with callback in a for loop, remember that the callback from callee will be fired in the first event completed inside the loop. In your case request lib call is an asynchronous process inside for loop, you need to handle all callback from all the request call before you want to callback the callee.
Please read:
How to write asynchronous functions for Node.js
Maybe it's time for you to start using Javascript Promise.
The async library for Node will help you for doing this kind of tasks.
Use async waterfall.It Runs an array of functions in series, each passing their results to the next in the array. However, if any of the functions pass an error to the callback, the next function is not executed and the main callback is immediately called with the error.
js
var create = function (req, res) {
async.waterfall([
_function1(req),
_function2,
_function3
], function (error, success) {
if (error) { alert('Something is wrong!'); }
return alert('Done!');
});
};
function _function1 (req) {
return function (callback) {
var something = req.body;
callback (null, something);
}
}
function _function2 (something, callback) {
return function (callback) {
var somethingelse = function () { // do something here };
callback (err, somethingelse);
}
}
function _function3 (something, callback) {
return function (callback) {
var somethingmore = function () { // do something here };
callback (err, somethingmore);
}
}
Reference
I am facing the following synchronization issue. I wouldn't be surprised if it has a simple solution/workaround. The BuildMenu() function is called from another block of code and it calls the CreateMenuData() which makes a request to a service which return some data. The problem is that since it is an async call to the service when the data variable is being used it is undefined. I have provided the js log that also shows my point.
BuildMenu: function () {
console.log("before call");
var data=this.CreateMenuData();
console.log("after call");
//Doing more stuff with data that fail.
}
CreateMenuData: function () {
console.log("func starts");
data = [];
dojo.forEach(config.layerlist, function (collection, colindex) {
var layersRequest = esriRequest({
url: collection.url,
handleAs: "json",
});
layersRequest.then(
function (response) {
dojo.forEach(response.records, function (value, key) {
console.log(key);
data.push(key);
});
}, function (error) {
});
});
console.log("func ends");
return data;
}
Console log writes:
before call
func starts
func ends
after call
0
1
2
3
4
FYI: using anything "dojo." is deprecated. Make sure you are pulling all the modules you need in "require".
Ken has pointed you the right direction, go through the link and get familiarized with the asynchronous requests.
However, I'd like to point out that you are not handling only one async request, but potentionally there might be more of them of which you are trying to fill the "data" with. To make sure you handle the results only when all of the requests are finished, you should use "dojo/promise/all".
CreateMenuData: function (callback) {
console.log("func starts");
requests = [];
data = [];
var scope = this;
require(["dojo/_base/lang", "dojo/base/array", "dojo/promise/all"], function(lang, array, all){
array.forEach(config.layerlist, function (collection, colindex) {
var promise = esriRequest({
url: collection.url,
handleAs: "json",
});
requests.push(promise);
});
// Now use the dojo/promise/all object
all(requests).then(function(responses){
// Check for all the responses and add whatever you need to the data object.
...
// once it's all done, apply the callback. watch the scope!
if (typeof callback == "function")
callback.apply(scope, data);
});
});
}
so now you have that method ready, call it
BuildMenu: function () {
console.log("before call");
var dataCallback = function(data){
// do whatever you need to do with the data or call other function that handles them.
}
this.CreateMenuData(dataCallback);
}
How can I assign value to variable in global from double callback?
First of, I read some value from file, when its done, I pass it to some fn in callback and want to use result value in initial scope, outside callback.
I can't wrap my head around this for some reason although at first it looks trivial.
var done = function(err, value) {
if (err) {
return;
}
var resultValue = someMethod(value);
};
loadFile(done);
var resultVal = ?? //result value needed here
function loadFile(done) {
fs.realpath(filePath, function (err, resolvedPath) {
if (err) {
return done(err);
}
fs.readFile(resolvedPath, function (err, value) {
if (err) {
return done(err);
}
return done(null, data);
});
});
}
As I said in my comment you are using an asynchronous call to load a file. You want the result of someMethod stored into the global variable resultVal. Except this isn't possible.
When you call loadFile(done) a asynchronous call is made to the server. This call is being resolved by an event. If the event returns 200 the server returned the expected answer. If their is an error it will be passed to done, if not the data will be passed. Let's say this takes about 250 ms to resolve.
In the mean time JavaScript continued parsing the code, because the call was asynchronous, running in a separate thread, thus not halting the execution of the main thread. The next line that gets parsed is returnVal. However the call isn't resolved yet because this line gets executed 1 ms after the function loadFile was called. This leaves a gap of 249 ms.
The solution is to rethink your code to cope with the asynchronous call.
var done = function(err, value) {
if (err) {
return;
}
var resultValue = callBack(value);
};
loadFile(done);
function someMethod(value)
{
//execute whatever you want to do here!
}
function loadFile(done) {
fs.realpath(filePath, function (err, resolvedPath) {
if (err) {
return done(err);
}
fs.readFile(resolvedPath, function (err, value) {
if (err) {
return done(err);
}
return done(null, data);
});
});
}
Of course you can provide the function done with the callback you want. Just look at this code:
var done = function(err, value, callBack) {
if (err) {
return;
}
var resultValue = someMethod(value);
};
loadFile(done, method1);
function method1(value)
{
//execute whatever you want to do here!
}
function loadFile(done, callBack) {
fs.realpath(filePath, function (err, resolvedPath) {
if (err) {
return done(err);
}
fs.readFile(resolvedPath, function (err, value) {
if (err) {
return done(err);
}
return done(null, data, callBack);
});
});
}
Instead of declaring resultValue as : var resultValue = someMethod(value);
You can do global.resultValue = someMethod(value);
This will make resultValue as a global variable.
You can access it anywhere using global.resultValue.
Similarly,instead of using global you can also use process.
global and process are global objects for nodejs just like window is for javascript.
I have this chunk of code
User.find({}, function(err, users) {
for (var i = 0; i < users.length; i++) {
pseudocode
Friend.find({
'user': curUser._id
}, function(err, friends) * * ANOTHER CALLBACK * * {
for (var i = 0; i < friends.length; i++) {
pseudocode
}
console.log("HERE I'm CHECKING " + curUser);
if (curUser.websiteaccount != "None") {
request.post({
url: 'blah',
formData: blah
}, function(err, httpResponse, body) { * * ANOTHER CALLBACK * *
pseudocode
sendMail(friendResults, curUser);
});
} else {
pseudocode
sendMail(friendResults, curUser);
}
});
console.log("finished friend");
console.log(friendResults);
sleep.sleep(15);
console.log("finished waiting");
console.log(friendResults);
}
});
There's a couple asynchronous things happening here. For each user, I want to find their relevant friends and concat them to a variable. I then want to check if that user has a website account, and if so, make a post request and grab some information there. Only thing is, that everything is happening out of order since the code isn't waiting for the callbacks to finish. I've been using a sleep but that doesn't solve the problem either since it's still jumbled.
I've looked into async, but these functions are intertwined and not really separate, so I wasn't sure how it'd work with async either.
Any suggestions to get this code to run sequentially?
Thanks!
I prefer the promise module to q https://www.npmjs.com/package/promise because of its simplicity
var Promises = require('promise');
var promise = new Promises(function (resolve, reject) {
// do some async stuff
if (success) {
resolve(data);
} else {
reject(reason);
}
});
promise.then(function (data) {
// function called when first promise returned
return new Promises(function (resolve, reject) {
// second async stuff
if (success) {
resolve(data);
} else {
reject(reason);
}
});
}, function (reason) {
// error handler
}).then(function (data) {
// second success handler
}, function (reason) {
// second error handler
}).then(function (data) {
// third success handler
}, function (reason) {
// third error handler
});
As you can see, you can continue like this forever. You can also return simple values instead of promises from the async handlers and then these will simply be passed to the then callback.
I rewrote your code so it was a bit easier to read. You have a few choices of what to do if you want to guarantee synchronous execution:
Use the async library. It provides some helper functions that run your code in series, particularly, this: https://github.com/caolan/async#seriestasks-callback
Use promises to avoid making callbacks, and simplify your code APIs. Promises are a new feature in Javascript, although, in my opinion, you might not want to do this right now. There is still poor library support for promises, and it's not possible to use them with a lot of popular libraries :(
Now -- in regards to your program -- there's actually nothing wrong with your code at all right now (assuming you don't have async code in the pseucode blocks). Your code right now will work just fine, and will execute as expected.
I'd recommend using async for your sequential needs at the moment, as it works both server and client side, is essentially guaranteed to work with all popular libraries, and is well used / tested.
Cleaned up code below
User.find({}, function(err, users) {
for (var i = 0; i < users.length; i++) {
Friend.find({'user':curUser._id}, function(err, friends) {
for (var i = 0; i < friends.length; i++) {
// pseudocode
}
console.log("HERE I'm CHECKING " + curUser);
if (curUser.websiteaccount != "None") {
request.post({ url: 'blah', formData: 'blah' }, function(err, httpResponse, body) {
// pseudocode
sendMail(friendResults, curUser);
});
} else {
// pseudocode
sendMail(friendResults, curUser);
}
});
console.log("finished friend");
console.log(friendResults);
sleep.sleep(15);
console.log("finished waiting");
console.log(friendResults);
}
});
First lets go a bit more functional
var users = User.find({});
users.forEach(function (user) {
var friends = Friend.find({
user: user._id
});
friends.forEach(function (friend) {
if (user.websiteaccount !== 'None') {
post(friend, user);
}
sendMail(friend, user);
});
});
Then lets async that
async.waterfall([
async.apply(Users.find, {}),
function (users, cb) {
async.each(users, function (user, cb) {
async.waterfall([
async.apply(Friends.find, { user, user.id}),
function (friends, cb) {
if (user.websiteAccount !== 'None') {
post(friend, user, function (err, data) {
if (err) {
cb(err);
} else {
sendMail(friend, user, cb);
}
});
} else {
sendMail(friend, user, cb);
}
}
], cb);
});
}
], function (err) {
if (err) {
// all the errors in one spot
throw err;
}
console.log('all done');
});
Also, this is you doing a join, SQL is really good at those.
You'll want to look into something called promises. They'll allow you to chain events and run them in order. Here's a nice tutorial on what they are and how to use them http://strongloop.com/strongblog/promises-in-node-js-with-q-an-alternative-to-callbacks/
You can also take a look at the Async JavaScript library: Async It provides utility functions for ordering the execution of asynchronous functions in JavaScript.
Note: I think the number of queries you are doing within a handler is a code smell. This problem is probably better solved at the query level. That said, let's proceed!
It's hard to know exactly what you want, because your psuedocode could use a cleanup IMHO, but I'm going to what you want to do is this:
Get all users, and for each user
a. get all the user's friends and for each friend:
send a post request if the user has a website account
send an email
Do something after the process has finished
You can do this many different ways. Vanilla callbacks or async work great; I'm going to advocate for promises because they are the future, and library support is quite good. I'll use rsvp, because it is light, but any Promise/A+ compliant library will do the trick.
// helpers to simulate async calls
var User = {}, Friend = {}, request = {};
var asyncTask = User.find = Friend.find = request.post = function (cb) {
setTimeout(function () {
var result = [1, 2, 3];
cb(null, result);
}, 10);
};
User.find(function (err, usersResults) {
// we reduce over the results, creating a "chain" of promises
// that we can .then off of
var userTask = usersResults.reduce(function (outerChain, outerResult) {
return outerChain.then(function (outerValue) {
// since we do not care about the return value or order
// of the asynchronous calls here, we just nest them
// and resolve our promise when they are done
return new RSVP.Promise(function (resolveFriend, reject){
Friend.find(function (err, friendResults) {
friendResults.forEach(function (result) {
request.post(function(err, finalResult) {
resolveFriend(outerValue + '\n finished user' + outerResult);
}, true);
});
});
});
});
}, RSVP.Promise.resolve(''));
// handle success
userTask.then(function (res) {
document.body.textContent = res;
});
// handle errors
userTask.catch(function (err) {
console.log(error);
});
});
jsbin
like many other peoples, I want to turn a async function of a third party module (Patio) into a sync function.
function get_user_message_list(parameters, array, response)
{
var new_array = [];
for (var i in array) {
var json = array[i];
json['content']['users_seen'] = ["1757842565"];
json['content']['users_not_seen'] = [];
new_array.push(json);
}
console.log("NEW ARRAY :");
console.log(new_array);
response.writeHeader(200, {'Content-Type':'application/json'});
response.end(JSON.stringify(new_array));
}
function get_json_message(parameters, json)
{
console.log("JSON OBJECT :");
console.log(json);
var dataset = db.from(TABLES.USER).join(TABLES.MOVIE_LIST, {MLUserId: sql.URId}).join(TABLES.MOVIE, {MVId: sql.MLMovieId});
dataset.where('MLSeen={seen} AND MVSourceId={movie} AND MVSource={source} AND URId!={user}', {seen: 1, movie: json['content']['movie_id'], source: json['content']['movie_source'], user:parameters.FACEBOOK_ID}).all().then(function(users){
if (users) {
for (var j in users) {
json['content']['users_seen'].push(users[j].URId);
}
}
//console.log(json['content']['users_seen']);
dataset.where('MLSeen={seen} AND MVSourceId={movie} AND MVSource={source} AND URId!={user}', {seen: 0, movie: json['content']['movie_id'], source: json['content']['movie_source'], user:parameters.FACEBOOK_ID}).all().then(function(users){
if (users) {
for (var j in users) {
json['content']['users_not_seen'].push(users[j].URId);
}
}
console.log(json);
}, errorHandler);
}, errorHandler);
}
In the get_user_message_list function I iterate into an array and for each iteration I calling the async function. In this async function I'm using Patio module to make request to MySQL database. But like you can see, I must wait for the query result to be get after sending a result to the previous function.
How can I wait for the query result to be got before I send it to the next function?
You CAN and you SHOULD turn async functions into something that behaves like sync functions when a problem needs to be fixed. You can't is never the correct answer, the shouldn't is for the programmer to answer.
So, I recently found some code in the nodeunit module which may help you. It fires the async functions, keep track of which are ready. After all requests are in, fires the callback. This could be the idea behind the solution to your problem (so no, this is not the final solution).
async.forEachSeries = function (arr, iterator, callback) {
if (!arr.length) {
return callback();
}
var completed = 0;
var iterate = function () {
iterator(arr[completed], function (err) {
if (err) {
callback(err);
callback = function () {};
}
else {
completed += 1;
if (completed === arr.length) {
callback();
}
else {
iterate();
}
}
});
};
iterate();
};
This test triggered me to see how it was done:
exports['series'] = function (test) {
var call_order = [];
async.series([
function (callback) {
setTimeout(function () {
call_order.push(1);
callback(null, 1);
}, 25);
},
function (callback) {
setTimeout(function () {
call_order.push(2);
callback(null, 2);
}, 50);
},
function (callback) {
setTimeout(function () {
call_order.push(3);
callback(null, 3, 3);
}, 15);
}
],
function (err, results) {
test.equals(err, null);
test.same(results, [1, 2, [3, 3]]);
test.same(call_order, [1, 2, 3]);
test.done();
});
};
Happy programming!
You can't and you shouldn't. This would effectively block your Node.JS server and you would loose every advantage Node.JS provides. Also it is against the whole asynchronous idea behind Node.JS.
Just pass callback parameter to your function:
function get_json_message(parameters, json, callback){ // <---- note the callback
// some other stuff
dataset.where( ...
// some other stuff
dataset.where( ...
// some other stuff
// I've finished the job, call the callback
callback(); // <--- you can pass additional params here
});
});
}
and call it like this:
get_json_message( params, json, function() {
console.log('Hello world!');
// do whatever you like inside callback
});
I've been using syncrhonize.js with great success. There's even a pending pull request (which works quite well) to support async functions which have multiple parameters. Far better and easier to use than node-sync imho. Added bonus that it has easy-to-understand and thorough documentation, whereas node-sync does not.