I'm new to the world of callback hell, and have gotten a little stuck. I'm doing some analytics work, so every time a function is called, I call a function which does a request.post to log the request on our server. In each of those original functions, however, a request.get is made to return some information. So, I am essentially calling a function that makes a request inside another function that needs to make a request after that one. The code itself is a lot more complicated (and by complicated, I mean long and annoying) than I just described, so here's the dumbed-down version:
function myFunction(x, y, callback) {
// variable declarations
postFunction(a, b);
// other logic for the function
request.get(url, function(error, response, body) {
// request code here
callback(somedata, otherFunctionThatDoesntMatterHere(blah));
});
}
function postFunction(a, b) {
// other logic for the function
var dataToSend = "XML HERE";
request.post({
url: url,
body: dataToSend,
headers: {'Content-Type' : 'text/xml'},
function(error, response, body) {
if (error) {
console.log(error);
}
else if (response.statusCode !== 200) {
console.log('Invalid Status Code Returned:', response.statusCode);
}
else if (!error && response.statusCode == 200) {
console.log('POST RESPONSE: ', body);
}
}
);
}
What's happening right now is that the post function doesn't return before the end of the main function. I know this is part of node's asynchronous-ness, but I would like these to be executed synchronously. I am pretty sure I can solve this by adding a callback to the post function, but I'm not sure how. I've tried many different approaches, but I'm afraid I'm not entirely understanding how this all works. Thank you in advance for any help you can provide!
If you truly want it to be synchronous, then yes you would need to add a callback to the postFunction.
For example:
function myFunction(x, y, callback) {
// variable declarations
postFunction(a, b, function(err) {
// other logic for the function
//maybe check if "err" has a value here
request.get(url, function(error, response, body) {
// request code here
callback(somedata, otherFunctionThatDoesntMatterHere(blah));
});
});
}
function postFunction(a, b, next) {
// other logic for the function
var dataToSend = "XML HERE";
request.post({
url: url,
body: dataToSend,
headers: {'Content-Type' : 'text/xml'},
function(error, response, body) {
if (error) {
console.log(error);
next(error);
}
else if (response.statusCode !== 200) {
console.log('Invalid Status Code Returned:', response.statusCode);
next('Invalid Status Code Returned:' + response.statusCode);
}
else if (!error && response.statusCode == 200) {
console.log('POST RESPONSE: ', body);
next(null);
}
}
);
}
This puts the rest of the code you want to fire within the callback of the postFunction, requiring it to complete first before anything else fires. Of course this will increase overall runtime of the function, since it has to now wait before continuing to run. Be careful of locking the event loop if you're doing CPU intensive tasks.
Also, to combat the ugliness of callback hell, you can use something like async-waterfall to chain functions one after another: https://www.npmjs.com/package/async-waterfall
Related
I checked a few thread in StackOverflow, but nothing works for me.
I have this request call and I need it to try to send the request until it succeed (but if it fails, it has to wait at least 3 seconds):
sortingKeywords.sortVerifiedPhrase = function(phrase) {
var URL = "an API URL"+phrase; //<== Obviously that in my program it has an actual API URL
request(URL, function(error, response, body) {
if(!error && response.statusCode == 200) {
var keyword = JSON.parse(body);
if(sortingKeywords.isKeyMeetRequirements(keyword)){ //Check if the data is up to a certain criteria
sortingKeywords.addKeyToKeywordsCollection(keyword); //Adding to the DB
} else {
console.log("doesn't meet rquirement");
}
} else {
console.log(phrase);
console.log("Error: "+ error);
}
});
};
Here's the weird part, if I call the same phrases in a row from the browser, it works almost without errors (it usually states: rate limit time esceeded).
Appreciate your help.
Thanks in advance.
Here is a working program that I've written for this request. It sends the request via a function, if the request fails, it returns error handler and calls the function again.
If the function succeeds the program returns a promise and quits executing.
NOTE: if you enter an invalid url the programs exits right away, that's something that's have to do with request module, which I like to be honest. It lets you know that you have an invalid url. So you must include https:// or http:// in the url
var request = require('request');
var myReq;
//our request function
function sendTheReq(){
myReq = request.get({
url: 'http://www.google.com/',
json: true
}, (err, res, data) => {
if (err) {
console.log('Error:', err)
} else if (res.statusCode !== 200) {
console.log('Status:', res.statusCode)
} else {
// data is already parsed as JSON:
//console.log(data);
}
})
}
sendTheReq();
//promise function
function resolveWhenDone(x) {
return new Promise(resolve => {
myReq.on('end', function(){
resolve(x)
})
myReq.on('error', function(err){
console.log('there was an error: ---Trying again');
sendTheReq(); //sending the request again
f1(); //starting the promise again
})
});
}
//success handler
async function f1() {
var x = await resolveWhenDone(100);
if(x == 100){
console.log("request has been completed");
//handle other stuff
}
}
f1();
On error run this code
setTimeout(function(){}, 3000);
See this https://www.w3schools.com/jsref/met_win_settimeout.asp
Also you can make a code like this
var func1 = function(){};
setTimeout(func1, 3000);
Inside this bit of code, the req parameter is defined, but when I get inside the callback, it isn't anymore. I can still get to it via res.req (weird), though. What happened? I thought that the function would close over its environment.
function addDocument(req, res) {
// in scope
Request.findById(req.body._id, function(error, request){
if (request) {
// disappeared
} else {
res.send404('Couldn\'t find a request with that ID.');
}
});
}
In javascript you d do that, all simply
function addDocument(req, res) {
// in scope
Request.findById(req.body._id, function(error, request){
if (request) {
req.whatever; // What is Request.findById for ?
} else {
res.send404('Couldn\'t find a request with that ID.');
}
});
}
I am working with javascript for quite some time now and recently started using Nodejs in my project. I have a requirement to make some http calls from my nodejs app and then when all of them are done, need to consolidate and send the response.
In this regards, I was searching and came across async module for Nodejs. (https://github.com/caolan/async)
I also found a blog which nicely explains how to use this feature. (http://justinklemm.com/node-js-async-tutorial/)
The below code snippet is what I am planning to use for my task.
// Include the async package
// Make sure you add "async" to your package.json
async = require("async");
// 1st para in async.each() is the array of items
async.each(items,
// 2nd param is the function that each item is passed to
function(item, callback){
// Call an asynchronous function, often a save() to DB
item.someAsyncCall(function (){
// Async call is done, alert via callback
callback();
});
},
// 3rd param is the function to call when everything's done
function(err){
// All tasks are done now
doSomethingOnceAllAreDone();
}
);
In the above I need to make http call instead of item.someAsyncCall section. Specifically the code that I have is
var formaatedResponse=[];
request("http://" + API_HOST + "/app/v1/customers/" + element,
function (error, response, body) {
console.log('Ajax call response');
formaatedResponse.push(JSON.parse(response.body));
});
How to accommodate my changes since when I tried adding the above code it does not work as intended.
My code:
function consolidateResponse(){
console.log("Everything is done.");
console.log(formaatedResponse);
}
// 1st para in async.each() is the array of items
async.each(result,
// 2nd param is the function that each item is passed to
function(element, callback){
// Call an asynchronous function, often a save() to DB
request("http://" + API_HOST + "/app/v1/customers/" + element, function (error, response, body) {
console.log('Ajax call response');
formaatedResponse.push(JSON.parse(response.body));
});
callback();
},
// 3rd param is the function to call when everything's done
function(err){
// All tasks are done now
consolidateResponse();
}
);
Regards,
Pradeep
You should call callback() inside request()'s callback:
request("http://" + API_HOST + "/app/v1/customers/" + element,
function (error, response, body) {
console.log('Ajax call response');
formaatedResponse.push(JSON.parse(response.body));
callback();
});
This way you're signalling you're done with the particular item when http request actually finishes instead of right after it starts.
I expect that moving the callback into your call should resolve the issue:
// 1st para in async.each() is the array of items
async.each(result,
// 2nd param is the function that each item is passed to
function (element, callback) {
// Call an asynchronous function, often a save() to DB
element.someAsyncCall(function () {
request("http://" + GLASS_API_HOST + "/glass/v1/vmware/vm/" + element, function (error, response, body) {
console.log('Ajax call response');
formaatedResponse.push(JSON.parse(response.body));
/*Changed:*/ callback();
});
});
},
// 3rd param is the function to call when everything's done
function (err) {
// All tasks are done now
consolidateResponse();
}
);
Explanation:
In the previous code, the callback was called immediately after a request was issued, without waiting for the response. By moving the callback to the internal function of the request, you make sure it will not be called before the response is returned.
Also note async.each() works in parallel for all items, so if your result set is extemely large you may want to consider using async.eachLimit which will limit parallel requests to a defined value, or async.eachSeries which will make sure execution of requests is in series.
Another suggestion:
You can utilize the callback to handle errors, this way:
request("http://" + GLASS_API_HOST + "/glass/v1/vmware/vm/" + element, function (error, response, body) {
if(error){
callback(error);
return
}
console.log('Ajax call response');
formaatedResponse.push(JSON.parse(response.body));
callback();
});
And then async.each callback:
// 3rd param is the function to call when everything's done
function (err) {
if (err) {
// Handle error here: log it, report it, and return response with err code. example:
console.error(err);
// Halt execution;
return;
}
// All tasks are done now, and no error occurred:
consolidateResponse();
}
So your final code might look like:
// 1st para in async.each() is the array of items
async.each(result,
// 2nd param is the function that each item is passed to
function (element, callback) {
// Call an asynchronous function, often a save() to DB
element.someAsyncCall(function () {
request("http://" + GLASS_API_HOST + "/glass/v1/vmware/vm/" + element, function (error, response, body) {
if (error) {
callback(error);
return
}
console.log('Ajax call response');
formaatedResponse.push(JSON.parse(response.body));
callback();
});
});
},
// 3rd param is the function to call when everything's done
function (err) {
if (err) {
// Handle error here: log it, report it, and return response with err code. example:
console.log(err);
// Halt execution;
return;
}
// All tasks are done now, and no error occurred:
consolidateResponse();
}
);
I am trying to understand control flow in Node.js applications. Specifically does control returns to the original function once callback method completes (like a callback stack in recursive calls). I wrote a simple program that make a GET call and return the data. Here is the program:
Code:
var async = require('async');
var http = require('http');
function getGoogleData(url, callback) {
http.get(url, function(response) {
if (response.statusCode == 200) {
var googleInfo = '';
response.on('data', function(chunk) {
console.log("receiving data... ");
googleInfo += chunk;
return;
});
response.on('end', function() {
console.log("End of data receive... ");
response.setEncoding('utf8');
return callback(null, googleInfo);
});
}
console.log("I am here but why!");
//callback(new Error("GET called failed status_code=" + response.statusCode));
});
console.log("Return from get google data");
}
async.waterfall([
function(callback) {
console.log("In func 1");
getGoogleData("http://www.google.com", callback);
},
function(data, callback) {
console.log("In func 2");
callback(data);
}],
function (err, res) {
console.log("In err fn");
});
Here is output of the program:
Output:
In func 1
Return from get google data
I am here but why!
receiving data...
receiving data...
End of data receive...
In func 2
In err fn
Can someone help me understand why 'I am here but why!' line gets printed as the second output line in console log even after returning from 'data' event emitter? What is the overall control flow here?
The reason you're seeing that message logged first is that all that the code inside the if block is doing is adding event handlers. Those events are emitted some time in the future, after your console.log has already executed.
It's a similar reason why "Return from get google data" gets printed before the request finishes, because the http request is asynchronous.
I'm looking to execute a callback upon the full completion of a recursive function that can go on for an undetermined amount of time. I'm struggling with async issues and was hoping to get some help here. The code, using the request module, is as follows:
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, callback);
});
};
var recurse = function(startingPlace, callback) {
request.get({
url: 'bbb'
}, function(error, response, body) {
// store body somewhere outside these funtions
// make second request
request.get({
url: 'ccc'
}, function(error, response, body) {
var anArray = JSON.parse(body).stuff;
if (anArray) {
anArray.forEach(function(thing) {
request.get({
url: 'ddd'
}, function(error, response, body) {
var nextPlace = JSON.parse(body).place;
recurse(nextPlace);
});
})
}
});
});
callback();
};
start(function() {
// calls final function to print out results from storage that gets updated each recursive call
finalFunction();
});
It seems that once my code goes past the for loop in the nested requests, it continues out of the request and ends the initial function call while the recursive calls are still going on. I want it to not finish the highest-level iteration until all the nested recursive calls have completed (which I have no way of knowing how many there are).
Any help is GREATLY appreciated!
In your example you have no recursive calls. If I understand correctly you want to say that recurse(point, otherFunc); is the beginning of a recursive call.
Then just go back to the definition of the recursive call (which you have not shown in your post) and do this (add a third argument for a callback function to be called in the end of recursion; the caller will pass it as a parameter):
function recurse(startingPlace, otherFunc, callback_one) {
// code you may have ...
if (your_terminating_criterion === true) {
return callback_one(val); // where val is potentially some value you want to return (or a json object with results)
}
// more code you may have
}
Then in the original code that you posted, make this call instead (in the inner-most part):
recurse(startingPlace, otherFunc, function (results) {
// results is now a variable with the data returned at the end of recursion
console.log ("Recursion finished with results " + results);
callback(); // the callback that you wanted to call right from the beginning
});
Just spend some time and try to understand my explanation. When you understand, then you will know node. This is the node philosophy in one post. I hope it is clear. Your very first example should look like this:
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, otherFunc, function (results) {
console.log ("Recursion finished with results " + results);
callback();
});
});
};
Below is only additional information in case you are interested. Otherwise you are set with the above.
Typically in node.js though, people return an error value as well, so that the caller knows if the function that was called has finished successfully. There is no big mystery here. Instead of returning just results people make a call of the form
return callback_one(null, val);
Then in the other function you can have:
recurse(startingPlace, otherFunc, function (recError, results) {
if (recErr) {
// treat the error from recursion
return callback(); // important: use return, otherwise you will keep on executing whatever is there after the if part when the callback ends ;)
}
// No problems/errors
console.log ("Recursion finished with results " + results);
callback(); // writing down `return callback();` is not a bad habit when you want to stop execution there and actually call the callback()
});
Update with my suggestion
This is my suggestion for the recursive function, but before that, it looks like you need to define your own get:
function myGet (a, callback) {
request.get(a, function (error, response, body) {
var nextPlace = JSON.parse(body).place;
return callback(null, nextPlace); // null for no errors, and return the nextPlace to async
});
}
var recurse = function(startingPlace, callback2) {
request.get({
url: 'bbb'
}, function(error1, response1, body1) {
// store body somewhere outside these funtions
// make second request
request.get({
url: 'ccc'
}, function(error2, response2, body2) {
var anArray = JSON.parse(body2).stuff;
if (anArray) {
// The function that you want to call for each element of the array is `get`.
// So, prepare these calls, but you also need to pass different arguments
// and this is where `bind` comes into the picture and the link that I gave earlier.
var theParallelCalls = [];
for (var i = 0; i < anArray.length; i++) {
theParallelCalls.push(myGet.bind(null, {url: 'ddd'})); // Here, during the execution, parallel will pass its own callback as third argument of `myGet`; this is why we have callback and callback2 in the code
}
// Now perform the parallel calls:
async.parallel(theParallelCalls, function (error3, results) {
// All the parallel calls have returned
for (var i = 0; i < results.length; i++) {
var nextPlace = results[i];
recurse(nextPlace, callback2);
}
});
} else {
return callback2(null);
}
});
});
};
Note that I assume that the get request for 'bbb' is always followed by a get request for 'ccc'. In other words, you have not hidden a return point for the recursive calls where you have the comments.
Typically when you write a recursive function it will do something and then either call itself or return.
You need to define callback in the scope of the recursive function (i.e. recurse instead of start), and you need to call it at the point where you would normally return.
So, a hypothetical example would look something like:
get_all_pages(callback, page) {
page = page || 1;
request.get({
url: "http://example.com/getPage.php",
data: { page_number: 1 },
success: function (data) {
if (data.is_last_page) {
// We are at the end so we call the callback
callback(page);
} else {
// We are not at the end so we recurse
get_all_pages(callback, page + 1);
}
}
}
}
function show_page_count(data) {
alert(data);
}
get_all_pages(show_page_count);
I think you might find caolan/async useful. Look especially into async.waterfall. It will allow you to pass results from a callback from another and when done, do something with the results.
Example:
async.waterfall([
function(cb) {
request.get({
url: 'aaa.com'
}, function(err, res, body) {
if(err) {
return cb(err);
}
cb(null, JSON.parse(body).id);
});
},
function(id, cb) {
// do that otherFunc now
// ...
cb(); // remember to pass result here
}
], function (err, result) {
// do something with possible error and result now
});
If your recursive function is synchronous, just call the callback on the next line:
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, otherFunc);
// Call output function AFTER recursion has completed
callback();
});
};
Else you need to keep a reference to the callback in your recursive function.
Pass the callback as an argument to the function and call it whenever it is finished.
var start = function(callback) {
request.get({
url: 'aaa.com'
}, function (error, response, body) {
var startingPlace = JSON.parse(body).id;
recurse(startingPlace, otherFunc, callback);
});
};
Build your code from this example:
var udpate = function (callback){
//Do stuff
callback(null);
}
function doUpdate() {
update(updateDone)
}
function updateDone(err) {
if (err)
throw err;
else
doUpdate()
}
doUpdate();
With ES6, 'es6-deferred' & 'q'. You could try as following,
var Q = require('q');
var Deferred = require('es6-deferred');
const process = (id) => {
var request = new Deferred();
const ids =//do something and get the data;
const subPromises = ids.map(id => process(id));
Q.all(subPromises).then(function () {
request.resolve();
})
.catch(error => {
console.log(error);
});
return request.promise
}
process("testId").then(() => {
console.log("done");
});