I use the node request ajax package. So, i have an loop, in every iteration it makes an request to my server.
// realItems needs the complete value of items assigned
var realItems;
var items = [];
_.forEach(JSON.parse(body), (value, key) => {
request('myurl/' + id, (error, response, body) => {
items = JSON.parse(body)
});
});
How can i bundle all my requests from request package, so I can assign the value of items variable to the realItems at the end?
// edit:
I use react js, so in this case realItems is an state, and i can't trigger it in every loop iteration, because render triggers on every setState
There are a number of ways to approach this. Here's a brute force method that does not preserve the order of the results:
var items = [];
var cnt = 0;
_.forEach(JSON.parse(body), (value, key) => {
++cnt;
request('myurl/' + value.id, (error, response, body) => {
items.push(JSON.parse(body));
// if all requesets are done
if (--cnt === 0) {
// process items here as all results are done now
}
});
});
Here's a version that uses Bluebird promises:
var Promise = require('bluebird');
var request = Promise.promisify(require("request"));
Promise.promisifyAll(request);
var promises = [];
_.forEach(JSON.parse(body), (value, key) => {
promises.push(request('myurl/' + value.id));
});
Promise.all(promises).then(function(results) {
// all requests are done, data from all requests is in the results array
// and are in the order that the requests were originally made
});
And, here's a little bit simpler Bluebird promises method that uses a Bluebird iterator:
var Promise = require('bluebird');
var request = Promise.promisify(require("request"));
Promise.promisifyAll(request);
Promise.map(JSON.parse(body), function(value) {
return request('myurl/' + value.id);
}).then(function(results) {
// all requests are done, data is in the results array
});
Is it a requirement that you use the request package? I use async which is similar and comes with a parallel method which does exactly what you're asking -
https://github.com/caolan/async#parallel
example:
async.parallel([
function(callback){
setTimeout(function(){
callback(null, 'one');
}, 200);
},
function(callback){
setTimeout(function(){
callback(null, 'two');
}, 100);
}
],
// optional callback
function(err, results){
// the results array will equal ['one','two'] even though
// the second function had a shorter timeout.
});
Related
I have a couple of APIs I need to call to collect and merge information.
I make the first API call and, based on the result, I make several calls to the second one (in a loop).
Since http requests are asynchronous I'm loosing the information. By the time the second step is finished the server (nodejs) already sent the response back to the client.
I've already tried to, somehow, use the callback functions. This managed to keep the response to the client waiting but the information of the second call was still lost. I guess somehow the variables are not being synchronized.
I also did a quick test with away/async but my Javascript mojo was not enough to make it run without errors.
/* pseudo code */
function getData(var1, callback){
url= "http://test.server/bla?param="+var1;
request.get(url, function (error, response, body){
var results = [];
for(var item of JSON.parse(body).entity.resultArray) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function(secondStepData){
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
});
results.push(o);
}
callback(results);
});
}
function getSecondStep(object, callback){
url = "http://othertest.server/foobar?param=" + object.data1;
request.get(url, function (error, response, body){
var results = [];
if(response.statusCode == 200){
for(var item of JSON.parse(body).object.array) {
var o = {}
o['data4'] = item.data4;
o['data5'] = item.data5;
results.push(o);
}
callback(results);
}
});
}
What I would like is to be able to collect all the information into one JSON object to return it back to the client.
The client will then be responsible for rendering it in a nice way.
I recommend using the async / await pattern with the request-promise-native library.
This makes API calls really easy to make and the code is cleaner when using this pattern.
In the example below I'm just calling a httpbin API to generate a UUID but the principle applies for any API.
const rp = require('request-promise-native');
async function callAPIs() {
let firstAPIResponse = await rp("https://httpbin.org/uuid", { json: true });
console.log("First API response: ", firstAPIResponse);
// Call several times, we can switch on the first API response if we like.
const callCount = 3;
let promiseList = [...Array(callCount).keys()].map(() => rp("https://httpbin.org/uuid", { json: true }));
let secondAPIResponses = await Promise.all(promiseList);
return { firstAPIResponse: firstAPIResponse, secondAPIResponses: secondAPIResponses };
}
async function testAPIs() {
let combinedResponse = await callAPIs();
console.log("Combined response: " , combinedResponse);
}
testAPIs();
In this simple example we get a combined response like so:
{
{
firstAPIResponse: { uuid: '640858f8-2e69-4c2b-8f2e-da8c68795f21' },
secondAPIResponses: [
{ uuid: '202f9618-f646-49a2-8d30-4fe153e3c78a' },
{ uuid: '381b57db-2b7f-424a-9899-7e2f543867a8' },
{ uuid: '50facc6e-1d7c-41c6-aa0e-095915ae3070' }
]
}
}
I suggest you go over to a library that supports promises (eg: https://github.com/request/request-promise) as the code becomes much easier to deal with than the callback method.
Your code would look something like:
function getData(var1){
var url = "http://test.server/bla?param="+var1;
return request.get(url).then(result1 => {
var arr = JSON.parse(body).entity.resultArray;
return Promise.all( arr.map(x => request.get("http://othertest.server/foobar?param=" + result1.data1)))
.then(result2 => {
return {
data1: result1.data1,
data2: result1.data2,
data3: result1.data3,
secondStepData: result2.map(x => ({data4:x.data4, data5:x.data5}))
}
})
});
}
And usage would be
getData("SomeVar1").then(result => ... );
The problem is that you are calling the callback while you still have async calls going on. Several approaches are possible, such us using async/await, or reverting to Promises (which I would probably do in your case).
Or you can, well, call the callback only when you have all the information available. Pseudo code follows:
function getData(var1, callback){
url= "http://test.server/bla?param="+var1;
request.get(url, function (error, response, body){
var results = [];
var items = JSON.parse(body).entity.resultArray;
var done = 0, max = items.length;
for(var item of items) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function(secondStepData){
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
results.push(o);
done += 1;
if(done === max) callback(results);
});
}
});
}
(note that since this is pseudo code, I am not checking for errors or handling a possible empty result from request.get(...))
You need to call the callback of first function only when all the second callback functions have been called. Try this changes:
function getData(var1, callback) {
url = "http://test.server/bla?param=" + var1;
request.get(url, function (error, response, body) {
var results = [],count=0;
var arr = JSON.parse(body).entity.resultArray;
for (let [index, value] of arr.entries()) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function (secondStepData) {
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
results[index] = o;
count++;
if (count === arr.length) {
callback(results);
}
});
}
});
}
I am trying to perform sql queries based on the callback results in if conditions but i am unable to write the code .so please provide som information in code
app.get('/resell-property', function(req, res) {
var data = {}
data.unit_price_id = 1;
function callback(error, result) {
if (result.count == 0) {
return hp_property_sell_request.create(data)
}
else if (result.count > 0) {
return hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
}
}
hp_property_sell_request.findAndCountAll({
where: {
unit_price_id: data.unit_price_id
}
}).then(function (result) {
if (result) {
callback(null, result);
}
});
});
In this how can i write the callbacks for
hp_property_sell_request.create(data) ,hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
In that after returning result again i have to handle callbacks and perform this query
if(result.request_id){
return hp_unit_price.findAll({
where:{
unit_price_id:result.unit_price_id,
hp_property_id:result.property_id,
hp_unit_details_id:result.unit_details_id
}
}).then(function (result){
if(result.is_resale_unit==0 && result.sold_out==0){
return Sequelize.query('UPDATE hp_unit_price SET resale_unit_status=1 WHERE hp_unit_details_id='+result.unit_details_id+' and hp_property_id='+result.property_id)
}
})
}
The promise resolve function takes only one input argument, so if you need to pass in multiple stuff, you have to enclose them in a single object. Like, if you have to go with something like:
database.openCollection()
.then(function(collection){
var result = collection.query(something);
var resultObject = { result: result, collection: collection };
})
.then(function(resultObject){
doSomethingSyncronousWithResult(resultObject.result);
resultObject.collection.close();
});
You can't use Promise all if all of your stuff isn't a result of a promise resolve, you might need to go with something like this.
Disclaimer: The code example is a very poor one, but it explains the concept.
I would suggest you to learn about Promises, particularly Bluebird.
You can promisify traditional callback methods.
I would also create model level functions in different files. Here's an example.
parent.js
const db = require("./connections/database"); // connection to database
const getChildForParent = function (parentId, childId, callback) {
db.find({parent: parentId, child_id: childId}, "childrenTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
children.js
const db = require("./connections/database"); // connection to database
const getToysForChild = function (childId, callback) {
db.find({toy_belongs_to: parentId}, "toysTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
Then in controller you can do something like this:
const Bluebird = require("bluebird");
const Parent = require("./parent.js");
const Child = require("./child.js");
// Promisifying adds "Async" at the end of your methods' names (these are promisified)
Bluebird.promisifyAll(Parent);
Bluebird.promisifyAll(Child);
// Just an example.
app.get("/parent/:parentId/children/:childId", function(req, res) {
return Bluebird.try(function() {
return User.getChildForParentAsync(req.params.parentId, req.params.childId);
}).then(function(child) {
return Child.getToysForChildAsync(child.child_id);
}).then(function(toys) {
// Do something with toys.
});
});
Of course you can do much more with this and this is not the only way.
Also you can use Promise.all(). This method is useful for when you want to wait for more than one promise to complete.
Let's say you have a list of urls that you want to fetch and process the results after all the data has been fetched.
var urls = [url1, url2, url3, url4, url5 .......... ];
var Bluebird = require("bluebird");
var request = require("request"); // callback version library
Bluebird.promisifyAll(request);
// create a list which will keep all the promises
var promises = [];
urls.forEach(function(url) {
promises.push(request.getAsync(url1));
});
// promises array has all the promises
// Then define what you want to do on completion.
Bluebird.all(promises).then(function(results) {
// results is an array with result a url in an index
// process results.
});
I would recommend to use Promises to solve that. If you need all results of all Requests, when they are all done Promise.all() will do that for you. Your basic could look like that:
var req1 = new Promise(function(res, rej){
var req = new XMLHttpRequest()
…
req.addEventListener('load', function (e) {
res(e);
})
var req2 = //similar to the above
Promise.all([req1, req2, …]).then(function(values){
//all requests are done here and you can do your stuff
});
You can also use the new fetch api, which creates Promises like so:
var req1 = fetch(…);
var req2 = fetch(…);
Promise.all([req1, re2, …]).then(…);
I am trying to get resource from the some url, that resource is separated as multiple pages, the number of page is over 500, but the order of resource have to be guaranteed, so I decided to use Async module.
function getData(page, callback){
var url = "http://someurl.com/document?page="+page;
// get data from the url ...
// when success,
callback();
};
So, above function is get resource from some url, I have to iterate this function to many times, but I don't know how can I iterate this with async waterfall. What point I should push the for iteration?
async.waterfall([
// page 1
function(callback){
getData(1, function(){
callback(null);
});
},
// page 2
function(callback){
getData(2, function(){
callback(null);
});
},
// page 3, 4, 5..... 100, ... 500
],function(err, result){
if(err) return next();
console.log('finish !');
});
Why don't you use Promises:
function getData(page, callback){
var url = "http://someurl.com/document?page="+page;
// var request = call to get data from the url ...
return request;
};
var promises = [];
for (var page = 1; page < totalPages; page++) {
promises.push(getData(page));
}
Promise.all(promises).then(function (listOfResults) {
// Do callback here
});
Just make sure that your method of sending the get request returns a promise. If not you can use a function like this:
function usesCallback(data, callback) {
// Do something with data
callback();
}
function makePromise(data) {
return new Promise(function (resolve, reject) {
usesCallback(data, resolve);
});
}
Here is some more info on Promises
If you do want to use async, async.map() and async.mapLimit() are better choices since they apply the iteratee to each item in parallel and guarantee the results array to be in the same order as the original collection.
async.mapLimit(_.range(1, 500), 10, getData, function (err, results) {
// do something with results
});
The code above means to get data from page 1 to page 500, with no more than 10 async operations at a time.
_.range(1, 500) is a function from underscore to generate an array [1, 2, 3, ..., 500]. If you do not like to use underscore in your project, you can simply rewrite it such as:
function range(lower, upper) {
return Array.apply(null, Array(upper - lower + 1))
.map(function (_, i) { return lower + i; });
}
You can use async's whilst for this check here.
I need to make a request to get a list of html, and I need to scan it and loop through it and make more requests for each item in the list found, and those might have lists inside them, and so on till theres none left.
I need a way to keep track of all the requests called and call another function when they're done. The tricky bit is the function calls itself over and over for any list items found in the HTML.
The problem I'm having is using Q promises the only promises it waits for are from first request made, and I cant understand why assuming node works like I think it does, see code:
var _ = require('underscore');
var request = require('request');
var allPromises = [];
var finalArray = [];
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function() {
_.each(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore) {
start(item.id);
}
}
deferred.resolve();
});
allPromises.push(deferred.promise);
}
console.log('Starting');
start(1);
Q.all(allPromises).done(function (values) {
console.log('All Done');
});
What I thought happens is:
1 - starts() is called for the first time and the first deferred var is created
2 - the first request is made and the first created deferred variable is pushed to the promises array
3 - Q.all is called and waits
4 - The first request's callback is called
5 - if the request contains body.x, start() is called again with a new id
6 - a new promises is created and pushed and a new request is made
7 - the first promise is resolved
assuming this only went one level deep
8 - the second promise is resolved
9 - Q.all calls its callback
but in practice, Q.all calls its callback after the first promise, it doesn't wait for any others even though the second promise is pushed before the first promise is resolved.
Why? And how can I make this work?
Update forgot to add the loop inside the request callback.
Answer to edited question:
var request = require('request');
var finalArray = [];
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function() {
var subitems = [];
_.each(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore) {
subitems.push(start(item.id));
}
}
if (subitems.length) {
deferred.resolve(Q.all(subitems)); // resolve result of Q.all
} else {
deferred.resolve();
}
});
return deferred.promise;
}
start(1).done(function() {
console.log('All Done');
});
#Bergi's code
var request = require('request');
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function(err, body) {
if (err) deferred.reject(err);
else deferred.resolve(body);
});
return deferred.promise.then(function(body) {
var finalArray = [];
return Q.all(_.map(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore)
return start(item.id);
else
return [];
})).then(function(moreResults) {
return finalArray.concat.apply(finalArray, moreResults);
});
});
}
start(1).then(function(finalArray) {
console.log('All Done');
});
I have a strange problem, when I push my result in my array, the result isn't at the right position in my array (for example the result instead of being at the index number 1 is at the index 3), and when I re-run my module results change of position randomly in the array .
var cote = function(links, callback) {
var http = require('http');
var bl = require('bl');
var coteArgus = [];
for (i = 0; i < links.length; i ++) {
http.get('http://www.website.com/' + links[i], function(response) {
response.pipe(bl(function(err, data) {
if (err) {
callback(err + " erreur");
return;
}
var data = data.toString()
newcoteArgus = data.substring(data.indexOf('<div class="tx12">') + 85, data.indexOf(';</span>') - 5);
myresult.push(newcoteArgus);
callback(myresult);
}));
});
}
};
exports.cote = cote;
The problem lies in the fact that although the for is synchronous the http.get and the pipe operation are not (I/O is async in nodejs) so the order of the array depends on which request and pipe finishes first which is unknown.
Try to avoid making async operations in a loop, instead use libraries like async for flow control.
I think this can be done in the right order, using async map
Here a sample with map and using request module.
// There's no need to make requires inside the function,
// is better just one time outside the function.
var request = require("request");
var async = require("async");
var cote = function(links, callback) {
var coteArgus = [];
async.map(links, function(link, nextLink) {
request("http://www.website.com/" + link, function(err, response, body) {
if (err) {
// if error so, send to line 28 with a error, exit from loop.
return nextLink(err);
}
var newcoteArgus = body.substring(
body.indexOf("<div class='tx12'>") + 85,
body.indexOf(";</span>") - 5
);
// pass to next link, and add newcoteArgus to the final result
nextLink(null, newcoteArgus);
});
},
function(err, results) {
// if there's some errors, so call with error
if(err) return callback(err);
// there's no errors so get results as second arg
callback(null, results);
});
};
exports.cote = cote;
One more thing, i'm not sure, really what you are doing in the part where you search html content in the responses but there's a really good library to work with JQuery selectors from server side maybe can be useful for you.
Here's how you should call the function
// Call function sample.
var thelinks = ["features", "how-it-works"];
cote(thelinks, function(err, data) {
if(err) return console.log("Error: ", err);
console.log("data --> ", data);
});