I am making a call to a 3rd party api and I am having problems collecting all the returns and returning them as 1 array In my API. I can see that I am successfully making the calls and they are returning. Due to asynch the final array is returning before it is populated. Is there an elegant solution to handling this?
var itemIds = ['1','2','3','4','5','6']
exports.getItemData = function getItemData(req, res) {
var items = [];
var errors = [];
for(var itemId in itemIds) {
var options = {
uri: itemEndpoint + itemIds[itemId] +'/',
json: true
};
RequestPromise(options).then(function (item){
console.log(item);
items.push(item);
}).catch(function(err){
console.log(err)
errors.push(err);
});
};
res.type('application/json');
res.json(items);
};
Felix is right. You need to create an array of RequestPromise(options) Promises and then use the Promise.all([array-of-promises]).then(function (<array-of-result-arrays>){}).
So your refactored code will look like this:
var allPromises = [];
for(var itemId in itemIds) {
var options = {
uri: itemEndpoint + itemIds[itemId] +'/',
json: true
};
allPromises .push(RequestPromise(options));
}
//so now you have an array of promises in allPromises. Now when they all resolve:
Promise.all(allPromises).then(function (allResults){
console.log(allResults);
//do whatever with the results...
}).catch(function(err){
console.log(err)
errors.push(err);
});
I hope this helps.
Related
I have a couple of APIs I need to call to collect and merge information.
I make the first API call and, based on the result, I make several calls to the second one (in a loop).
Since http requests are asynchronous I'm loosing the information. By the time the second step is finished the server (nodejs) already sent the response back to the client.
I've already tried to, somehow, use the callback functions. This managed to keep the response to the client waiting but the information of the second call was still lost. I guess somehow the variables are not being synchronized.
I also did a quick test with away/async but my Javascript mojo was not enough to make it run without errors.
/* pseudo code */
function getData(var1, callback){
url= "http://test.server/bla?param="+var1;
request.get(url, function (error, response, body){
var results = [];
for(var item of JSON.parse(body).entity.resultArray) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function(secondStepData){
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
});
results.push(o);
}
callback(results);
});
}
function getSecondStep(object, callback){
url = "http://othertest.server/foobar?param=" + object.data1;
request.get(url, function (error, response, body){
var results = [];
if(response.statusCode == 200){
for(var item of JSON.parse(body).object.array) {
var o = {}
o['data4'] = item.data4;
o['data5'] = item.data5;
results.push(o);
}
callback(results);
}
});
}
What I would like is to be able to collect all the information into one JSON object to return it back to the client.
The client will then be responsible for rendering it in a nice way.
I recommend using the async / await pattern with the request-promise-native library.
This makes API calls really easy to make and the code is cleaner when using this pattern.
In the example below I'm just calling a httpbin API to generate a UUID but the principle applies for any API.
const rp = require('request-promise-native');
async function callAPIs() {
let firstAPIResponse = await rp("https://httpbin.org/uuid", { json: true });
console.log("First API response: ", firstAPIResponse);
// Call several times, we can switch on the first API response if we like.
const callCount = 3;
let promiseList = [...Array(callCount).keys()].map(() => rp("https://httpbin.org/uuid", { json: true }));
let secondAPIResponses = await Promise.all(promiseList);
return { firstAPIResponse: firstAPIResponse, secondAPIResponses: secondAPIResponses };
}
async function testAPIs() {
let combinedResponse = await callAPIs();
console.log("Combined response: " , combinedResponse);
}
testAPIs();
In this simple example we get a combined response like so:
{
{
firstAPIResponse: { uuid: '640858f8-2e69-4c2b-8f2e-da8c68795f21' },
secondAPIResponses: [
{ uuid: '202f9618-f646-49a2-8d30-4fe153e3c78a' },
{ uuid: '381b57db-2b7f-424a-9899-7e2f543867a8' },
{ uuid: '50facc6e-1d7c-41c6-aa0e-095915ae3070' }
]
}
}
I suggest you go over to a library that supports promises (eg: https://github.com/request/request-promise) as the code becomes much easier to deal with than the callback method.
Your code would look something like:
function getData(var1){
var url = "http://test.server/bla?param="+var1;
return request.get(url).then(result1 => {
var arr = JSON.parse(body).entity.resultArray;
return Promise.all( arr.map(x => request.get("http://othertest.server/foobar?param=" + result1.data1)))
.then(result2 => {
return {
data1: result1.data1,
data2: result1.data2,
data3: result1.data3,
secondStepData: result2.map(x => ({data4:x.data4, data5:x.data5}))
}
})
});
}
And usage would be
getData("SomeVar1").then(result => ... );
The problem is that you are calling the callback while you still have async calls going on. Several approaches are possible, such us using async/await, or reverting to Promises (which I would probably do in your case).
Or you can, well, call the callback only when you have all the information available. Pseudo code follows:
function getData(var1, callback){
url= "http://test.server/bla?param="+var1;
request.get(url, function (error, response, body){
var results = [];
var items = JSON.parse(body).entity.resultArray;
var done = 0, max = items.length;
for(var item of items) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function(secondStepData){
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
results.push(o);
done += 1;
if(done === max) callback(results);
});
}
});
}
(note that since this is pseudo code, I am not checking for errors or handling a possible empty result from request.get(...))
You need to call the callback of first function only when all the second callback functions have been called. Try this changes:
function getData(var1, callback) {
url = "http://test.server/bla?param=" + var1;
request.get(url, function (error, response, body) {
var results = [],count=0;
var arr = JSON.parse(body).entity.resultArray;
for (let [index, value] of arr.entries()) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function (secondStepData) {
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
results[index] = o;
count++;
if (count === arr.length) {
callback(results);
}
});
}
});
}
I am trying to perform sql queries based on the callback results in if conditions but i am unable to write the code .so please provide som information in code
app.get('/resell-property', function(req, res) {
var data = {}
data.unit_price_id = 1;
function callback(error, result) {
if (result.count == 0) {
return hp_property_sell_request.create(data)
}
else if (result.count > 0) {
return hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
}
}
hp_property_sell_request.findAndCountAll({
where: {
unit_price_id: data.unit_price_id
}
}).then(function (result) {
if (result) {
callback(null, result);
}
});
});
In this how can i write the callbacks for
hp_property_sell_request.create(data) ,hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
In that after returning result again i have to handle callbacks and perform this query
if(result.request_id){
return hp_unit_price.findAll({
where:{
unit_price_id:result.unit_price_id,
hp_property_id:result.property_id,
hp_unit_details_id:result.unit_details_id
}
}).then(function (result){
if(result.is_resale_unit==0 && result.sold_out==0){
return Sequelize.query('UPDATE hp_unit_price SET resale_unit_status=1 WHERE hp_unit_details_id='+result.unit_details_id+' and hp_property_id='+result.property_id)
}
})
}
The promise resolve function takes only one input argument, so if you need to pass in multiple stuff, you have to enclose them in a single object. Like, if you have to go with something like:
database.openCollection()
.then(function(collection){
var result = collection.query(something);
var resultObject = { result: result, collection: collection };
})
.then(function(resultObject){
doSomethingSyncronousWithResult(resultObject.result);
resultObject.collection.close();
});
You can't use Promise all if all of your stuff isn't a result of a promise resolve, you might need to go with something like this.
Disclaimer: The code example is a very poor one, but it explains the concept.
I would suggest you to learn about Promises, particularly Bluebird.
You can promisify traditional callback methods.
I would also create model level functions in different files. Here's an example.
parent.js
const db = require("./connections/database"); // connection to database
const getChildForParent = function (parentId, childId, callback) {
db.find({parent: parentId, child_id: childId}, "childrenTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
children.js
const db = require("./connections/database"); // connection to database
const getToysForChild = function (childId, callback) {
db.find({toy_belongs_to: parentId}, "toysTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
Then in controller you can do something like this:
const Bluebird = require("bluebird");
const Parent = require("./parent.js");
const Child = require("./child.js");
// Promisifying adds "Async" at the end of your methods' names (these are promisified)
Bluebird.promisifyAll(Parent);
Bluebird.promisifyAll(Child);
// Just an example.
app.get("/parent/:parentId/children/:childId", function(req, res) {
return Bluebird.try(function() {
return User.getChildForParentAsync(req.params.parentId, req.params.childId);
}).then(function(child) {
return Child.getToysForChildAsync(child.child_id);
}).then(function(toys) {
// Do something with toys.
});
});
Of course you can do much more with this and this is not the only way.
Also you can use Promise.all(). This method is useful for when you want to wait for more than one promise to complete.
Let's say you have a list of urls that you want to fetch and process the results after all the data has been fetched.
var urls = [url1, url2, url3, url4, url5 .......... ];
var Bluebird = require("bluebird");
var request = require("request"); // callback version library
Bluebird.promisifyAll(request);
// create a list which will keep all the promises
var promises = [];
urls.forEach(function(url) {
promises.push(request.getAsync(url1));
});
// promises array has all the promises
// Then define what you want to do on completion.
Bluebird.all(promises).then(function(results) {
// results is an array with result a url in an index
// process results.
});
I would recommend to use Promises to solve that. If you need all results of all Requests, when they are all done Promise.all() will do that for you. Your basic could look like that:
var req1 = new Promise(function(res, rej){
var req = new XMLHttpRequest()
…
req.addEventListener('load', function (e) {
res(e);
})
var req2 = //similar to the above
Promise.all([req1, req2, …]).then(function(values){
//all requests are done here and you can do your stuff
});
You can also use the new fetch api, which creates Promises like so:
var req1 = fetch(…);
var req2 = fetch(…);
Promise.all([req1, re2, …]).then(…);
Ive seen that there are questions about chaining promises, but this one is a little bit different.
I'm making http get requests in my code. The first call returns an array. For each object in the array, i need to make another http call which returns another array and so on (this chains 3 levels deep).
The problem is, I need to keep track of which array element was used for to make each http call, and I dont know how to do this using promises.
I also want to end the chain by returning a promise.
I have the code for what I want to do written in nodejs without promises:
var https = require('https');
var fs = require('fs');
function makeRequest(options){
var httpopts = {
host: 'soc.courseoff.com',
path: '/gatech/terms/201601/majors/' + options.p,
method: 'GET'
};
var response = "";
var req = https.request(httpopts, function(res) {
res.on('data', function(d) {
response += d;
});
res.on('end',function(){
options.cb(response,options)
})
});
req.end();
req.on('error', function(e) {
console.error(e);
});
}
var classData = {};
function getCourses(m){
var majors = JSON.parse(m);
majors.forEach(function(maj){
classData[maj] = {};
var options = {
p:maj.ident +'/courses',
cb:getSections,
major:maj
};
makeRequest(options);
});
}
var classCount = 0;
function getSections(c,opts){
var courses = JSON.parse(c);
courses.forEach(function(course){
classCount++;
var options = JSON.parse(JSON.stringify(opts));
options.p += '/'+course.ident+'/sections';
options.course = course
options.cb = buildData
makeRequest(options)
});
}
var sectionCount = 0;
function buildData(r, options){
var major = options.major.ident;
sectionCount++;
if(!classData[major]){
classData[major] = {
name: options.major.name,
classes:{}
};
}
classData[major].classes[options.course.ident] = {
name:options.course.name,
sections:JSON.parse(r)
};
console.log('classCount-sectionCount '+classCount + '---'+sectionCount);
if(classCount === sectionCount){
writeIt();
}
}
makeRequest({
p:'',
cb:getCourses
});
function writeIt(){
fs.writeFileSync('./classData.js', 'module.exports = ' + JSON.stringify(classData));
}
EDIT:
I managed to get the promises to nest while keeping track of the data, but how can i return a promise that eventually resolves with the final data object?
My code:
Thanks four your help! I've managed to code it so that the promises work, my only problem now is in returning the final data as a promise
fact.factory('ClassFactory', ['$http',function ($http) {
var eventData = {};
var promise;
var courseData = [];
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
eventData.getClasses = function (event) {
if(!promise){
promise = $http.get(baseURL).then(
function(majors){
Promise.all(majors.data.map(m => $http.get(baseURL + m.ident+'/courses')
.then(
function(courses){
if(!m.courses) m.courses = [];
courses.data.map(c => $http.get(baseURL+ m.ident+'/courses/' +c.ident+'/sections' )
.then(
function(sections){
c.sections = sections.data;
m.courses.push(c);
}
));
courseData.push(m);
}
)));
}
)
}
return promise;
}
return eventData;
}]);
Almost certainly, each time you deal with an array of Promises, you'll want to use Promise.all in order to connect and merge your promises into a new promise. That promise will then contain an array of the results from each call. Nested Promise.alls can thus return Arrays of Arrays with all your levels of results as long as you use something like a map and a closure to capture the outer levels.
var fakeCall = x => Promise.resolve(x||Math.random());
Promise.all([fakeCall(1),fakeCall(2)])
.then(
results => Promise.all(results.map( x => fakeCall(5).then( results2 => [x, results2]) ))
)
.then( x => console.log(x));//-> [[1,5],[2,5]]
The first array of calls generates an array of results, and mapping over those with a function that makes yet more calls will return a single result that can be paired with its parent.
Explicitly nesting things in this way will work for even deeper levels, but is not going to be pretty. There's probably an abstraction you can create using Array.reduce which can generalize this pattern.
You forgot some returns in your code. The function you pass to .then should always return something. Also you are modifying majors but then throw it away without using it. When working with promises - especially when they are complex and nested - it's not a good idea to modify any data structures contained in those promises unless you are sure nothing bad can possibly happen.
I would split it into several functions.
e.g.
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
function getSections(major, course) {
return $http.get(baseURL+ major.ident+'/courses/' +course.ident+'/sections')
.then(sections => sections.data)
.catch(e => []);
}
function getCourses(major) {
return $http.get(baseURL + major.ident+'/courses')
.then(courses => Promise.all(courses.data.map(course =>
getSections(major, course).then(sections => ({[course.ident]: {name: course.name, sections: sections}})))))
.then(courses => angular.extend({}, ...courses))
.catch(e => ({}));
}
function getClassData() {
return $http.get(baseURL)
.then(majors => Promise.all(majors.data.map(major =>
getCourses(major).then(courses => ({[major.ident]: {name: major.name, classes: courses}})))))
.then(majors => angular.extend({}, ...majors))
.catch(e => ({}));
}
getClassData().then(data => console.log(data));
I have Meteor method with a following fragment (below) and it has asynchronous loop inside. As far as i know, this should be wrapped with meteor wrapAsync, but their documentation isn't clear enough to me, so i'm not sure how to implement it properly. What would be the easiest way to use wrapAsync on this example?
if(Meteor.isServer){
Meteor.methods({
listCollections: function(){
// list collections in database
db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
collections = db.listCollections();
// convert cursor with collections to javascript array:
// method should wait for this loop to finish before returning result
var collectionsArray = [];
collections.each(function(n, collection){
if(collection){
collectionsArray.push(collection.name);
}
});
// return ignores the each loop and empty array is passed as a result
return collectionsArray;
}
});
Meteor.call("listCollections", function(err, res){
console.log(res);
});
}
It is quite hard to understand from your context, but normally, the wrapAsync could be used like this
Meteor.methods({
'yourCurrentMethod': function(sentCollections) {
var processArray = function (collections) {
var collectionsArray = [];
collections.each(function(n, collection){
if(collection){
collectionsArray.push(collection.name);
}
});
// return ignores the each loop and empty array is passed as a result
return collectionsArray;
};
var processArrayCalling = Meteor.wrapAsync(processArray);
var result = processArrayCalling(sentCollections);
// process with the result
}
});
I have resolved this problem using Future, which was really easy to use.
if(Meteor.isServer){
var Future = Npm.require( 'fibers/future' ); // future code
Meteor.methods({
listCollections: function(){
// list collections in database
db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
collections = db.listCollections();
// create our future instance
var future = new Future(); // future code
// count number of collections inside collection list
var collectionsCount = 0;
collections.each(function(n, collection){
collectionsCount++;
});
// method should wait for this loop to finish before returning result
var collectionsArray = [];
var i = 0;
collections.each(function(n, collection){
if(collection){
collectionsArray.push(collection.name);
}
i++;
if( i == collectionsCount ){
future.return( collectionsArray ); // future code
}
});
return future.wait(); // future code
}
});
Meteor.call("listCollections", function(err, res){
console.log(res);
});
}
I know this has been asked quite a few times already but after a day of search I still don't get it to work, although it's just like what is shown as a solution everywhere...
I have a async request to a database which returns an array of data. For each object in this array I need to start another async request to the database and as soon as ALL of these async requests resolve, I want to return them. I read you could do it with $q.all(...)
So here's the code:
Factory.firstAsyncRequest(id).then(function (arrayWithObjects) {
var promises = [];
var dataArr = [];
angular.forEach(arrayWithObjects, function (object, key) {
var deferred = $q.defer();
promises.push(deferred);
Factory.otherAsyncRequest(key).then(function (objectData) {
dataArr.push({
name: objectData.name,
key: key,
status: objectData.status
});
deferred.resolve();
console.info('Object ' + key + ' resolved');
});
});
$q.all(promises).then(function () {
$rootScope.data = dataArr;
console.info('All resolved');
});});
From the console I see that the $q.all is resolved BEFORE each object. Did I get something wrong? This seems to work for everyone...
Your help is highly appreciated, been looking the whole night, it's 5:30am now lol..
Cheers
EDIT:
So for anyone who's coming here later: It was just the promises.push(deferred.PROMISE) bit. Tho, I read that anguar.forEach is actually not a recommended method to loop through array because it was originally not constructed to be used by the end-user. Don't know if that's correct but I figured out another way if you don't want to use angular.forEach:
Users.getAll(uid).then(function (users) {
var uids = ObjHandler.getKeys(users); //own function just iterating through Object.keys and pushing them to the array
var cntr = 0;
function next() {
if (cntr < uids.length) {
Users.getProfile(uids[cntr]).then(function (profile) {
var Profile = {
name: profile.name,
key: uids[cntr],
status: profile.status
});
dataArr[uids[cntr]] = Profile;
if(cntr===uids.length-1) {
defer.resolve();
console.info('Service: query finished');
} else {cntr++;next}
});
}
}
next();
});
And the getKey function:
.factory('ObjHandler', [
function () {
return {
getKeys: function(obj) {
var r = [];
for (var k in obj) {
if (!obj.hasOwnProperty(k))
continue;
r.push(k)
}
return r
}
};
}])
Instead of
promises.push(deferred);
Try this:
promises.push(deferred.promise);