Develop Reference

node.js: structure multiple API requests, work on them and combine them

currently I am struggeling a little bit with node.js (I am new to it) doing different API requests (Usabilla API), work on the results and then combine them in order to work on the whole set (e.g. export).
Requesting the API is not the problem but I can't get the results out to do some other stuff on it (asynchronous code drives me crazy).
Attached please find a overview how I thought to do this. Maybe I am totally wrong about this or maybe you have other more elegant suggestions.
My code works until I have to request the two different API "adresses" (they are provided) and then extract the results to do some other stuff.
My problem here is that there are nested functions with a promise and I cant figure out how to pass this through the parent function inside waterfall to get handled by the next function.
In the code, of course there is nothing parallel as shown in the diagram.
Thats another point, how to do that ? Simply nest parallel and series/ another waterfall inside waterfall ?
I am a little bit confused because that gets more and more complex for a simple problem when this would be done with synchronous code.
Here I build up all my request querys (at the moment 4):
function buildQuery(IDs,callback){
var i = 0;
var max = Object.keys(IDs).length;
async.whilst(
function(){return i < max},
function(callback){
FeedbackQuery[i] =
{
identifier: IDs[i].identifier,
query:
{id: IDs[i].id,
params: {since:sinceDate,}
}
};
i++;
callback(null,i);
})
console.log(FeedbackQuery);
callback (null,FeedbackQuery);
};
I then have to decide which type of query it is and add it to an object which should contain all the items of this identifier type:
function FeedbackRequest(FeedbackQuery,callback)
{
var i = 0;
var max = Object.keys(FeedbackQuery).length;
async.whilst(
function(){return i < max},
function (callback){
identifier = FeedbackQuery[i].identifier;
APIquery = FeedbackQuery[i].query;
switch(identifier)
{
case 'mobilePortal':
console.log(FeedbackQuery[i].identifier, 'aktiviert!');
var result = api.websites.buttons.feedback.get(APIquery);
result.then(function(feedback)
{
var item = Object.keys(feedbackResults).length;
feedbackResultsA[item] = feedback;
callback(null, feedbackResultsA);
})
break;
case 'apps':
console.log(FeedbackQuery[i].identifier, 'aktiviert!');
var result = api.apps.forms.feedback.get(APIquery);
result.then(function(feedback)
{
var item = Object.keys(feedbackResults).length;
feedbackResultsB[item] = feedback;
callback(null, feedbackResultsB);
})
break;
}
i++;
callback(null,i);
})
};
Currently the functions are bundled in an async waterfall:
async.waterfall([
async.apply(buildQuery,IDs2request),
FeedbackRequest,
// a function to do something on the whole feedbackResults array
],function (err, result) {
// result now equals 'done'
if (err) { console.log('Something is wrong!'); }
return console.log('Done!');
})
How it actually should be:
Structure
Thank you very much for any tips or hints!

I'm not proficient with async, and I believe if you'r new to this, it's harder than a simple Promise library like bluebird combine with lodash for helpers.
What I would do based on your schemas :
var firstStepRequests = [];
firstStepRequests.push(buildQuery());// construct your first steps queries, can be a loop, goal is to have firstStepRequests to be an array of promise.
Promise.all(firstStepRequests)
.then((allResults) => {
var type1 = _.filter(allResults, 'request_type_1');
var type2 = _.filter(allResults, 'request_type_2');
return {
type1: type1,
type2: type2
};
})
.then((result) => {
result.type1 = //do some work
result.type2 = //do some work
return result;
})
.then((result) => {
//export or merge or whatever.
});
Goal is to have a simple state machine.
UPDATE
If you want to keep identifier for a request, you can use props to have :
var props = {
id_1:Promise,
id_2:Promise,
id_3:Promise
};
Promise.props(props).then((results) => {
// results is {
id_1:result of promise,
id_2:result of promise,
etc...
}
})
You could do something like :
var type1Promises = getType1Requests(); //array of type 1
var type2Promises = getType2Requests(); // array of type 2
var props = {
type_1: Promise.all(type1Promises),
type_2: Promise.all(type2Promises)
}
Promise.props(props).then((result) => {
//result is : {
type_1: array of result of promises of type 1
type_2: array of result of promises of type 2
}
})

A promise within a function

My program downloads a large amount of data, processes it, and makes it available through a returned function. The program gets ahread of the download, so I am adding promises to make it wait for the data to arrive.
function dataSource(...) {
var _data = null;
// download: a promise that returns data for the _data object.
let download = function() { ... }
return function(...) {
if (!_data) {
download(...).then(data => _data = data).done();
}
var datum = _data[key];
var outbound = doSomethingWithData(datum);
return outbound;
}
}
My code is structured like this because the function that Engine returns makes my code very neat.
var generate = dataSource(param1,param2);
var fullName = generate("malename")+" "+generate("malename")+" "+generate("surname");
The specific requirements are:
Download the data only once.
Query the data by key any number of times without downloading the data again.
Do not change the existing interface.
I could have dataSource return a promise rather than a function. I know what the pattern for using promises looks like. But that will force me to rewrite the code that consumes this function. This pattern is used extensively throughout the code, and changing it isn't an acceptable solution.
How can I structure this to ensure that my function doesn't return until it has the data, without returning the promise?

This should fix it
function dataSource(){
return function(){
return download(...).then(data=>doSomethingWithData(data[key]));
};
}
var generate = dataSource();
Promise
.all(["malename","malename","surname"].map((name)=>return generate(name)))
.spread((name1,name2,name3)=>{
return [name1,name2,name2].join(" ");
});
the spread is not needed, but it helps for illustration purposes

Have dataSource return a promise rather than the data function. The revised dataSource looks like this:
function dataSource(...) {
var _data = null;
// download: a promise that returns data for the _data object.
let download = function() { ... }
function _generate(...) {...}
return download(group,subgroup,options).then(data => _data = data).then(() => _generate);
}
Then, where the code is consumed, get the generate function from the returned promise:
let generate = function() {};
dataSource.then(fn => generate = fn).done();

How do I wait for a promise to fill and then return a generator function?

I know this is wrong, but essentially I want to
connect to a db/orm via a promise
wait on that promise to fulfill and get the models (the return from the promise)
use the results for form a middleware generator function to place the models on the request
I suspect that this isn't the best approach, so essentially I have two questions:
Should I be rewriting my db/orm connect to a generator function (I have a feeling that is more inline with koa style)
Back to the original question (as I am sure I will not get a chance to rewrite all my business logic) - how do I wait on a promise to fulfill and to then return a generator function?
This is my poor attempt - which is not working, and honestly I didn't expect it to, but I wanted to start by writing code, to have something to work with to figure this out:
var connectImpl = function() {
var qPromise = q.nbind(object.method, object);
return qPromise ;
}
var domainMiddlewareImpl = function() {
let connectPromise = connectImpl()
return connectPromise.then(function(models){
return function *(next){
this.request.models = models ;
}
})
}
var app = koa()
app.use(domainMiddlewareImpl())

According to this, you can do the following:
var domainMiddlewareImpl = function() {
return function *(){
this.request.models = yield connectImpl();
};
};

A context sensitive answer based on the info provided by Hugo (thx):
var connectImpl = function() {
var qPromise = q.nbind(object.method, object);
return qPromise ;
}
var domainMiddlewareImpl = function () {
let models = null ;
return function *(next) {
if(models == null){
//////////////////////////////////////////////////////
// we only want one "connection", if that sets up a
// pool so be it
//////////////////////////////////////////////////////
models = yield connectImpl() ;
}
this.request.models = models.collections;
this.request.connections = models.connections;
yield next
};
};
My example the connectImpl is setting up domain models in an ORM (waterline for now), connecting to a database (pooled), and returning a promise for the ORM models and DB connections. I only want that to happen once, and then for every request through my Koa middleware, add the objects to the request.

Structuring promises within angularjs

I have done a lot of reading around this, but ultimately the tutorials and guides I have found differ too much for me to get a decent grasp on this concept.
This is what I want to achieve:
1) Simple http request from our server [Any API for demonstration]
2) Run a function with data from (1). [Remove a property from the object]
3) Use result and length of (2) to run a loop of $http requests to our server. [Or any server]
4) This will result in 6 different objects. Run a function on these 6 objects. [Add a property]
5) Once ALL of this is done, run a separate function [Log "finished"]
How can this be achieved using promises? How do I pass data from (1) via a promise to (2)? Is this the right way to achieve what I need to do?
If anyone can show me how this should be structured it would be immensely helpful; I have kept the functions as simple as possible for this question.

Yes, promises are very nice to structure solutions for this kind of problems.
Simplified solution (more or less pseudo-code):
$http(...)
.then(function(response) {
// do something with response, for example:
var list = reponse.data.list;
// return it so that you can use it in the next 'then'.
return list;
})
.then(function(list) {
var promises = [];
angular.forEach(list, function(item) {
// perform a request for each item
var promise = $http(...).then(function(itemResponse) {
itemResponse.extraProperty = true;
return itemResponse;
});
// we make an array of promises
promises.push(promise);
});
// combine all promises into one and return it for the next then()
return $q.all(promises);
})
.then(function(itemsList) {
// itemsList is now an array of all parsed item responses.
console.log(itemsList);
});
(Hopefully this is right, I did not tested it.)
As you can see, you can return values in a callback to pass it to the next then(), or you can pass a promise, and this will result in calling the next callback when it resolves. $q.all() is used to combine multiple promises into one and resolve if all are resolved.
Edit: I realised that you can optionally leave out these three lines:
return list;
})
.then(function(list) {
But it is nice syntax though, because the separation of tasks is more visible.

Check code below, it could contains syntax error, the important is the structure. Step3 contains multiple(6) $http requests, it waits until the last request response to return a unique response object (array) containing response for each $http requets.
//Step 1
var Step1 = function () {
$http.get('api/controller').success(function (resp) {
var object1 = resp;
Step2(object1);
Step3(object1).then(function (resp) {
//resp.data is an array containing the response of each $http request
Step4(resp);
Step5();
});
});
}
//Step2
var Step2 = function(obj){
//do whatever with the object
}
//Step3
var Step3 = function (object1) {
var call = $q.defer();
var get1 = $http.get(object1[0].url);
var get2 = $http.get(object[1].url2);
//...
var get6 = $http.get(object[5].url6);
$q.all([get1, get2,..get6]).then(function (resp) {
call.resolve(resp);
});
return call.promise;
}
//Step4
var Step4 = function (resp) {
for (var i=0; i<resp.data.lenght;i++){
DoWhatEver(resp.data[i]);
};
}
//Step5
var Step5 = function () {
alert("Finished");
}
Step1(); //Call Step1 function

Don't know why you have difficulty implementing this, but maybe $q.all() is what you're missing:
var config1={method:'GET',url:'/api/...'};
$http(config1).success(function(resultsFrom1){
functionForResultsOf1(resultsFrom1);
})
var functionForResultsOf1 = function(resultsOf1){
//remove something from the result, assuming this is a synchronous operation
resultsOf1.splice()...;
var promises=makePromises(*pass whatever you want*);
$q.all(promises).then(function(aggregateOfAllCallsToServer){
angular.forEach(aggregateOfAllCallsToServer,function(data){
//do something to data from each call to the server
})
console.log("finished");
})
}
var makePromises = function(serverUrls){
var promises = [];
angular.forEach(serverUrls, function(url) {
var promise=$http({
url : '/api/'+url,
method: 'GET',
})
promises.push(promise);
});
return $q.all(promises);
}

Return an object from a function that has nested ajax calls

I would like to write a javascript function that returns informations from youtube videos; to be more specific I would like to get the ID and the length of videos got by a search, in a json object. So I took a look at the youtube API and I came out with this solution:
function getYoutubeDurationMap( query ){
var youtubeSearchReq = "https://gdata.youtube.com/feeds/api/videos?q="+ query +
"&max-results=20&duration=long&category=film&alt=json&v=2";
var youtubeMap = [];
$.getJSON(youtubeSearchReq, function(youtubeResult){
var youtubeVideoDetailReq = "https://gdata.youtube.com/feeds/api/videos/";
for(var i =0;i<youtubeResult.feed.entry.length;i++){
var youtubeVideoId = youtubeResult.feed.entry[i].id.$t.substring(27);
$.getJSON(youtubeVideoDetailReq + youtubeVideoId + "?alt=json&v=2",function(videoDetails){
youtubeMap.push({id: videoDetails.entry.id.$t.substring(27),runtime: videoDetails.entry.media$group.media$content[0].duration});
});
}
});
return youtubeMap;
}
The logic is ok, but as many of you have already understood because of ajax when I call this function I get an empty array. Is there anyway to get the complete object? Should I use a Deferred object? Thanks for your answers.

Yes, you should use deferred objects.
The simplest approach here is to create an array into which you can store the jqXHR result of your inner $.getJSON() calls.
var def = [];
for (var i = 0; ...) {
def[i] = $.getJSON(...).done(function(videoDetails) {
... // extract and store in youtubeMap
});
}
and then at the end of the whole function, use $.when to create a new promise that will be resolved only when all of the inner calls have finished:
return $.when.apply($, def).then(function() {
return youtubeMap;
});
and then use .done to handle the result from your function:
getYoutubeDurationMap(query).done(function(map) {
// map contains your results
});
See http://jsfiddle.net/alnitak/8XQ4H/ for a demonstration using this YouTube API of how deferred objects allow you to completely separate the AJAX calls from the subsequent data processing for your "duration search".
The code is a little long, but reproduced here too. However whilst the code is longer than you might expect note that the generic functions herein are now reusable for any calls you might want to make to the YouTube API.
// generic search - some of the fields could be parameterised
function youtubeSearch(query) {
var url = 'https://gdata.youtube.com/feeds/api/videos';
return $.getJSON(url, {
q: query,
'max-results': 20,
duration: 'long', category: 'film', // parameters?
alt: 'json', v: 2
});
}
// get details for one YouTube vid
function youtubeDetails(id) {
var url = 'https://gdata.youtube.com/feeds/api/videos/' + id;
return $.getJSON(url, {
alt: 'json', v: 2
});
}
// get the details for *all* the vids returned by a search
function youtubeResultDetails(result) {
var details = [];
var def = result.feed.entry.map(function(entry, i) {
var id = entry.id.$t.substring(27);
return youtubeDetails(id).done(function(data) {
details[i] = data;
});
});
return $.when.apply($, def).then(function() {
return details;
});
}
// use deferred composition to do a search and then get all details
function youtubeSearchDetails(query) {
return youtubeSearch(query).then(youtubeResultDetails);
}
// this code (and _only_ this code) specific to your requirement to
// return an array of {id, duration}
function youtubeDetailsToDurationMap(details) {
return details.map(function(detail) {
return {
id: detail.entry.id.$t.substring(27),
duration: detail.entry.media$group.media$content[0].duration
}
});
}
// and calling it all together
youtubeSearchDetails("after earth").then(youtubeDetailsToDurationMap).done(function(map) {
// use map[i].id and .duration
});

As you have discovered, you can't return youtubeMap directly as it's not yet populated at the point of return. But you can return a Promise of a fully populated youtubeMap, which can be acted on with eg .done(), .fail() or .then().
function getYoutubeDurationMap(query) {
var youtubeSearchReq = "https://gdata.youtube.com/feeds/api/videos?q=" + query + "&max-results=20&duration=long&category=film&alt=json&v=2";
var youtubeVideoDetailReq = "https://gdata.youtube.com/feeds/api/videos/";
var youtubeMap = [];
var dfrd = $.Deferred();
var p = $.getJSON(youtubeSearchReq).done(function(youtubeResult) {
$.each(youtubeResult.feed.entry, function(i, entry) {
var youtubeVideoId = entry.id.$t.substring(27);
//Build a .then() chain to perform sequential queries
p = p.then(function() {
return $.getJSON(youtubeVideoDetailReq + youtubeVideoId + "?alt=json&v=2").done(function(videoDetails) {
youtubeMap.push({
id: videoDetails.entry.id.$t.substring(27),
runtime: videoDetails.entry.media$group.media$content[0].duration
});
});
});
});
//Add a terminal .then() to resolve dfrd when all video queries are complete.
p.then(function() {
dfrd.resolve(query, youtubeMap);
});
});
return dfrd.promise();
}
And the call to getYoutubeDurationMap() would be of the following form :
getYoutubeDurationMap("....").done(function(query, map) {
alert("Query: " + query + "\nYouTube videos found: " + map.length);
});
Notes:
In practice, you would probably loop through map and display the .id and .runtime data.
Sequential queries is preferable to parallel queries as sequential is kinder to both client and server, and more likely to succeed.
Another valid approach would be to return an array of separate promises (one per video) and to respond to completion with $.when.apply(..), however the required data would be more awkward to extract.