Develop Reference

Javascript: Making sure one async function doesn't run until the other one is complete; working with promises

I'm working with fetching information from a github repository. I want to get the list of pull requests within that repo, get the list of commits associated with each pull request, then for each commit I want to get information such as the author of the commit, the number of files associated with each commit and the number of additions and deletions made to each file. I'm using axios and the github API to accomplish this. I know how to work with the API, but the promises and async functions are keeping me from accomplishing my task. I have the following code:
const axios = require('axios');
var mapOfInformationObjects = new Map();
var listOfCommits = [];
var listOfSHAs = [];
var gitApiPrefix = link I'll use to start fetching data;
var listOfPullRequestDataObjects = [];
var listOfPullRequestNumbers = [];
var mapOfPullNumberToCommits = new Map();
function getAllPullRequests(gitPullRequestApiLink) {
return new Promise((resolve, reject) => {
axios.get(gitPullRequestApiLink).then((response) =>{
listOfPullRequestDataObjects = response['data'];
var k;
for (k = 0; k < listOfPullRequestDataObjects.length; k++){
listOfPullRequestNumbers.push(listOfPullRequestDataObjects[k]['number']);
}
resolve(listOfPullRequestNumbers);
}).catch((error) => {
reject(error);
})
})
}
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
var j;
for (j = 0; j < listOfPRNumbers.length; j++) {
currPromise = new Promise((resolve, reject) => {
currentGitApiLink = gitApiPrefix + listOfPRNumbers[j] + "/commits";
axios.get(currentGitApiLink).then((response) => {
mapOfPullNumberToCommits.set(listOfPRNumbers[j], response['data']);
resolve("Done with Pull Request Number: " + listOfPRNumbers[j]);
}).catch((error) => {
reject(error);
})
})
}
}
function getListOfCommits(gitCommitApiLink){
return new Promise((resolve, reject) => {
axios.get(gitCommitApiLink).then((response) => {
resolve(response);
}).catch((error) => {
reject(error);
})
})
}
So far, I made some functions that I would like to call sequentially.
First I'd like to call getAllPullRequestNumbers(someLink)
Then I'd like to call getCommitsForEachPullRequestNumber(listofprnumbers)
Then getListOfCommits(anotherLink)
So it would look something like
getAllPullRequestNumbers(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
But two problems arise:
1) I'm not sure if this is how you would call the functions so that the first function in the sequence completes before the other.
2) Because I'm not familiar with Javascript, I'm not sure, especially with the getCommitsForEachPullRequestNumber function since you run a loop and call axios.get() on each iteration of the loop, if this is how you work with promises within the functions.
Would this be how you would go about accomplishing these two tasks? Any help is much appreciated. Thanks!

When you a number of asynchronous operations (represented by promises) that you can run all together and you want to know when they are all done, you use Promise.all(). You collect an array of promises and pass it to Promise.all() and it will tell you when they have all completed or when one of them triggers an error. If all completed, Promise.all() will return a promise that resolves to an array of results (one for each asynchronous operation).
When you're iterating an array to do your set of asynchronous operations, it then works best to use .map() because that helps you create a parallel array of promises that you can feed to Promise.all(). Here's how you do that in getCommitsForEachPullRequestNumber():
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
let mapOfPullNumberToCommits = new Map();
return Promise.all(listOfPRNumbers.map(item => {
let currentGitApiLink = gitApiPrefix + item + "/commits";
return axios.get(currentGitApiLink).then(response => {
// put data into the map
mapOfPullNumberToCommits.set(item, response.data);
});
})).then(() => {
// make resolved value be the map we created, now that everything is done
return mapOfPullNumberToCommits;
});
}
// usage:
getCommitsForEachPullRequestNumber(list).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
Then, in getListOfCommits(), since axios already returns a promise, there is no reason to wrap it in a manually created promise. That is, in fact, consider a promise anti-pattern. Instead, just return the promise that axios already returns. In fact, there's probably not even a reason to have this as a function since one can just use axios.get() directly to achieve the same result:
function getListOfCommits(gitCommitApiLink){
return axios.get(gitCommitApiLink);
}
Then, in getAllPullRequests() it appears you are just doing one axios.get() call and then processing the results. That can be done like this:
function getAllPullRequests(gitPullRequestApiLink) {
return axios.get(gitPullRequestApiLink).then(response => {
let listOfPullRequestDataObjects = response.data;
return listOfPullRequestDataObjects.map(item => {
return item.number;
});
});
}
Now, if you're trying to execute these three operations sequentially in this order:
getAllPullRequests(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
You can chain the promises from those three operations together to sequence them:
getAllPullRequests(someLink)
.then(getCommitsForEachPullRequestNumber)
.then(mapOfPullNumberToCommits => {
// not entirely sure what you want to do here, perhaps
// call getListOfCommits on each item in the map?
}).catch(err => {
console.log(err);
});
Or, if you put this code in an async function, then you can use async/awit:
async function getAllCommits(someLink) {
let pullRequests = await getAllPullRequests(someLink);
let mapOfPullNumberToCommits = await getCommitsForEachPullRequestNumber(pullRequests);
// then use getlistOfCommits() somehow to process mapOfPullNumberToCommits
return finalResults;
}
getAllCommits.then(finalResults => {
console.log(finalResults);
}).catch(err => {
console.log(err);
});

not as clean as jfriend00 solution,
but I played with your code and it finally worked
https://repl.it/#gui3/githubApiPromises
you get the list of commits in the variable listOfCommits
I don't understand the purpose of your last function, so I dropped it

Chained and Nested promises with For Loop

I am trying to get each property of my games within chained promises (Each of the property is coming from a different async calls).
Logic of my algorithm:
Check the Network and Get the smart contract address
Register the contract containing the addresses of all the Games
Get the number of Games
For each game, perform one aSync call
per property
Print all the games and details (here I am not able
to get the updated object)
Code:
var games = [];
window.addEventListener('load', function() {
// Check the Network and assign the smart contract address
web3.eth.net.getId()
.then(function(networkId) {
let contractAddressRegistry;
if (networkId == 1) {
contractAddressRegistry = "0xQWERTYUIOPQWERTYUIOPQWERTY"
} else {
contractAddressRegistry = "0x12345678901234567890123456"
}
return contractAddressRegistry;
})
.then(function(contractAddressRegistry) {
let contractRegistry = new web3.eth.Contract(contractAbiRegistry, contractAddressRegistry);
contractRegistry.methods.numberOfGames().call()
.then(function(numberOfGames) {
for (let i = 0; i < numberOfGames; i++) {
let game = {};
game.propertyA = aSyncCallGetPropertyA(i); // Promise
game.propertyB = aSyncCallGetPropertyB(i); // Promise
game.propertyC = aSyncCallGetPropertyC(i); // Promise
}
games.push(game);
})
})
.then(function() {
console.log(games) // Empty
})
})
I tried used Promises.all() but I am not able to sync it properly as some async calls are within a then().
How can I make sure to get the object Games filled with all its properties?

You should use Promise.all like this. Basically, you need to wrap all three aSyncCallGetProperty async calls in Promise.all for waiting until they really finish then assign the results to object game.
whatever
.then(function(contractAddressRegistry) {
let contractRegistry = new web3.eth.Contract(contractAbiRegistry, contractAddressRegistry);
return contractRegistry.methods.numberOfGames().call();
})
.then(function(numberOfGames) {
return Promise.all(numberOfGames.map(() => {
return Promise.all([
aSyncCallGetPropertyA(),
aSyncCallGetPropertyB(),
aSyncCallGetPropertyC()
]).then(results => {
let game = {};
game.propertyA = results[0];
game.propertyB = results[1];
game.propertyC = results[2];
return game;
});
}));
})
.then(function(games) {
console.log(JSON.stringify(games));
})

#Lewis' code seems right but I can not make sure what numberOfGames is. Assuming that it's an integer as used in your question (not an array as treated in the other answer) here is a further rephrased version without nested .then()s.
window.addEventListener('load', function() {
web3.eth.net.getId()
.then(networkId => networkId === 1 ? "0xQWERTYUIOPQWERTYUIOPQWERTY"
: "0x12345678901234567890123456")
.then(contractAddressRegistry => new web3.eth.Contract(contractAbiRegistry, contractAddressRegistry).methods.numberOfGames().call())
.then(numberOfGames => Promise.all(Array(numberOfGames).fill()
.map(_ => Promise.all([aSyncCallGetPropertyA(),
aSyncCallGetPropertyB(),
aSyncCallGetPropertyC()]))))
.then(function(games){
games = games.map(game => ({propertyA: game[0],
propertyB: game[1],
propertyC: game[2]}));
doSomethingWith(games);
});
});

calling a promise within a promise , with $q.all [duplicate]

This question already has an answer here:
multiple promises running in parallel, $q.all needs chaining?
(1 answer)
Closed 5 years ago.
I have multiple api requests which can run in parallel.
getlocations, getstates, get territories can make a request in parallel. however once i have the states list from getstates() call, i need to make an api request for each individual state and parse the state object.
Here is the code that i have so far, Is it ok to call a promise within a promise this way ?
getAllLocations() {
//make a promise call for all here .
var promise = [];
//first prmise
promise.push(this.getAllLocations(Id).then(
(locationsData) => {
this.locations = locationsData;
}));
//second promise
promise.push(this.getAllStates(Id).then(
(resp) => {
this.states = resp.data;
//for each state , need to make an api call and add to the state info
angular.forEach(this.states, function(state){
var nodeId = state.Id;
this.getStateSummary(nodeId).then((resp) => {
state.healthStatus = resp.data.operationalStatus;
});
},this)
}));
//third promise
promise.push(this.getAllterritories(Id).then(
(resp) => {
this.territories = resp.data;
}));
//after all promises parse teh location data
$q.all(promise).then(() => {
for (var i = 0; i < this.locations.length; i++) {
//do something with with location here
}
this.gridData = this.locations;
});
}
if not , whats the best way to make teh getStatesummary() call ?

I'd simplify each of the 3 promises (especially 1 and 3), so that this.locations, this.states and this.territories are all updated in the $q.all .then - you don't have to, but I think it makes the code a little more readable
Next, assign all 3 promises to a variable (p1, p2, p3 in the code below)
The next issue is waiting on all the states API calls to complete - another $q.all required there
All up, with a little extra ES6 goodness, you get
getAllLocations() {
//first prmise
const p1 = this.getAllLocations(Id);
//second promise
const p2 = this.getAllStates(Id).then(resp =>
//for each state , need to make an api call and add to the state info
$q.all(resp.map(state => this.getStateSummary(state.Id).then(resp => state.healthStatus = resp.data.operationalStatus)))
.then(() => resp) // so we return data for this.states
);
//third promise
const p3 = this.getAllterritories(Id).then(resp => resp.data);
//after all promises parse teh location data
$q.all([p1, p2, p3]).then(([locations, states, territories]) => {
this.locations = locations;
this.states = states;
this.territories = territories;
for (var i = 0; i < locations.length; i++) {
//do something with with location here
}
this.gridData = this.locations;
});
}
If the getAllStates promise returns a Object instead of my assumption that it's an array - then replace the inner $q.all with
$q.all(Object.entries(resp).map(([key, state]) => this.getStateSummary(state.Id).then(resp => state.healthStatus = resp.data.operationalStatus)))
Actually, the above code would work for Array or Object equally :p

you can use promise chain to send catch promises one after the another
getAllLocations() {
//make a promise call for all here .
var promise = [];
//first prmise
this.getAllLocations(Id)
.then(
(locationsData) => {
this.locations = locationsData;
promise.push(this.locations);
return this.getAllStates(Id)
}).then(
(resp) => {
this.states = resp;
//for each state , need to make an api call and add to the state info
angular.forEach(this.states, function(state) {
var nodeId = state.Id;
return this.getStateSummary(nodeId);
}, this)
return this.getAllterritories(Id);
}).then((resp) => {
state.healthStatus = resp.data.operationalStatus;
promise.push(state.healthStatus);
}).then(
(resp) => {
this.territories = resp.data;
promise.push(this.territories);
this.callAll(promise)
})
}
callAll(promise) {
//after all promises parse teh location data
this.$q.all(promise).then(() => {
for (var i = 0; i < this.locations.length; i++) {
//do something with with location here
}
this.gridData = this.locations;
});
}

Promises structure misunderstood

I have a problem with understanding Promises syntax.
So, what I am trying to do:
getPosts() gets some data from a DB then, I want to get some metadata for each row with another promise call, addMetadata(). Then, once all the metadata is fetched, I want to console it out.
See my attempt below:
var getPosts = function(){
return new Promise(function(resolve, reject){
postModel.find()
.exec(function(err, posts) {
resolve(posts);
);
});
};
var addMetadata = function(posts){
var options = {
host: 'localhost',
port: 3000,
path: '',
method: 'GET'
};
var postPromises = posts.map(function(post) {
return new Promise(function(resolve) {
options.path = '/api/user?id=' + post.profileID;
var req = http.get(options, function(res) {
var bodyChunks = [];
res.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
var parsedBody = JSON.parse(body);
post.fullname = parsedBody.user.fullname;
post.profilePic = parsedBody.user.profilePic;
// resolve the promise with the updated post
resolve(post);
});
});
});
});
// Is this the right place to put Promise.all???
Promise.all(postPromises)
.then(function(posts) {
//What should I put here
});
};
getPosts()
.then(function(posts){
return addMetadata(posts);
})
.then(function(posts){//I get a little lost here
console.log();//posts is undefined
});
Of course, my understanding is wrong but I thought I was going the right way. Can someone please guide me to the right direction?
Thanks

Change
// Is this the right place to put Promise.all???
Promise.all(postPromises)
.then(function (posts) {
//What should I put here
});
into
// Is this the right place to put Promise.all???
return Promise.all(postPromises);
This way your addMetadata function will return Promise that resolve when all promises from postPromises resolves or reject if any of postPromises rejects.

The key point to understand the async concept of it and what time the content is available.
Reading this will help to put you in the right direction.
For instance:
var promise = new Promise(function(resolve, reject) {
resolve(1);
});
promise
.then(function(val) {
console.log(val); // 1
return val + 2;
})
.then(function(val) {
console.log(val); // 3
})
After as per your scenario, in order to have all the metadata Promise.all is the way to go.
Promise.all(arrayOfPromises).then(function(arrayOfResults) {
// One result per each promise of AddMetadata
})

What you wanna do here, if I am correct, is called streams, as you wanna call multiple paralel promises as your concept of looping through list of posts using map is not going to work this way
Take a look at this short video introducing streams Streams - FunFunFunction, he is using library for workin with streams called Baconjs
Here is a short example on streams
const stupidNumberStream = {
each: (callback) => {
setTimeout( () => callback(1), 3000 )
setTimeout( () => callback(2), 2000 )
setTimeout( () => callback(3), 1000 )
}
}
stupidNumberStream.each(console.log)

Your getPosts function is good in the sense that its only job is to promisfy the database find. (Though, I think if it's mongo, the exec produces a promise for you).
Your addMetadataToAPost is less good, because it mixes up processing an array of posts and "promisifying" the http.get. Use the same pattern you applied correctly in the first function and return a promise to do a single get and add metadata. (It would be even better to just wrap the get, which you can reuse, and build a simple add-metadata function that returns - rather than creates - a promise)
// renamed pedantically
var addMetadataToAPost = function(post) {
return new Promise(function(resolve) {
options.path = '/api/user?id=' + post.profileID;
var req = http.get(options, function(res) {
var bodyChunks = [];
res.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
var parsedBody = JSON.parse(body);
post.fullname = parsedBody.user.fullname;
post.profilePic = parsedBody.user.profilePic;
// resolve the promise with the updated post
resolve(post);
});
});
});
}
Now your batching function is simple:
// also renamed pedantically
var addMetadataToAllPosts = function(posts){
var postPromises = posts.map(function(post) {
return addMetadataToAPost(post)
})
return Promise.all(postPromises)
};
Your original code should work...
getPosts().then(function(posts){
return addMetadataToAllPosts(posts);
})
.then(function(posts){
console.log(posts);//posts should be an array of posts with metadata added
});

node.js: structure multiple API requests, work on them and combine them

currently I am struggeling a little bit with node.js (I am new to it) doing different API requests (Usabilla API), work on the results and then combine them in order to work on the whole set (e.g. export).
Requesting the API is not the problem but I can't get the results out to do some other stuff on it (asynchronous code drives me crazy).
Attached please find a overview how I thought to do this. Maybe I am totally wrong about this or maybe you have other more elegant suggestions.
My code works until I have to request the two different API "adresses" (they are provided) and then extract the results to do some other stuff.
My problem here is that there are nested functions with a promise and I cant figure out how to pass this through the parent function inside waterfall to get handled by the next function.
In the code, of course there is nothing parallel as shown in the diagram.
Thats another point, how to do that ? Simply nest parallel and series/ another waterfall inside waterfall ?
I am a little bit confused because that gets more and more complex for a simple problem when this would be done with synchronous code.
Here I build up all my request querys (at the moment 4):
function buildQuery(IDs,callback){
var i = 0;
var max = Object.keys(IDs).length;
async.whilst(
function(){return i < max},
function(callback){
FeedbackQuery[i] =
{
identifier: IDs[i].identifier,
query:
{id: IDs[i].id,
params: {since:sinceDate,}
}
};
i++;
callback(null,i);
})
console.log(FeedbackQuery);
callback (null,FeedbackQuery);
};
I then have to decide which type of query it is and add it to an object which should contain all the items of this identifier type:
function FeedbackRequest(FeedbackQuery,callback)
{
var i = 0;
var max = Object.keys(FeedbackQuery).length;
async.whilst(
function(){return i < max},
function (callback){
identifier = FeedbackQuery[i].identifier;
APIquery = FeedbackQuery[i].query;
switch(identifier)
{
case 'mobilePortal':
console.log(FeedbackQuery[i].identifier, 'aktiviert!');
var result = api.websites.buttons.feedback.get(APIquery);
result.then(function(feedback)
{
var item = Object.keys(feedbackResults).length;
feedbackResultsA[item] = feedback;
callback(null, feedbackResultsA);
})
break;
case 'apps':
console.log(FeedbackQuery[i].identifier, 'aktiviert!');
var result = api.apps.forms.feedback.get(APIquery);
result.then(function(feedback)
{
var item = Object.keys(feedbackResults).length;
feedbackResultsB[item] = feedback;
callback(null, feedbackResultsB);
})
break;
}
i++;
callback(null,i);
})
};
Currently the functions are bundled in an async waterfall:
async.waterfall([
async.apply(buildQuery,IDs2request),
FeedbackRequest,
// a function to do something on the whole feedbackResults array
],function (err, result) {
// result now equals 'done'
if (err) { console.log('Something is wrong!'); }
return console.log('Done!');
})
How it actually should be:
Structure
Thank you very much for any tips or hints!

I'm not proficient with async, and I believe if you'r new to this, it's harder than a simple Promise library like bluebird combine with lodash for helpers.
What I would do based on your schemas :
var firstStepRequests = [];
firstStepRequests.push(buildQuery());// construct your first steps queries, can be a loop, goal is to have firstStepRequests to be an array of promise.
Promise.all(firstStepRequests)
.then((allResults) => {
var type1 = _.filter(allResults, 'request_type_1');
var type2 = _.filter(allResults, 'request_type_2');
return {
type1: type1,
type2: type2
};
})
.then((result) => {
result.type1 = //do some work
result.type2 = //do some work
return result;
})
.then((result) => {
//export or merge or whatever.
});
Goal is to have a simple state machine.
UPDATE
If you want to keep identifier for a request, you can use props to have :
var props = {
id_1:Promise,
id_2:Promise,
id_3:Promise
};
Promise.props(props).then((results) => {
// results is {
id_1:result of promise,
id_2:result of promise,
etc...
}
})
You could do something like :
var type1Promises = getType1Requests(); //array of type 1
var type2Promises = getType2Requests(); // array of type 2
var props = {
type_1: Promise.all(type1Promises),
type_2: Promise.all(type2Promises)
}
Promise.props(props).then((result) => {
//result is : {
type_1: array of result of promises of type 1
type_2: array of result of promises of type 2
}
})