Develop Reference

Javascript: Making sure one async function doesn't run until the other one is complete; working with promises

I'm working with fetching information from a github repository. I want to get the list of pull requests within that repo, get the list of commits associated with each pull request, then for each commit I want to get information such as the author of the commit, the number of files associated with each commit and the number of additions and deletions made to each file. I'm using axios and the github API to accomplish this. I know how to work with the API, but the promises and async functions are keeping me from accomplishing my task. I have the following code:
const axios = require('axios');
var mapOfInformationObjects = new Map();
var listOfCommits = [];
var listOfSHAs = [];
var gitApiPrefix = link I'll use to start fetching data;
var listOfPullRequestDataObjects = [];
var listOfPullRequestNumbers = [];
var mapOfPullNumberToCommits = new Map();
function getAllPullRequests(gitPullRequestApiLink) {
return new Promise((resolve, reject) => {
axios.get(gitPullRequestApiLink).then((response) =>{
listOfPullRequestDataObjects = response['data'];
var k;
for (k = 0; k < listOfPullRequestDataObjects.length; k++){
listOfPullRequestNumbers.push(listOfPullRequestDataObjects[k]['number']);
}
resolve(listOfPullRequestNumbers);
}).catch((error) => {
reject(error);
})
})
}
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
var j;
for (j = 0; j < listOfPRNumbers.length; j++) {
currPromise = new Promise((resolve, reject) => {
currentGitApiLink = gitApiPrefix + listOfPRNumbers[j] + "/commits";
axios.get(currentGitApiLink).then((response) => {
mapOfPullNumberToCommits.set(listOfPRNumbers[j], response['data']);
resolve("Done with Pull Request Number: " + listOfPRNumbers[j]);
}).catch((error) => {
reject(error);
})
})
}
}
function getListOfCommits(gitCommitApiLink){
return new Promise((resolve, reject) => {
axios.get(gitCommitApiLink).then((response) => {
resolve(response);
}).catch((error) => {
reject(error);
})
})
}
So far, I made some functions that I would like to call sequentially.
First I'd like to call getAllPullRequestNumbers(someLink)
Then I'd like to call getCommitsForEachPullRequestNumber(listofprnumbers)
Then getListOfCommits(anotherLink)
So it would look something like
getAllPullRequestNumbers(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
But two problems arise:
1) I'm not sure if this is how you would call the functions so that the first function in the sequence completes before the other.
2) Because I'm not familiar with Javascript, I'm not sure, especially with the getCommitsForEachPullRequestNumber function since you run a loop and call axios.get() on each iteration of the loop, if this is how you work with promises within the functions.
Would this be how you would go about accomplishing these two tasks? Any help is much appreciated. Thanks!

When you a number of asynchronous operations (represented by promises) that you can run all together and you want to know when they are all done, you use Promise.all(). You collect an array of promises and pass it to Promise.all() and it will tell you when they have all completed or when one of them triggers an error. If all completed, Promise.all() will return a promise that resolves to an array of results (one for each asynchronous operation).
When you're iterating an array to do your set of asynchronous operations, it then works best to use .map() because that helps you create a parallel array of promises that you can feed to Promise.all(). Here's how you do that in getCommitsForEachPullRequestNumber():
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
let mapOfPullNumberToCommits = new Map();
return Promise.all(listOfPRNumbers.map(item => {
let currentGitApiLink = gitApiPrefix + item + "/commits";
return axios.get(currentGitApiLink).then(response => {
// put data into the map
mapOfPullNumberToCommits.set(item, response.data);
});
})).then(() => {
// make resolved value be the map we created, now that everything is done
return mapOfPullNumberToCommits;
});
}
// usage:
getCommitsForEachPullRequestNumber(list).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
Then, in getListOfCommits(), since axios already returns a promise, there is no reason to wrap it in a manually created promise. That is, in fact, consider a promise anti-pattern. Instead, just return the promise that axios already returns. In fact, there's probably not even a reason to have this as a function since one can just use axios.get() directly to achieve the same result:
function getListOfCommits(gitCommitApiLink){
return axios.get(gitCommitApiLink);
}
Then, in getAllPullRequests() it appears you are just doing one axios.get() call and then processing the results. That can be done like this:
function getAllPullRequests(gitPullRequestApiLink) {
return axios.get(gitPullRequestApiLink).then(response => {
let listOfPullRequestDataObjects = response.data;
return listOfPullRequestDataObjects.map(item => {
return item.number;
});
});
}
Now, if you're trying to execute these three operations sequentially in this order:
getAllPullRequests(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
You can chain the promises from those three operations together to sequence them:
getAllPullRequests(someLink)
.then(getCommitsForEachPullRequestNumber)
.then(mapOfPullNumberToCommits => {
// not entirely sure what you want to do here, perhaps
// call getListOfCommits on each item in the map?
}).catch(err => {
console.log(err);
});
Or, if you put this code in an async function, then you can use async/awit:
async function getAllCommits(someLink) {
let pullRequests = await getAllPullRequests(someLink);
let mapOfPullNumberToCommits = await getCommitsForEachPullRequestNumber(pullRequests);
// then use getlistOfCommits() somehow to process mapOfPullNumberToCommits
return finalResults;
}
getAllCommits.then(finalResults => {
console.log(finalResults);
}).catch(err => {
console.log(err);
});

not as clean as jfriend00 solution,
but I played with your code and it finally worked
https://repl.it/#gui3/githubApiPromises
you get the list of commits in the variable listOfCommits
I don't understand the purpose of your last function, so I dropped it

Promise 2 resolves before the Promise 1

I am doing a search in the textfield and as I type, there is a call going to the backend after say 100ms.
For example, if we search "5041" and immediately search for "50" and again make it "5041", then there are 3 calls made to the backend.
1."5041" -> Promise 1
2."50" -> Promise 2
3."5041" -> Promise 3
However, promise 3 (web call takes 200ms) resolves before promise 2 (web call takes 500ms) which makes the screen reflect results for promise 2 ("50") when all I have in the textfield is "5041".
I need some way to let user type in the textfield without blocking the user along with the ability to show results for only the last call.
This is something that can be achieved using switchMap from rxjs in an angular app. However I need a way to achieve the same in vanilla JS.

First you can wrap your fetchData function into a something like fetchLatestSearchResults function which notes the time when network call was made and return the latest result from all the network calls(irrespective of what data was returned from server)
const generateLatestSearchFetch = function(fetchFunc){
let mostRecentResult = null;
let mostRecentResultFetchTime = Date.now();
return (...args) => {
const myFetchStartTime = Date.now();
return fetchFunc(...args)
.then(data => {
if (myFetchStartTime > mostRecentResultFetchTime) {
mostRecentResult = data;
mostRecentResultFetchTime = myFetchStartTime
}
return mostRecentResult;
});
}
};
Use Like:
fetchData = generateLatestSearchFetch(fetchData);
fetchData('10'); // resolves first and returns result for 10
fetchData('102'); // resolves third and returns result for 1024
fetchData('1024'); // resolves second and returns result for 1024
Last but not the least, use debounce more on this to optimize number of network calls made for every type event.

You need a "last" function:
// takes a function returning a promise and only reports the last resort
function last(fn) {
let p;
return function(...args) {
let current = fn(); // call the function
p = current; // mark it as the last call
return p.then(result => {
// ask am I still the last call?
if (p === current) return result;
else return new Promise(() => {}); // never resolve
});
}
}
let onlyLastSearch = last((name) => fetch('/api?name=' + name));
onlyLastSearch('a'); // will be ignored
onlyLastSearch('b'); // will be ignored
onlyLastSearch('c'); // only relevant result

You can use observer pattern for this.
const createWrapper = (fn) => {
let counter = 0;
let lastFetchId = 0;
const listeners = [];
return {
fetch: (str) => {
let id = ++counter;
fn(str).then((data) => {
if(id > lastFetchId) {
listeners.forEach(fn => {
fn(data);
});
lastFetchId = id;
}
});
},
listen: (fn) => {
listeners.push(fn);
return () => {
const index = listeners.indexOf(fn);
listeners.splice(index, 1);
};
}
}
}
const SearchWrapper = createWrapper(fetchData);
SearchWrapper.fetch('a');
SearchWrapper.fetch('b');
SearchWrapper.fetch('c');
SearchWrapper.listen((data) => {
console.log(data);
})

Async await with a forEach loop still running asynchronously [duplicate]

This question already has answers here:
Using async/await with a forEach loop
(33 answers)
Closed 4 years ago.
First of all I did read through similar questions, and still cannot see where I'm making my mistake.
Here's my code:
async function validateWebsites(website) {
var result = url.parse(`http://${website}`);
console.log(result.hostname);
return await fetch(`http://www.${result.hostname}`)
.then(() => console.log(true))
.catch(() => console.log(false));
}
var wrongWebsites = [];
var wrongWebsites = [];
var i = 0;
websites.forEach(website => {
i++;
if (validateWebsites(website) === false
) {
wrongWebsites.push(i);
}
});
console.log(wrongWebsites);
How it works:
The user passes an array of websites, and I want to validate if they're valid websites, not to waste resources and block other errors. Now to the console:
digitlead.com
google.com
georgiancollege.ca
youtube.com
[]
true
true
true
true
So as you see, it prints out first the websites array, and then the response. So it's still async. How do I make it wait? I changed the loop from a for to forEach as suggested by many posts, I used the await and I am returning a promise. So what else do I have to do?
Edit:
I tried to do this:
async function validateWebsites(website) {
var result = url.parse(`http://${website}`); // TODO figure out if filtering all the subpages is a good idea.
console.log(result.hostname);
return await fetch(`http://www.${result.hostname}`)
.then(()=>console.log(true))
.catch(()=>console.log(false));
}
But it doesn't change anything
I found a function called readFileSync. That's more or less what I'm looking for, but with the ability to call a different website.

Here is how you can get all valid websites.
Problem with your validateWebsites function is that it returns promise wchih is resolving to undefined thanks to promise chaining and your loging
Also using forEach to filter array is unnesesery.
But if you wanted you could do something like this
websites.forEach(async website => {
i++;
if (await validateWebsites(website) === false) { // now value is Boolean instead of Promise
wrongWebsites.push(i);
}
});
Also note that if you use global i with asyncronous functions to keep track of index this can lead to many errors.
However I think this soultion should satisfy you
async function validateWebsites(website) {
var result = url.parse(`http://${website}`)
return fetch(`http://www.${result.hostname}`)
.then(() => true) // async function returns promise
.catch(() => false)
}
const websites = ['digitlead.com',
'google.com',
'georgiancollege.ca',
'youtube.com',
'111.1',
'foobarbaz']
async function filter(array, func) {
const tmp = await Promise.all( // waits for all promises to resolve
array.map(func) // evecutes async function and stores it result in new array then returns array of promises
)
return array.filter((_, i) => tmp[i]) // removes invalid websites
}
const validWebsites = filter(websites, validateWebsites)
validWebsites.then(console.log)
Get the indexes of the non-valid sites
async function filter(array, func) {
const tmp = await Promise.all(array.map(func))
return tmp
.map((x, i) => !x && i) // flip true to false and asign index when x is false
.filter(x => x !== false) // return indexes
}

destoryeris saying you should do something like this:
websites.forEach(async website => {
i++;
if (await validateWebsites(website) === false
) {
wrongWebsites.push(i);
}
});
But that alone is problematic because you have wrap your async functions in a try/catch to handle their errors. So something more like this:
websites.forEach(async website => {
i++;
try {
const validSites = await validateWebsites(website);
if (validSites === false) {
wrongWebsites.push(i);
}
} catch(e) {
// handle e
}
})

calling a promise within a promise , with $q.all [duplicate]

This question already has an answer here:
multiple promises running in parallel, $q.all needs chaining?
(1 answer)
Closed 5 years ago.
I have multiple api requests which can run in parallel.
getlocations, getstates, get territories can make a request in parallel. however once i have the states list from getstates() call, i need to make an api request for each individual state and parse the state object.
Here is the code that i have so far, Is it ok to call a promise within a promise this way ?
getAllLocations() {
//make a promise call for all here .
var promise = [];
//first prmise
promise.push(this.getAllLocations(Id).then(
(locationsData) => {
this.locations = locationsData;
}));
//second promise
promise.push(this.getAllStates(Id).then(
(resp) => {
this.states = resp.data;
//for each state , need to make an api call and add to the state info
angular.forEach(this.states, function(state){
var nodeId = state.Id;
this.getStateSummary(nodeId).then((resp) => {
state.healthStatus = resp.data.operationalStatus;
});
},this)
}));
//third promise
promise.push(this.getAllterritories(Id).then(
(resp) => {
this.territories = resp.data;
}));
//after all promises parse teh location data
$q.all(promise).then(() => {
for (var i = 0; i < this.locations.length; i++) {
//do something with with location here
}
this.gridData = this.locations;
});
}
if not , whats the best way to make teh getStatesummary() call ?

I'd simplify each of the 3 promises (especially 1 and 3), so that this.locations, this.states and this.territories are all updated in the $q.all .then - you don't have to, but I think it makes the code a little more readable
Next, assign all 3 promises to a variable (p1, p2, p3 in the code below)
The next issue is waiting on all the states API calls to complete - another $q.all required there
All up, with a little extra ES6 goodness, you get
getAllLocations() {
//first prmise
const p1 = this.getAllLocations(Id);
//second promise
const p2 = this.getAllStates(Id).then(resp =>
//for each state , need to make an api call and add to the state info
$q.all(resp.map(state => this.getStateSummary(state.Id).then(resp => state.healthStatus = resp.data.operationalStatus)))
.then(() => resp) // so we return data for this.states
);
//third promise
const p3 = this.getAllterritories(Id).then(resp => resp.data);
//after all promises parse teh location data
$q.all([p1, p2, p3]).then(([locations, states, territories]) => {
this.locations = locations;
this.states = states;
this.territories = territories;
for (var i = 0; i < locations.length; i++) {
//do something with with location here
}
this.gridData = this.locations;
});
}
If the getAllStates promise returns a Object instead of my assumption that it's an array - then replace the inner $q.all with
$q.all(Object.entries(resp).map(([key, state]) => this.getStateSummary(state.Id).then(resp => state.healthStatus = resp.data.operationalStatus)))
Actually, the above code would work for Array or Object equally :p

you can use promise chain to send catch promises one after the another
getAllLocations() {
//make a promise call for all here .
var promise = [];
//first prmise
this.getAllLocations(Id)
.then(
(locationsData) => {
this.locations = locationsData;
promise.push(this.locations);
return this.getAllStates(Id)
}).then(
(resp) => {
this.states = resp;
//for each state , need to make an api call and add to the state info
angular.forEach(this.states, function(state) {
var nodeId = state.Id;
return this.getStateSummary(nodeId);
}, this)
return this.getAllterritories(Id);
}).then((resp) => {
state.healthStatus = resp.data.operationalStatus;
promise.push(state.healthStatus);
}).then(
(resp) => {
this.territories = resp.data;
promise.push(this.territories);
this.callAll(promise)
})
}
callAll(promise) {
//after all promises parse teh location data
this.$q.all(promise).then(() => {
for (var i = 0; i < this.locations.length; i++) {
//do something with with location here
}
this.gridData = this.locations;
});
}

Chaining Promises while maintaining data (angular js)

Ive seen that there are questions about chaining promises, but this one is a little bit different.
I'm making http get requests in my code. The first call returns an array. For each object in the array, i need to make another http call which returns another array and so on (this chains 3 levels deep).
The problem is, I need to keep track of which array element was used for to make each http call, and I dont know how to do this using promises.
I also want to end the chain by returning a promise.
I have the code for what I want to do written in nodejs without promises:
var https = require('https');
var fs = require('fs');
function makeRequest(options){
var httpopts = {
host: 'soc.courseoff.com',
path: '/gatech/terms/201601/majors/' + options.p,
method: 'GET'
};
var response = "";
var req = https.request(httpopts, function(res) {
res.on('data', function(d) {
response += d;
});
res.on('end',function(){
options.cb(response,options)
})
});
req.end();
req.on('error', function(e) {
console.error(e);
});
}
var classData = {};
function getCourses(m){
var majors = JSON.parse(m);
majors.forEach(function(maj){
classData[maj] = {};
var options = {
p:maj.ident +'/courses',
cb:getSections,
major:maj
};
makeRequest(options);
});
}
var classCount = 0;
function getSections(c,opts){
var courses = JSON.parse(c);
courses.forEach(function(course){
classCount++;
var options = JSON.parse(JSON.stringify(opts));
options.p += '/'+course.ident+'/sections';
options.course = course
options.cb = buildData
makeRequest(options)
});
}
var sectionCount = 0;
function buildData(r, options){
var major = options.major.ident;
sectionCount++;
if(!classData[major]){
classData[major] = {
name: options.major.name,
classes:{}
};
}
classData[major].classes[options.course.ident] = {
name:options.course.name,
sections:JSON.parse(r)
};
console.log('classCount-sectionCount '+classCount + '---'+sectionCount);
if(classCount === sectionCount){
writeIt();
}
}
makeRequest({
p:'',
cb:getCourses
});
function writeIt(){
fs.writeFileSync('./classData.js', 'module.exports = ' + JSON.stringify(classData));
}
EDIT:
I managed to get the promises to nest while keeping track of the data, but how can i return a promise that eventually resolves with the final data object?
My code:
Thanks four your help! I've managed to code it so that the promises work, my only problem now is in returning the final data as a promise
fact.factory('ClassFactory', ['$http',function ($http) {
var eventData = {};
var promise;
var courseData = [];
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
eventData.getClasses = function (event) {
if(!promise){
promise = $http.get(baseURL).then(
function(majors){
Promise.all(majors.data.map(m => $http.get(baseURL + m.ident+'/courses')
.then(
function(courses){
if(!m.courses) m.courses = [];
courses.data.map(c => $http.get(baseURL+ m.ident+'/courses/' +c.ident+'/sections' )
.then(
function(sections){
c.sections = sections.data;
m.courses.push(c);
}
));
courseData.push(m);
}
)));
}
)
}
return promise;
}
return eventData;
}]);

Almost certainly, each time you deal with an array of Promises, you'll want to use Promise.all in order to connect and merge your promises into a new promise. That promise will then contain an array of the results from each call. Nested Promise.alls can thus return Arrays of Arrays with all your levels of results as long as you use something like a map and a closure to capture the outer levels.
var fakeCall = x => Promise.resolve(x||Math.random());
Promise.all([fakeCall(1),fakeCall(2)])
.then(
results => Promise.all(results.map( x => fakeCall(5).then( results2 => [x, results2]) ))
)
.then( x => console.log(x));//-> [[1,5],[2,5]]
The first array of calls generates an array of results, and mapping over those with a function that makes yet more calls will return a single result that can be paired with its parent.
Explicitly nesting things in this way will work for even deeper levels, but is not going to be pretty. There's probably an abstraction you can create using Array.reduce which can generalize this pattern.

You forgot some returns in your code. The function you pass to .then should always return something. Also you are modifying majors but then throw it away without using it. When working with promises - especially when they are complex and nested - it's not a good idea to modify any data structures contained in those promises unless you are sure nothing bad can possibly happen.
I would split it into several functions.
e.g.
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
function getSections(major, course) {
return $http.get(baseURL+ major.ident+'/courses/' +course.ident+'/sections')
.then(sections => sections.data)
.catch(e => []);
}
function getCourses(major) {
return $http.get(baseURL + major.ident+'/courses')
.then(courses => Promise.all(courses.data.map(course =>
getSections(major, course).then(sections => ({[course.ident]: {name: course.name, sections: sections}})))))
.then(courses => angular.extend({}, ...courses))
.catch(e => ({}));
}
function getClassData() {
return $http.get(baseURL)
.then(majors => Promise.all(majors.data.map(major =>
getCourses(major).then(courses => ({[major.ident]: {name: major.name, classes: courses}})))))
.then(majors => angular.extend({}, ...majors))
.catch(e => ({}));
}
getClassData().then(data => console.log(data));