Develop Reference

Javascript: Making sure one async function doesn't run until the other one is complete; working with promises

I'm working with fetching information from a github repository. I want to get the list of pull requests within that repo, get the list of commits associated with each pull request, then for each commit I want to get information such as the author of the commit, the number of files associated with each commit and the number of additions and deletions made to each file. I'm using axios and the github API to accomplish this. I know how to work with the API, but the promises and async functions are keeping me from accomplishing my task. I have the following code:
const axios = require('axios');
var mapOfInformationObjects = new Map();
var listOfCommits = [];
var listOfSHAs = [];
var gitApiPrefix = link I'll use to start fetching data;
var listOfPullRequestDataObjects = [];
var listOfPullRequestNumbers = [];
var mapOfPullNumberToCommits = new Map();
function getAllPullRequests(gitPullRequestApiLink) {
return new Promise((resolve, reject) => {
axios.get(gitPullRequestApiLink).then((response) =>{
listOfPullRequestDataObjects = response['data'];
var k;
for (k = 0; k < listOfPullRequestDataObjects.length; k++){
listOfPullRequestNumbers.push(listOfPullRequestDataObjects[k]['number']);
}
resolve(listOfPullRequestNumbers);
}).catch((error) => {
reject(error);
})
})
}
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
var j;
for (j = 0; j < listOfPRNumbers.length; j++) {
currPromise = new Promise((resolve, reject) => {
currentGitApiLink = gitApiPrefix + listOfPRNumbers[j] + "/commits";
axios.get(currentGitApiLink).then((response) => {
mapOfPullNumberToCommits.set(listOfPRNumbers[j], response['data']);
resolve("Done with Pull Request Number: " + listOfPRNumbers[j]);
}).catch((error) => {
reject(error);
})
})
}
}
function getListOfCommits(gitCommitApiLink){
return new Promise((resolve, reject) => {
axios.get(gitCommitApiLink).then((response) => {
resolve(response);
}).catch((error) => {
reject(error);
})
})
}
So far, I made some functions that I would like to call sequentially.
First I'd like to call getAllPullRequestNumbers(someLink)
Then I'd like to call getCommitsForEachPullRequestNumber(listofprnumbers)
Then getListOfCommits(anotherLink)
So it would look something like
getAllPullRequestNumbers(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
But two problems arise:
1) I'm not sure if this is how you would call the functions so that the first function in the sequence completes before the other.
2) Because I'm not familiar with Javascript, I'm not sure, especially with the getCommitsForEachPullRequestNumber function since you run a loop and call axios.get() on each iteration of the loop, if this is how you work with promises within the functions.
Would this be how you would go about accomplishing these two tasks? Any help is much appreciated. Thanks!

When you a number of asynchronous operations (represented by promises) that you can run all together and you want to know when they are all done, you use Promise.all(). You collect an array of promises and pass it to Promise.all() and it will tell you when they have all completed or when one of them triggers an error. If all completed, Promise.all() will return a promise that resolves to an array of results (one for each asynchronous operation).
When you're iterating an array to do your set of asynchronous operations, it then works best to use .map() because that helps you create a parallel array of promises that you can feed to Promise.all(). Here's how you do that in getCommitsForEachPullRequestNumber():
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
let mapOfPullNumberToCommits = new Map();
return Promise.all(listOfPRNumbers.map(item => {
let currentGitApiLink = gitApiPrefix + item + "/commits";
return axios.get(currentGitApiLink).then(response => {
// put data into the map
mapOfPullNumberToCommits.set(item, response.data);
});
})).then(() => {
// make resolved value be the map we created, now that everything is done
return mapOfPullNumberToCommits;
});
}
// usage:
getCommitsForEachPullRequestNumber(list).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
Then, in getListOfCommits(), since axios already returns a promise, there is no reason to wrap it in a manually created promise. That is, in fact, consider a promise anti-pattern. Instead, just return the promise that axios already returns. In fact, there's probably not even a reason to have this as a function since one can just use axios.get() directly to achieve the same result:
function getListOfCommits(gitCommitApiLink){
return axios.get(gitCommitApiLink);
}
Then, in getAllPullRequests() it appears you are just doing one axios.get() call and then processing the results. That can be done like this:
function getAllPullRequests(gitPullRequestApiLink) {
return axios.get(gitPullRequestApiLink).then(response => {
let listOfPullRequestDataObjects = response.data;
return listOfPullRequestDataObjects.map(item => {
return item.number;
});
});
}
Now, if you're trying to execute these three operations sequentially in this order:
getAllPullRequests(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
You can chain the promises from those three operations together to sequence them:
getAllPullRequests(someLink)
.then(getCommitsForEachPullRequestNumber)
.then(mapOfPullNumberToCommits => {
// not entirely sure what you want to do here, perhaps
// call getListOfCommits on each item in the map?
}).catch(err => {
console.log(err);
});
Or, if you put this code in an async function, then you can use async/awit:
async function getAllCommits(someLink) {
let pullRequests = await getAllPullRequests(someLink);
let mapOfPullNumberToCommits = await getCommitsForEachPullRequestNumber(pullRequests);
// then use getlistOfCommits() somehow to process mapOfPullNumberToCommits
return finalResults;
}
getAllCommits.then(finalResults => {
console.log(finalResults);
}).catch(err => {
console.log(err);
});

not as clean as jfriend00 solution,
but I played with your code and it finally worked
https://repl.it/#gui3/githubApiPromises
you get the list of commits in the variable listOfCommits
I don't understand the purpose of your last function, so I dropped it

Can't promisify callback based function

I want to use the library astro-js where a typical call in their docs looks like this:
const aztroJs = require("aztro-js");
//Get all horoscope i.e. today's, yesterday's and tomorrow's horoscope
aztroJs.getAllHoroscope(sign, function(res) {
console.log(res);
});
For several reasons, I would like to use it using async/await style and leverage try/catch. So I tried promisify like this:
const aztroJs = require("aztro-js");
const {promisify} = require('util');
const getAllHoroscopeAsync = promisify(aztroJs.getAllHoroscope);
async function handle() {
let result, sign = 'libra';
try {
result = await getAllHoroscopeAsync(sign);
}
catch (err) {
console.log(err);
}
console.log("Result: " + result);
}
However, when I log result it comes as undefined. I know the call worked since the library is automatically logging a response via console.log and I see a proper response in the logs.
How can I "await" on this call? (even by other means if this one is not "promisifyable")

util.promisify() expects the callback function to accept two arguments, the first is an error that must be null when there is no error and non-null when there is an error and the second is the value (if no error). It will only properly promisify a function if the callback follows that specific rule.
To work around that, you will have to manually promisify your function.
// manually promisify
aztroJs.getAllHoroscopePromise = function(sign) {
return new Promise(resolve => {
aztroJs.getAllHoroscope(sign, function(data) {
resolve(data);
});
});
};
// usage
aztroJs.getAllHoroscopePromise(sign).then(results => {
console.log(results);
});
Note, it's unusual for an asynchronous function that returns data not to have a means of returning errors so the aztroJs.getAllHoroscope() interface seems a little suspect in that regard.
In fact, if you look at the code for this function, you can see that it is making a network request using the request() library and then trying to throw in the async callback when errors. That's a completely flawed design since you (as the caller) can't catch exceptions thrown asynchronously. So, this package has no reasonable way of communicating back errors. It is designed poorly.

Try custom promisified function
aztroJs.getAllHoroscope[util.promisify.custom] = (sign) => {
return new Promise((resolve, reject) => {
aztroJs.getAllHoroscope(sign, resolve);
});
};
const getAllHoroscopeAsync = util.promisify(aztroJs.getAllHoroscope);

You could change your getAllHoroscopeAsync to a promise function
Example:
const getAllHoroscopeAsync = (sign) =>
new Promise(resolve =>
aztroJs.getAllHoroscope(sign, (res) => resolve(res)));

Sequential execution of Promise.all

Hi I need to execute promises one after the other how do I achieve this using promise.all any help would be awesome. Below is the sample of my code I am currently using but it executes parallel so the search will not work properly
public testData: any = (req, res) => {
// This method is called first via API and then promise is triggerd
var body = req.body;
// set up data eg 2 is repeated twice so insert 2, 5 only once into DB
// Assuming we cant control the data and also maybe 3 maybe inside the DB
let arrayOfData = [1,2,3,2,4,5,5];
const promises = arrayOfData.map(this.searchAndInsert.bind(this));
Promise.all(promises)
.then((results) => {
// we only get here if ALL promises fulfill
console.log('Success', results);
res.status(200).json({ "status": 1, "message": "Success data" });
})
.catch((err) => {
// Will catch failure of first failed promise
console.log('Failed:', err);
res.status(200).json({ "status": 0, "message": "Failed data" });
});
}
public searchAndInsert: any = (data) => {
// There are database operations happening here like searching for other
// entries in the JSON and inserting to DB
console.log('Searching and updating', data);
return new Promise((resolve, reject) => {
// This is not an other function its just written her to make code readable
if(dataExistsInDB(data) == true){
resolve(data);
} else {
// This is not an other function its just written her to make code readable
insertIntoDB(data).then() => resolve(data);
}
});
}
I looked up in google and saw the reduce will help I would appreciate any help on how to convert this to reduce or any method you suggest (Concurrency in .map did not work)

the Promises unfortunatelly does not allow any control of their flow. It means -> once you create new Promise, it will be doing its asynchronous parts as they like.
The Promise.all does not change it, its only purpose is that it checks all promises that you put into it and it is resolved once all of them are finished (or one of them fail).
To be able to create and control asynchronous flow, the easiest way is to wrap the creation of Promise into function and create some kind of factory method. Then instead of creating all promises upfront, you just create only one promise when you need it, wait until it is resolved and after it continue in same behaviour.
async function doAllSequentually(fnPromiseArr) {
for (let i=0; i < fnPromiseArr.length; i++) {
const val = await fnPromiseArr[i]();
console.log(val);
}
}
function createFnPromise(val) {
return () => new Promise(resolve => resolve(val));
}
const arr = [];
for (let j=0; j < 10; j++) {
arr.push(createFnPromise(Math.random()));
}
doAllSequentually(arr).then(() => console.log('finished'));
PS: It is also possible without async/await using standard promise-chains, but it requires to be implemented with recursion.

If anyone else cares about ESLint complaining about the use of "for" and the "no await in loop" here is a typescript ESLint friendly version of the above answer:
async function runPromisesSequentially<T>(promises: Array<Promise<T>>):Promise<Array<T>> {
if (promises.length === 0) return [];
const [firstElement, ...rest] = promises;
return [await firstElement, ...(await runPromisesSequentially(rest))];
}
You can then just replace Promise.all by runPromisesSequentially.

#lmX2015's answer is close but it's taking in promises that have already started executing.
A slight modification fixes it
export async function runPromisesSequentially<T>(functions: (() => Promise<T>)[]): Promise<T[]> {
if (functions.length === 0) {
return [];
}
const [first, ...rest] = functions;
return [await first(), ...(await runPromisesSequentially(rest))];
}

await for function with callback

I'm playing with streams and async/await functionality. What I have so far is:
let logRecord = ((record, callback) => {
console.log(record);
return callback();
});
let importCSVfromPath = async((csv_path) => {
return new Promise(function(resolve, reject) {
var parser = parse();
var input = fs.createReadStream(csv_path);
var transformer = transform(logRecord, {parallel: 1});
input.on('error', (err) => {
reject(err);
});
input.on('finish', ()=> {
resolve();
});
input.pipe(parser).pipe(transformer);
});
});
Now I want to replace logRecord with importRecord. The problem is that this function has to use functions that are already part of the async stack.
let importRecord = async( (record) => {
.......
await(insertRow(row));
});
What's the right way to do this?

It's slightly more complicated than this - node.js streams are not adapted (at least not yet) to the es7 async/await methods.
If you'd like to develop this on your own, consider writing a class derived from Readable stream. Implementing a promise based interface is quite a task, but it is possible.
If you're however fine with using a permissive licensed framework - take a look at Scramjet. With it your code will look like this (most of the example is parsing the CSV - I'll add a helper in the next version):
fs.createReadStream("file.csv") // open your file
.pipe(new StringStream()) // pass to scramjet
.split("\n") // split by line
.parse((line) => line.split(",")) // convert lines to arrays
.map(async (line) => { // run asynchrounous mapping
await importRecord(line); // import log to DB
return logRecord(line); // return some log for the output
})
.pipe(process.stdout); // pipe the output wherever you like
I believe it's exactly what you're looking for and it will run your record imports in parallel, while keeping the output order.

Chaining Promises while maintaining data (angular js)

Ive seen that there are questions about chaining promises, but this one is a little bit different.
I'm making http get requests in my code. The first call returns an array. For each object in the array, i need to make another http call which returns another array and so on (this chains 3 levels deep).
The problem is, I need to keep track of which array element was used for to make each http call, and I dont know how to do this using promises.
I also want to end the chain by returning a promise.
I have the code for what I want to do written in nodejs without promises:
var https = require('https');
var fs = require('fs');
function makeRequest(options){
var httpopts = {
host: 'soc.courseoff.com',
path: '/gatech/terms/201601/majors/' + options.p,
method: 'GET'
};
var response = "";
var req = https.request(httpopts, function(res) {
res.on('data', function(d) {
response += d;
});
res.on('end',function(){
options.cb(response,options)
})
});
req.end();
req.on('error', function(e) {
console.error(e);
});
}
var classData = {};
function getCourses(m){
var majors = JSON.parse(m);
majors.forEach(function(maj){
classData[maj] = {};
var options = {
p:maj.ident +'/courses',
cb:getSections,
major:maj
};
makeRequest(options);
});
}
var classCount = 0;
function getSections(c,opts){
var courses = JSON.parse(c);
courses.forEach(function(course){
classCount++;
var options = JSON.parse(JSON.stringify(opts));
options.p += '/'+course.ident+'/sections';
options.course = course
options.cb = buildData
makeRequest(options)
});
}
var sectionCount = 0;
function buildData(r, options){
var major = options.major.ident;
sectionCount++;
if(!classData[major]){
classData[major] = {
name: options.major.name,
classes:{}
};
}
classData[major].classes[options.course.ident] = {
name:options.course.name,
sections:JSON.parse(r)
};
console.log('classCount-sectionCount '+classCount + '---'+sectionCount);
if(classCount === sectionCount){
writeIt();
}
}
makeRequest({
p:'',
cb:getCourses
});
function writeIt(){
fs.writeFileSync('./classData.js', 'module.exports = ' + JSON.stringify(classData));
}
EDIT:
I managed to get the promises to nest while keeping track of the data, but how can i return a promise that eventually resolves with the final data object?
My code:
Thanks four your help! I've managed to code it so that the promises work, my only problem now is in returning the final data as a promise
fact.factory('ClassFactory', ['$http',function ($http) {
var eventData = {};
var promise;
var courseData = [];
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
eventData.getClasses = function (event) {
if(!promise){
promise = $http.get(baseURL).then(
function(majors){
Promise.all(majors.data.map(m => $http.get(baseURL + m.ident+'/courses')
.then(
function(courses){
if(!m.courses) m.courses = [];
courses.data.map(c => $http.get(baseURL+ m.ident+'/courses/' +c.ident+'/sections' )
.then(
function(sections){
c.sections = sections.data;
m.courses.push(c);
}
));
courseData.push(m);
}
)));
}
)
}
return promise;
}
return eventData;
}]);

Almost certainly, each time you deal with an array of Promises, you'll want to use Promise.all in order to connect and merge your promises into a new promise. That promise will then contain an array of the results from each call. Nested Promise.alls can thus return Arrays of Arrays with all your levels of results as long as you use something like a map and a closure to capture the outer levels.
var fakeCall = x => Promise.resolve(x||Math.random());
Promise.all([fakeCall(1),fakeCall(2)])
.then(
results => Promise.all(results.map( x => fakeCall(5).then( results2 => [x, results2]) ))
)
.then( x => console.log(x));//-> [[1,5],[2,5]]
The first array of calls generates an array of results, and mapping over those with a function that makes yet more calls will return a single result that can be paired with its parent.
Explicitly nesting things in this way will work for even deeper levels, but is not going to be pretty. There's probably an abstraction you can create using Array.reduce which can generalize this pattern.

You forgot some returns in your code. The function you pass to .then should always return something. Also you are modifying majors but then throw it away without using it. When working with promises - especially when they are complex and nested - it's not a good idea to modify any data structures contained in those promises unless you are sure nothing bad can possibly happen.
I would split it into several functions.
e.g.
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
function getSections(major, course) {
return $http.get(baseURL+ major.ident+'/courses/' +course.ident+'/sections')
.then(sections => sections.data)
.catch(e => []);
}
function getCourses(major) {
return $http.get(baseURL + major.ident+'/courses')
.then(courses => Promise.all(courses.data.map(course =>
getSections(major, course).then(sections => ({[course.ident]: {name: course.name, sections: sections}})))))
.then(courses => angular.extend({}, ...courses))
.catch(e => ({}));
}
function getClassData() {
return $http.get(baseURL)
.then(majors => Promise.all(majors.data.map(major =>
getCourses(major).then(courses => ({[major.ident]: {name: major.name, classes: courses}})))))
.then(majors => angular.extend({}, ...majors))
.catch(e => ({}));
}
getClassData().then(data => console.log(data));