I'm working with fetching information from a github repository. I want to get the list of pull requests within that repo, get the list of commits associated with each pull request, then for each commit I want to get information such as the author of the commit, the number of files associated with each commit and the number of additions and deletions made to each file. I'm using axios and the github API to accomplish this. I know how to work with the API, but the promises and async functions are keeping me from accomplishing my task. I have the following code:
const axios = require('axios');
var mapOfInformationObjects = new Map();
var listOfCommits = [];
var listOfSHAs = [];
var gitApiPrefix = link I'll use to start fetching data;
var listOfPullRequestDataObjects = [];
var listOfPullRequestNumbers = [];
var mapOfPullNumberToCommits = new Map();
function getAllPullRequests(gitPullRequestApiLink) {
return new Promise((resolve, reject) => {
axios.get(gitPullRequestApiLink).then((response) =>{
listOfPullRequestDataObjects = response['data'];
var k;
for (k = 0; k < listOfPullRequestDataObjects.length; k++){
listOfPullRequestNumbers.push(listOfPullRequestDataObjects[k]['number']);
}
resolve(listOfPullRequestNumbers);
}).catch((error) => {
reject(error);
})
})
}
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
var j;
for (j = 0; j < listOfPRNumbers.length; j++) {
currPromise = new Promise((resolve, reject) => {
currentGitApiLink = gitApiPrefix + listOfPRNumbers[j] + "/commits";
axios.get(currentGitApiLink).then((response) => {
mapOfPullNumberToCommits.set(listOfPRNumbers[j], response['data']);
resolve("Done with Pull Request Number: " + listOfPRNumbers[j]);
}).catch((error) => {
reject(error);
})
})
}
}
function getListOfCommits(gitCommitApiLink){
return new Promise((resolve, reject) => {
axios.get(gitCommitApiLink).then((response) => {
resolve(response);
}).catch((error) => {
reject(error);
})
})
}
So far, I made some functions that I would like to call sequentially.
First I'd like to call getAllPullRequestNumbers(someLink)
Then I'd like to call getCommitsForEachPullRequestNumber(listofprnumbers)
Then getListOfCommits(anotherLink)
So it would look something like
getAllPullRequestNumbers(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
But two problems arise:
1) I'm not sure if this is how you would call the functions so that the first function in the sequence completes before the other.
2) Because I'm not familiar with Javascript, I'm not sure, especially with the getCommitsForEachPullRequestNumber function since you run a loop and call axios.get() on each iteration of the loop, if this is how you work with promises within the functions.
Would this be how you would go about accomplishing these two tasks? Any help is much appreciated. Thanks!
When you a number of asynchronous operations (represented by promises) that you can run all together and you want to know when they are all done, you use Promise.all(). You collect an array of promises and pass it to Promise.all() and it will tell you when they have all completed or when one of them triggers an error. If all completed, Promise.all() will return a promise that resolves to an array of results (one for each asynchronous operation).
When you're iterating an array to do your set of asynchronous operations, it then works best to use .map() because that helps you create a parallel array of promises that you can feed to Promise.all(). Here's how you do that in getCommitsForEachPullRequestNumber():
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
let mapOfPullNumberToCommits = new Map();
return Promise.all(listOfPRNumbers.map(item => {
let currentGitApiLink = gitApiPrefix + item + "/commits";
return axios.get(currentGitApiLink).then(response => {
// put data into the map
mapOfPullNumberToCommits.set(item, response.data);
});
})).then(() => {
// make resolved value be the map we created, now that everything is done
return mapOfPullNumberToCommits;
});
}
// usage:
getCommitsForEachPullRequestNumber(list).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
Then, in getListOfCommits(), since axios already returns a promise, there is no reason to wrap it in a manually created promise. That is, in fact, consider a promise anti-pattern. Instead, just return the promise that axios already returns. In fact, there's probably not even a reason to have this as a function since one can just use axios.get() directly to achieve the same result:
function getListOfCommits(gitCommitApiLink){
return axios.get(gitCommitApiLink);
}
Then, in getAllPullRequests() it appears you are just doing one axios.get() call and then processing the results. That can be done like this:
function getAllPullRequests(gitPullRequestApiLink) {
return axios.get(gitPullRequestApiLink).then(response => {
let listOfPullRequestDataObjects = response.data;
return listOfPullRequestDataObjects.map(item => {
return item.number;
});
});
}
Now, if you're trying to execute these three operations sequentially in this order:
getAllPullRequests(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
You can chain the promises from those three operations together to sequence them:
getAllPullRequests(someLink)
.then(getCommitsForEachPullRequestNumber)
.then(mapOfPullNumberToCommits => {
// not entirely sure what you want to do here, perhaps
// call getListOfCommits on each item in the map?
}).catch(err => {
console.log(err);
});
Or, if you put this code in an async function, then you can use async/awit:
async function getAllCommits(someLink) {
let pullRequests = await getAllPullRequests(someLink);
let mapOfPullNumberToCommits = await getCommitsForEachPullRequestNumber(pullRequests);
// then use getlistOfCommits() somehow to process mapOfPullNumberToCommits
return finalResults;
}
getAllCommits.then(finalResults => {
console.log(finalResults);
}).catch(err => {
console.log(err);
});
not as clean as jfriend00 solution,
but I played with your code and it finally worked
https://repl.it/#gui3/githubApiPromises
you get the list of commits in the variable listOfCommits
I don't understand the purpose of your last function, so I dropped it
This question already has answers here:
Using async/await with a forEach loop
(33 answers)
Closed last year.
In node.js I need to use a function procesMultipleCandidates () which contains Array.foreach which process insert every element into db. but the entire function should return response after completing all insertion operation
JavaScript Code
async function procesMultipleCandidates (data) {
let generatedResponse = []
await data.forEach(async (elem) => {
try {
// here candidate data is inserted into
let insertResponse = await insertionInCandidate(elem)
//and response need to be added into final response array
generatedResponse.push(insertResponse)
} catch (error) {
console.log('error'+ error);
}
})
console.log('complete all') // gets loged first
return generatedResponse // return without waiting for process of
}
And as described above last return statement not waiting for the foreach execution to complete first.
Use Array.prototype.map and Promise.all:
async function procesMultipleCandidates (data) {
let generatedResponse = []
await Promise.all(data.map(async (elem) => {
try {
// here candidate data is inserted into
let insertResponse = await insertionInCandidate(elem)
// and response need to be added into final response array
generatedResponse.push(insertResponse)
} catch (error) {
console.log('error'+ error);
}
}))
console.log('complete all') // gets loged first
return generatedResponse // return without waiting for process of
}
Or use a for/of loop if you don't want the loop run concurrently:
async function procesMultipleCandidates (data) {
let generatedResponse = []
for(let elem of data) {
try {
// here candidate data is inserted into
let insertResponse = await insertionInCandidate(elem)
// and response need to be added into final response array
generatedResponse.push(insertResponse)
} catch (error) {
console.log('error'+ error);
}
}
console.log('complete all') // gets loged first
return generatedResponse // return without waiting for process of
}
Array.prototype.forEach() tries to execute sequentially and you may not always get the expected result.
Array.prototype.map() is the widely used for creating Promises using data and then resolved using await Promise.all([])
using .map() function, all the promises are resolved in parallel and if you happen to call an API you may end up getting error 429: Too many Requests
To execute all the promises sequentially, you can use for ...of.
const array1 = ['a', 'b', 'c'];
for (const element of array1) {
console.log(element);
}
MDN Reference
As far as I'm concern, for of works better with asynchrony than forEach. Anyway, in this case, if you only care for the aggregated result, I think this could be a solution.
async function procesMultipleCandidates(data) {
const promises = data.map(insertionInCandidate);
return await Promise.all(promises);
}
Use Promise.all instead, which will resolve once all Promises in the array passed to it have resolved. The syntax will probably be simplified if you use and return a plain Promise chain rather than async/await:
const procesMultipleCandidates = data => Promise.all(
data.map(insertionInCandidate)
)
.catch(err => {
console.log('err: ' + err);
});
Note that if there's an error, procesMultipleCandidates will currently return a resolved Promise rather than rejecting - it might be a better idea to have the consumer of procesMultipleCandidates handle errors, so that it can handle them appropriately (rather than simply console.logging it).
function getMentionedUsers(str, next){
var array = getUsernamesFromString(str); //['john','alex','jess'];
if(array.length > 0){
var users = [];
var pending = array.length;
array.forEach(function(username){
getUserByUsername(username).then(function(model){
users.push(model.key);
--pending || next(users); //this is a callback model
});
});
}
};
function getUserByUsername(username){
return admin.database().ref('/users').orderByChild('username').equalTo(username).once('value').then(function(snapshot) {
return snapshot.val(); //this is the firebase example of a promise
});
};
Right now, I'm doing this:
getMentionedUsers(model.body, function(users){
console.log("Mentions", users);
});
However, I'd like to turn getMentionedUsers into a promise. How can I do that? I'm new to Promises
You could use Promise.all and Array#map:
function getMentionedUsers(str) {
return Promise.all(getUsernamesFromString(str).map((username) => {
return getUserByUsername(username).then((model) => model.key);
}));
}
A more readable version broken into two functions:
function getUserKeyByUsername(username) {
return getUserByUsername(username).then((user) => user.key);
}
function getMentionedUsers(str) {
const promises = getUsernamesFromString(str).map(getUserKeyByUsername);
return Promise.all(promises);
}
Use Promise.all.
const getMentionedUsers = str =>
Promise.all(
getUsernamesFromString(str).map(
username => getUserByUsername(username)
.then(model => model.key)
)
);
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
You can have the best of both. If you pass a next function, it will get called with the results. If not, your method will return a promise.
function getMentionedUsers(str, next){
var array = getUsernamesFromString(str); //['john','alex','jess'];
var promise = Promise.resolve([]); // default
var hasNext = typeof next === 'function';
if(array.length > 0){
promise = Promise.all(array.map(function(username){
return getUserByUsername(username);
}));
}
promise = promise.then(models => {
var users = models.map(model => model.key);
if (hasNext) next(null, users);
return users;
});
if (hasNext) promise.catch(next);
else return promise;
};
UPDATE: Though not part of your original question, this is still a good point and worth pointing out. Your existing code is using a non-standard callback technique. The standard callback technique expects an error as the first parameter and results as a second parameter:
next(new Error(...)); //-> when something fails
next(null, results); //-> when something succeeds
As such, I have updated my code to show the "standard" callback behavior alongside promises. Using the hybrid approach above allows for existing code to stay in place while allowing new code to use the new Promise technique. This would be considered a "non-breaking change".
using native ES6 promises, written in a functional style:
// Returns array of usernames
function getUsernamesFromString(str = '') {
return str.split(',').map(s => s.trim())
}
// returns promise of user
function getUserByUserName(username) {
// Lets say this is a slow async function and returns a promise
return Promise.resolve({
id: (Math.random() * 10 ** 10).toFixed(0),
username
});
}
function getMentionedUsers(str) {
return Promise.all(
getUsernamesFromString(str).map(getUserByUserName)
);
}
getMentionedUsers('kai, ava, mia, nova').then(console.log.bind(console))
However, there are also libraries like bluebird that can promisify objects and functions automatically as long as the follow the NODE convention of (err, result) as the callback arguments.
You can also just return a new Promise((resolve, reject) => { /* all your code */ }) and just call resolve(dataToResolveWith) if it succeeds and reject(new Error()) if it fails, but you rarely have to do that and in fact, it's an anti-pattern.
I have some asynchronous code running inside a JavaScript forEach loop. I want to wait until the code inside the asynchronous process has finished running before proceeding after the loop.
Example below:
ids is an array of strings. db is a node module I created to work with MongoDB
var appIdsNotFound = "";
var count = 0;
ids.forEach(function(id) {
output[count] = {};
//console.log(id);
db.findApp(id, function(error, result) {
if(error) {
fatalError = true;
console.log(error);
} else {
if (result) {
output[count] = result;
//console.log(output[count]);
count++;
} else {
appNotFound = true;
appIdsNotFound += id + ", ";
console.log(appIdsNotFound);
}
}
});
});
//more code that we want to wait before executing
Is there a way to wait before executing the rest of the code that is outside the loop, and if so, how would I go about doing that.
Assuming db is some module to access your DB, try to look for the synchronous version. This assumes you are ok with synchronous, since you're attempting to write it that way, waiting for everything before proceeding.
If your db library uses promises, you can use it in conjunction with Promise.all. Fire a request for each item, collect all their promises in an array, feed them to Promise.all. The promise from Promise.all will resolve when all promises resolve.
const promises = ids.map(id => db.promiseReturningFindApp(id));
const allRequests = Promise.all(promises).then(responses => {
// responses is an array of all results
});
If you don't have a promise-returning version of your API, wrap db.findApp in a promise, do suggestion #2.
function promiseReturningFindApp(id){
return new Promise((resolve, reject) => {
db.findApp(id, (error, result) => {
if(error) reject(error);
else resolve(result);
});
});
}
Options 2 and 3 are asynchronous, and as such, you technically don't "wait". Therefore, code that needs to execute after can only reside in a callback.
You could make every item into an function and use async:
var async = require('async');
var output = [], appsNotFound = [];
var appRequests = ids.map((id) => (cb) => {
db.findApp(id, (error, result) => {
if (error) {
appsNotFound.push(id);
return cb();
}
output.push(id);
return cb();
})
})
async.parallel(appRequests, () => {
console.log('N# of Apps found',output.length);
console.log("Ids not found:",appIdsNotFound.join(','))
console.log("N# Apps not found:",appIdsNotFound.length)
})
If the DB don't handle it, try to use async.serial.
You can make something similar with promises if you prefer, but this requires less lines of code.
I would like to clarify this point, as the documentation is not too clear about it;
Q1: Is Promise.all(iterable) processing all promises sequentially or in parallel? Or, more specifically, is it the equivalent of running chained promises like
p1.then(p2).then(p3).then(p4).then(p5)....
or is it some other kind of algorithm where all p1, p2, p3, p4, p5, etc. are being called at the same time (in parallel) and results are returned as soon as all resolve (or one rejects)?
Q2: If Promise.all runs in parallel, is there a convenient way to run an iterable sequencially?
Note: I don't want to use Q, or Bluebird, but all native ES6 specs.
Is Promise.all(iterable) executing all promises?
No, promises cannot "be executed". They start their task when they are being created - they represent the results only - and you are executing everything in parallel even before passing them to Promise.all.
Promise.all does only await multiple promises. It doesn't care in what order they resolve, or whether the computations are running in parallel.
is there a convenient way to run an iterable sequencially?
If you already have your promises, you can't do much but Promise.all([p1, p2, p3, …]) (which does not have a notion of sequence). But if you do have an iterable of asynchronous functions, you can indeed run them sequentially. Basically you need to get from
[fn1, fn2, fn3, …]
to
fn1().then(fn2).then(fn3).then(…)
and the solution to do that is using Array::reduce:
iterable.reduce((p, fn) => p.then(fn), Promise.resolve())
In parallel
await Promise.all(items.map(async (item) => {
await fetchItem(item)
}))
Advantages: Faster. All iterations will be started even if one fails later on. However, it will "fail fast". Use Promise.allSettled, to complete all iterations in parallel even if some throw. Technically, these are concurrent invocations not in parallel.
In sequence
for (const item of items) {
await fetchItem(item)
}
Advantages: Variables in the loop can be shared by each iteration. Behaves like normal imperative synchronous code.
NodeJS does not run promises in parallel, it runs them concurrently since it’s a single-threaded event loop architecture. There is a possibility to run things in parallel by creating a new child process to take advantage of the multiple core CPU.
Parallel Vs Concurent
In fact, what Promise.all does is, stacking the promises function in the appropriate queue (see event loop architecture) running them concurrently (call P1, P2,...) then waiting for each result, then resolving the Promise.all with all the promises results.
Promise.all will fail at the first promise which fails unless you have to manage the rejection yourself.
There is a major difference between parallel and concurrent, the first one will run a different computation in a separate process at exactly the same time and they will progress at their rhythm, while the other one will execute the different computation one after another without waiting for the previous computation to finish and progress at the same time without depending on each other.
Finally, to answer your question, Promise.all will execute neither in parallel nor sequentially but concurrently.
Bergi's answer got me on the right track using Array.reduce.
However, to actually get the functions returning my promises to execute one after another I had to add some more nesting.
My real use case is an array of files that I need to transfer in order one after another due to limits downstream...
Here is what I ended up with:
getAllFiles().then( (files) => {
return files.reduce((p, theFile) => {
return p.then(() => {
return transferFile(theFile); //function returns a promise
});
}, Promise.resolve()).then(()=>{
console.log("All files transferred");
});
}).catch((error)=>{
console.log(error);
});
As previous answers suggest, using:
getAllFiles().then( (files) => {
return files.reduce((p, theFile) => {
return p.then(transferFile(theFile));
}, Promise.resolve()).then(()=>{
console.log("All files transferred");
});
}).catch((error)=>{
console.log(error);
});
didn't wait for the transfer to complete before starting another and also the "All files transferred" text came before even the first file transfer was started.
Not sure what I did wrong, but wanted to share what worked for me.
Edit: Since I wrote this post I now understand why the first version didn't work. then() expects a function returning a promise. So, you should pass in the function name without parentheses! Now, my function wants an argument so then I need to wrap in in a anonymous function taking no argument!
You can also process an iterable sequentially with an async function using a recursive function. For example, given an array a to process with asynchronous function someAsyncFunction():
var a = [1, 2, 3, 4, 5, 6]
function someAsyncFunction(n) {
return new Promise((resolve, reject) => {
setTimeout(() => {
console.log("someAsyncFunction: ", n)
resolve(n)
}, Math.random() * 1500)
})
}
//You can run each array sequentially with:
function sequential(arr, index = 0) {
if (index >= arr.length) return Promise.resolve()
return someAsyncFunction(arr[index])
.then(r => {
console.log("got value: ", r)
return sequential(arr, index + 1)
})
}
sequential(a).then(() => console.log("done"))
Just to elaborate on #Bergi's answer (which is very succinct, but tricky to understand ;)
This code will run each item in the array and add the next 'then chain' to the end:
function eachorder(prev,order) {
return prev.then(function() {
return get_order(order)
.then(check_order)
.then(update_order);
});
}
orderArray.reduce(eachorder,Promise.resolve());
Using async await an array of promises can easily be executed sequentially:
let a = [promise1, promise2, promise3];
async function func() {
for(let i=0; i<a.length; i++){
await a[i]();
}
}
func();
Note: In above implementation, if a promise is rejected, the rest wouldn't be executed.If you want all your promises to be executed, then wrap your await a[i](); inside try catch
parallel
see this example
const resolveAfterTimeout = async i => {
return new Promise(resolve => {
console.log("CALLED");
setTimeout(() => {
resolve("RESOLVED", i);
}, 5000);
});
};
const call = async () => {
const res = await Promise.all([
resolveAfterTimeout(1),
resolveAfterTimeout(2),
resolveAfterTimeout(3),
resolveAfterTimeout(4),
resolveAfterTimeout(5),
resolveAfterTimeout(6)
]);
console.log({ res });
};
call();
by running the code it'll console "CALLED" for all six promises and when they are resolved it will console every 6 responses after timeout at the same time
I stumbled across this page while trying to solve a problem in NodeJS: reassembly of file chunks. Basically:
I have an array of filenames.
I need to append all those files, in the correct order, to create one large file.
I must do this asynchronously.
Node's 'fs' module does provide appendFileSync but I didn't want to block the server during this operation. I wanted to use the fs.promises module and find a way to chain this stuff together. The examples on this page didn't quite work for me because I actually needed two operations: fsPromises.read() to read in the file chunk, and fsPromises.appendFile() to concat to the destination file. Maybe if I was better with JavaScript I could have made the previous answers work for me. ;-)
I stumbled across this and I was able to hack together a working solution:
/**
* sequentially append a list of files into a specified destination file
*/
exports.append_files = function (destinationFile, arrayOfFilenames) {
return arrayOfFilenames.reduce((previousPromise, currentFile) => {
return previousPromise.then(() => {
return fsPromises.readFile(currentFile).then(fileContents => {
return fsPromises.appendFile(destinationFile, fileContents);
});
});
}, Promise.resolve());
};
And here's a jasmine unit test for it:
const fsPromises = require('fs').promises;
const fsUtils = require( ... );
const TEMPDIR = 'temp';
describe("test append_files", function() {
it('append_files should work', async function(done) {
try {
// setup: create some files
await fsPromises.mkdir(TEMPDIR);
await fsPromises.writeFile(path.join(TEMPDIR, '1'), 'one');
await fsPromises.writeFile(path.join(TEMPDIR, '2'), 'two');
await fsPromises.writeFile(path.join(TEMPDIR, '3'), 'three');
await fsPromises.writeFile(path.join(TEMPDIR, '4'), 'four');
await fsPromises.writeFile(path.join(TEMPDIR, '5'), 'five');
const filenameArray = [];
for (var i=1; i < 6; i++) {
filenameArray.push(path.join(TEMPDIR, i.toString()));
}
const DESTFILE = path.join(TEMPDIR, 'final');
await fsUtils.append_files(DESTFILE, filenameArray);
// confirm "final" file exists
const fsStat = await fsPromises.stat(DESTFILE);
expect(fsStat.isFile()).toBeTruthy();
// confirm content of the "final" file
const expectedContent = new Buffer('onetwothreefourfive', 'utf8');
var fileContents = await fsPromises.readFile(DESTFILE);
expect(fileContents).toEqual(expectedContent);
done();
}
catch (err) {
fail(err);
}
finally {
}
});
});
You can do it by for loop.
async function return promise:
async function createClient(client) {
return await Client.create(client);
}
let clients = [client1, client2, client3];
if you write following code then client are created parallelly:
const createdClientsArray = yield Promise.all(clients.map((client) =>
createClient(client);
));
But if you want to create client sequentially then you should use for loop:
const createdClientsArray = [];
for(let i = 0; i < clients.length; i++) {
const createdClient = yield createClient(clients[i]);
createdClientsArray.push(createdClient);
}
Bergi's answer helped me to make the call synchronous. I have added an example below where we call each function after the previous function is called:
function func1 (param1) {
console.log("function1 : " + param1);
}
function func2 () {
console.log("function2");
}
function func3 (param2, param3) {
console.log("function3 : " + param2 + ", " + param3);
}
function func4 (param4) {
console.log("function4 : " + param4);
}
param4 = "Kate";
//adding 3 functions to array
a=[
()=>func1("Hi"),
()=>func2(),
()=>func3("Lindsay",param4)
];
//adding 4th function
a.push(()=>func4("dad"));
//below does func1().then(func2).then(func3).then(func4)
a.reduce((p, fn) => p.then(fn), Promise.resolve());
I've been using for of in order to solve sequential promises. I'm not sure if it helps here but this is what I've been doing.
async function run() {
for (let val of arr) {
const res = await someQuery(val)
console.log(val)
}
}
run().then().catch()
Yes, you can chain an array of promise returning functions as follows
(this passes the result of each function to the next). You could of course edit it to pass the same argument (or no arguments) to each function.
function tester1(a) {
return new Promise(function(done) {
setTimeout(function() {
done(a + 1);
}, 1000);
})
}
function tester2(a) {
return new Promise(function(done) {
setTimeout(function() {
done(a * 5);
}, 1000);
})
}
function promise_chain(args, list, results) {
return new Promise(function(done, errs) {
var fn = list.shift();
if (results === undefined) results = [];
if (typeof fn === 'function') {
fn(args).then(function(result) {
results.push(result);
console.log(result);
promise_chain(result, list, results).then(done);
}, errs);
} else {
done(results);
}
});
}
promise_chain(0, [tester1, tester2, tester1, tester2, tester2]).then(console.log.bind(console), console.error.bind(console));
see this sample
Promise.all working parallel
const { range, random, forEach, delay} = require("lodash");
const run = id => {
console.log(`Start Task ${id}`);
let prom = new Promise((resolve, reject) => {
delay(() => {
console.log(`Finish Task ${id}`);
resolve(id);
}, random(2000, 15000));
});
return prom;
}
const exec = () => {
let proms = [];
forEach(range(1,10), (id,index) => {
proms.push(run(id));
});
let allPromis = Promise.all(proms);
allPromis.then(
res => {
forEach(res, v => console.log(v));
}
);
}
exec();