I am working on a Chrome extension where I need to contact from the page extension code with background script using Chrome messages. This adds a fair level asynch fiddling to my code. The general purpose is to process text macros ie parse text, using data stored in the database accessible from the background script.
at some point, I have:
findMacro(key)
.then(result => {
processMacro(key, result);
}};
The processor roughly looks like:
function processMacro(shortcut, text) {
text = macroBuilder(text);
}
macroBuilder function processes various types of macros, pushing processed text to an array then running join(''). My problem is that, I need to support nested macros and when there are such macros, I call findMacro again, which inside does chrome sendMessage to the background process. The processor does something like:
function macroBuilder(text) {
let pieces = [];
macroProcessorCode(text).forEach(res => {
res.visit({
text(txt, ctx) {
pieces.push(txt);
},
dates(obj, ctx) {
pieces.push(processDates(obj))
},
...
nested(obj,ctx) {
let fragments = [];
fragments.push(
findMacro(obj.name)
.then(res => {
return macroBuilder(res);
})
);
console.log('promised results:', fragments);
Promise.all(fragments)
.then(fragments => {
console.log('resolved results:', fragments);
pieces.push(fragments);
});
}
});
});
return pieces.join('');
}
For some reason my function returns before resolving, so promised results happens before it returns, and resolved results after. In short, I return from the code processing text with the result, before nested macros are processed. This only happens with nested, other types of macros are processed correctly.
Any ideas?
macroBuilder creates a bunch of promises but never does anything with them. Instead, it needs to wait for them and return its own promise that will resolve/reject based on the promises for the pieces/fragments.
This is somewhat off-the-cuff and probably needs tweaking, but should get you going the right direction. See *** comments:
function macroBuilder(text) {
// *** An array for all the promises below
const piecePromises = [];
macroProcessorCode(text).forEach(res => {
res.visit({
text(txt, ctx) {
pieces.push(txt);
},
dates(obj, ctx) {
pieces.push(processDates(obj))
},
//...
nested(obj, ctx) {
let fragments = [];
fragments.push(
findMacro(obj.name)
.then(res => macroBuilder) // *** Note we can just pass `macroBuilder` directly
);
console.log('promised results:', fragments);
// *** Add this promise to the array
pieces.push(Promise.all(fragments)
// (*** This whole then handler can be removed
// once you don't need the logging anymore;
// just push the result of Promise.all)
.then(fragments => {
console.log('resolved results:', fragments);
return fragments;
})
);
}
});
});
// *** Wait for all of those and then join; ultimate result
// is a promise that resolves with the pieces joined together
return Promise.all(piecePromises).then(pieces => pieces.join(''));
}
At that point, there's not much point to processMacro; it would just look like this (note that it returns a promise):
function processMacro(shortcut, text) {
return macroBuilder(text);
}
...unless there's something you do with shortcut that you haven't shown.
Assuming you need processMacro, you'd call it like this if you're propagating the promise to the caller:
return findMacro(key)
.then(result => processMacro(key, result));
...or like this if you're not propagating the promise:
findMacro(key)
.then(result => processMacro(key, result))
.catch(err => {
// Deal with the fact an error occurred
});
Since one of the rules of promises is that you either propagate the promise or handle errors from it.
Related
I'm working with fetching information from a github repository. I want to get the list of pull requests within that repo, get the list of commits associated with each pull request, then for each commit I want to get information such as the author of the commit, the number of files associated with each commit and the number of additions and deletions made to each file. I'm using axios and the github API to accomplish this. I know how to work with the API, but the promises and async functions are keeping me from accomplishing my task. I have the following code:
const axios = require('axios');
var mapOfInformationObjects = new Map();
var listOfCommits = [];
var listOfSHAs = [];
var gitApiPrefix = link I'll use to start fetching data;
var listOfPullRequestDataObjects = [];
var listOfPullRequestNumbers = [];
var mapOfPullNumberToCommits = new Map();
function getAllPullRequests(gitPullRequestApiLink) {
return new Promise((resolve, reject) => {
axios.get(gitPullRequestApiLink).then((response) =>{
listOfPullRequestDataObjects = response['data'];
var k;
for (k = 0; k < listOfPullRequestDataObjects.length; k++){
listOfPullRequestNumbers.push(listOfPullRequestDataObjects[k]['number']);
}
resolve(listOfPullRequestNumbers);
}).catch((error) => {
reject(error);
})
})
}
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
var j;
for (j = 0; j < listOfPRNumbers.length; j++) {
currPromise = new Promise((resolve, reject) => {
currentGitApiLink = gitApiPrefix + listOfPRNumbers[j] + "/commits";
axios.get(currentGitApiLink).then((response) => {
mapOfPullNumberToCommits.set(listOfPRNumbers[j], response['data']);
resolve("Done with Pull Request Number: " + listOfPRNumbers[j]);
}).catch((error) => {
reject(error);
})
})
}
}
function getListOfCommits(gitCommitApiLink){
return new Promise((resolve, reject) => {
axios.get(gitCommitApiLink).then((response) => {
resolve(response);
}).catch((error) => {
reject(error);
})
})
}
So far, I made some functions that I would like to call sequentially.
First I'd like to call getAllPullRequestNumbers(someLink)
Then I'd like to call getCommitsForEachPullRequestNumber(listofprnumbers)
Then getListOfCommits(anotherLink)
So it would look something like
getAllPullRequestNumbers(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
But two problems arise:
1) I'm not sure if this is how you would call the functions so that the first function in the sequence completes before the other.
2) Because I'm not familiar with Javascript, I'm not sure, especially with the getCommitsForEachPullRequestNumber function since you run a loop and call axios.get() on each iteration of the loop, if this is how you work with promises within the functions.
Would this be how you would go about accomplishing these two tasks? Any help is much appreciated. Thanks!
When you a number of asynchronous operations (represented by promises) that you can run all together and you want to know when they are all done, you use Promise.all(). You collect an array of promises and pass it to Promise.all() and it will tell you when they have all completed or when one of them triggers an error. If all completed, Promise.all() will return a promise that resolves to an array of results (one for each asynchronous operation).
When you're iterating an array to do your set of asynchronous operations, it then works best to use .map() because that helps you create a parallel array of promises that you can feed to Promise.all(). Here's how you do that in getCommitsForEachPullRequestNumber():
function getCommitsForEachPullRequestNumber(listOfPRNumbers) {
let mapOfPullNumberToCommits = new Map();
return Promise.all(listOfPRNumbers.map(item => {
let currentGitApiLink = gitApiPrefix + item + "/commits";
return axios.get(currentGitApiLink).then(response => {
// put data into the map
mapOfPullNumberToCommits.set(item, response.data);
});
})).then(() => {
// make resolved value be the map we created, now that everything is done
return mapOfPullNumberToCommits;
});
}
// usage:
getCommitsForEachPullRequestNumber(list).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
Then, in getListOfCommits(), since axios already returns a promise, there is no reason to wrap it in a manually created promise. That is, in fact, consider a promise anti-pattern. Instead, just return the promise that axios already returns. In fact, there's probably not even a reason to have this as a function since one can just use axios.get() directly to achieve the same result:
function getListOfCommits(gitCommitApiLink){
return axios.get(gitCommitApiLink);
}
Then, in getAllPullRequests() it appears you are just doing one axios.get() call and then processing the results. That can be done like this:
function getAllPullRequests(gitPullRequestApiLink) {
return axios.get(gitPullRequestApiLink).then(response => {
let listOfPullRequestDataObjects = response.data;
return listOfPullRequestDataObjects.map(item => {
return item.number;
});
});
}
Now, if you're trying to execute these three operations sequentially in this order:
getAllPullRequests(someLink)
getCommitsForEachPullRequestNumber(listofprnumbers)
getListOfCommits(anotherlink)
You can chain the promises from those three operations together to sequence them:
getAllPullRequests(someLink)
.then(getCommitsForEachPullRequestNumber)
.then(mapOfPullNumberToCommits => {
// not entirely sure what you want to do here, perhaps
// call getListOfCommits on each item in the map?
}).catch(err => {
console.log(err);
});
Or, if you put this code in an async function, then you can use async/awit:
async function getAllCommits(someLink) {
let pullRequests = await getAllPullRequests(someLink);
let mapOfPullNumberToCommits = await getCommitsForEachPullRequestNumber(pullRequests);
// then use getlistOfCommits() somehow to process mapOfPullNumberToCommits
return finalResults;
}
getAllCommits.then(finalResults => {
console.log(finalResults);
}).catch(err => {
console.log(err);
});
not as clean as jfriend00 solution,
but I played with your code and it finally worked
https://repl.it/#gui3/githubApiPromises
you get the list of commits in the variable listOfCommits
I don't understand the purpose of your last function, so I dropped it
Hi I need to execute promises one after the other how do I achieve this using promise.all any help would be awesome. Below is the sample of my code I am currently using but it executes parallel so the search will not work properly
public testData: any = (req, res) => {
// This method is called first via API and then promise is triggerd
var body = req.body;
// set up data eg 2 is repeated twice so insert 2, 5 only once into DB
// Assuming we cant control the data and also maybe 3 maybe inside the DB
let arrayOfData = [1,2,3,2,4,5,5];
const promises = arrayOfData.map(this.searchAndInsert.bind(this));
Promise.all(promises)
.then((results) => {
// we only get here if ALL promises fulfill
console.log('Success', results);
res.status(200).json({ "status": 1, "message": "Success data" });
})
.catch((err) => {
// Will catch failure of first failed promise
console.log('Failed:', err);
res.status(200).json({ "status": 0, "message": "Failed data" });
});
}
public searchAndInsert: any = (data) => {
// There are database operations happening here like searching for other
// entries in the JSON and inserting to DB
console.log('Searching and updating', data);
return new Promise((resolve, reject) => {
// This is not an other function its just written her to make code readable
if(dataExistsInDB(data) == true){
resolve(data);
} else {
// This is not an other function its just written her to make code readable
insertIntoDB(data).then() => resolve(data);
}
});
}
I looked up in google and saw the reduce will help I would appreciate any help on how to convert this to reduce or any method you suggest (Concurrency in .map did not work)
the Promises unfortunatelly does not allow any control of their flow. It means -> once you create new Promise, it will be doing its asynchronous parts as they like.
The Promise.all does not change it, its only purpose is that it checks all promises that you put into it and it is resolved once all of them are finished (or one of them fail).
To be able to create and control asynchronous flow, the easiest way is to wrap the creation of Promise into function and create some kind of factory method. Then instead of creating all promises upfront, you just create only one promise when you need it, wait until it is resolved and after it continue in same behaviour.
async function doAllSequentually(fnPromiseArr) {
for (let i=0; i < fnPromiseArr.length; i++) {
const val = await fnPromiseArr[i]();
console.log(val);
}
}
function createFnPromise(val) {
return () => new Promise(resolve => resolve(val));
}
const arr = [];
for (let j=0; j < 10; j++) {
arr.push(createFnPromise(Math.random()));
}
doAllSequentually(arr).then(() => console.log('finished'));
PS: It is also possible without async/await using standard promise-chains, but it requires to be implemented with recursion.
If anyone else cares about ESLint complaining about the use of "for" and the "no await in loop" here is a typescript ESLint friendly version of the above answer:
async function runPromisesSequentially<T>(promises: Array<Promise<T>>):Promise<Array<T>> {
if (promises.length === 0) return [];
const [firstElement, ...rest] = promises;
return [await firstElement, ...(await runPromisesSequentially(rest))];
}
You can then just replace Promise.all by runPromisesSequentially.
#lmX2015's answer is close but it's taking in promises that have already started executing.
A slight modification fixes it
export async function runPromisesSequentially<T>(functions: (() => Promise<T>)[]): Promise<T[]> {
if (functions.length === 0) {
return [];
}
const [first, ...rest] = functions;
return [await first(), ...(await runPromisesSequentially(rest))];
}
I'm faced with a small issue when trying to chain complex function calls with Promises and callbacks.
I have a main function, which calls subroutines. In these routines API calls are made.
For Example:
function handle(){
new Promise(function(resolve, reject){
let result = doAPICall1()
if (result === true) resolve(true);
reject(JSON.stringify(result))
}).then(function(){
let result = doAPICall2()
if (result === true) return true
throw new Error(JSON.stringify(result))
}).catch(error){
console.error(JSON.stringify(error))
}
}
function doAPICall1(){
axios.get('...').then(function(){
return true
}).catch(function(error){
return error
})
}
function doAPICall2(){
axios.get('...').then(function(){
return true
}).catch(function(error){
return error
})
}
But when I execute this example, doAPICall2 will be executed, while doAPICall1 is still running.
It only occures when long running calls are made.
Does anyone can give me a hint? Thank you!
You're overdoing manually a lot of things that Promises already do for you:
axios.get already returns a Promise, so there is no point in return a the response when it resolves and return a false when it rejects. A catch handler at the end of a Promise chain already handles all errors that may arise during the chain, so you don't need to catch every Promise.
I would do something like:
function doAPICall1(){
return axios.get('...');
}
function doAPICall2(){
return axios.get('...');
}
function handle(){
// in case you would use the api calls results.
let firstResult = null;
let secondResult = null;
return doAPICall1()
.then(res => {firstResult = res})
.then(() => doAPICall2())
.then(res => {
secondResult = res;
return []
})
}
I guess you will use the Api calls results for something. With the code above, you could consume the handle() function like follows:
function someSortOfController(){
handle().then(results => {
console.log(results[0]); // first api call result
console.log(results[1]); // second api call result
})
.catch(err => {
// here you will find any error, either it fires from the first api call or from the second.
// there is *almomst* no point on catch before
console.log(err);
})
}
There, you will access any error, either it came from the first api call or the second. (And, due to how Promises work, if the first call fails, the second won't fire).
For more fine grained error control, you may want to catch after every Promise so you can add some extra logs, like:
function doAPICall1(){
return axios.get('...')
.catch(err => {
console.log('the error came from the first call');
throw err;
});
}
function doAPICall2(){
return axios.get('...')
.catch(err => {
console.log('the error came from the second call');
throw err;
});
}
Now, if the first api call fails, everything will work as before (since you're throwing the error again in the catch), but you have more control over error handling (maybe the error returning from API calls is not clear at all and you want this kind of control mechanism).
Disclaimer
This answer doesn't answer why your code acts like it does. However, there are so much things wrong in your code, so I think providing you with an example about using Promises is more valuable.
Don't worry and take some time to understand Promises better. In the example code below, doAPICall function return a Promise which resolves to a value, not the value itself.
function handle() {
doAPICall().then(result => {
//do something with the result
}).catch(error => {
//catch failed API call
console.error(error)
})
}
doAPICall() {
// this returns a Promise
return axios.get(...)
}
I got a little bit of a problem. Here is the code:
Situation A:
var foundRiders = [];
riders.forEach(function(rider){
Rider.findOne({_id: rider}, function(err, foundRider){
if(err){
console.log("program tried to look up rider for the forEach loop finalizing the results, but could not find");
} else {
foundRiders.push(foundRider);
console.log(foundRiders);
}
});
});
Situation B
var foundRiders = [];
riders.forEach(function(rider){
Rider.findOne({_id: rider}, function(err, foundRider){
if(err){
console.log("program tried to look up rider for the forEach loop finalizing the results, but could not find");
} else {
foundRiders.push(foundRider);
}
});
});
console.log(foundRiders);
So in Situation A when I console log I get that foundRiders is an array filled with objects. In situation B when I put the console.log outside the loop, my roundRiders array is completely empty...
How come?
As others have said, your database code is asynchronous. That means that the callbacks inside your loop are called sometime later, long after your loop has already finishes. There are a variety of ways to program for an async loop. In your case, it's probably best to move to the promise interface for your database and then start using promises to coordinate your multiple database calls. You can do that like this:
Promise.all(riders.map(rider => {
return Rider.findOne({_id: rider}).exec();
})).then(foundRiders => {
// all found riders here
}).catch(err => {
// error here
});
This uses the .exec() interface to the mongoose database to run your query and return a promise. Then, riders.map() builds and returns an array of these promises. Then,Promise.all()monitors all the promises in the array and calls.then()when they are all done or.catch()` when there's an error.
If you want to ignore any riders that aren't found in the database, rather than abort with an error, then you can do this:
Promise.all(riders.map(rider => {
return Rider.findOne({_id: rider}).exec().catch(err => {
// convert error to null result in resolved array
return null;
});
})).then(foundRiders => {
foundRiders = foundRiders.filter(rider => rider !== null);
console.log(founderRiders);
}).catch(err => {
// handle error here
});
To help illustrate what's going on here, this is a more old fashioned way of monitoring when all the database callbacks are done (with a manual counter):
riders.forEach(function(rider){
let cntr = 0;
Rider.findOne({_id: rider}, function(err, foundRider){
++cntr;
if(err){
console.log("program tried to look up rider for the forEach loop finalizing the results, but could not find");
} else {
foundRiders.push(foundRider);
}
// if all DB requests are done here
if (cntr === riders.length) {
// put code here that wants to process the finished foundRiders
console.log(foundRiders);
}
});
});
The business of maintaining a counter to track multiple async requests is what Promise.all() has built in.
The code above assumes that you want to parallelize your code and to run the queries together to save time. If you want to serialize your queries, then you could use await in ES6 with a for loop to make the loop "wait" for each result (this will probably slow things down). Here's how you would do that:
async function lookForRiders(riders) {
let foundRiders = [];
for (let rider of riders) {
try {
let found = await Rider.findOne({_id: rider}).exec();
foundRiders.push(found);
} catch(e) {
console.log(`did not find rider ${rider} in database`);
}
}
console.log(foundRiders);
return foundRiders;
}
lookForRiders(riders).then(foundRiders => {
// process results here
}).catch(err => {
// process error here
});
Note, that while this looks like it's more synchronous code like you may be used to in other languages, it's still using asynchronous concepts and the lookForRiders() function is still returning a promise who's result you access with .then(). This is a newer feature in Javascript which makes some types of async code easier to write.
I am using [ssh2-sftp-client][1] package to recursively read all the directories inside a given remote path.
Here is the code.
const argv = require('yargs').argv;
const client = require('ssh-sftp-client');
const server = new Client();
const auth = {
host: '192.168.1.11',
username: argv.u,
password: argv.p
};
const serverRoot = '/sites/';
const siteName = 'webmaster.com';
// list of directories on the server will be pushed to this array
const serverPaths = [];
server.connect(auth).then(() => {
console.log(`connected to ${auth.host} as ${auth.username}`);
}).catch((err) => {
if (err) throw err;
});
server.list('/sites/').then((dirs) => {
redursiveDirectorySearch(dirs, `${serverRoot}${siteName}/`);
})
.catch((err) => {
if (err) throw err;
});
function recursiveDirectorySearch(dirs, prevPath) {
let paths = dirs.filter((dir) => {
// returns directories only
return dir.type === 'd';
});
if (paths.length > 0) {
paths.forEach((path) => {
server
.list(`${prevPath}${path.name}`)
.then((dirs) => {
console.log(`${prevPath}${path.name}`);
recursiveDirectorySearch(dirs, `${prevPath}${path.name}`);
serverPaths.push(`${prevPath}${path.name}`);
})
}
}
}
At first, a connection will be made to the server and then list whatever is under '/sites/' directory, which will then be passed to 'recursiveDirectorySearch' function. This function will receive an array of whatever is found under '/sites/' directory on the server as the first parameter, which will be filtered out so it only has directories. If one or more directory was found, a call to the server for each directory in the array will be made in order to retrieve everything under '/sites/'+'name of the directory in the array'. This same function will be called again with whatever is returned by the call to the server until no other directory is found.
Whenever a directory is found, its name in string will be pushed to 'serverPaths' array. As far as I can tell, this search is working and successfully pushing all the directory names to the array.
However, I can't think of a way to detect when this recursive search for all the directories is complete so I can do something with the 'serverPaths' array.
I tried to take advantage of Promise.all() but don't know how to use it when how many function calls are made is unknown.
You're simply lacking a couple of returns, add a Promise.all, and an Array#map and you're done
Note: not using Promise.all on serverPaths, but rather, using the fact that returning a Promise in .then will result in the Promise that is returned by .then taking on the Promise that is returned (hmmm, that isn't very well explained, is it, but it's Promises 101 stuff really!
server.list('/sites/').then((dirs) => {
// added a return here
return recursiveDirectorySearch(dirs, `${serverRoot}${siteName}/`);
})
.then(() => {
// everything is done at this point,
// serverPaths should be complete
})
.catch((err) => {
if (err) throw err;
});
function recursiveDirectorySearch(dirs, prevPath) {
let paths = dirs.filter((dir) => {
// returns directories only
return dir.type === 'd';
});
// added a return, Promise.all and changed forEach to map
return Promise.all(paths.map((path) => {
//added a return here
return server
.list(`${prevPath}${path.name}`)
.then((dirs) => {
console.log(`${prevPath}${path.name}`);
// swapped the next two lines
serverPaths.push(`${prevPath}${path.name}`);
// added a return here, push the path before
return recursiveDirectorySearch(dirs, `${prevPath}${path.name}`);
})
}));
}
One of the main things that is jumping out at me is your initial if statement. (if paths.length > 0) { run recursion } This appears to work really well for the first call because you know that the data coming back will be populated with an array full of directories.
Your function however, does not appear to have logic built out for an array with a length of 0. In this scenario it would be possible for you to get all of the directory names you are looking for. Presented in the manner that you are looking for. It would also mean that your calls on the higher parts of the tree are never able to resolve.
Try to add logic to handle cases for an array with a length of zero | if (paths.length === 0) return; | This would be a hard break out of the recursive calls on the higher parts of the stack.