Related
I am trying to create a chain of promises where each promise waits for the previous promise before getting executed.
const syncStatusChanges = () => {
return new Promise((resolve, reject) => {
console.log("in")
setTimeout(() => {
console.log("done")
resolve({ done: true })
}, 2000);
});
}
const run = () => {
const promises = [syncStatusChanges(), syncStatusChanges()]
promises[0].then(res => {
console.log("done 1")
promises[1].then(res => {
console.log("done 2")
})
})
}
run()
In this example the output is:
in
in
done
done 1
done
done 2
But I want it to be:
in
done
done 1
in
done
done 2
I also want it to work for any n number of functions. I saw this answer but the output is the same!
var promise = statusChangeCalls[0];
for (var i = 1; i < statusChangeCalls.length; i++)
promise = promise.then(statusChangeCalls[i]);
As it is written in comments. You are executing the functions in the array itself. What i understood by seeing your output. below run function can help you.
const run = () => {
const promise = syncStatusChanges();
promise.then(res => {
console.log("done 1")
syncStatusChanges().then(res => {
console.log("done 2")
})
})
}
Promise executes eagerly. It does not wait to register the then function. You can look for observable, they are lazy in execution. Basically they wait until you subscribe them.
For your second doubt about loop. You can use async await keyword to achieve chaining. Just pass number as an parameter in runInLoop function to execute promise that many times.
const runInLoop = async(numberOfPromisesCall)=> {
for (let i = 0; i < numberOfPromisesCall; i++){
await syncStatusChanges();
console.log(`done ${i + 1}`);
}
}
runInLoop(5)
In JavaScript, I have an array of objects being some tasks to do. I iterate through this array with a for loop with await, calling a function doOneTask with returns a Promise.
That works pretty well as long as the code inside doOneTask works as expected. However, those things often fail. Trying again helps almost all the time. So, I'd like to implement a procedure for auto-retrying inside the JavaScript code.
My idea was a recursive function: In case of a failure, doOneTask call itself till the promise if finally resolved.
My code looks like this:
var tasks = [{label: 'task0'},{label: 'task1'},{label: 'task2'}];
async function mainFunction() {
for(let k = 0; k < tasks.length; k++) {
await doOneTask(tasks[k]);
console.log("doOneTask done for index " + k);
}
console.log("End reached!");
}
function doOneTask(task) {
return new Promise(async function (resolve,reject) {
console.log("Starting with: " + task.label);
let checkIfDoeSomeStuffWorked = await doSomeAsyncStuff();
if(checkIfDoeSomeStuffWorked == false) {
console.log(task.label + ": FAILED");
return doOneTask(task);
}
else {
console.log(task.label + ": SUCCESS");
resolve(true);
}
});
}
function doSomeAsyncStuff() {
return new Promise(function (resolve,reject) {
var myRandom = Math.random();
if(myRandom < 0.3) {
resolve(true);
}
else {
resolve(false);
}
});
}
mainFunction();
(In real life, doSomeAsyncStuff is a backend call which often fails. The random() part is just for demonstration. In reality, I also limit the number of trials, before stopping the script.)
However, it doesn't work. In case of a failure, the script stops after having reached the SUCCESS console log. I never get back to the loop and the next items in the loop never get executed.
You have no use for the q library dependency. async functions always return a Promise, so you can simplify your code quite a bit -
async function doOneTask (task) {
const result = await doSomeAsyncStuff()
if (result === false) {
console.log(`${task} failed`)
return doOneTask(task)
}
else {
console.log(`${task} passed`)
return true
}
}
Your fake function doSomeAsyncStuff can be cleaned up too -
async function doSomeAsyncStuff () {
return Math.random() < 0.3
}
But let's add a fake delay of 1 second so that we can show things working 100% -
async function doSomeAsyncStuff () {
return new Promise(resolve =>
setTimeout(resolve, 1000, Math.random() < 0.3)
)
}
Last, your main function uses a really old looping convention. As you're using modern JavaScript, you might as well use for-of syntax -
async function main (tasks = []) {
for (const t of tasks) {
await doOneTask(t)
}
return "done"
}
Finally we run the program -
const tasks =
[ 'task0', 'task1', 'task2' ]
main(tasks).then(console.log, console.error)
// task0 failed
// task0 passed
// task1 failed
// task1 failed
// task1 passed
// task2 passed
// done
Expand the snippet below to verify the results in your own browser -
async function doOneTask (task) {
const result = await doSomeAsyncStuff()
if (result === false) {
console.log(`${task} failed`)
return doOneTask(task)
}
else {
console.log(`${task} passed`)
return true
}
}
async function doSomeAsyncStuff () {
return new Promise(resolve =>
setTimeout(resolve, 1000, Math.random() < 0.3)
)
}
async function main (tasks = []) {
for (const t of tasks) {
await doOneTask(t)
}
return "done"
}
const tasks =
[ 'task0', 'task1', 'task2' ]
main(tasks).then(console.log, console.error)
// task0 failed
// task0 passed
// task1 failed
// task1 failed
// task1 passed
// task2 passed
// done
After having completed the question, but just before posting, something came into my mind: In the setup above, I don't resolve the very same promise object when I finally reach success, but for each function call a new promise object is generated. My solution/workaround is quite simple: use q the promise library passing the promise from one function call to the next function call:
var q = require('q');
async function doOneTask(task,promiseObj) {
if(!promiseObj) {
var promiseObj = q.defer();
}
console.log("Starting with: " + task.label);
let checkIfDoeSomeStuffWorked = await doSomeAsyncStuff();
if(checkIfDoeSomeStuffWorked == false) {
console.log(task.label + ": FAILED");
return doOneTask(task,promiseObj);
}
else {
console.log(task.label + ": SUCCESS");
promiseObj.resolve(true);
}
return promiseObj.promise;
}
That way, we make sure that the very same promise object which is generated at the first call of doOneTask is resolved in the end - event after the 20th execution.
I have such a loop :
var i,j,temparray,chunk = 200;
for (i=0,j=document.mainarray.length; i<j; i+=chunk) {
temparray = document.mainarray.slice(i,i+chunk);
var docs = collection.find({ id: { "$in": temparray}}).toArray();
docs.then(function(singleDoc)
{
if(singleDoc)
{
console.log("single doc length : " + singleDoc.length);
var t;
for(t = 0, len = singleDoc.length; t < len;t++)
{
fs.appendFile("C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n", function(err) {
if(err) {
return console.log(err);
}
});
}
}
});
}
The loop iterates for two times. In first iteration it gets 200 elements, in second, it gets 130 elements. And when I open the .txt file, I see only 130 names. I guess because of the async nature of Node.js, only second part of the array is processed. What should I do to get all parts of the array to be processed? Thanks in advance.
EDIT : I finally turned the code to this :
var generalArr = [];
var i,j,temparray,chunk = 200;
for (i=0,j=document.mainarray.length; i<j; i+=chunk) {
temparray = document.mainarray.slice(i,i+chunk);
generalArr.push(temparray);
}
async.each(generalArr, function(item, callback)
{
var docs = collection.find({ id: { "$in": item}}).toArray();
docs.then(function(singleDoc)
{
if(singleDoc)
{
console.log("single doc length : " + singleDoc.length);
var t;
for(t = 0, len = singleDoc.length; t < len;t++)
{
fs.appendFile("C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n", function(err) {
if(err) {
return console.log(err);
}
});
}
}
});
callback(null);
})
When I change this line :
var docs = collection.find({ id: { "$in": item}}).toArray();
To this line :
var docs = collection.find({ id: { "$in": item}}).project({ name: 1 }).toArray();
It works, I'm able to print all names. I guess there is a problem with memory when I try without .project(). How can I make this work without using project? Should I change some memory limits? Thanks in advance.
I think your code is unnecessary complicated and appending file in a loop is very expensive when compared to in-memory computation. A better way would be to write to file just once.
var i, j, temparray, chunk = 200;
for (i = 0, j = document.mainarray.length; i < j; i += chunk) {
temparray = document.mainarray.slice(i, i + chunk);
generalArr.push(temparray);
}
const queryPromises = [];
generalArr.forEach((item, index) => {
queryPromises.push(collection.find({ id: { "$in": item } }).toArray());
});
let stringToWrite = '';
Promise.all(queryPromises).then((result) => {
result.forEach((item) => {
item.forEach((element) => {
//create a single string which you want to write
stringToWrite = stringToWrite + "\n" + element.name;
});
});
fs.appendFile("C:/Users/x/Desktop/names.txt", stringToWrite, function (err) {
if (err) {
return console.log(err);
} else {
// call your callback or return
}
});
});
In the code above, I do the following.
Wait for all the db queries to finish
Lets iterate over this list and create one string that we need to write to the file
Write to the file
Once you go asynchronous you cannot go back - all your code needs to be asynchronous. In node 8 you handle this with async and await keywords. In older versions you can use Promise - async/await are just syntax sugar for it anyway.
However, most of the API in node are older than Promise, and so they use callbacks instead. There is a promisify function to update callback functions to promises.
There are two ways to handle this, you can let all the asynchronous actions happen at the same time, or you can chain them one after another (which preserves order but takes longer).
So, collection.find is asynchronous, it either takes a callback function or returns a Promise. I'm going to assume that the API you're using does the latter, but your problem could be the former (in which case look up promisify).
var findPromise = collection.find({ id: { "$in": item}});
Now, at this point findPromise holds the running find action. We say this is a promise that resolves (completes successfully) or rejects (throws an error). We want to queue up an action to do once it completes, and we do that with then:
// The result of collection.find is the collection of matches
findPromise.then(function(docs) {
// Any code we run here happens asynchronously
});
// Code here will run first
Inside the promise we can return further promises (allowing them to be chained - complete one async, then complete the next, then fire the final resolve once all done) or use Promise.all to let them all happen in parallel and resolve once done:
var p = new Promise(function(resolve, reject) {
var findPromise = collection.find({ id: { "$in": item}});
findPromise.then(function(docs) {
var singleDocNames = [];
for(var i = 0; i < docs.length; i++) {
var singleDoc = docs[i];
if(!singleDoc)
continue;
for(var t = 0; t < singleDoc.length; t++)
singleDocNames.push(singleDoc[t].name);
}
// Resolve the outer promise with the final result
resolve(singleDocNames);
});
});
// When the promise finishes log it to the console
p.then(console.log);
// Code inline here will fire before the promise
This is much easier in node 8 with async/await:
async function p() {
// Await puts the rest of this function in the .then() of the promise
const docs = await collection.find({ id: { "$in": item}});
const singleDocNames = [];
for(var i = 0; i < docs.length; i++) {
// ... synchronous code unchanged ...
}
// Resolve the outer promise with the final result
return singleDocNames;
});
// async functions can be treated like promises
p().then(console.log);
If you need to write the results to a text file asynchronously there are a couple of ways to do it - you can wait until the end and write all of them, or chain a promise to write them after each find, though I find parallel IO operations tend to be at more risk of deadlocks.
Code above have multiple issues about asynchronous control flow. Similar code possible can exists, but only if case of using ES7 async/await operators on all async operation.
Of course, you can easily achieve solution by promises sequence. Solution:
let flowPromise = Promise.resolve();
const chunk = 200;
for (let i=0,j=document.mainarray.length; i<j; i+=chunk) {
flowPromise = flowPromise.then(() => {
const temparray = document.mainarray.slice(i,i+chunk);
const docs = collection.find({ id: { "$in": temparray}}).toArray();
return docs.then((singleDoc) => {
let innerFlowPromise = Promise.resolve();
if(singleDoc) {
console.log("single doc length : " + singleDoc.length);
for(let t = 0, len = singleDoc.length; t < len;t++) {
innerFlowPromise = innerFlowPromise.then(() => new Promise((resolve, reject) =>
fs.appendFile(
"C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n",
err => (err ? reject(err) : resolve())
)
));
}
}
return innerFlowPromise;
}
});
}
flowPromise.then(() => {
console.log('Done');
}).catch((err) => {
console.log('Error: ', err);
})
When use async-like control flow, based on Promises, always remember that every loop and function call sequence will not pause execution till async operation be done, so include all then sequences manually. Or use async/await syntax.
Which version of nodejs are you using? You should use the native async/await support which is built into newer versions nodejs (no libraries required). Also note, fs.appendFile is asyncronous so you need to either use a library like promisify to transform the callback into a promise or just use the appendFileSync and suffer the blocking IO (but might be okay for you, depending on the use case.)
async function(){
...
for(var item of generalArr) {
var singleDoc = await collection.find({ id: { "$in": item}}).toArray();
// if(singleDoc) { this won't do anything, since collection.find will always return something even if its just an empty array
console.log("single doc length : " + singleDoc.length);
var t;
for(t = 0, len = singleDoc.length; t < len;t++){
fs.appendFileSync("C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n");
}
};
}
var docs = collection.find({ id: { "$in": document.mainarray}}), // returns a cursor
doc,
names = [],
toInsert;
function saveToFile(cb) {
toInsert = names.splice(0,100);
if(!toInsert.length) return cb();
fs.appendFile("C:/Users/x/Desktop/names.txt", toInsert.join("\n"), cb);
}
(function process() {
if(docs.hasNext()) {
doc = docs.next();
doc.forEach(function(d) {
names.push(d.name);
});
if(names.length === 100) {
// save when we have 100 names in memory and clear the memory
saveToFile(function(err) {
process();
});
} else {
process();
}
} else {
saveToFile(function(){
console.log('All done');
});
}
}()); // invoke the function
If you can't solve your issue using core modules and basic nodejs, there is most likely a lack of understanding of how things work or insufficient knowledge about a library (in this case FileSystem module).
Here is how you can solve your issue, without 3th party libraries and such.
'use strict';
const
fs = require('fs');
let chunk = 200;
// How many rounds of array chunking we expect
let rounds = Math.ceil(mainArray.length/chunk);
// copy to temp (for the counter)
let tempRounds = rounds;
// set file name
let filePath = './names.txt'
// Open writable Stream
let myFileStream = fs.createWriteStream(filePath);
// from round: 0-${rounds}
for (let i = 0; i < rounds; i++) {
// assume array has ${chunk} elements left in this round
let tempChunk = chunk;
// if ${chunk} is to big i.e. i=3 -> chunk = 600 , but mainArray.length = 512
// This way we adjust the last round for "the leftovers"
if (mainArray.length < i*chunk) tempChunk = Math.abs(mainArray.length - i*chunk);
// slice it for this round
let tempArray = mainArray.slice(i*chunk, i*chunk + tempChunk);
// get stuff from DB
let docs = collection.find({ id: { "$in": tempArray}}).toArray();
docs.then(function(singleDoc){
// for each name in the doc
for (let j = 0; j < singleDoc.length; j++) {
// write to stream
myFileStream.write(singleDoc[t].name + "\n");
}
// declare round done (reduce tempRounds) and check if it hits 0
if (!--tempRounds) {
// if all rounds are done, end the stream
myFileStream.end();
// BAM! you done
console.log("Done")
}
});
}
The key is to use fs.WritableStreams :)
link here to docs
I'm tackling a project that requires me to use JavaScript with an API method call. I'm a Java programmer who has never done web development before so I'm having some trouble with it.
This API method is asynchronous and it's in a while loop. If it returns an empty array, the while loop finishes. Otherwise, it loops. Code:
var done = true;
do
{
async_api_call(
"method.name",
{
// Do stuff.
},
function(result)
{
if(result.error())
{
console.error(result.error());
}
else
{
// Sets the boolean to true if the returned array is empty, or false otherwise.
done = (result.data().length === 0) ? true : false;
}
}
);
} while (!done);
This doesn't work. The loop ends before the value of "done" is updated. I've done some reading up on the subject and it appears I need to use promises or callbacks because the API call is asynchronous, but I can't understand how to apply them to the code I have above.
Help would be appreciated!
edit: see the bottom, there is the real answer.
I encourage you yo use the Promise API. Your problem can be solved using a Promise.all call:
let promises = [];
while(something){
promises.push(new Promise((r, j) => {
YourAsyncCall(() => r());
});
}
//Then this returns a promise that will resolve when ALL are so.
Promise.all(promises).then(() => {
//All operations done
});
The syntax is in es6, here is the es5 equivalent (Promise API may be included externally):
var promises = [];
while(something){
promises.push(new Promise(function(r, j){
YourAsyncCall(function(){ r(); });
});
}
//Then this returns a promise that will resolve when ALL are so.
Promise.all(promises).then(function(){
//All operations done
});
You can also make your api call return the promise and push it directly to the promise array.
If you don't want to edit the api_call_method you can always wrap your code in a new promise and call the method resolve when it finishes.
edit: I have seen now the point of your code, sorry. I've just realized that Promise.all will not solve the problem.
You shall put what you posted (excluding the while loop and the control value) inside a function, and depending on the condition calling it again.
Then, all can be wraped inside a promise in order to make the external code aware of this asynchronous execution. I'll post some sample code later with my PC.
So the good answer
You can use a promise to control the flow of your application and use recursion instead of the while loop:
function asyncOp(resolve, reject) {
//If you're using NodeJS you can use Es6 syntax:
async_api_call("method.name", {}, (result) => {
if(result.error()) {
console.error(result.error());
reject(result.error()); //You can reject the promise, this is optional.
} else {
//If your operation succeeds, resolve the promise and don't call again.
if (result.data().length === 0) {
asyncOp(resolve); //Try again
} else {
resolve(result); //Resolve the promise, pass the result.
}
}
});
}
new Promise((r, j) => {
asyncOp(r, j);
}).then((result) => {
//This will call if your algorithm succeeds!
});
/*
* Please note that "(...) => {}" equivals to "function(...){}"
*/
sigmasoldier's solution is correct, just wanted to share the ES6 version with async / await:
const asyncFunction = (t) => new Promise(resolve => setTimeout(resolve, t));
const getData = async (resolve, reject, count) => {
console.log('waiting');
await asyncFunction(3000);
console.log('finshed waiting');
count++;
if (count < 2) {
getData(resolve, reject, count);
} else {
return resolve();
}
}
const runScript = async () => {
await new Promise((r, j) => getData(r, j, 0));
console.log('finished');
};
runScript();
If you don't want to use recursion you can change your while loop into a for of loop and use a generator function for maintaining done state. Here's a simple example where the for of loop will wait for the async function until we've had 5 iterations and then done is flipped to true. You should be able to update this concept to set your done variable to true when your webservice calls have buffered all of your data rows.
let done = false;
let count = 0;
const whileGenerator = function* () {
while (!done) {
yield count;
}
};
const asyncFunction = async function(){
await new Promise(resolve => { setTimeout(resolve); });
};
const main = new Promise(async (resolve)=>{
for (let i of whileGenerator()){
console.log(i);
await asyncFunction();
count++;
if (count === 5){
done = true;
}
}
resolve();
});
main.then(()=>{
console.log('all done!');
});
Also you may try recursion solution.
function asyncCall(cb) {
// Some async operation
}
function responseHandler(result) {
if (result.error()) {
console.error(result.error());
} else if(result.data() && result.data().length) {
asyncCall(responseHandler);
}
}
asyncCall(responseHandler);
Here is a solution I came up with. Place this in an async function.
let finished = false;
const loop = async () => {
return new Promise(async (resolve, reject) => {
const inner = async () => {
if (!finished) {
//insert loop code here
if (xxx is done) { //insert this in your loop code after task is complete
finshed = true;
resolve();
} else {
return inner();
}
}
}
await inner();
})
}
await loop();
If you don't want to use Promises you can restructure your code like so:
var tasks = [];
var index = 0;
function processNextTask()
{
if(++index == tasks.length)
{
// no more tasks
return;
}
async_api_call(
"method.name",
{
// Do stuff.
},
function(result)
{
if(result.error())
{
console.error(result.error());
}
else
{
// process data
setTimeout(processNextTask);
}
}
);
}
Your loop won't work, because it is sync, your async task is async, so the loop will finish before the async task can even respond. I'd reccomend you to use Promises to manage async tasks:
//first wrapping your API into a promise
var async_api_call_promise = function(methodName, someObject){
return new Promise((resolve, reject) => {
async_api_call(methodName, someObject, function(result){
if(result.error()){
reject( result.error() )
}else{
resolve( result.data() )
}
});
})
}
now to your polling code:
//a local utility because I don't want to repeat myself
var poll = () => async_api_call_promise("method.name", {/*Do stuff.*/});
//your pulling operation
poll().then(
data => data.length === 0 || poll(), //true || tryAgain
err => {
console.error(err);
return poll();
}
).then((done) => {
//done === true
//here you put the code that has to wait for your "loop" to finish
});
Why Promises? Because they do state-management of async operations. Why implement that yourself?
let taskPool = new Promise(function(resolve, reject) {
resolve("Success!");
});
let that = this;
while (index < this.totalPieces) {
end = start + thisPartSize;
if (end > filesize) {
end = filesize;
thisPartSize = filesize - start;
}
taskPool.then(() => {
that.worker(start, end, index, thisPartSize);
});
index++;
start = end;
}
I have an array that contains an array of promises, and each inner array could have either 4k, 2k or 500 promises.
In total there are around 60k promises and I may test it with other values as well.
Now I need to execute the Promise.all(BigArray[0]).
Once the first inner array is done, I need to execute the next Promise.all(BigArray[1]) and so on and so on.
If I try to execute a Promise.all(BigArray) it's throwing:
fatal error call_and_retry_2 allocation failed - process out of memory
I need to execute it each of promises in series, not in parallel which I think that’s what Node its doing. I shouldn't use new libs however am willing to consider the answer!.
Edit:
Here is an example piece of code:
function getInfoForEveryInnerArgument(InnerArray) {
const CPTPromises = _.map(InnerArray, (argument) => getDBInfo(argument));
return Promise.all(CPTPromises)
.then((results) => {
return doSomethingWithResults(results);
});
}
function mainFunction() {
BigArray = [[argument1, argument2, argument3, argument4], [argument5, argument6, argument7, argument8], ....];
//the summ of all arguments is over 60k...
const promiseArrayCombination = _.map(BigArray, (InnerArray, key) => getInfoForEveryInnerArgument(InnerArray));
Promise.all(promiseArrayCombination).then((fullResults) => {
console.log(fullResults);
return fullResults;
})
}
An answer from October 2020. Async/await makes it short: only 10 code lines+JSDoc.
/**
* Same as Promise.all(items.map(item => task(item))), but it waits for
* the first {batchSize} promises to finish before starting the next batch.
*
* #template A
* #template B
* #param {function(A): B} task The task to run for each item.
* #param {A[]} items Arguments to pass to the task for each call.
* #param {int} batchSize
* #returns {Promise<B[]>}
*/
async function promiseAllInBatches(task, items, batchSize) {
let position = 0;
let results = [];
while (position < items.length) {
const itemsForBatch = items.slice(position, position + batchSize);
results = [...results, ...await Promise.all(itemsForBatch.map(item => task(item)))];
position += batchSize;
}
return results;
}
Your question is a bit misnamed which may have confused some folks in this question and in the previous version of this question. You are trying to execute a batch of async operations in series, one batch of operations, then when that is done execute another batch of operations. The results of those async operations are tracked with promises. Promises themselves represent async operations that have already been started. "Promises" aren't executed themselves. So technically, you don't "execute a batch of promises in series". You execute a set of operations, track their results with promises, then execute the next batch when the first batch is all done.
Anyway, here's a solution to serializing each batch of operations.
You can create an inner function which I usually call next() that lets you process each iteration. When the promise resolves from processing one innerArray, you call next() again:
function mainFunction() {
return new Promise(function(resolve, reject) {
var bigArray = [[argument1, argument2, argument3, argument4], [argument5, argument6, argument7, argument8], ....];
//the summ of all arguments is over 60k...
var results = [];
var index = 0;
function next() {
if (index < bigArray.length) {
getInfoForEveryInnerArgument(bigArray[index++]).then(function(data) {
results.push(data);
next();
}, reject);
} else {
resolve(results);
}
}
// start first iteration
next();
});
}
This also collects all the sub-results into a results array and returns a master promise who's resolved value is this results array. So, you could use this like:
mainFunction().then(function(results) {
// final results array here and everything done
}, function(err) {
// some error here
});
You could also use the .reduce() design pattern for iterating an array serially:
function mainFunction() {
var bigArray = [[argument1, argument2, argument3, argument4], [argument5, argument6, argument7, argument8], ....];
return bigArray.reduce(function(p, item) {
return p.then(function(results) {
return getInfoForEveryInnerArgument(item).then(function(data) {
results.push(data);
return results;
})
});
}, Promise.resolve([]));
}
This creates more simultaneous promises than the first option and I don't know if that is an issue for such a large set of promises (which is why I offered the original option), but this code is cleaner and the concept is convenient to use for other situations too.
FYI, there are some promise add-on features built for doing this for you. In the Bluebird promise library (which is a great library for development using promises), they have Promise.map() which is made for this:
function mainFunction() {
var bigArray = [[argument1, argument2, argument3, argument4], [argument5, argument6, argument7, argument8], ....];
return Promise.map(bigArray, getInfoForEveryInnerArgument);
}
#jfriend00 Just adding to your answer using async/await with reduce:
function runPromisesInSeries(bigArray, getInfoForEveryInnerArgument) {
try {
return bigArray.reduce(async (acc, cItem) => {
const results = await acc
const data = await getInfoForEveryInnerArgument(cItem)
results.push(data)
return results
}, Promise.resolve([]))
} catch (err) {
throw err
}
}
In addition, if original array is not of promises but of objects that should be processed, batch processing can be done without an external dependency using combination of Array.prototype.map(), Array.prototype.slice() and Promise.all():
// Main batch parallelization function.
function batch(tasks, pstart, atonce, runner, pos) {
if (!pos) pos = 0;
if (pos >= tasks.length) return pstart;
var p = pstart.then(function() {
output('Batch:', pos / atonce + 1);
return Promise.all(tasks.slice(pos, pos + atonce).map(function(task) {
return runner(task);
}));
});
return batch(tasks, p, atonce, runner, pos + atonce);
}
// Output function for the example
function output() {
document.getElementById("result").innerHTML += Array.prototype.slice.call(arguments).join(' ') + "<br />";
window.scrollTo(0, document.body.scrollHeight);
}
/*
* Example code.
* Note: Task runner should return Promise.
*/
function taskrunner(task) {
return new Promise(function(resolve, reject) {
setTimeout(function() {
output('Processed:', task.text, 'Delay:', task.delay);
resolve();
}, task.delay);
});
}
var taskarray = [];
function populatetasks(size) {
taskarray = [];
for (var i = 0; i < size; i++) {
taskarray.push({
delay: 500 + Math.ceil(Math.random() * 50) * 10,
text: 'Item ' + (i + 1)
});
}
}
function clean() {
document.getElementById("result").innerHTML = '';
}
var init = Promise.resolve();
function start() {
var bsize = parseInt(document.getElementById("batchsize").value, 10),
tsize = parseInt(document.getElementById("taskssize").value, 10);
populatetasks(tsize);
init = batch(taskarray.slice() /*tasks array*/ , init /*starting promise*/ , bsize /*batch size*/ , taskrunner /*task runner*/ );
}
<input type="button" onclick="start()" value="Start" />
<input type="button" onclick="clean()" value="Clear" /> Batch size:
<input id="batchsize" value="4" size="2"/> Tasks:
<input id="taskssize" value="10" size="2"/>
<pre id="result" />
You can do it recursively, for example here I needed to put about 60k documents in mongo, but it was too big, to do it in one step, therefore I take 1k documents, send them to the mongo, after it is finished I take another 1k documents etc.
exports.rawRecursive = (arr, start) => {
//ending condition
if (start > arr.length) {
return;
}
Rawmedicament.insertManyAsync(_.slice(arr, start, start + 1000)).then(() => {
//recursive
exports.rawRecursive(arr, start + 1000);
});
};
If you want to notice, when everything is done, you can in ending condition put callback or if you like Promises you can call resolve() there.
Dynamically batching more promises
A simple implementation where you can have a queue of tasks batched to run in parallel and add more dynamically:
class TaskQueue {
constructor ({
makeTask,
initialData = [],
getId = data => data.id,
batchSize = 15,
onComplete = () => {},
}) {
if (!makeTask) throw new Error('The "makeTask" parameter is required');
this.makeTask = makeTask;
this.getId = getId;
this.batchSize = batchSize;
this.onComplete = onComplete;
this.queue = new Map();
this.add(initialData);
}
add(...data) {
data.forEach(item => {
const id = this.getId(item);
if (this.queue.has(id)) return;
this.queue.set(id, item);
});
// running automatically on create or additional items added
this.runNextBatch();
}
runNextBatch () {
if (this.queueStarted) return;
if (this.queue.size === 0) return;
this.queueStarted = true;
const currentBatchData = Array.from(this.queue.values()).slice(0, this.batchSize);
const tasks = currentBatchData.map(data => {
const id = this.getId(data);
// Have some error handling implemented in `makeTask`
this.makeTask(data)
.finally(() => this.queue.delete(id));
});
return Promise.all(tasks)
.then(() => {
this.queueStarted = false;
this.runNextBatch();
})
.finally(() => {
this.queueStarted = false;
if (this.queue.size === 0) this.onComplete();
});
}
}
// Usage
const lotOfFilesForUpload = [{ uri: 'file://some-path' }, { uri: 'file://some-other-path' }];
const upload = (file) => console.log('fake uploading file: ', file);
const taskQueue = new TaskQueue({
initialData: lotOfFilesForUpload,
getId: file => file.uri,
makeTask: file => upload(file),
onComplete: () => console.log('Queue completed'),
});
// You can add more tasks dynamically
taskQueue.add({ uri: 'file://yet-another-file' });