There's a async call I'm making that queries a database on a service, but this service has a limit of how many it can output at once, so I need to check if it hit its limit through the result it sends and repeat the query until it doesn't.
Synchronous mockup :
var query_results = [];
var limit_hit = true; #While this is true means that the query hit the record limit
var start_from = 0; #Pagination parameter
while (limit_hit) {
Server.Query(params={start_from : start_from}, callback=function(result){
limit_hit = result.limit_hit;
start_from = result.results.length;
query_result.push(result.results);
}
}
Obviously the above does not work, I've seen some other questions here about the issue, but they don't mention what to do when you need each iteration to wait for the last one to finish and you don't know before hand the number of iterations.
How can I turn the above asynchronous? I'm open to answers using promise/deferred-like logic, but preferably something clean.
I can probably think of a monstruous and horrible way of doing this using waits/timeouts, but there has to be a clean, clever and modern way to solve it.
Another way is to make a "pre-query" to know the number of features before hand so you know the number of loops, I'm not sure if this is the correct way.
Here we use Dojo sometimes, but the examples I found does not explain what to do when you have an unknown amount of loops https://www.sitepen.com/blog/2015/06/10/dojo-faq-how-can-i-sequence-asynchronous-operations/
although many answers already, still I believe async/await is the cleanest way.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
and you might need babel
https://babeljs.io/
JS async logic syntax changed from callback to promise then to async/await, they all do the same thing, when callback nests a lot we need something like a chain, then promise come, when promise goes in loop, we need something make the chain more plain more simple, then async/await come. But not all browsers support the new syntax, so babel come to compile new syntax to old syntax, then you can always code in new syntax.
getData().then((data) => {
//do something with final data
})
async function getData() {
var query_results = [];
var limit_hit = true;
var start_from = 0;
//when you use await, handle error with try/catch
try {
while (limit_hit) {
const result = await loadPage(start_from)
limit_hit = result.limit_hit;
start_from = result.results.length;
query_result.push(result.results);
}
} catch (e) {
//when loadPage rejects
console.log(e)
return null
}
return query_result
}
async function loadPage(start_from) {
//when you use promise, handle error with reject
return new Promise((resolve, reject) => Server.Query({
start_from
}, (result, err) => {
//error reject
if (err) {
reject(err)
return
}
resolve(result)
}))
}
If you want to use a loop then I think there is no (clean) way to do it without Promises.
A different approach would be the following:
var query_results = [];
var start_from = 0;
funciton myCallback(result) {
if(!result) {
//first call
Server.Query({ start_from: start_from}, myCallback);
} else {
//repeated call
start_from = result.results.length
query_result.push(result.results);
if(!result.limit_hit) {
//limit has not been hit yet
//repeat the query with new start value
Server.Query({ start_from: start_from}, myCallback);
} else {
//call some callback function here
}
}
}
myCallback(null);
You could call this recursive, but since the Query is asynchronous you shouldn't have problems with call stack limits etc.
Using promises in an ES6 environment you could make use of async/await. Im not sure if this is possible with dojo.
You don't understand callbacks until you have written a rate limiter or queue ;) The trick is to use a counter: Increment the counter before the async request, and decrement it when you get the response, then you will know how many requests are "in flight".
If the server is choked you want to put the item back in the queue.
There are many things you need to take into account:
What will happen to the queue if the process is killed ?
How long to wait before sending another request ?
Make sure the callback is not called many times !
How many times should you retry ?
How long to wait before giving up ?
Make sure there are no loose ends ! (callback is never called)
When all edge cases are taken into account you will have a rather long and not so elegant solution. But you can abstract it into one function! (that returns a Promise or whatever you fancy).
If you have a user interface you also want to show a loading bar and some statistics!
You must await for the server response every time. Here a encapsulated method
var query = (function(){
var results = [];
var count = 0;
return function check(fun){
Server.Query({ start_from: count}, function(d){
count = d.results.length;
results.push(d.results);
if (d.limit_hit && fun) fun(results);
else check(fun);
});
};
})();
// Call here
var my_query = query(function(d){
// --> retrive all data when limit_hit is true)
});
You can use a generator function Generators to achieve this
For POC:
some basics
- You define a generator with an asterick *
- it exposes a next function which returns the next value
- generators can pause with yield statement internally and can resume externally by calling the next()
- While (true) will ensure that the generator is not done until limit has reached
function *limitQueries() {
let limit_hit = false;
let start_from = 0;
const query_result = [];
while (true) {
if (limit_hit) {break;}
yield Server.Query(params={start_from : start_from},
callback=function* (result) {
limit_hit = result.limit_hit;
start_from = result.results.length;
yield query_result.push(result.results);
}
}
}
So apparently, the generator function maintains its own state. Generator function exposes two properties { value, done } and you can call it like this
const gen = limitQueries();
let results = [];
let next = gen.next();
while(next.done) {
next = gen.next();
}
results = next.value;
You might have to touch your Server.Query method to handle generator callback. Hope this helps! Cheers!
I know, it is a newbie question:
I created a mobile application reading an information feed from a bluetooth serial device.
Data are retrieved using a promise in this way:
myServiceClass.getRemoteValue(valueId).then(reply: number) {
...
}
I need to read multiple parameters coming from this feed and I have to wait the previous call to finish before requesting the new value.
If I run:
let requiredValues = [1, 2, 3, 4, ..., n];
for (let i=0; i<requiredValues.length; i++) {
myServiceClass.getRemoteValue(valueId).then(reply: number) {
...
}
}
In this way request will run in parallel, but I need them to run in sequence one after the other. Is there any solution to subsequentially chain an array of promises someway?
In other words I need to run the n-th promise only after the previous promise has been resolved.
Thank you very much for your time.
Well, you can use a recursive method to achieve that... Please take a look at this plunker (when running the plunker, please notice that the values are being printed in the console)
I'm just using some fake data, but I guess it's enough to give you the overall idea:
public start(): void {
this.getSeveralRemoteValues([1,2,3,4,5,6,7,8,9]);
}
private getSeveralRemoteValues(array): Promise<boolean> {
if(array && array.length) {
return this.getRemoteValueFromService(array[0]).then(() => {
array.shift(); // remove the first item of the array
this.getSeveralRemoteValues(array); // call the same method with the items left
})
} else {
this.logEnd();
}
}
private logEnd(): void {
alert('All promises are done!');
}
private getRemoteValueFromService(value: number): Promise<boolean> {
// this simulates the call to the service
return new Promise((resolve, reject) => {
setTimeout(() => {
console.log(`Promise: ${value}`);
resolve(true);
}, 1000);
});
}
I'm writing an application in javascript where I make a CORS request to the server to grab a data array.
Then, for each item in the array, I need to make another CORS call to get additional info on that element.
I originally thought I could return values from my CORS request like:
data = getData(param);
But apparently you can't mix synchronous and asynchronous code.
What's the best way to accomplish this?
Promises. Here's how you might use them using your requirements, and a setTimeout to mimic an AJAX request.
getData returns a new promise. In this case if the function is called with no params an array is sent back after a second (your first request). If a param is passed into the function 100 is added to the param before resolving - the later requests.
function getData(param) {
return new Promise(function(resolve, reject) {
if (param) {
setTimeout(() => resolve(param + 100), 500);
} else {
setTimeout(() => resolve([1, 2, 3, 4, 5]), 1000)
}
});
}
Call getData without a param and [1, 2, 3, 4, 5] is returned. then we map over the array elements and return new promises for each of them. then we use Promise.all to resolve those promises and then we output the final array [101, 102, 103, 104, 105].
getData()
.then((arr) => arr.map(el => getData(el)))
.then(arr => Promise.all(arr))
.then(arr => console.log(arr));
DEMO
So you can see that you can run one AJAX request and then run more based on the result of the value that's returned until all requests have been made.
You can use async.series. checkout https://github.com/caolan/async . Very good library to solve problem like this - process an array data asynchronously(My favourite).
Or
You can use js promise from https://www.promisejs.org/
Or play with callbacks... like below
Note: Below functions are indicative functions just to show how you can approach the problem as you haven't shared any code. Change them accordingly. Also there might be syntactical/spell error as the code is written directly here.
function ajaxRequester(method,uri, data, onSuccess, onError){ // you can make this function as per requirement.
$.ajax({
type: method,
url: uri,
data: data
success: function(response){
onSuccess(response);
}
});
}
function yourFunction(){
ajaxRequester('GET',urlOf1stRequest,dataToSend,function(resp){
// assuming resp is the array received from server. we'll start with 0th element
processArray(0,resp, function(){
// do your final stuff
});
});
}
function processArray(index, arr, onComplete){
if(index < arr.lenght){
var objToProcess = arr[index]; // get your data to process
ajaxRequester(yourMethod,obj.url, obj.data, function(resp){
// do work with your response variable resp
processArray(++index, arr); // process next element of array after completion of current
});
} else {
onComplete(); // all elements are processed call final callback
}
}
I have an array and i need to send values of array to webservice through http post request one by one . For the node.js , i'm using "async" package to do that for ex: async.eachSeries doing it well , how can i do that same thing for angular.js , my normal async code;
//this code sends all queries of array (maybe 5.000 request at same time , it is hard to process for webservice :=) ) at same time and wait for all responses.
//it works but actually for me , responses should wait others at end of loop should work one by one
//like async.eachSeries module!
for (var i = 0; i < myArr.lenght; i++) {
(function (i) {
var data = {
"myQuery": myArr[i].query
};
$http.post("/myServiceUrl", data).success(function (result) {
console.log(result);
});
})(i);
}
Both Matt Way and Chris L answers Correct , you can investigate Chris's answer for understanding about async to sync functions in for loops.
You can use $q to create a similar requirement by chaining promises together. For example:
var chain = $q.when();
angular.forEach(myArr, function(item){
chain = chain.then(function(){
var data = {
myQuery: item.query
};
return $http.post('/myServiceUrl', data).success(function(result){
console.log(result);
});
});
});
// the final chain object will resolve once all the posts have completed.
chain.then(function(){
console.log('all done!');
});
Essentially you are just running the next promise once the previous one has completed. Emphasis here on the fact that each request will wait until the previous one has completed, as per your question.
function logResultFromWebService(value)
{
$http.post("/myServiceUrl", value).success(console.log);
}
angular.forEach(myArray, logResultFromWebService);
If I understand your question correctly. You want to run a for loop in a synchronized manner such that the next iteration only occurs once the previous iteration is completed. For that, you can use a synchronized loop/callbacks. Especially if the order matters.
var syncLoop = function (iterations, process, exit) {
var index = 0,
done = false,
shouldExit = false;
var loop = {
next: function () {
if (done) {
if (shouldExit && exit) {
return exit(); // Exit if we're done
}
}
// If we're not finished
if (index < iterations) {
index++; // Increment our index
process(loop); // Run our process, pass in the loop
// Otherwise we're done
} else {
done = true; // Make sure we say we're done
if (exit) exit(); // Call the callback on exit
}
},
iteration: function () {
return index - 1; // Return the loop number we're on
},
break: function (end) {
done = true; // End the loop
shouldExit = end; // Passing end as true means we still call the exit callback
}
};
console.log('running first time');
loop.next();
return loop;
}
For your particular implementation:
syncLoop(myArray.length, function (loop) {
var index = loop.iteration();
var data = {
"myQuery": myArray[index].query
};
$http.post("/myServiceUrl", data).success(function (result) {
console.log(result);
loop.next();
});
}, function () {
console.log('done');
});
If you intend on doing something with the data once returned (such as perform calculations) you can do so with this method because you will return the data in a specified order.
I implemented something similar in a statistical calculation web app I built.
EDIT:
To illustrate the problem I had when using $q.when I have set up a fiddle. Hopefully this will help illustrate why I did this the way I did.
https://jsfiddle.net/chrislewispac/6atp3w8o/
Using the following code from Matt's answer:
var chain = $q.when(promise.getResult());
angular.forEach(myArr, function (item) {
chain = chain.then(function () {
$rootScope.status = item;
console.log(item);
});
});
// the final chain object will resolve once all the posts have completed.
chain.then(function () {
console.log('all done!');
});
And this fiddle is an example of my solution:
https://jsfiddle.net/chrislewispac/Lgwteone/3/
Compare the $q version to my version. View the console and imagine those being delivered to the user interface for user intervention in the process and/or performing statistical operations on the sequential returns.
You will see that it does not sequentially give the numbers 1,2,3,4 etc. either in the console or in the view in Matt's answer. It 'batches' the responses and then returns them. Therefore, if step 3 is not to be run depending on the response in step 2 there is not, at least in the answer provided, a way to break out or explicitly control the synchronous operation here. This presents a significant problem when attempting to perform sequential calculations and/or allow the user to control break points, etc.
Now, I am digging through both the $q libraries and the Q library to see if there is a more elegant solution for this problem. However, my solution does work as requested and is very explicit which allows me to place the function in a service and manipulate for certain use cases at my will because I completely understand what it is doing. For me, that is more important than using a library (at least at this stage in my development as a programmer and I am sure there are lots of other people at the same stage on StackOverflow as well).
If the order doesn't matter in which they are sent
var items = [/* your array */];
var promises = [];
angular.forEach(items, function(value, key){
var promise = $http.post("/myServiceUrl", { "myQuery": value.query });
promises.push(promise);
});
return $q.all(promises);
I would like to clarify this point, as the documentation is not too clear about it;
Q1: Is Promise.all(iterable) processing all promises sequentially or in parallel? Or, more specifically, is it the equivalent of running chained promises like
p1.then(p2).then(p3).then(p4).then(p5)....
or is it some other kind of algorithm where all p1, p2, p3, p4, p5, etc. are being called at the same time (in parallel) and results are returned as soon as all resolve (or one rejects)?
Q2: If Promise.all runs in parallel, is there a convenient way to run an iterable sequencially?
Note: I don't want to use Q, or Bluebird, but all native ES6 specs.
Is Promise.all(iterable) executing all promises?
No, promises cannot "be executed". They start their task when they are being created - they represent the results only - and you are executing everything in parallel even before passing them to Promise.all.
Promise.all does only await multiple promises. It doesn't care in what order they resolve, or whether the computations are running in parallel.
is there a convenient way to run an iterable sequencially?
If you already have your promises, you can't do much but Promise.all([p1, p2, p3, …]) (which does not have a notion of sequence). But if you do have an iterable of asynchronous functions, you can indeed run them sequentially. Basically you need to get from
[fn1, fn2, fn3, …]
to
fn1().then(fn2).then(fn3).then(…)
and the solution to do that is using Array::reduce:
iterable.reduce((p, fn) => p.then(fn), Promise.resolve())
In parallel
await Promise.all(items.map(async (item) => {
await fetchItem(item)
}))
Advantages: Faster. All iterations will be started even if one fails later on. However, it will "fail fast". Use Promise.allSettled, to complete all iterations in parallel even if some throw. Technically, these are concurrent invocations not in parallel.
In sequence
for (const item of items) {
await fetchItem(item)
}
Advantages: Variables in the loop can be shared by each iteration. Behaves like normal imperative synchronous code.
NodeJS does not run promises in parallel, it runs them concurrently since it’s a single-threaded event loop architecture. There is a possibility to run things in parallel by creating a new child process to take advantage of the multiple core CPU.
Parallel Vs Concurent
In fact, what Promise.all does is, stacking the promises function in the appropriate queue (see event loop architecture) running them concurrently (call P1, P2,...) then waiting for each result, then resolving the Promise.all with all the promises results.
Promise.all will fail at the first promise which fails unless you have to manage the rejection yourself.
There is a major difference between parallel and concurrent, the first one will run a different computation in a separate process at exactly the same time and they will progress at their rhythm, while the other one will execute the different computation one after another without waiting for the previous computation to finish and progress at the same time without depending on each other.
Finally, to answer your question, Promise.all will execute neither in parallel nor sequentially but concurrently.
Bergi's answer got me on the right track using Array.reduce.
However, to actually get the functions returning my promises to execute one after another I had to add some more nesting.
My real use case is an array of files that I need to transfer in order one after another due to limits downstream...
Here is what I ended up with:
getAllFiles().then( (files) => {
return files.reduce((p, theFile) => {
return p.then(() => {
return transferFile(theFile); //function returns a promise
});
}, Promise.resolve()).then(()=>{
console.log("All files transferred");
});
}).catch((error)=>{
console.log(error);
});
As previous answers suggest, using:
getAllFiles().then( (files) => {
return files.reduce((p, theFile) => {
return p.then(transferFile(theFile));
}, Promise.resolve()).then(()=>{
console.log("All files transferred");
});
}).catch((error)=>{
console.log(error);
});
didn't wait for the transfer to complete before starting another and also the "All files transferred" text came before even the first file transfer was started.
Not sure what I did wrong, but wanted to share what worked for me.
Edit: Since I wrote this post I now understand why the first version didn't work. then() expects a function returning a promise. So, you should pass in the function name without parentheses! Now, my function wants an argument so then I need to wrap in in a anonymous function taking no argument!
You can also process an iterable sequentially with an async function using a recursive function. For example, given an array a to process with asynchronous function someAsyncFunction():
var a = [1, 2, 3, 4, 5, 6]
function someAsyncFunction(n) {
return new Promise((resolve, reject) => {
setTimeout(() => {
console.log("someAsyncFunction: ", n)
resolve(n)
}, Math.random() * 1500)
})
}
//You can run each array sequentially with:
function sequential(arr, index = 0) {
if (index >= arr.length) return Promise.resolve()
return someAsyncFunction(arr[index])
.then(r => {
console.log("got value: ", r)
return sequential(arr, index + 1)
})
}
sequential(a).then(() => console.log("done"))
Just to elaborate on #Bergi's answer (which is very succinct, but tricky to understand ;)
This code will run each item in the array and add the next 'then chain' to the end:
function eachorder(prev,order) {
return prev.then(function() {
return get_order(order)
.then(check_order)
.then(update_order);
});
}
orderArray.reduce(eachorder,Promise.resolve());
Using async await an array of promises can easily be executed sequentially:
let a = [promise1, promise2, promise3];
async function func() {
for(let i=0; i<a.length; i++){
await a[i]();
}
}
func();
Note: In above implementation, if a promise is rejected, the rest wouldn't be executed.If you want all your promises to be executed, then wrap your await a[i](); inside try catch
parallel
see this example
const resolveAfterTimeout = async i => {
return new Promise(resolve => {
console.log("CALLED");
setTimeout(() => {
resolve("RESOLVED", i);
}, 5000);
});
};
const call = async () => {
const res = await Promise.all([
resolveAfterTimeout(1),
resolveAfterTimeout(2),
resolveAfterTimeout(3),
resolveAfterTimeout(4),
resolveAfterTimeout(5),
resolveAfterTimeout(6)
]);
console.log({ res });
};
call();
by running the code it'll console "CALLED" for all six promises and when they are resolved it will console every 6 responses after timeout at the same time
I stumbled across this page while trying to solve a problem in NodeJS: reassembly of file chunks. Basically:
I have an array of filenames.
I need to append all those files, in the correct order, to create one large file.
I must do this asynchronously.
Node's 'fs' module does provide appendFileSync but I didn't want to block the server during this operation. I wanted to use the fs.promises module and find a way to chain this stuff together. The examples on this page didn't quite work for me because I actually needed two operations: fsPromises.read() to read in the file chunk, and fsPromises.appendFile() to concat to the destination file. Maybe if I was better with JavaScript I could have made the previous answers work for me. ;-)
I stumbled across this and I was able to hack together a working solution:
/**
* sequentially append a list of files into a specified destination file
*/
exports.append_files = function (destinationFile, arrayOfFilenames) {
return arrayOfFilenames.reduce((previousPromise, currentFile) => {
return previousPromise.then(() => {
return fsPromises.readFile(currentFile).then(fileContents => {
return fsPromises.appendFile(destinationFile, fileContents);
});
});
}, Promise.resolve());
};
And here's a jasmine unit test for it:
const fsPromises = require('fs').promises;
const fsUtils = require( ... );
const TEMPDIR = 'temp';
describe("test append_files", function() {
it('append_files should work', async function(done) {
try {
// setup: create some files
await fsPromises.mkdir(TEMPDIR);
await fsPromises.writeFile(path.join(TEMPDIR, '1'), 'one');
await fsPromises.writeFile(path.join(TEMPDIR, '2'), 'two');
await fsPromises.writeFile(path.join(TEMPDIR, '3'), 'three');
await fsPromises.writeFile(path.join(TEMPDIR, '4'), 'four');
await fsPromises.writeFile(path.join(TEMPDIR, '5'), 'five');
const filenameArray = [];
for (var i=1; i < 6; i++) {
filenameArray.push(path.join(TEMPDIR, i.toString()));
}
const DESTFILE = path.join(TEMPDIR, 'final');
await fsUtils.append_files(DESTFILE, filenameArray);
// confirm "final" file exists
const fsStat = await fsPromises.stat(DESTFILE);
expect(fsStat.isFile()).toBeTruthy();
// confirm content of the "final" file
const expectedContent = new Buffer('onetwothreefourfive', 'utf8');
var fileContents = await fsPromises.readFile(DESTFILE);
expect(fileContents).toEqual(expectedContent);
done();
}
catch (err) {
fail(err);
}
finally {
}
});
});
You can do it by for loop.
async function return promise:
async function createClient(client) {
return await Client.create(client);
}
let clients = [client1, client2, client3];
if you write following code then client are created parallelly:
const createdClientsArray = yield Promise.all(clients.map((client) =>
createClient(client);
));
But if you want to create client sequentially then you should use for loop:
const createdClientsArray = [];
for(let i = 0; i < clients.length; i++) {
const createdClient = yield createClient(clients[i]);
createdClientsArray.push(createdClient);
}
Bergi's answer helped me to make the call synchronous. I have added an example below where we call each function after the previous function is called:
function func1 (param1) {
console.log("function1 : " + param1);
}
function func2 () {
console.log("function2");
}
function func3 (param2, param3) {
console.log("function3 : " + param2 + ", " + param3);
}
function func4 (param4) {
console.log("function4 : " + param4);
}
param4 = "Kate";
//adding 3 functions to array
a=[
()=>func1("Hi"),
()=>func2(),
()=>func3("Lindsay",param4)
];
//adding 4th function
a.push(()=>func4("dad"));
//below does func1().then(func2).then(func3).then(func4)
a.reduce((p, fn) => p.then(fn), Promise.resolve());
I've been using for of in order to solve sequential promises. I'm not sure if it helps here but this is what I've been doing.
async function run() {
for (let val of arr) {
const res = await someQuery(val)
console.log(val)
}
}
run().then().catch()
Yes, you can chain an array of promise returning functions as follows
(this passes the result of each function to the next). You could of course edit it to pass the same argument (or no arguments) to each function.
function tester1(a) {
return new Promise(function(done) {
setTimeout(function() {
done(a + 1);
}, 1000);
})
}
function tester2(a) {
return new Promise(function(done) {
setTimeout(function() {
done(a * 5);
}, 1000);
})
}
function promise_chain(args, list, results) {
return new Promise(function(done, errs) {
var fn = list.shift();
if (results === undefined) results = [];
if (typeof fn === 'function') {
fn(args).then(function(result) {
results.push(result);
console.log(result);
promise_chain(result, list, results).then(done);
}, errs);
} else {
done(results);
}
});
}
promise_chain(0, [tester1, tester2, tester1, tester2, tester2]).then(console.log.bind(console), console.error.bind(console));
see this sample
Promise.all working parallel
const { range, random, forEach, delay} = require("lodash");
const run = id => {
console.log(`Start Task ${id}`);
let prom = new Promise((resolve, reject) => {
delay(() => {
console.log(`Finish Task ${id}`);
resolve(id);
}, random(2000, 15000));
});
return prom;
}
const exec = () => {
let proms = [];
forEach(range(1,10), (id,index) => {
proms.push(run(id));
});
let allPromis = Promise.all(proms);
allPromis.then(
res => {
forEach(res, v => console.log(v));
}
);
}
exec();