Develop Reference

Processing array of data with promise

I'm seeking guidance on how best to handle the following scenario. I'm fairly new to JS and async development, so I'm trying to understand the best way to handle this. I call one api (callAPI) which returns an array of items. I need to take those items and send them to another api (callAPI2) but that api doesn't have a bulk method, so I have to call the api for each item in the array. Below is how I have it structured: getArray promise returns the first array, I send the array to promise 2 (getIndividualData) where I loop and call the second api. I'm wondering if there are better ways to structure this? If I pass in a large array, I may need to pace the calls to the api so I don't get throttled... so maybe I need some version of Promise.all?
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = function(arrayOfItems) {
return new Promise(function(resolve,reject) {
var responseArray = [];
for(var i = 0; i < arrayOfItems.length; i++) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
responseArray.push(result);
});
}
resolve(responseArray);
});
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response);
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);

You can make a request for each item in a large array without getting throttled by implementing a concurrency throttler with a Set of Promises and async/await syntax. I've duplicated your code below, modifying the implementation of getIndividualData and passing in concurrency as an option.
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = async function(arrayOfItems, { concurrency }) {
var promiseSet = new Set(),
responseArray = [],
i = 0;
while (i < arrayOfItems.length) {
if (promiseSet.size >= concurrency) {
await Promise.race(promiseSet)
}
const promise = new Promise(function(resolve,reject) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
resolve(result)
})
})
responseArray.push(promise.then(result => {
promiseSet.delete(promise)
return result
}))
i += 1
}
return Promise.all(responseArray)
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response, { concurrency: 10 });
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);

Reformulating this to a helper that promisifies an object that has an .on('success') event handler and an async function for the top-level .then()ish code gives us something like this...
To pace the API calls, add in p-limit or similar to getIndividualData.
function promisifyOnSuccessObj(onSuccessObj) {
return new Promise((resolve) => {
onSuccessObj.on("success", (result, response) => resolve(result));
// TODO: what about `response`?
// TODO: onSuccessObj.on('error')..?
});
}
function getIndividualData(arrayOfItems) {
// Returns an array of promises
return arrayOfItems.map((item) =>
promisifyOnSuccessObj(callApi2.get(item)),
);
}
async function doThings() {
const result = await promisifyOnSuccessObj(callApi.get());
const individualDatas = await Promise.all(getIndividualData(result));
console.log(`The final array is ${JSON.stringify(individualDatas)}`);
}

You could combine Promise.all, map and async...await syntax and in the end get one array of resolved individual promises based on the previous resolved array promise.
const mockApi = {
request: (response) => {
return new Promise((resolve, reject) => {
setTimeout(() => resolve(response), 1000)
})
},
getArray() {
return this.request(['foo', 'bar', 'baz'])
},
getItem(item) {
return this.request(`Resolved: ${item}`)
}
}
async function getData() {
const array = await mockApi.getArray();
const final = await Promise.all(array.map(e => mockApi.getItem(e)));
console.log(final)
}
getData()

How to alter result of callback/promise function?

I have a function that can return result in both callback and promise:
function foo(args, cb) {
// do stuff
const promise = getSomePromise();
if (cb) {
promise.then((result) => {
cb(null, result);
}, (err) => {
cb(err);
});
} else {
return promise;
}
}
I want to alter the result of promise before returning it. How to do this in a way that introduces the least amount of spaghetti code?

Add a function to alter and return modified result
function alterResult(result){
const alteredResult = /// do something to result
return alteredResult;
}
Then add it in a then()to:
const promise = getSomePromise().then(alterResult);

You can write a function which returns something and pass it to the callback when initiating it like this
function foo(cb) {
const promise = getSomePromise(); // return some promise
promise.then((result) => { // result is here just "Hello"
cb(doSomething(result)); // return the altered value here to the callback
})
}
function getSomePromise() {
return new Promise((resolve, _) => resolve("Hello")) // make a new promise
}
function doSomething(res) { // define a function that alters your result
return res + ", World"
}
foo((val) => console.log(val)) // log it

Who can explain nodejs promises in a loop to a PHP programmer

I have searched high and low but can't get my head around promises. What I do understand is how to define one promise and use its result by using .then.
What I do not understand is how I can create a loop to query the database for different blocks of records. This is needed due to a limit set on the number of records to query.
The predefined promise api call is used like this:
let getRecords = (name) => {
return new Promise((resolve,reject) => {
xyz.api.getRecords(name, 1000, 1000, function(err, result){
// result gets processed here.
resolve(//somevariables here);
});
)};
going with what I am used to, I tried:
for (let i=1; i<90000; i+=500) {
xyz.api.getRecords('james', i, 500, function(err, result){
// result gets processed here.
});
}
But then I can't access the information (could be my wrong doing)
Also tried something like this:
function getRecords(name,i){
xyz.api.getRecords(name, i, 500, function(err, result){
// result gets processed here.
});
};
for (let i=1; i<90000; i+=500) {
var someThing = getRecords('james', i);
}
All tutorials only seem to use one query, and process the data.
How do I call the api function multiple times with different arguments, collect the data and process it once everything is retrieved?
Only thing I can think of is, to write info to a file, terrible thought.

Using async/await
(async () => {
function getRecords(name,i){
// create new Promise so you can await for it later
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, function(err, result){
if(err) {
return reject(err);
}
resolve(result);
});
});
}
for (let i = 1; i < 90000; i += 500) {
// wait for the result in every loop iteration
const someThing = await getRecords('james', i);
}
})();
To handle errors you need to use try/catch block
try {
const someThing = await getRecords('james', i);
} catch(e) {
// handle somehow
}
Using only Promises
function getRecords(name, i) {
// create Promise so you can use Promise.all
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, function (err, result) {
if (err) {
return reject(err);
}
resolve(result);
});
});
}
const results = [];
for (let i = 1; i < 90000; i += 500) {
// push Promise's to array without waiting for results
results.push(getRecords("james", i));
}
// wait for all pending Promise's
Promise.all(results).then((results) => {
console.log(results);
});
let count = 0;
function getRecords(name, i) {
return new Promise((resolve, reject) => {
setTimeout(() => {
// example results
resolve((new Array(10)).fill(0).map(() => ++count));
}, 100);
});
}
const results = [];
for (let i = 1; i < 9000; i += 500) {
results.push(getRecords("james", i));
}
Promise.all(results).then((results) => {
console.log("Results:", results);
console.log("Combined results:",[].concat(...results));
});
To handle errors you need to use .catch() block
Promise.all(results).then((results) => { ... }).catch((error) => {
// handle somehow
});

By returning a promise and calling your asynchronous function inside, you can resolve the result and then use it this way:
function getRecords (name, i) {
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
for (let i = 1; i < 90000; i * 500) {
getRecords('james', i)
.then(result => {
// do stuff with result
})
}
Or, using async / await syntax:
async function getRecords (name, i) {
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
// this needs to happen inside a function, node does not allow top level `await`
for (let i = 1; i < 90000; i *= 500) {
const result = await getRecords('james', i);
// do stuff
}
Get all of your records at once
const requests = [];
for (let i = 1; i < 90000; i * 500) {
requests.push(getRecords('james', i));
}
const results = await Promise.all(requests);

One Promise calling an unknown (dynamic) number of async methods in sequence [duplicate]

This question already has answers here:
ES6 promises with timeout interval
(6 answers)
Closed 5 years ago.
I want to create one Promise that, internally, calls a series of asynchronous methods in sequence and returns the results of these methods, concatenated in an ordered Array, once the last method returns.
I was trying this:
const getArray = function (thisArray) {
return new Promise ( function (resolve, reject) {
if (thisArray.length < 3) {
setTimeout(function() {
console.log('adding element');
thisArray.push(thisArray.length);
getArray(thisArray);
}, 1000);
} else {
console.log(thisArray);
console.log('resolving');
resolve(thisArray);
}
});
}
getArray([]).then(function(data) {
console.log('thened');
console.log(data);
})
but it doesn't ever calls the then(). How can I do this?

You're not far off at all, you just need to be sure to call resolve for each new Promise. If you're still building the array, you pass the resulting promise from your recursive getArray call; otherwise, you pass the array:
const getArray = function(thisArray) {
return new Promise((resolve, reject) => {
if (thisArray.length >= 3) {
// All done
resolve(thisArray);
} else {
// Do something async...
setTimeout(() => {
// ...add the result
console.log('adding element');
thisArray.push(thisArray.length);
// ...and recurse
resolve(getArray(thisArray));
}, 1000);
}
});
}
getArray([]).then(data => {
console.log('done');
console.log(data);
});
That's a bit cleaner if you separate the function doing the async work from getArray, and ensure that the function doing the async work returns a promise. For our purposes, we can just wrap setTimeout in a Promise wrapper:
const setTimeoutP = (cb, delay) => {
return new Promise(resolve => {
setTimeout(() => {
resolve(cb());
}, delay);
});
};
then:
const getArray = thisArray => {
if (thisArray.length >= 3) {
return Promise.resolve(thisArray);
}
return setTimeoutP(() => {
console.log("adding element");
thisArray.push(thisArray.length);
});
};
Live copy:
const setTimeoutP = (cb, delay) => {
return new Promise(resolve => {
setTimeout(() => {
resolve(cb());
}, delay);
});
};
const getArray = thisArray => {
if (thisArray.length >= 3) {
return Promise.resolve(thisArray);
}
return setTimeoutP(() => {
console.log("adding element");
thisArray.push(thisArray.length);
return getArray(thisArray);
}, 1000);
};
getArray([]).then(data => {
console.log('done');
console.log(data);
});

Iterate array and wait for promises

How can I iterate through an array of data using Promises and returning data? I have seen some promises.push(asyncFunc) methods but some of the entries from my array will fail so from what I gather I can't use that.
var filesFromDisk = [
'41679_4_2015-09-06_17-02-12.mp4',
'41679_4_2015-09-06_17-02-12.smil',
'41680_4_2015-09-09_10-44-05.mp4'
];
start(filesFromDisk)
.then((data) => {
console.log(data); // Want my data here
});
I start start(dbFiles) from another file which is why I want the data returned there.
function start(dbFiles) {
var listOfFiles = [],
promises = [];
return new Promise((fulfill, reject) => {
for (var i = 0; i < dbFiles.length; i++) {
getMp4(dbFiles[i])
.then((data) => {
listOfFiles = listOfFiles.concat(data);
console.log(listOfFiles);
})
}
fulfill(listOfFiles) // Need to happen AFTER for loop has filled listOfFiles
});
}
So for every entry in my array I want to check if the file with the new extension exists and read that file. If the file with extension does not exist I fulfill the original file. My Promise.all chain works and all the data is returned in for loop above (getMp4(dbFiles[i]))
function getMp4(filename) {
var mp4Files = [];
var smil = privateMethods.setSmileExt(localData.devPath + filename.toString());
return new Promise((fulfill, reject) => {
Promise.all([
privateMethods.fileExists(smil),
privateMethods.readTest(smil)
]).then(() => {
readFile(filename).then((files) => {
fulfill(files)
});
}).catch((err) => {
if (!err.exists) fulfill([filename]);
});
});
}
function readFile(filename){
var filesFromSmil = [];
return new Promise((fulfill, reject) => {
fs.readFile(localData.devPath + filename, function (err, res){
if (err) {
reject(err);
}
else {
xmlParser(res.toString(), {trim: true}, (err, result) => {
var entry = JSON.parse(JSON.stringify(result.smil.body[0].switch[0].video));
for (var i = 0; i < entry.length; i++) {
filesFromSmil.push(privateMethods.getFileName(entry[i].$.src))
}
});
fulfill(filesFromSmil);
}
});
});
};
Methods in the Promise.all chain in getMp4 - have no problems with these that I know.
var privateMethods = {
getFileName: (str) => {
var rx = /[a-zA-Z-1\--9-_]*.mp4/g;
var file = rx.exec(str);
return file[0];
},
setSmileExt: (videoFile) => {
return videoFile.split('.').shift() + '.smil';
},
fileExists: (file) => {
return new Promise((fulfill, reject) => {
try {
fs.accessSync(file);
fulfill({exists: true})
} catch (ex) {
reject({exists: false})
}
})
},
readTest: (file) => {
return new Promise((fulfill, reject) => {
fs.readFile(file, (err, res) => {
if (err) reject(err);
else fulfill(res.toString());
})
})
}
}

If you need them to run in parallel, Promise.all is what you want:
function start(dbFiles) {
return Promise.all(dbFiles.map(getMp4));
}
That starts the getMp4 operation for all of the files and waits until they all complete, then resolves with an array of the results. (getMp4 will receive multiple arguments — the value, its index, and a a reference to the dbFiles arary — but since it only uses the first, that's fine.)
Usage:
start(filesFromDisk).then(function(results) {
// `results` is an array of the results, in order
});
Just for completeness, if you needed them to run sequentially, you could use the reduce pattern:
function start(dbFiles) {
return dbFiles.reduce(function(p, file) {
return p.then(function(results) {
return getMp4(file).then(function(data) {
results.push(data);
return results;
});
});
}, Promise.resolve([]));
}
Same usage. Note how we start with a promise resolved with [], then queue up a bunch of then handlers, each of which receives the array, does the getMp4 call, and when it gets the result pushes the result on the array and returns it; the final resolution value is the filled array.