I have an API call with a forEach loop which I need to be finished before another function is called. It looks like this:
var getTypes = function() {
var stations = [];
stationservice.getCount('/stations')
.then(succCB, errorCB);
function succCB(data) {
data.data.forEach(function(station) {
stations.push({
id: station._id,
})
})
};
// This should only be called once the forEach Loop is done
processStations(stations);
}
I can't find an understandable example of how I can make sure the processStations() gets called once the loop is done. How can I create a promise for this such that it does what I want to achieve?
As soon as you use promises you have to chain everything that depends on that promise (or use await and async if your environment supports it):
function getTypes() {
return stationservice.getCount('/stations')
.then(function(data) {
var stations = [];
data.data.forEach(function(station) {
stations.push({
id: station._id,
})
})
return stations;
})
.then(processStations);
}
And you should return the Promise chain from your getTypes at least if the getTypes should return something that depends on the stationservice.getCount.
Instead of the forEach you might want to use map because this is what you actually do:
function getTypes() {
return stationservice.getCount('/stations')
.then(function(data) {
return data.data.map(function(station) {
return {
id: station._id,
};
})
})
.then(processStations);
}
If you want a "modern code" answer
var getTypes = function() {
return stationservice.getCount('/stations')
.then(data => data.data.map(({_id: id}) =>({id})))
.then(processStations);
}
this is equal to
var getTypes = function getTypes() {
return stationservice.getCount('/stations').then(function (data) {
return data.data.map(function (_ref) {
return { id: _ref._id };
});
}).then(processStations);
};
Though, since the map isn't asynchronous at all
const getTypes = () => stationservice.getCount('/stations').then(data => processStations(data.data.map(({_id: id}) =>({id}))));
is just fine - and in pre modern browser
var getTypes = function getTypes() {
return stationservice.getCount('/stations').then(function (data) {
return processStations(data.data.map(function (_ref) {
return { id: _ref._id };
}));
});
};
Using Async library
async.forEach(data.data, function (item, callback){
stations.push({
id: item._id,
})
callback();
}, function(err) {
processStations(stations);
});
Related
I'm seeking guidance on how best to handle the following scenario. I'm fairly new to JS and async development, so I'm trying to understand the best way to handle this. I call one api (callAPI) which returns an array of items. I need to take those items and send them to another api (callAPI2) but that api doesn't have a bulk method, so I have to call the api for each item in the array. Below is how I have it structured: getArray promise returns the first array, I send the array to promise 2 (getIndividualData) where I loop and call the second api. I'm wondering if there are better ways to structure this? If I pass in a large array, I may need to pace the calls to the api so I don't get throttled... so maybe I need some version of Promise.all?
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = function(arrayOfItems) {
return new Promise(function(resolve,reject) {
var responseArray = [];
for(var i = 0; i < arrayOfItems.length; i++) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
responseArray.push(result);
});
}
resolve(responseArray);
});
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response);
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);
You can make a request for each item in a large array without getting throttled by implementing a concurrency throttler with a Set of Promises and async/await syntax. I've duplicated your code below, modifying the implementation of getIndividualData and passing in concurrency as an option.
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = async function(arrayOfItems, { concurrency }) {
var promiseSet = new Set(),
responseArray = [],
i = 0;
while (i < arrayOfItems.length) {
if (promiseSet.size >= concurrency) {
await Promise.race(promiseSet)
}
const promise = new Promise(function(resolve,reject) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
resolve(result)
})
})
responseArray.push(promise.then(result => {
promiseSet.delete(promise)
return result
}))
i += 1
}
return Promise.all(responseArray)
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response, { concurrency: 10 });
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);
Reformulating this to a helper that promisifies an object that has an .on('success') event handler and an async function for the top-level .then()ish code gives us something like this...
To pace the API calls, add in p-limit or similar to getIndividualData.
function promisifyOnSuccessObj(onSuccessObj) {
return new Promise((resolve) => {
onSuccessObj.on("success", (result, response) => resolve(result));
// TODO: what about `response`?
// TODO: onSuccessObj.on('error')..?
});
}
function getIndividualData(arrayOfItems) {
// Returns an array of promises
return arrayOfItems.map((item) =>
promisifyOnSuccessObj(callApi2.get(item)),
);
}
async function doThings() {
const result = await promisifyOnSuccessObj(callApi.get());
const individualDatas = await Promise.all(getIndividualData(result));
console.log(`The final array is ${JSON.stringify(individualDatas)}`);
}
You could combine Promise.all, map and async...await syntax and in the end get one array of resolved individual promises based on the previous resolved array promise.
const mockApi = {
request: (response) => {
return new Promise((resolve, reject) => {
setTimeout(() => resolve(response), 1000)
})
},
getArray() {
return this.request(['foo', 'bar', 'baz'])
},
getItem(item) {
return this.request(`Resolved: ${item}`)
}
}
async function getData() {
const array = await mockApi.getArray();
const final = await Promise.all(array.map(e => mockApi.getItem(e)));
console.log(final)
}
getData()
Sorry if my title is not very explicit I dont know how to explain this properly.
I am trying to use the distinct function for my app that uses loopback 3 and mongodb. It seems to work right but my endpoint wont hit the return inside my function.
This is my code
const distinctUsers = await sellerCollection.distinct('userId',{
hostId : host.id,
eventId:{
"$ne" : eventId
}
}, async function (err, userIds) {;
if(!userIds || userIds.length ==0)
return [];
const filter = {
where:{
id: {
inq: userIds
}
}
};
console.log("should be last")
return await BPUser.find(filter);
});
console.log(distinctUsers);
console.log("wtf??");
//return [];
If I uncomment the return [] it will send the return and later it will show the should be last, so even when I dont have the return it seems to finish. It is now waiting for the response. I dont like the way my code looks so any pointer of how to make this look better I will take it.
It looks like the sellerCollection.distinct takes a callback as one of it's parameters, therefore, you cannot use async/await with a callback-style function, since it's not a promise.
I would suggest turning this call into a promise if you'd like to use async/await:
function findDistinct(hostId, eventId) {
return new Promise((resolve, reject) => {
sellerCollection.distinct(
'userId',
{ hostId, eventId: { "$ne": eventId } },
function (error, userIds) {
if (error) {
reject(error);
return;
}
if (!userIds || userIds.length === 0) {
resolve([]);
return;
}
resolve(userIds);
}
)
})
}
Then, you can use this new function with async/await like such:
async function getDistinctUsers() {
try {
const hostId = ...
const eventId = ...
const distinctUsers = await findDistinct(hostId, eventId)
if (distinctUsers.length === 0) {
return
}
const filter = {
where: {
id: { inq: userIds }
}
}
const bpUsers = await BPUser.find(filter) // assuming it's a promise
console.log(bpUsers)
} catch (error) {
// handle error
}
}
When I console.log this.state.events and this.state.meets the terminal logs that these two states are empty arrays. This suggests that this function executes before function 2 and 3 finish and that I am doing my promises wrong. I can't seem to figure out my issue though and I am very confused. I know that the setState is working as well since once the page renders I have a button whose function console.logs those states and it executes properly.
I have tried formatting my promises in various ways but I have no idea how they resolve before my functions finish.
obtainEventsAndMeetsRefs = () => {
return Promise.resolve(
this.userRef.get().then(
doc => {
this.setState(
{
eventRefs: doc.data().Events,
meetingsRef: doc.data().Meets
}
)
}
)
)
}
obtainEvents() {
var that = this
return new Promise(function (resolve, reject) {
for (i = 0; i < that.state.eventRefs.length; i++) {
docRef = that.state.eventRefs[i]._documentPath._parts[1];
firebase.firestore().collection('All Events').doc(docRef).get().then(
doc => {
that.setState({
events: update(that.state.events, {
$push: [
{
eventName: doc.data().eventName,
eventDescription: doc.data().eventDescription,
startDate: doc.data().startDate,
endDate: doc.data().endDate,
location: doc.data().location,
privacy: doc.data().privacy,
status: doc.data().status,
attending: doc.data().attending
}
]
})
})
}
)
if (i == that.state.eventRefs.length - 1) {
console.log('1')
resolve(true)
}
}
})
}
obtainMeetings() {
var that = this
return new Promise(function (resolve, reject) {
for (i = 0; i < that.state.meetingsRef.length; i++) {
userRef = that.state.meetingsRef[i]._documentPath._parts[1];
meetRef = that.state.meetingsRef[i]._documentPath._parts[3];
ref = firebase.firestore().collection('Users').doc(userRef)
.collection('Meets').doc(meetRef)
ref.get().then(
doc => {
that.setState({
meets: update(that.state.meets, {
$push: [
{
headline: doc.data().headline,
agenda: doc.data().agenda,
startTime: doc.data().startTime,
endTime: doc.data().endTime,
date: doc.data().date,
name: doc.data().name
}
]
})
})
}
)
if (i == that.state.meetingsRef.length - 1) {
console.log('2')
resolve(true)
}
}
})
}
orderByDate = () => {
console.log(this.state.events)
console.log(this.state.meets)
}
componentDidMount() {
this.obtainEventsAndMeetsRefs().then(
() => this.obtainEvents().then(
() => this.obtainMeetings().then(
() => this.orderByDate()
)
)
)
}
I expected the output to be an array filled with data that I inputted using setState, not an empty array.
The crux of the issue is that obtainMeetings() and obtainEvents() don't return promises that resolve when their asynchronous operations are done. Thus, you can't know when their work is done and thus you can't sequence them properly.
There are all sorts of things wrong in this code. To start with, you don't need to wrap existing promises in another promise (the promise construction anti-pattern) because that leads to errors (which you are making). Instead, you can just return the existing promise.
For example, change this:
obtainEventsAndMeetsRefs = () => {
return Promise.resolve(
this.userRef.get().then(
doc => {
this.setState(
{
eventRefs: doc.data().Events,
meetingsRef: doc.data().Meets
}
)
}
)
)
}
to this:
obtainEventsAndMeetsRefs = () => {
return this.userRef.get().then(doc => {
this.setState({
eventRefs: doc.data().Events,
meetingsRef: doc.data().Meets
});
});
}
Then, chain rather than nest and return a promise that is linked to the completion/error of the operations.
So change this:
componentDidMount() {
this.obtainEventsAndMeetsRefs().then(
() => this.obtainEvents().then(
() => this.obtainMeetings().then(
() => this.orderByDate()
)
)
)
}
to this:
componentDidMount() {
return this.obtainEventsAndMeetsRefs()
.then(this.obtainEvents.bind(this))
.then(this.obtainMeetings.bind(this))
.then(this.orderByDate.bind(this))
}
or if you prefer:
componentDidMount() {
return this.obtainEventsAndMeetsRefs()
.then(() => this.obtainEvents())
.then(() => this.obtainMeetings())
.then(() => this.orderByDate())
}
These chain instead of nest and they return a promise that tracks the whole chain so that caller knows when the chain completed and/or if there was an error.
Then, if you want your for loop to execute serially, you can make it async and use await on the promise inside the loop like this:
async obtainMeetings() {
for (let i = 0; i < this.state.meetingsRef.length; i++) {
let userRef = this.state.meetingsRef[i]._documentPath._parts[1];
let meetRef = this.state.meetingsRef[i]._documentPath._parts[3];
let ref = firebase.firestore().collection('Users').doc(userRef)
.collection('Meets').doc(meetRef)
// make the for loop wait for this operation to complete
await ref.get().then(doc => {
this.setState({
meets: update(this.state.meets, {
$push: [{
headline: doc.data().headline,
agenda: doc.data().agenda,
startTime: doc.data().startTime,
endTime: doc.data().endTime,
date: doc.data().date,
name: doc.data().name
}]
});
});
});
// this return here isn't making a whole lot of sense
// as your for loop already stops things when this index reaches
// the end value
if (i == this.state.meetingsRef.length - 1) {
console.log('2')
return true;
}
}
// it seems like you probably want to have a return value here
return true;
}
This also looks like there's a problem with the missing declarations of i, userRef, meetRef and ref in your implementation. Those should be locally declared variables.
obtainEvents() needs the same redesign.
If you can't use async/await, then you can transpile or you'd have to design the loop to work differently (more of a manual asynchronous iteration).
If you can run all the async operations in the loop in parallel and just want to know when they are all done, you can do something like this:
obtainMeetings() {
return Promise.all(this.state.meetingsRef.map(item => {
let userRef = item._documentPath._parts[1];
let meetRef = item._documentPath._parts[3];
let ref = firebase.firestore().collection('Users').doc(userRef)
.collection('Meets').doc(meetRef)
// make the for loop wait for this operation to complete
return ref.get().then(doc => {
this.setState({
meets: update(this.state.meets, {
$push: [{
headline: doc.data().headline,
agenda: doc.data().agenda,
startTime: doc.data().startTime,
endTime: doc.data().endTime,
date: doc.data().date,
name: doc.data().name
}]
});
});
});
})).then(allResults => {
// process all results, decide what the resolved value of the promise should be here
return true;
});
}
I have multiple loop in transaction chain code, but after loop done transaction immediately commit the chain and not continue to next query. my code is something like this:
return sm.sequelize.transaction(function (t) {
return Room.create({
room_type: req.body.room_type
},{transaction: t}).then(function(roomtype){
for (i=0;i<room.length;i++){
return Roomtype.create({
name : req.body.roomtype[i]
},{transaction: t}); //my transaction only work untill here and commit after that
}
for (i=0;i<another.length;i++){ //I need to continue my query to second loop here
return User.create({
email :req.body.email[i]
},{transaction:t});
}
}).then(function(result){
}).catch (function(err){
})
})
so how to make the transaction after loop go to next loop?
Remember that as soon as you use a return statement, the function is over. No more iterations of your for loop will run, and neither will the second loop. You need to do this without returning in the middle of your loops.
As long as the transaction order isn't important, the best way to do this is to use Promise.all, which takes an array of promises, and returns the results of all of them once they are all done. Keeping the rest of the logic you have above, it would look like this:
return Room.create({
room_type: req.body.room_type
}, { transaction: t })
.then(function (roomtype) {
const promises = []
for (let i = 0; i < room.length; i++) {
const promise = Roomtype.create({
name : req.body.roomtype[i]
}, { transaction: t })
promises.push(promise)
}
for (let i = 0; i < another.length; i++){
const promise = User.create({
email :req.body.email[i]
}, { transaction: t })
promises.push(promise)
}
return Promise.all(promises)
})
.then(function (results) {
. . .
})
I would go a step further though and replace the for loops with map. This is not strictly necessary, but will clean up your code a lot:
return Room.create({ room_type: req.body.room_type }, { transaction: t })
.then(function (roomtype) {
return Promise.all(req.body.roomtype.map(function (type) {
return Roomtype.create({ name: type }, { transaction: t })
}))
})
.then(function (roomResults) {
// Do the same thing for creating users here
})
Alternatively, if order is important, you are probably going to have to do something like a recursive function that will go through each Promise in order.
const createRooms = function (transaction, types, index = 0) {
return Roomtype.create({ name: types[index] }, { transaction })
.then(function (result) {
if (index < types.length - 1) {
return createRooms(transaction, types, index + 1)
}
return result
})
}
You would then call that function in your code:
return Room.create({ room_type: req.body.room_type }, { transaction: t })
.then(function (roomtype) {
return createRooms(t, req.body.roomtype)
})
.then(function (result) {
// Do the same thing for creating Users here
})
In an attempt to grasp Q.js, I'd like to convert the following code using async.series in Q.js. Basically I create a folder if it doesn't exist (using mkdirp), move a file into a backup folder and save a file into a main folder.
var async = require('async');
var fs = require('fs');
var path = require('path');
var sessiondId = new Date().getTime() % 2 == 0 ? new Date().getTime().toString() : '_1234';
var backupFolder = path.join(__dirname,sessiondId);
var backupFullPath = path.join(backupFolder,'a.txt');
var fullPath = path.join(__dirname,'main','a.txt');
var mkdirp = require('mkdirp');
async.series({
createOrSkip: function(callback) {
mkdirp(backupFolder, function (err, dir) {
if(err) {
callback(err, null);
} else {
callback(null, {created: !!dir, folderAt: backupFolder});
}
});
},
move: function(callback) {
fs.rename(fullPath, backupFullPath, function(err) {
if(err) {
callback(err, null);
} else {
callback(null, {backupAt: backupFullPath});
}
});
},
write: function(callback) {
fs.writeFile(fullPath, 'abc', function(err) {
if (err) {
callback(err, null);
} else {
callback(null, {saveAt: fullPath});
}
});
}
}, function(err, result) {
console.log(result);
});
Actually I don't know where to start. Thanks for your help.
R.
The key is to convert the node.js functions to return promises using Q.denodeify before you start, this means the header of your file should look like:
var Q = require('q')
var fs = require('fs');
var path = require('path');
var sessiondId = new Date().getTime() % 2 == 0 ? new Date().getTime().toString() : '_1234';
var backupFolder = path.join(__dirname,sessiondId);
var backupFullPath = path.join(backupFolder,'a.txt');
var fullPath = path.join(__dirname,'main','a.txt');
var mkdirp = Q.denodeify(require('mkdirp'));
var rename = Q.denodeify(fs.rename);
var writeFile = Q.denodeify(fs.writeFile);
That change wouldn't be needed if node.js natively supported promises.
Option 1
// createOrSkip
mkdirp(backupFolder)
.then(function (dir) {
// move
return rename(fullPath, backupFullPath);
})
.then(function () {
// write
return writeFile(fullPath, 'abc');
})
.done(function () {
console.log('operation complete')
});
I don't think it gets much simpler than that. Like #Bergi said though, it's more similar to "waterfall". If you want the exact behavior of series (but with promises) you'll have to use something like Option 2 or Option 3.
Option 2
You could write out the code manually to save the results. I usually find that, although this requires a little extra writing, it's by far the easiest to read:
var result = {}
mkdirp(backupFolder)
.then(function (dir) {
result.createOrSkip = {created: !!dir, folderAt: backupFolder};
return rename(fullPath, backupFullPath);
})
.then(function () {
result.move = {backupAt: backupFullPath};
return writeFile(fullPath, 'abc');
})
.then(function () {
result.write = {saveAt: fullPath};
return result;
})
.done(function (result) {
console.log(result);
});
Option 3
If you find yourself using this sort of code all the time, you could write a very simple series helper (I've never found the need to do this personally):
function promiseSeries(series) {
var ready = Q(null);
var result = {};
Object.keys(series)
.forEach(function (key) {
ready = ready.then(function () {
return series[key]();
}).then(function (res) {
result[key] = res;
});
});
return ready.then(function () {
return result;
});
}
promiseSeries({
createOrSkip: function () {
return mkdirp(backupFolder).then(function (dir) {
return {created: !!dir, folderAt: backupFolder};
});
},
move: function () {
return rename(fullPath, backupFullPath)
.thenResolve({backupAt: backupFullPath});
},
write: function () {
return writeFile(fullPath, 'abc')
.thenResolve({saveAt: fullPath});
}
}).done(function (result) {
console.log(result);
});
I'd say once you've written the helper, the code is a lot clearer for promises than with all the error handling cruft required to work with callbacks. I'd say it's clearer still when you either write it by hand or don't keep track of all those intermediate results.
Summing Up
You may or may not think these examples are clearer than the async.series version. Consider how well you might know that function though. It's actually doing something pretty complex in a very opaque manner. I initially assumed that only the last result would be returned (ala waterfall) and had to look it up in the documentation of Async. I almost never have to look something up int the documentation of a Promise library.
Make each of your functions return a promise. Construct them with a Deferred:
function createOrSkip(folder) {
var deferred = Q.defer();
mkdirp(folder, function (err, dir) {
if(err) {
deferred.reject(err);
} else {
deferred.resolve({created: !!dir, folderAt: backupFolder});
}
});
return deferred.promise;
}
However, there are helper functions for node-style callbacks so that you don't need to check for the err yourself everytime. With Q.nfcall it becomes
function createOrSkip(folder) {
return Q.nfcall(mkdirp, folder).then(function transform(dir) {
return {created: !!dir, folderAt: backupFolder};
});
}
The transform function will map the result (dir) to the object you expect.
If you have done this for all your functions, you can chain them with then:
createOrSkip(backupfolder).then(function(createResult) {
return move(fullPath, backupFullPath);
}).then(function(moveResult) {
return write(fullPath, 'abc');
}).then(function(writeResult) {
console.log("I'm done");
}, function(err) {
console.error("Something has failed:", err);
});
Notice that this works like async's waterfall, not series, i.e. the intermediate results will be lost. To achieve that, you would need to nest them:
createOrSkip(backupfolder).then(function(createResult) {
return move(fullPath, backupFullPath).then(function(moveResult) {
return write(fullPath, 'abc');.then(function(writeResult) {
return {
createOrSkip: createResult,
move: moveResult,
write: writeResult
};
});
});
}).then(function(res){
console.log(res);
}, function(err) {
console.error("Something has failed:", err);
});