Develop Reference

Calling recursive function in loop with async/await and Promise.all

I have a use case where I'm trying to loop through an array of objects, where I need to make some GraphQL requests that may have some pagination for a given object in the array. I'm trying to speed up performance by pushing the recursive function to an array of promises, and then use Promse.all to resolve all of those.
I'm running into an issue though where I'm getting an undefined response from Promise.all - The end goal is to have the following response for each unique object in the array:
[{
account: test1,
id: 1,
high: 2039,
critical: 4059
},
{
account: test2,
id: 2,
high: 395,
critical: 203
}]
...where I'm only returning anAccount object after recursion is done paginating/making all requests for a given account object.
Here is the sample code:
const fetch = require('isomorphic-fetch');
const API_KEY = '<key>';
async function main() {
let promises = [];
let accounts = [{'name': 'test1', 'id': 1}, {'name': 'test2' , 'id': 2}];
for (const a of accounts) {
let cursor = null;
let anAccountsResults = [];
promises.push(getCounts(a, anAccountsResults, cursor));
}
let allResults = await Promise.all(promises);
console.log(allResults);
}
async function getCounts(acct, results, c) {
var q = ``;
if (c == null) {
q = `{
actor {
account(id: ${acct.id}) {
aiIssues {
issues(filter: {states: ACTIVATED}) {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`
} else {
q = `{
actor {
account(id: ${acct.id}) {
aiIssues {
issues(filter: {states: ACTIVATED}, cursor: "${c}") {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`
}
const resp = await fetch('https://my.api.com/graphql', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'API-Key': API_KEY
},
body: JSON.stringify({
query: q,
variables: ''}),
});
let json_resp = await resp.json();
let aSingleResult = json_resp.data.actor.account.aiIssues.issues.issues;
let nextCursor = json_resp.data.actor.account.aiIssues.issues.nextCursor;
console.log(nextCursor);
if (nextCursor == null) {
results = results.concat(aSingleResult);
} else {
results = results.concat(aSingleResult);
await getCounts(acct, results, nextCursor);
}
let criticalCount = results.filter(i => i.priority == 'CRITICAL').length;
let highCount = results.filter(i => i.priority == 'HIGH').length;
let anAccount = {
account: acct.name,
id: acct.id,
high: highCount,
critical: criticalCount
};
return anAccount;
}
main();
logging anAccount in function getCounts has the correct detail, but when returning it, logging the output of Promise.all(promises) yields undefined. Is there a better way to handle this in a way where I can still asynchronously run multiple recursive functions in parallel within the loop with Promise.all?

Your main problem appears to be that results = results.concat(aSingleResult); does not mutate the array you passed, but only reassigns the local variable results inside the function, so the anAccount only will use the aSingleResult from the current call.
Instead of collecting things into a results array that you pass an a parameter, better have every call return a new array. Then in the recursive await getCounts(acct, results, nextCursor) call, do not ignore the return value.
async function main() {
let promises = [];
const accounts = [{'name': 'test1', 'id': 1}, {'name': 'test2' , 'id': 2}];
const promises = accounts.map(async acct => {
const results = await getIssues(acct);
const criticalCount = results.filter(i => i.priority == 'CRITICAL').length;
const highCount = results.filter(i => i.priority == 'HIGH').length;
return {
account: acct.name,
id: acct.id,
high: highCount,
critical: criticalCount
};
});
const allResults = await Promise.all(promises);
console.log(allResults);
}
const query = `query ($accountId: ID!, $cursor: IssuesCursor) {
actor {
account(id: $accountId) {
aiIssues {
issues(filter: {states: ACTIVATED}, cursor: $cursor) {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`;
async function getIssues(acct, cursor) {
const resp = await fetch('https://my.api.com/graphql', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'API-Key': API_KEY
},
body: JSON.stringify({
query: q,
variables: {
accountId: acct.id,
cursor,
}
}),
});
if (!resp.ok) throw new Error(resp.statusText);
const { data, error } = await resp.json();
if (error) throw new Error('GraphQL error', {cause: error});
const { nextCursor, issues } = data.actor.account.aiIssues.issues;
if (nextCursor == null) {
return issues;
} else {
return issues.concat(await getIssues(acct, nextCursor));
}
}

What is the reason the Async function that I have below is returning Promise { <pending> } instead of returning the proper value?

When I console.log() paymentsResults and pendingResults from the code below, I can see the data being logged properly. However, right at the last moment when I am trying to return payments and pending I see the results as Promise { <pending> } in my backend. What is the reason and how can I fix the issue here?
const query = await pool.query(q);
const payments = query.rows.filter(o => getCategory(o) === "payment");
const pending = query.rows.filter(o => getCategory(o) === "pending");
const inactive = query.rows.filter(o => getCategory(o) === "inactive");
const paymentsResults = payments.map(async order => {
const A = ({
date: getDate(order),
item: await getItem(order),
total: getTotal(order),
purchaser: getPurchaser(order),
email: getEmail(order),
paymentMethod: getPaidUsing(order),
status: getStatus(order)
});
return A;
});
const pendingResults = pending.map(async order => {
const B = {
date: getDate(order),
item: await getItem(order),
total: getTotal(order),
purchaser: getPurchaser(order),
email: getEmail(order),
status: getStatus(order)
};
return B;
});
return {
payments: paymentsResults,
pending: pendingResults,
inactive: inactive
};
};

const paymentsResults = await Promise.all(payments.map(async order => {
const A = {
date: getDate(order),
item: await getItem(order),
total: getTotal(order),
purchaser: getPurchaser(order),
email: getEmail(order),
paymentMethod: getPaidUsing(order),
status: getStatus(order)
};
return A;
}));
const pendingResults = await Promise.all(pending.map(async order => {
const B = {
date: getDate(order),
item: await getItem(order),
total: getTotal(order),
purchaser: getPurchaser(order),
email: getEmail(order),
status: getStatus(order)
};
return B;
}));
array of Promise should be resolved by Promise.all before using.
async function always returns promise.

Asynchronous Callback to Array.map()

I've just started learning Node.js a few weeks ago.... I am not able to understand why the 'products' array contains null instead of the desired objects....
On Line 13, when I am console logging the object, I get the desired object but I don't understand why they are null when I console log them on Line 40 after the map function has completed it's execution....
If the array length is 2 (which should imply successful pushing) why are the objects stored inside still null instead of the objects that I wanted to store ?
Console Output
Order Schema
exports.getOrders = async (req, res, next) => {
const userOrders = [];
Order.find({ 'user.userId': req.user._id }).then((orders) => {
console.log(orders); // Line 4
async.eachSeries(
orders,
function (order, callback) {
const products = order.products.map((p) => {
const seriesId = p.product.seriesId;
const volumeId = p.product.volumeId;
Series.findById(seriesId).then((series) => {
const volume = series.volumes.id(volumeId);
console.log(volume, p.quantity); // Line 13
return {
seriesTitle: volume.parent().title,
volume: volume,
quantity: p.quantity
};
});
});
console.log('Product Array Length: ', products.length); // Line 21
if (products.length !== 0) {
const data = {
productData: products,
orderData: {
date: order.date,
totalPrice: order.totalPrice
}
};
userOrders.push(data);
callback(null);
} else {
callback('Failed');
}
},
function (err) {
if (err) {
console.log('Could not retrieve orders');
} else {
console.log(userOrders); // Line 40
res.render('shop/orders', {
docTitle: 'My Orders',
path: 'orders',
orders: userOrders,
user: req.user
});
}
}
);
});
};

In line 8, order.products.map returns an array of null. Because this is an asynchronous mapping. For each product, you are calling Series.findById which is a promise and it only returns the value when it resolves. As you are not waiting for the promise to resolve, so it returns null on each iteration.
You have to map all the promises first then call Promise.all to resolve them and after that, you will get the expected value.
exports.getOrders = async (req, res, next) => {
const userOrders = [];
Order.find({ 'user.userId': req.user._id }).then((orders) => {
console.log(orders); // Line 4
async.eachSeries(
orders,
function (order, callback) {
const productPromise = order.products.map((p) => {
const seriesId = p.product.seriesId;
const volumeId = p.product.volumeId;
//ANSWER: Return the following promise
return Series.findById(seriesId).then((series) => {
const volume = series.volumes.id(volumeId);
console.log(volume, p.quantity); // Line 13
return {
seriesTitle: volume.parent().title,
volume: volume,
quantity: p.quantity
};
});
});
// ANSWER: call all the promises
Promise.all(productPromise)
.then(function(products) {
console.log('Product Array Length: ', products.length);
if (products.length !== 0) {
const data = {
productData: products,
orderData: {
date: order.date,
totalPrice: order.totalPrice
}
};
userOrders.push(data);
callback(null);
} else {
callback('Failed');
}
});
},
function (err) {
if (err) {
console.log('Could not retrieve orders');
} else {
console.log(userOrders); // Line 40
res.render('shop/orders', {
docTitle: 'My Orders',
path: 'orders',
orders: userOrders,
user: req.user
});
}
}
);
});
};

Counter not increasing in async map function

I am working with mongodb and nodejs. I have an array of customers I have to create each inside database.
const promises2 = customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOne({ type: "Customer" });
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
await Counter.findOneAndUpdate({ type: "Customer" }, { $inc: { sequence_value: 1 } });
}
});
await Promise.all([...promises2]);
The issue is counter is not increasing every time. I am getting same counter in all the created customers. What is the issue here?
Issue is something like this but don't have an answer.

The problem is that all the calls overlap. Since the first thing they each do is get the current counter, they all get the same counter, then try to use it. Fundamentally, you don't want to do this:
const counter = await Counter.findOne({ type: "Customer" });
// ...
await Counter.findOneAndUpdate({ type: "Customer" }, { $inc: { sequence_value: 1 } });
...because it creates a race condition: overlapping asynchronous operations can both get the same sequence value and then both issue an update to it.
You want an atomic operation for incrementing and retrieving a new ID. I don't use MongoDB, but I think the findOneAndUpdate operation can do that for you if you add the returnNewDocument option. If so, the minimal change would be to swap over to using that:
const promises2 = customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOneAndUpdate(
{ type: "Customer" },
{ $inc: { sequence_value: 1 } },
{ returnNewDocument: true }
);
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
}
});
await Promise.all([...promises2]);
...but there's no reason to create an array and then immediately copy it, just use it directly:
await Promise.all(customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOneAndUpdate(
{ type: "Customer" },
{ $inc: { sequence_value: 1 } },
{ returnNewDocument: true }
);
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
}
}));
The overall operation will fail if anything fails, and only the first failure is reported back to your code (the other operations then continue and succeed or fail as the case may be). If you want to know everything that happened (which is probably useful in this case), you can use allSettled instead of all:
// Gets an array of {status, value/reason} objects
const results = await Promise.allSettled(customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOneAndUpdate(
{ type: "Customer" },
{ $inc: { sequence_value: 1 } },
{ returnNewDocument: true }
);
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
}
}));
const errors = results.filter(({status}) => status === "rejected").map(({reason}) => reason);
if (errors.length) {
// Handle/report errors here
}
Promise.allSettled is new in ES2021, but easily polyfilled if needed.
If I'm mistaken about the above use of findOneAndUpdate in some way, I'm sure MongoDB gives you a way to get those IDs without a race condition. But in the worst case, you can pre-allocate the IDs instead, something like this:
// Allocate IDs (in series)
const ids = [];
for (const customer of customers) {
if (!customer.customerId) {
const counter = await Counter.findOne({ type: "Customer" });
await Counter.findOneAndUpdate({ type: "Customer" }, { $inc: { sequence_value: 1 } });
ids.push(counter.sequence_value);
}
}
// Create customers (in parallel)
const results = await Promise.allSettled(customers.map(async(customer, index) => {
const customerId = ids[index];
try {
await Customer.create({
customerId
});
} catch (e) {
// Failed, remove the counter, but without allowing any error doing so to
// shadow the error we're already handling
try {
await Counter.someDeleteMethodHere(/*...customerId...*/);
} catch (e2) {
// ...perhaps report `e2` here, but don't shadow `e`
}
throw e;
}
});
// Get just the errors
const errors = results.filter(({status}) => status === "rejected").map(({reason}) => reason);
if (errors.length) {
// Handle/report errors here
}

Your map function is not returning a promise.
Try this :
const promises2 = [];
customers.map((customer) => {
return new Promise(async (resolve) => {
if (!customer.customerId) {
const counter = await Counter.findOne({ type: 'Customer' });
console.log({ counter });
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
await Counter.findOneAndUpdate({ type: 'Customer' }, { $inc: { sequence_value: 1 } });
}
resolve();
});
});
await Promise.all(promises2);

Async-await 'Promise {<pending>} with Array.prototype.map

I know there are many questions discuss about the same error and I saw most of them and they didn't fix my problem.
I wrote this code:
const userOrganizationGroups = (organizationGroupsList) => {
if (Array.isArray(organizationGroupsList) && organizationGroupsList.length) {
const result = organizationGroupsList.map(async (element) => {
const { organizationId, groupId } = element;
const { Organizations, Groups } = models;
const organization = await Organizations.findOne(
{ _id: organizationId },
{ name: 1, _id: 0 },
);
const group = await Groups.findOne({ _id: groupId });
return Object.assign({}, {
organizationName: organization.name,
group: group.name,
});
});
return result;
}
};
when I debug the code:
console.log('userOrganizationGroups : ',userOrganizationGroups(list))
I got such a result:
userOrganizationGroups: Promise { <pending> }
I found a similair question: Promise { } - Trying to await for .map and I used the solution mentioned in the question:
const userOrganizationGroups = async (organizationGroupsList) => {
if (Array.isArray(organizationGroupsList) && organizationGroupsList.length) {
const result = await Promise.all(organizationGroupsList.map(async (element) => {
const { organizationId, groupId } = element;
const { Organizations, Groups } = models;
const organization = await Organizations.findOne(
{ _id: organizationId },
{ name: 1, _id: 0 },
);
const group = await Groups.findOne({ _id: groupId });
return Object.assign({}, {
organizationName: organization.name,
group: group.name,
});
}));
return result;
}
How can I fix this issue?

instead of
console.log('userOrganizationGroups : ',userOrganizationGroups(list))
use
userOrganizationGroups(list).then( groups => console.log('userOrganizationGroups : ', groups)
or
(async () => {
const groups = await userOrganizationGroups(list);
console.log('userOrganizationGroups : ', groups);
})();

console.log() was called first because you didn't wait using await or then.
So You should write below instead of
console.log('userOrganizationGroups : ',userOrganizationGroups(list))
;(async () => {
const resultGroups = await userOrganizationGroups(list)
resultGroups.forEach(group => {
console.log(`group: ${JSON.stringfy(group, null, ' ')}`)
})
})()