I have a use case where I'm trying to loop through an array of objects, where I need to make some GraphQL requests that may have some pagination for a given object in the array. I'm trying to speed up performance by pushing the recursive function to an array of promises, and then use Promse.all to resolve all of those.
I'm running into an issue though where I'm getting an undefined response from Promise.all - The end goal is to have the following response for each unique object in the array:
[{
account: test1,
id: 1,
high: 2039,
critical: 4059
},
{
account: test2,
id: 2,
high: 395,
critical: 203
}]
...where I'm only returning anAccount object after recursion is done paginating/making all requests for a given account object.
Here is the sample code:
const fetch = require('isomorphic-fetch');
const API_KEY = '<key>';
async function main() {
let promises = [];
let accounts = [{'name': 'test1', 'id': 1}, {'name': 'test2' , 'id': 2}];
for (const a of accounts) {
let cursor = null;
let anAccountsResults = [];
promises.push(getCounts(a, anAccountsResults, cursor));
}
let allResults = await Promise.all(promises);
console.log(allResults);
}
async function getCounts(acct, results, c) {
var q = ``;
if (c == null) {
q = `{
actor {
account(id: ${acct.id}) {
aiIssues {
issues(filter: {states: ACTIVATED}) {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`
} else {
q = `{
actor {
account(id: ${acct.id}) {
aiIssues {
issues(filter: {states: ACTIVATED}, cursor: "${c}") {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`
}
const resp = await fetch('https://my.api.com/graphql', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'API-Key': API_KEY
},
body: JSON.stringify({
query: q,
variables: ''}),
});
let json_resp = await resp.json();
let aSingleResult = json_resp.data.actor.account.aiIssues.issues.issues;
let nextCursor = json_resp.data.actor.account.aiIssues.issues.nextCursor;
console.log(nextCursor);
if (nextCursor == null) {
results = results.concat(aSingleResult);
} else {
results = results.concat(aSingleResult);
await getCounts(acct, results, nextCursor);
}
let criticalCount = results.filter(i => i.priority == 'CRITICAL').length;
let highCount = results.filter(i => i.priority == 'HIGH').length;
let anAccount = {
account: acct.name,
id: acct.id,
high: highCount,
critical: criticalCount
};
return anAccount;
}
main();
logging anAccount in function getCounts has the correct detail, but when returning it, logging the output of Promise.all(promises) yields undefined. Is there a better way to handle this in a way where I can still asynchronously run multiple recursive functions in parallel within the loop with Promise.all?
Your main problem appears to be that results = results.concat(aSingleResult); does not mutate the array you passed, but only reassigns the local variable results inside the function, so the anAccount only will use the aSingleResult from the current call.
Instead of collecting things into a results array that you pass an a parameter, better have every call return a new array. Then in the recursive await getCounts(acct, results, nextCursor) call, do not ignore the return value.
async function main() {
let promises = [];
const accounts = [{'name': 'test1', 'id': 1}, {'name': 'test2' , 'id': 2}];
const promises = accounts.map(async acct => {
const results = await getIssues(acct);
const criticalCount = results.filter(i => i.priority == 'CRITICAL').length;
const highCount = results.filter(i => i.priority == 'HIGH').length;
return {
account: acct.name,
id: acct.id,
high: highCount,
critical: criticalCount
};
});
const allResults = await Promise.all(promises);
console.log(allResults);
}
const query = `query ($accountId: ID!, $cursor: IssuesCursor) {
actor {
account(id: $accountId) {
aiIssues {
issues(filter: {states: ACTIVATED}, cursor: $cursor) {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`;
async function getIssues(acct, cursor) {
const resp = await fetch('https://my.api.com/graphql', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'API-Key': API_KEY
},
body: JSON.stringify({
query: q,
variables: {
accountId: acct.id,
cursor,
}
}),
});
if (!resp.ok) throw new Error(resp.statusText);
const { data, error } = await resp.json();
if (error) throw new Error('GraphQL error', {cause: error});
const { nextCursor, issues } = data.actor.account.aiIssues.issues;
if (nextCursor == null) {
return issues;
} else {
return issues.concat(await getIssues(acct, nextCursor));
}
}
Related
I need to query some data from Redshift in my aws lambda code and I do it several times with different parameters, pretty soon I get the error:
ActiveStatementsExceededException: Active statements exceeded the allowed quota (200).
How can I send thousands of queries to Redshift without hitting a limit?
Here is my JS code:
...
let results = await redshiftQuery(connectionInfo, "SELECT * from db.games where game_id=:game_id", {game_id: 12345})
...
async function redshiftQuery(conInfo, query, params = {}) {
let rsParams = []
for(let key in params) {
rsParams.push( { name: key, value: params[key].toString() })
}
const executeStatementInput = {
ClusterIdentifier: conInfo.clusterId,
Database: conInfo.database,
SecretArn: conInfo.secret,
WithEvent: false,
Sql: query
}
if(rsParams.length) {
executeStatementInput.Parameters = rsParams
}
let results = []
try {
let exResponse = await redshiftDataApiClient.executeStatement(executeStatementInput).promise()
if(!exResponse.Id) return results
let describeStatementInfo = null
while(true) {
let { Status: queryStatus, ...rest } = await redshiftDataApiClient.describeStatement({ Id: exResponse.Id }).promise()
describeStatementInfo = rest
if(["FAILED", "FINISHED"].includes(queryStatus)) {
break
}
await sleepMillis(200)
}
if (describeStatementInfo && describeStatementInfo.HasResultSet) {
let result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id }).promise()
convertResult(result, results)
while(result.NextToken) {
result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id, NextToken: result.NextToken }).promise()
convertResult(result, results)
}
}
} catch (e) {
console.error("Redshift Error", e)
}
return results
}
This question already has answers here:
Promise.all is returning an array of undefined and resolves before being done
(3 answers)
Closed 25 days ago.
I'm trying to wrap my head around Promises, but my code doesn't work the way it should. My goal is to show that the table has loaded after all the promises are complete, but instead the code in then ("table loaded" and [undefined, undefined] 'res') get executed before. What is the way to make it work, waiting untill the table was filled?
My code
async function updateTable(accno, entity, start = null, funds) {
$table.bootstrapTable('removeAll')
prms = []
return Promise.all(
accno.map(
acc => {
let conf = {
method: 'get',
url: 'url',
params: {
account_number: acc,
start: start,
entity: entity,
}
}
axios(conf).then(function(response) {
let data = response.data
data_combined = []
for (let j = 0; j < data.return.length; j++) {
let row = data.return[j]
let temp = {
id: data.id,
// assigining more data here
}
data_combined.push(temp)
}
//another request deleted to simplify
fund_array = []
tableLoadAnimation('id-table-load', 'remove-all')
superset = [...superset, ...data_combined, ...fund_array]
$table.bootstrapTable('load', superset)
}).catch(function(error) {
console.log(error);
})
})
).then((a) => {
console.log('table loaded');
return a
}).then((res) => {
console.log(res, "res") // returns [undefined, undefined] 'res'
return res
})
}
Since axios returns promise, I had to return axios in map
async function updateTable(accno, entity, start = null, funds) {
$table.bootstrapTable('removeAll')
prms = []
prms = Promise.all(
accno.map(
acc => {
let conf = {
method: 'get',
url: 'url',
params: {
account_number: acc,
start: start,
entity: entity,
}
}
return axios(conf).then(function(response) {
let data = response.data
data_combined = []
for (let j = 0; j < data.return.length; j++) {
let row = data.return[j]
let temp = {
id: data.id,
// assigining more data here
}
data_combined.push(temp)
}
//another request deleted to simplify
fund_array = []
tableLoadAnimation('id-table-load', 'remove-all')
superset = [...superset, ...data_combined, ...fund_array]
$table.bootstrapTable('load', superset)
}).catch(function(error) {
console.log(error);
})
})
).then((a) => {
console.log('table loaded');
return a
})
}
I hope you are well. I am getting data from one API and sending to Shopify store API. As its working fine but its entering some products as when it iterates in the loop API is busy with index suppose 0,1,2 and then indexes 3,4,...10 bypassed. So , according to me I should delay the foreach loop with 10,15 seconds. Please help me to do this . I tried it many times with SETTimer etc but foreach loop structure is difficult for me as a new person. Please check the below code. Thanks
const request = require('request');
const { json } = require('express');
const { Parser } = require('json2csv');
const fastcsv = require('fast-csv');
//const csv = require('csv-parser');
const fs = require('fs');
const { privateDecrypt } = require('crypto');
const { time } = require('console');
const fields = ['Vendor', 'Price', 'SKU','error'];
const opts = { fields };
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
const csvWriter = createCsvWriter({
path: 'C:/Users/IT City/Desktop/arslan.csv',
header: [
{id: 'Vendor', title: 'Vendor'},
{id: 'Price', title: 'Price'},
{id: 'SKU', title: 'SKU'},
{id: 'error', title: 'error'},
]
});
let new_products = {
product_final: {
Vendor: String,
Price: String,
SKU: String,
Error: String,
}
};
//////////First API from which I am getting the data
const options = {
method: 'GET',
url: 'https://api.dexi.io/runs/f58f7795-c11a-478c-b670-c7ae5df8677b/latest/result',
headers: {
accept: 'application/json',
json: true,
'x-dexiio-access': '9d56e967dfXXXXXXX725e234b311655c96',
'x-dexiio-account': '5e597feb-99axxxxxxxx-37f1315723ab'
}
};
products = ['Vendor','Price','SKU','Error']
let product = {};
request(options, function (error, response, body) {
if (error) throw new Error(error);
pro = JSON.parse(body);
/////Looping through the each Item from the first API and sending data to Shopify. But its entering only 10-12 products
//// As handle is busy to entering the product.
/// I have to delay the foreach loop 10, 15 seconds
pro.rows.forEach(
row => {
for (let z = 0; z < row.length; z++)
{
product[pro.headers[z]] = row[z];
product_final2[pro.headers[z]] = row[z];
}
productssdata.push(product_final2)
products.push(product)
var Price = product.Price;
var SKU = product.SKU;
var Vendor = product.Vendor;
var body_html = "THISFSDFSDFSDFSDFSFSDF";
let new_products = {
product: {
title: Vendor,
body_html: Price,
vendor: Vendor,
product_type: SKU,
tags: Price
}
};
const options = {
method: 'POST',
url:
'https://0abcfsdfsdf4bb6532f3b#amjad.myshopify.com/admin/api/2020-07/products.json',
headers: {
accept: 'application/json',
'apiKey': '07649cABSDCCSD8ffbae7af02',
'password': 'sSDCSDFDF',
body: new_products,
json: true,
};
request(options, function (error, response, body) {
if (error) throw new Error(error);
console.log(body)
});
}
);
}
);
you don't need to use setTimeOut() to delay the loop. thats what async and await are for let me share you an example how to make the forEach loop delay with await!!.
step1 : return a function with promise and use await until it is complete.
const wait =async props => {
return new Promise((reslove,reject) => {
return reslove(Math.random());
})
}
const x = [1,2,3,4]
x.forEach(async number =>{
const num = await wait();
console.log('start')
console.log(num);
console.log('end');
})
Request is deprecated
It can't be used with await anyway, which makes it inconvenient. There used to be another module, request-promise, that as wrapping request and return a Promise, so one could await it, but it's still deprecated.
For these reasons, use Axios or Fetch instead
You can't use await or delay a .forEach() loop, but you can in a for loop.
You think you need to delay the calls, because they are asynchronous, but in reality you should simply await each call. Delaying calls with an arbitrary timeout is a dirty workaround.
In the end, you can do something like :
( async () => {
let options = {
url : "http://......"
};
const response = await axios.get(options); // Add try/catch block around this to manage errors
const pro = response.data;
for( let row of pro.rows) {
options = {
url : "http://some.other.url"
}
const response = await axios.get(options); // Each call will be done one by one and awaited in order
}
})()
setTimeout(pro.rows.forEach((row) => {...}), 10000)
This executes the forEach after 10 seconds.
I am working with mongodb and nodejs. I have an array of customers I have to create each inside database.
const promises2 = customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOne({ type: "Customer" });
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
await Counter.findOneAndUpdate({ type: "Customer" }, { $inc: { sequence_value: 1 } });
}
});
await Promise.all([...promises2]);
The issue is counter is not increasing every time. I am getting same counter in all the created customers. What is the issue here?
Issue is something like this but don't have an answer.
The problem is that all the calls overlap. Since the first thing they each do is get the current counter, they all get the same counter, then try to use it. Fundamentally, you don't want to do this:
const counter = await Counter.findOne({ type: "Customer" });
// ...
await Counter.findOneAndUpdate({ type: "Customer" }, { $inc: { sequence_value: 1 } });
...because it creates a race condition: overlapping asynchronous operations can both get the same sequence value and then both issue an update to it.
You want an atomic operation for incrementing and retrieving a new ID. I don't use MongoDB, but I think the findOneAndUpdate operation can do that for you if you add the returnNewDocument option. If so, the minimal change would be to swap over to using that:
const promises2 = customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOneAndUpdate(
{ type: "Customer" },
{ $inc: { sequence_value: 1 } },
{ returnNewDocument: true }
);
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
}
});
await Promise.all([...promises2]);
...but there's no reason to create an array and then immediately copy it, just use it directly:
await Promise.all(customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOneAndUpdate(
{ type: "Customer" },
{ $inc: { sequence_value: 1 } },
{ returnNewDocument: true }
);
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
}
}));
The overall operation will fail if anything fails, and only the first failure is reported back to your code (the other operations then continue and succeed or fail as the case may be). If you want to know everything that happened (which is probably useful in this case), you can use allSettled instead of all:
// Gets an array of {status, value/reason} objects
const results = await Promise.allSettled(customers.map(async customer => {
if (!customer.customerId) {
const counter = await Counter.findOneAndUpdate(
{ type: "Customer" },
{ $inc: { sequence_value: 1 } },
{ returnNewDocument: true }
);
console.log({counter});
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
}
}));
const errors = results.filter(({status}) => status === "rejected").map(({reason}) => reason);
if (errors.length) {
// Handle/report errors here
}
Promise.allSettled is new in ES2021, but easily polyfilled if needed.
If I'm mistaken about the above use of findOneAndUpdate in some way, I'm sure MongoDB gives you a way to get those IDs without a race condition. But in the worst case, you can pre-allocate the IDs instead, something like this:
// Allocate IDs (in series)
const ids = [];
for (const customer of customers) {
if (!customer.customerId) {
const counter = await Counter.findOne({ type: "Customer" });
await Counter.findOneAndUpdate({ type: "Customer" }, { $inc: { sequence_value: 1 } });
ids.push(counter.sequence_value);
}
}
// Create customers (in parallel)
const results = await Promise.allSettled(customers.map(async(customer, index) => {
const customerId = ids[index];
try {
await Customer.create({
customerId
});
} catch (e) {
// Failed, remove the counter, but without allowing any error doing so to
// shadow the error we're already handling
try {
await Counter.someDeleteMethodHere(/*...customerId...*/);
} catch (e2) {
// ...perhaps report `e2` here, but don't shadow `e`
}
throw e;
}
});
// Get just the errors
const errors = results.filter(({status}) => status === "rejected").map(({reason}) => reason);
if (errors.length) {
// Handle/report errors here
}
Your map function is not returning a promise.
Try this :
const promises2 = [];
customers.map((customer) => {
return new Promise(async (resolve) => {
if (!customer.customerId) {
const counter = await Counter.findOne({ type: 'Customer' });
console.log({ counter });
const payload = {
customerId: counter.sequence_value,
};
await Customer.create(payload);
await Counter.findOneAndUpdate({ type: 'Customer' }, { $inc: { sequence_value: 1 } });
}
resolve();
});
});
await Promise.all(promises2);
I know there are many questions discuss about the same error and I saw most of them and they didn't fix my problem.
I wrote this code:
const userOrganizationGroups = (organizationGroupsList) => {
if (Array.isArray(organizationGroupsList) && organizationGroupsList.length) {
const result = organizationGroupsList.map(async (element) => {
const { organizationId, groupId } = element;
const { Organizations, Groups } = models;
const organization = await Organizations.findOne(
{ _id: organizationId },
{ name: 1, _id: 0 },
);
const group = await Groups.findOne({ _id: groupId });
return Object.assign({}, {
organizationName: organization.name,
group: group.name,
});
});
return result;
}
};
when I debug the code:
console.log('userOrganizationGroups : ',userOrganizationGroups(list))
I got such a result:
userOrganizationGroups: Promise { <pending> }
I found a similair question: Promise { } - Trying to await for .map and I used the solution mentioned in the question:
const userOrganizationGroups = async (organizationGroupsList) => {
if (Array.isArray(organizationGroupsList) && organizationGroupsList.length) {
const result = await Promise.all(organizationGroupsList.map(async (element) => {
const { organizationId, groupId } = element;
const { Organizations, Groups } = models;
const organization = await Organizations.findOne(
{ _id: organizationId },
{ name: 1, _id: 0 },
);
const group = await Groups.findOne({ _id: groupId });
return Object.assign({}, {
organizationName: organization.name,
group: group.name,
});
}));
return result;
}
How can I fix this issue?
instead of
console.log('userOrganizationGroups : ',userOrganizationGroups(list))
use
userOrganizationGroups(list).then( groups => console.log('userOrganizationGroups : ', groups)
or
(async () => {
const groups = await userOrganizationGroups(list);
console.log('userOrganizationGroups : ', groups);
})();
console.log() was called first because you didn't wait using await or then.
So You should write below instead of
console.log('userOrganizationGroups : ',userOrganizationGroups(list))
;(async () => {
const resultGroups = await userOrganizationGroups(list)
resultGroups.forEach(group => {
console.log(`group: ${JSON.stringfy(group, null, ' ')}`)
})
})()