I am using an external api that I use as my datasource. That api gives its data back based on a pagination like type.
So I can get data for page 1, 2, 3 etc. But the problem that I have is that I need all data at once, the api doesn't support this so I write it myself.
I have made the actual call code:
function getTransactionPart(start){
return new Promise(function(resolve, reject) {
const options = {
url: 'myurl?limit=40&start='+start,
json: true
};
let result = {total_transactions : 0, transactions : []};
request(options, function (error, response, body) {
if (error) {
return reject(result);
}
body.data.forEach(function (transaction) {
result.transactions.push({
timestamp: transaction.timestamp,
amount: transaction.amount,
confirmed: transaction.confirmed
});
});
result.total_transactions = body.total
return resolve(result);
})
})
}
The above code returns me the expected results with the limit that I gave. Also I get a number back (result.total_transactions) when this is more than 40 I need to make another call with 40 as start etc etc.
The code where I need to combine it:
function getTransactions(start) {
return new Promise(function(resolve, reject) {
getTransactionPart(start).then(result => {
if(result.total_transactions > 40) {
//next call
} else {
// return all?
}
console.log(result)
}).catch(error => {
console.error(error);
return r.json({
})
});
})
}
So I do the first call with getTransactionPart(0) after that the method itself needs to combine the result form all the sub calls and return the whole result as expected. How can I do this with recursion and promises?
This is easier if you use an async function and await the request:
async function getTransactions(start) {
const result = [];
for(let pos = start; ; pos += 40) {
const { total_transactions, transactions } = await getTransactionPart(pos);
result.push(...transactions);
if(total_transactions < 40) break;
}
return result;
}
For sure you could also do this recursively, but do you really need that?
async function getTransactions(start) {
const { total_transactions, transactions } = await getTransactionPart(pos);
if(total_transactions < 40)
return transactions;
return transactions.concat(await getTransactions(start + 40));
}
Related
I'm seeking guidance on how best to handle the following scenario. I'm fairly new to JS and async development, so I'm trying to understand the best way to handle this. I call one api (callAPI) which returns an array of items. I need to take those items and send them to another api (callAPI2) but that api doesn't have a bulk method, so I have to call the api for each item in the array. Below is how I have it structured: getArray promise returns the first array, I send the array to promise 2 (getIndividualData) where I loop and call the second api. I'm wondering if there are better ways to structure this? If I pass in a large array, I may need to pace the calls to the api so I don't get throttled... so maybe I need some version of Promise.all?
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = function(arrayOfItems) {
return new Promise(function(resolve,reject) {
var responseArray = [];
for(var i = 0; i < arrayOfItems.length; i++) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
responseArray.push(result);
});
}
resolve(responseArray);
});
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response);
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);
You can make a request for each item in a large array without getting throttled by implementing a concurrency throttler with a Set of Promises and async/await syntax. I've duplicated your code below, modifying the implementation of getIndividualData and passing in concurrency as an option.
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = async function(arrayOfItems, { concurrency }) {
var promiseSet = new Set(),
responseArray = [],
i = 0;
while (i < arrayOfItems.length) {
if (promiseSet.size >= concurrency) {
await Promise.race(promiseSet)
}
const promise = new Promise(function(resolve,reject) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
resolve(result)
})
})
responseArray.push(promise.then(result => {
promiseSet.delete(promise)
return result
}))
i += 1
}
return Promise.all(responseArray)
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response, { concurrency: 10 });
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);
Reformulating this to a helper that promisifies an object that has an .on('success') event handler and an async function for the top-level .then()ish code gives us something like this...
To pace the API calls, add in p-limit or similar to getIndividualData.
function promisifyOnSuccessObj(onSuccessObj) {
return new Promise((resolve) => {
onSuccessObj.on("success", (result, response) => resolve(result));
// TODO: what about `response`?
// TODO: onSuccessObj.on('error')..?
});
}
function getIndividualData(arrayOfItems) {
// Returns an array of promises
return arrayOfItems.map((item) =>
promisifyOnSuccessObj(callApi2.get(item)),
);
}
async function doThings() {
const result = await promisifyOnSuccessObj(callApi.get());
const individualDatas = await Promise.all(getIndividualData(result));
console.log(`The final array is ${JSON.stringify(individualDatas)}`);
}
You could combine Promise.all, map and async...await syntax and in the end get one array of resolved individual promises based on the previous resolved array promise.
const mockApi = {
request: (response) => {
return new Promise((resolve, reject) => {
setTimeout(() => resolve(response), 1000)
})
},
getArray() {
return this.request(['foo', 'bar', 'baz'])
},
getItem(item) {
return this.request(`Resolved: ${item}`)
}
}
async function getData() {
const array = await mockApi.getArray();
const final = await Promise.all(array.map(e => mockApi.getItem(e)));
console.log(final)
}
getData()
I have a an array of chunked data that I need to upload one chunk at time. The current implementation I used it to encapsulate the logic in an Promise.all() since I need to return the result of the promise,
The problem with this approach is that all the upload is done asynchronously resulting in a Timeout error as the server can't process all the requests at the same time, How can I modify this method so that the upload is done one chunk at time ?.
My code:
var chunks = _.chunk(variableRecords, 30);
return Promise.all(
chunks.map(chunk => this.portalService.updateDataForChart(variableId, chunk)))
.then((updateRes: boolean[]) => {
if (updateRes.every(updateStatus => updateStatus)) {
return this.executeRequest<HealthDataSource, boolean>({
path: `/variable/user/datasources/${dataSource.identifier}`,
method: 'PUT',
body: {
libelle: dataSource.datasource.libelle,
type: dataSource.datasource.type,
lastSyncDate: Math.max(maxDate, dataSource.datasource.lastSyncDate)
},
headers: this.getHeaders()
});
} else {
return false;
}
});
You need them in SEQUENCE , for of is the way to go :
async function chunksSequence(chunks) {
for(const chunk of chunks) {
await // your other code here
}
};
If you need to return something
async function chunksSequence(chunks) {
let results = []
for(const chunk of chunks) {
let result = await // your other code here
results.push(result)
}
return results
};
Because of comment needed in a promise on return
async function chunksSequence(chunks) {
return new Promise((resolve, reject)=>{
let results = []
for(const chunk of chunks) {
let result = await // your other code here
results.push(result)
}
resolve(results)
}
};
You can do this with the help of Array.reduce()
const chunks = _.chunk(variableRecords, 30);
return tasks.reduce((promiseChain, currentTask) => {
return promiseChain.then(chainResults =>
currentTask.then(currentResult =>
[ ...chainResults, currentResult ]
)
);
}, Promise.resolve([])).then(arrayOfResults => {
// Do something with all results
});
Source : https://decembersoft.com/posts/promises-in-serial-with-array-reduce/
If you don't / can't use await you could use something like this
function runSequenceItem(chunks, index) {
return new Promise(chunks[index])
.then(res => {
index ++
if (index < chunks.length) {
return runSequence(chunks[index], index + 1)
} else {
// this is not needed actually
return 'done'
}
})
}
function runInSequence(chunks) {
return runSequenceItem(chunks, 0)
}
If you also need the results then you can return an array at the end of the recursion runInSequence
function runSequenceItem(chunks, index, results) {
return new Promise(chunks[index])
.then(res => {
results.push(res)
index ++
if (index < chunks.length) {
return runSequence(chunks[index], index + 1)
} else {
return results
}
})
}
function runInSequence(chunks) {
return runSequenceItem(chunks, 0, [])
}
and then retrieve it at the end
let results = runInSequence(chunks)
I am calling an API where I can only fetch 1000 records per request,
I was able to achieve this using recursion.
I am now trying to achieve the same using promises, I am fairly new to Node.js and JavaScript too.
I tried adding the recursion code in an if else block but failed
var requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
}
const callback = (body) => {
// some code
.
.
.//saving records to file
.
//some code
if (totlExtractedRecords < total) {
requestP(option, callback).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
}
}
requestP(option).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
I want to execute the method using promise and in a synchronous way,
i.e. I want to wait until the records are all exported to a file and continue with my code
I think its better to create your own promise and simply resolve it when your done with your recursion. Here's a simply example just for you to understand the approach
async function myRecursiveLogic(resolveMethod, ctr = 0) {
// This is where you do the logic
await new Promise((res) => setTimeout(res, 1000)); // wait - just for example
ctr++;
console.log('counter:', ctr);
if (ctr === 5) {
resolveMethod(); // Work done, resolve the promise
} else {
await myRecursiveLogic(resolveMethod, ctr); // recursion - continue work
}
}
// Run the method with a single promise
new Promise((res) => myRecursiveLogic(res)).then(r => console.log('done'));
Here's a clean and nice solution using the latest NodeJS features.
The recursive function will continue executing until a specific condition is met (in this example asynchronously getting some data).
const sleep = require('util').promisify(setTimeout)
const recursive = async () => {
await sleep(1000)
const data = await getDataViaPromise() // you can replace this with request-promise
if (!data) {
return recursive() // call the function again
}
return data // job done, return the data
}
The recursive function can be used as follows:
const main = async () => {
const data = await recursive()
// do something here with the data
}
Using your code, I'd refactored it as shown below. I hope it helps.
const requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
};
/*
NOTE: Add async to the function so you can udse await inside the function
*/
const callback = async (body) => {
// some code
//saving records to file
//some code
try {
const result = await requestP(option, callback).auth('api-reader', token, true);
if (totlExtractedRecords < total) {
return callback(result);
}
return result;
} catch (error) {
console.log('Error Observed ' + err);
return error;
}
}
Created this code using feed back from Amir Popovich
const rp = require('Request-Promise')
const fs = require('fs')
const pageSize = 200
const options = {
url: 'https://jira.xyz.com/rest/api/2/search',
json: true,
qs: {
jql: "project in (PROJECT_XYZ)",
maxResults: pageSize,
startAt: 0,
fields: '*all'
},
auth: {
user: 'api-reader',
pass: '<token>',
sendImmediately: true
}
}
const updateCSV = (elment) => {
//fs.writeFileSync('issuedata.json', JSON.stringify(elment.body, undefined, 4))
}
async function getPageinatedData(resolve, reject, ctr = 0) {
var total = 0
await rp(options).then((body) => {
let a = body.issues
console.log(a)
a.forEach(element => {
console.log(element)
//updateCSV(element)
});
total = body.total
}).catch((error) => {
reject(error)
return
})
ctr = ctr + pageSize
options.qs.startAt = ctr
if (ctr >= total) {
resolve();
} else {
await getPageinatedData(resolve, reject, ctr);
}
}
new Promise((resolve, reject) => getPageinatedData(resolve, reject))
.then(() => console.log('DONE'))
.catch((error) => console.log('Error observed - ' + error.name + '\n' + 'Error Code - ' + error.statusCode));
Sorry if my title is not very explicit I dont know how to explain this properly.
I am trying to use the distinct function for my app that uses loopback 3 and mongodb. It seems to work right but my endpoint wont hit the return inside my function.
This is my code
const distinctUsers = await sellerCollection.distinct('userId',{
hostId : host.id,
eventId:{
"$ne" : eventId
}
}, async function (err, userIds) {;
if(!userIds || userIds.length ==0)
return [];
const filter = {
where:{
id: {
inq: userIds
}
}
};
console.log("should be last")
return await BPUser.find(filter);
});
console.log(distinctUsers);
console.log("wtf??");
//return [];
If I uncomment the return [] it will send the return and later it will show the should be last, so even when I dont have the return it seems to finish. It is now waiting for the response. I dont like the way my code looks so any pointer of how to make this look better I will take it.
It looks like the sellerCollection.distinct takes a callback as one of it's parameters, therefore, you cannot use async/await with a callback-style function, since it's not a promise.
I would suggest turning this call into a promise if you'd like to use async/await:
function findDistinct(hostId, eventId) {
return new Promise((resolve, reject) => {
sellerCollection.distinct(
'userId',
{ hostId, eventId: { "$ne": eventId } },
function (error, userIds) {
if (error) {
reject(error);
return;
}
if (!userIds || userIds.length === 0) {
resolve([]);
return;
}
resolve(userIds);
}
)
})
}
Then, you can use this new function with async/await like such:
async function getDistinctUsers() {
try {
const hostId = ...
const eventId = ...
const distinctUsers = await findDistinct(hostId, eventId)
if (distinctUsers.length === 0) {
return
}
const filter = {
where: {
id: { inq: userIds }
}
}
const bpUsers = await BPUser.find(filter) // assuming it's a promise
console.log(bpUsers)
} catch (error) {
// handle error
}
}
So I have a method, which I want to call multiple times in a loop. This is the function:
function PageSpeedCall(callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[0]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
As you see this is an async function that calls the PageSpeed API. It then gets the response thanks to the callback and renders it in the view. Now how do I get this to be work in a for/while loop? For example
function PageSpeedCall(websites, i, callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[i]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
var websites = ['google.com','facebook.com','stackoverflow.com'];
for (let i = 0; i < websites.length; i++) {
PageSpeedCall(websites, i);
}
I want to get a raport for each of these sites. The length of the array will change depending on what the user does.
I am using async.parallel to call the functions like this:
let freeReportCalls = [PageSpeedCall, MozCall, AlexaCall];
async.parallel(freeReportCalls, (err, results) => {
if (err) {
console.log(err);
} else {
res.render('reports/report', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}
});
I tried to use promise chaining, but for some reason I cannot put it together in my head. This is my attempt.
return Promise.all([PageSpeedCall,MozCall,AlexaCall]).then(([ps,mz,al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
I tried doing this:
return Promise.all([for(let i = 0;i < websites.length;i++){PageSpeedCall(websites, i)}, MozCall, AlexaCall]).
then(([ps, mz, al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
But node just said it's stupid.
And this would work if I didn't want to pass the websites and the iterator into the functions. Any idea how to solve this?
To recap. So far the functions work for single websites. I'd like them to work for an array of websites.
I'm basically not sure how to call them, and how to return the responses.
It's much easier if you use fetch and async/await
const fetch = require('node-fetch');
async function PageSpeedCall(website) {
const pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
const result = await fetch(pagespeeddCall);
return await result.json();
}
async function callAllSites (websites) {
const results = [];
for (const website of websites) {
results.push(await PageSpeedCall(website));
}
return results;
}
callAllSites(['google.com','facebook.com','stackoverflow.com'])
.then(results => console.log(results))
.error(error => console.error(error));
Which is better with a Promise.all
async function callAllSites (websites) {
return await Promise.all(websites.map(website => PageSpeedCall(website));
}
Starting on Node 7.5.0 you can use native async/await:
async function PageSpeedCall(website) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
return await promisify(pagespeedCall);
}
async function getResults(){
const websites = ['google.com','facebook.com','stackoverflow.com'];
return websites.map(website => {
try {
return await PageSpeedCall(website);
}
catch (ex) {
// handle exception
}
})
}
Node http "callback" to promise function:
function promisify(url) {
// return new pending promise
return new Promise((resolve, reject) => {
// select http or https module, depending on reqested url
const lib = url.startsWith('https') ? require('https') : require('http');
const request = lib.get(url, (response) => {
// handle http errors
if (response.statusCode < 200 || response.statusCode > 299) {
reject(new Error('Failed to load page, status code: ' + response.statusCode));
}
// temporary data holder
const body = [];
// on every content chunk, push it to the data array
response.on('data', (chunk) => body.push(chunk));
// we are done, resolve promise with those joined chunks
response.on('end', () => resolve(body.join('')));
});
// handle connection errors of the request
request.on('error', (err) => reject(err))
})
}
Make PageSpeedCall a promise and push that promise to an array as many times as you need, e.g. myArray.push(PageSpeedCall(foo)) then myArray.push(PageSpeedCall(foo2)) and so on. Then you Promise.all the array.
If subsequent asynch calls require the result of a prior asynch call, that is what .then is for.
Promise.all()
Promise.all([promise1, promise2, promise3]).then(function(values) {
console.log(values);
});