I am calling an API where I can only fetch 1000 records per request,
I was able to achieve this using recursion.
I am now trying to achieve the same using promises, I am fairly new to Node.js and JavaScript too.
I tried adding the recursion code in an if else block but failed
var requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
}
const callback = (body) => {
// some code
.
.
.//saving records to file
.
//some code
if (totlExtractedRecords < total) {
requestP(option, callback).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
}
}
requestP(option).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
I want to execute the method using promise and in a synchronous way,
i.e. I want to wait until the records are all exported to a file and continue with my code
I think its better to create your own promise and simply resolve it when your done with your recursion. Here's a simply example just for you to understand the approach
async function myRecursiveLogic(resolveMethod, ctr = 0) {
// This is where you do the logic
await new Promise((res) => setTimeout(res, 1000)); // wait - just for example
ctr++;
console.log('counter:', ctr);
if (ctr === 5) {
resolveMethod(); // Work done, resolve the promise
} else {
await myRecursiveLogic(resolveMethod, ctr); // recursion - continue work
}
}
// Run the method with a single promise
new Promise((res) => myRecursiveLogic(res)).then(r => console.log('done'));
Here's a clean and nice solution using the latest NodeJS features.
The recursive function will continue executing until a specific condition is met (in this example asynchronously getting some data).
const sleep = require('util').promisify(setTimeout)
const recursive = async () => {
await sleep(1000)
const data = await getDataViaPromise() // you can replace this with request-promise
if (!data) {
return recursive() // call the function again
}
return data // job done, return the data
}
The recursive function can be used as follows:
const main = async () => {
const data = await recursive()
// do something here with the data
}
Using your code, I'd refactored it as shown below. I hope it helps.
const requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
};
/*
NOTE: Add async to the function so you can udse await inside the function
*/
const callback = async (body) => {
// some code
//saving records to file
//some code
try {
const result = await requestP(option, callback).auth('api-reader', token, true);
if (totlExtractedRecords < total) {
return callback(result);
}
return result;
} catch (error) {
console.log('Error Observed ' + err);
return error;
}
}
Created this code using feed back from Amir Popovich
const rp = require('Request-Promise')
const fs = require('fs')
const pageSize = 200
const options = {
url: 'https://jira.xyz.com/rest/api/2/search',
json: true,
qs: {
jql: "project in (PROJECT_XYZ)",
maxResults: pageSize,
startAt: 0,
fields: '*all'
},
auth: {
user: 'api-reader',
pass: '<token>',
sendImmediately: true
}
}
const updateCSV = (elment) => {
//fs.writeFileSync('issuedata.json', JSON.stringify(elment.body, undefined, 4))
}
async function getPageinatedData(resolve, reject, ctr = 0) {
var total = 0
await rp(options).then((body) => {
let a = body.issues
console.log(a)
a.forEach(element => {
console.log(element)
//updateCSV(element)
});
total = body.total
}).catch((error) => {
reject(error)
return
})
ctr = ctr + pageSize
options.qs.startAt = ctr
if (ctr >= total) {
resolve();
} else {
await getPageinatedData(resolve, reject, ctr);
}
}
new Promise((resolve, reject) => getPageinatedData(resolve, reject))
.then(() => console.log('DONE'))
.catch((error) => console.log('Error observed - ' + error.name + '\n' + 'Error Code - ' + error.statusCode));
Related
I am writing a node js program as follows. The purpose of this code is to parse from multiple pages of an API (variable number of pages thus scraping the first page to see how many pages are to be scraped) followed by uploading all the pages to MongoDB and then "analysing the pages" with a function in another file (manipulate keyword):
const MongoClient = require('mongodb').MongoClient
const fetch = require('node-fetch')
const config = require('./config.json')
const manipulate = require('./manipulateDB')
async function startAHLoop() {
async function getAuctionPage(page = 0) {
return fetch(`https://website.net/page/${page}`).then(res => {
return res.json()
}).catch (error => console.error("Faced an error: " + error))
}
async function getFullAH() {
try {
let ah = []
let completedPages = 0
let firstPage = await getAuctionPage(0)
for (let i = 1; i <= firstPage.totalPages; i++) {
getAuctionPage(i).then((page) => {
if (completedPages !== firstPage.totalPages - 1) {
completedPages++
}
if (page.success) {
for (auction of page.auctions) {
ah.push(auction)
if (completedPages == firstPage.totalPages - 1) {
completedPages++
}
}
} else if (completedPages == firstPage.totalPages - 1) {
completedPages++
}
})
}
// Wait for the whole ah to download
while (completedPages !== firstPage.totalPages)
await new Promise((resolve) => setTimeout(resolve, 10))
return ah
} catch (e) {
console.log('Failed to update auctions', e)
return
}
}
async function main() {
let startTime = Date.now()
if (!db.isConnected()) await connectToDB()
let auctionCollection = data.collection('auctions')
let ah = await getFullAH()
let timeTaken = Date.now() - startTime
if (typeof ah.ok == 'undefined') {
auctionCollection.drop()
auctionCollection.insertMany(ah)
console.log(`Auction update complete in ${timeTaken} ms ${Date().toLocaleString()}`)
console.log("Starting analysis")
await auctionCollection.insertMany(ah)
manipulate.printAHInfos()
} else {
console.log(`Auction update failed in ${timeTaken} ms ${Date().toLocaleString()}`)
}
// This essentially is the delay instead of every 60000 ms
setTimeout(main, 60000 - timeTaken)
}
main()
}
async function connectToDB(isFirstConnect) {
console.log('Connecting to db...')
MongoClient.connect(
config.mongoSRV,
{ useNewUrlParser: true, useUnifiedTopology: true },
(err, DB) => {
if (err) return connectToDB()
db = DB
skyblock = DB.db('skyblock')
}
)
while (typeof db == 'undefined') {
await new Promise((resolve) => setTimeout(resolve, 10))
}
if (!db.isConnected()) {
console.log('Something weird happened... re-starting db connection')
return connectToDB()
}
console.log('Successful connection to database')
if (isFirstConnect) startAHLoop()
return db
}
connectToDB(true)
I am looking for a way to wait until collection.insertMany(ah) has finished before doing manipulate.AHdata
The issue I get is that manipulate.AHdata is invoked before collection.insertMany(ah) is finished. Resulting as follows when manipulate.AHdata outputs "Invoked":
Invoked
Connecting to db...
I tried using the following:
collection.insertMany(ah)
await collection.insertMany(ah)
manipulate.AHdata
But it doesn't work tho...
Any idea of what I could do?
Thanks for the help and have a great day!
Following up with all my comments and points, here's what I believe is a better (but obviously untested) version of the code :
const MongoClient = require('mongodb').MongoClient
const fetch = require('node-fetch')
const config = require('./config.json')
const manipulate = require('./manipulateDB')
let auctionCollection
async function getAuctionPage(page = 0) {
try {
const response = await fetch(`https://website.net/page/${page}`)
const data = await response.json()
return data
} catch (err) {
console.error("Faced an error: " + err)
}
}
async function getFullAH() {
try {
let ah = []
let firstPage = await getAuctionPage(0)
for (let i = 1; i <= firstPage.totalPages; i++) {
const page = await getAuctionPage(i);
for (let auction of page.auctions) ah.push(auction);
}
return ah
} catch (e) {
console.log('Failed to update auctions', e)
return
}
}
async function main() {
let startTime = Date.now()
let ah = await getFullAH()
let timeTaken = Date.now() - startTime
auctionCollection.drop()
auctionCollection.insertMany(ah)
console.log(`Auction update complete in ${timeTaken} ms ${Date().toLocaleString()}`)
console.log("Starting analysis")
await auctionCollection.insertMany(ah)
manipulate.printAHInfos()
// This essentially is the delay instead of every 60000 ms
setTimeout(main, 60000 - timeTaken)
}
async function connectToDB() {
console.log('Connecting to db...')
let db
try {
db = await MongoClient.connect(
config.mongoSRV,
{ useNewUrlParser: true, useUnifiedTopology: true });
} catch (err) {
return connectToDB()
}
auctionCollection = db.collection('auctions');
console.log('Successful connection to database')
main() // You don't need hacks and verifications to check if the DB is connected. If there was a problem, it got caught in the catch()
}
connectToDB()
I am using a library call to connect to my vendor. The libary call requires a callback in the call. Without a callback in the function, I can easily make this synchronous. With the Callback, everything I do is stuck in the callback and never bubbles it way out.
I have literally tried 100 different ways to get this to work.
function removeFromDNC(emailAddress, accessToken_in)
{
return new Promise( function(resolve, reject)
{
try{
const options =
{
auth: {
accessToken: accessToken_in
}
, soapEndpoint: 'https://webservice.XXX.exacttarget.com/Service.asmx'
};
var co = {
"CustomerKey": "DNC",
"Keys":[
{"Key":{"Name":"Email Address","Value": emailAddress}}]
};
var uo = {
SaveOptions: [{"SaveOption":{PropertyName:"DataExtensionObject",SaveAction:"Delete"}}]
};
const soapClient = new FuelSoap(options);
//again, I don't control the structure of the next call.
let res = soapClient.delete('DataExtensionObject', co, uo, async function( err, response ) {
if ( err ) {
// I can get here, but my reject, or if I use return, does nothing
reject();
}else{
// I can get here, but my reject, or if I use return, does nothing
resolve();
}
});
console.log("res value " + res); // undefined - of course
}catch(err){
console.log("ALERT: Bad response back for removeFromDNC for email: " + emailAddress + " error: " + err);
console.log("removeFromDNC promise fulfilled in catch");
reject();
}
});
}
Both methods resolve and reject expect parameters, which are res and err in your case.
As far as removeFromDNC returns a Promise instance, you should call it using either async/await syntax:
const res = await removeFromDNC(...);
or chaining then/catch calls:
removeFromDNC(...)
.then((res) => { ... }) // resolve
.catch((err) => { ... }) // reject
EDIT:
If you want to avoid usage of callbacks inside removeFromDNC, consider promisifying of soapClient.delete call. Refer to util.promisify() if you working in Node.js or use own implementation.
Here is the example for demonstration:
const promisify = (fun) => (...args) => {
return new Promise((resolve, reject) => {
fun(...args, (err, result) => {
if(err) reject(err);
else resolve(result);
})
})
}
const soapClient = {
delete: (value, cb) => {
setTimeout(() => cb(null, value), 10);
}
};
async function removeFromDNC(emailAddress, accessToken_in) {
const soapDelete = promisify(soapClient.delete.bind(soapClient));
const res = await soapDelete('Soap Responce');
//You can use res here
return res;
}
removeFromDNC().then(res => console.log(res))
I have a 2 identical Firebase functions that batch write data to Firestore. One is wrapped in a scheduled/onRun trigger, and the other is a HTTP onRequest trigger.
Both functions work fine and throw no errors.
They have the same amount of memory and timeout as well.
When invoking the http trigger, the function runs through and completes in about 30 seconds.
When invoking the scheduled onRun trigger, the function takes 5+ minutes to complete.
Is there something different about the runtimes that is not documented or something?
Edit: It works now - I made processMentions await totalMentions and return null.
processMentions does not have to return a promise, only a value because the actual scheduledPull/onRun function is returning the processMentions async function, which resolves the promise by returning a value.
Cheers for the help #dougstevenson
Triggers:
/**
* Get manual mentions
*/
exports.get = functions.https.onRequest((req, res) => {
const topic = 'topic'
const query = 'queryString'
processMentions(res, query, topic)
})
/**
* Get schedule mentions
*/
exports.scheduledPull = functions.pubsub.schedule('every day 1:00').onRun((context) => {
const topic = 'topic'
const query = 'queryString'
return processMentions('sched', query, topic)
})
Logic:
const functions = require('firebase-functions')
const admin = require('firebase-admin')
admin.initializeApp()
const db = admin.firestore()
const axios = require('axios')
const moment = require('moment')
// Globals
const auth = 'token'
const url = 'https://apiurl.com/'
async function totalMentions(nextPage, start, end, query) {
try {
let config = {
headers: {
Authorization: auth,
Accept: 'text/html',
}
}
const response = await axios.get(url, config)
const total = response.data.results.total
const loops = Math.ceil(total / 500)
return loops
} catch (error) {
console.log('error 1', error)
}
}
async function allMentions(nextPage, start, end, query) {
try {
let config = {
headers: {
Authorization: auth,
Accept: 'text/html',
},
}
const response = await axios.get(url, config)
return response
} catch (error) {
console.log('error 2', error)
}
}
async function saveData(response, end, topic) {
try {
let data = await response.data.results.clips
let batch = db.batch()
data.forEach((c) => {
delete c.localTime
let reff = db.collection(collection).doc(date).collection(collection).doc(c.id.toString())
batch.set(reff, c)
})
let batches = await batch.commit()
return batches
} catch (error) {
console.log('error3 ', error)
}
}
async function processMentions(res, query, topic) {
try {
totalMentions(1, start, end, query)
.then(async (loops) => {
let endbatch = 0
for (let i = 1; i <= loops; i++) {
await allMentions(i, start, end, query)
.then(async (response) => {
await saveData(response, end, topic)
return ++endbatch
})
.catch((err) => {
console.log('error 4 ' + err)
})
if (endbatch === loops) {
if (res !== 'sched') {
console.log('http trigger finished')
return res.status(200).end()
} else {
return console.log('schedule finished')
}
}
}
})
.catch((err) => {
console.log('error5 ' + err)
})
} catch (error) {
console.log('error6 ' + error)
}
}
For the pubsub trigger to work correctly, processMentions needs to return a promise that resovles when all of the async work is complete. Right now, it's returning nothing, which (since it's declared async) translates into a promise that's resolved immediately with no value. Calling then/catch on a promise isn't doing what you expect - you need to return a promise chain from your async work.
I'm not sure why you have it declared async, without also using await inside of it to manage the promises much more easily.
Am using sequilizer and struggling because the method is forever in pending state.
The following is a simplified version of what I am trying to do. Basically, an API makes use of the below methods, by calling BatchProcessor, which was supposed to process the provided json.
I basically want BatchProcessor to get themeprice and themegate from FinalTheme method but the promise is forever pending.
export default {
async FinalTheme(id) {
return db.Themes.findOne({
where: {
ID: id
},
attributes: ["ThemeCost","ThemeGate"],
limit: 1
})
.then(data => {
if (data == null) {
return -1;
}
return {
cost: data["ThemeCost"],
gate: data["ThemeGate"]
};
})
.catch(err => {
return false;
});
},
async BatchProcessor(record, index_number) {
const SQL ="SELECT * FROM themes";
return db.sequelize
.query(SQL, {
type: db.sequelize.QueryTypes.SELECT
})
.then(themes => {
// do we have data here?
const totalThemes = themes.length;
let lastAmount = record["Amount"];
for (
let counter = 0;
counter < totalThemes - 1;
counter++
) {
const CustomerFinalTheme = this.FinalTheme(record["CustomerID"]); // FOREVER PENDING
}
})
.catch(err => {
console.log(JSON.stringify(err));
});
},
};
What am I doing wrong exaclty?
this.FinalTheme(... returns a promise and not the value you have to do:
this.FinalTheme(record["CustomerId"]) // where is the record assigned?
.then(data => {
const CustomerFinalTheme = data;
})
also no need to use async when declaring the functions ie the following is ok:
FinalTheme(id) {
return db.Themes.findOne({
[...]
}
You are running a loop inside then block of BatchProcessor. you can await inside for loop.
async BatchProcessor(record, index_number) {
const SQL ="SELECT * FROM themes";
const themes = await db.sequelize.query(SQL, { type: db.sequelize.QueryTypes.SELECT });
const totalThemes = themes.length;
let lastAmount = record["Amount"];
for (let counter = 0; counter < totalThemes - 1; counter++) {
const CustomerFinalTheme = await this.FinalTheme(record["CustomerID"]);
}
return 'ALL DONE';
}
So I have a method, which I want to call multiple times in a loop. This is the function:
function PageSpeedCall(callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[0]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
As you see this is an async function that calls the PageSpeed API. It then gets the response thanks to the callback and renders it in the view. Now how do I get this to be work in a for/while loop? For example
function PageSpeedCall(websites, i, callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[i]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
var websites = ['google.com','facebook.com','stackoverflow.com'];
for (let i = 0; i < websites.length; i++) {
PageSpeedCall(websites, i);
}
I want to get a raport for each of these sites. The length of the array will change depending on what the user does.
I am using async.parallel to call the functions like this:
let freeReportCalls = [PageSpeedCall, MozCall, AlexaCall];
async.parallel(freeReportCalls, (err, results) => {
if (err) {
console.log(err);
} else {
res.render('reports/report', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}
});
I tried to use promise chaining, but for some reason I cannot put it together in my head. This is my attempt.
return Promise.all([PageSpeedCall,MozCall,AlexaCall]).then(([ps,mz,al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
I tried doing this:
return Promise.all([for(let i = 0;i < websites.length;i++){PageSpeedCall(websites, i)}, MozCall, AlexaCall]).
then(([ps, mz, al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
But node just said it's stupid.
And this would work if I didn't want to pass the websites and the iterator into the functions. Any idea how to solve this?
To recap. So far the functions work for single websites. I'd like them to work for an array of websites.
I'm basically not sure how to call them, and how to return the responses.
It's much easier if you use fetch and async/await
const fetch = require('node-fetch');
async function PageSpeedCall(website) {
const pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
const result = await fetch(pagespeeddCall);
return await result.json();
}
async function callAllSites (websites) {
const results = [];
for (const website of websites) {
results.push(await PageSpeedCall(website));
}
return results;
}
callAllSites(['google.com','facebook.com','stackoverflow.com'])
.then(results => console.log(results))
.error(error => console.error(error));
Which is better with a Promise.all
async function callAllSites (websites) {
return await Promise.all(websites.map(website => PageSpeedCall(website));
}
Starting on Node 7.5.0 you can use native async/await:
async function PageSpeedCall(website) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
return await promisify(pagespeedCall);
}
async function getResults(){
const websites = ['google.com','facebook.com','stackoverflow.com'];
return websites.map(website => {
try {
return await PageSpeedCall(website);
}
catch (ex) {
// handle exception
}
})
}
Node http "callback" to promise function:
function promisify(url) {
// return new pending promise
return new Promise((resolve, reject) => {
// select http or https module, depending on reqested url
const lib = url.startsWith('https') ? require('https') : require('http');
const request = lib.get(url, (response) => {
// handle http errors
if (response.statusCode < 200 || response.statusCode > 299) {
reject(new Error('Failed to load page, status code: ' + response.statusCode));
}
// temporary data holder
const body = [];
// on every content chunk, push it to the data array
response.on('data', (chunk) => body.push(chunk));
// we are done, resolve promise with those joined chunks
response.on('end', () => resolve(body.join('')));
});
// handle connection errors of the request
request.on('error', (err) => reject(err))
})
}
Make PageSpeedCall a promise and push that promise to an array as many times as you need, e.g. myArray.push(PageSpeedCall(foo)) then myArray.push(PageSpeedCall(foo2)) and so on. Then you Promise.all the array.
If subsequent asynch calls require the result of a prior asynch call, that is what .then is for.
Promise.all()
Promise.all([promise1, promise2, promise3]).then(function(values) {
console.log(values);
});