Develop Reference

How to wait for an async query to finish in a for loop?

I want to get the result of an async function - depending on the result I either want to execute another function or continue the for loop. Here is my Cloud Function:
return query.get().then(geoSnapshot => {
// If GeoquerySnapshot returns 0 docs, createDoc()
let docs = geoSnapshot.docs;
if (docs.length === 0) return createDoc(data);
for(let [index, doc] of docs.entries()){
// If age preferences are valid, joinDoc
let currentDoc = docs[index].data();
if (validAge(currentDoc, data)){
let matchedBefore = await(matchedBefore(currentDoc, data))
if (!matchedBefore){
return joinDoc(docs[index], data);
} else if (index === (docs.length - 1)) {
return createDoc(data);
}
}
return
}
}).catch( error => {
console.log("error query: " + error);
return { error: error };
})
async function matchedBefore(currentDoc, data){
return db.collection('users').doc(data.uid).get().then( doc => {
if ( !doc.exists ) return false;
// If user1 in matchedUsers
let matchedUsers = doc.get('matchedUsers');
if (matchedUsers === undefined) return true
let matchedBefore = matchedUsers.includes(currentDoc.user1);
console.log('matchedBefore: ' + matchedBefore);
if (matchedBefore) {
return false
} else {
return true
}
})
}
I'm getting this following error on let matchedBefore = await(matchedBefore(currentDoc, data)):
Each then() should return a value or throw
How can I ensure the function matchedBefore() finishes before the for loop continues?

I think you are too confused with the implementation of async-await. The correct way would be:
return query.get().then(async geoSnapshot => {
// If GeoquerySnapshot returns 0 docs, createDoc()
let docs = geoSnapshot.docs;
if (docs.length === 0) return createDoc(data);
/* eslint-disable no-await-in-loop */
for(let [index, doc] of docs.entries()) {
// If age preferences are valid, joinDoc
let currentDoc = docs[index].data();
if (validAge(currentDoc, data)){
let matchedBefore = await(matchedBefore(currentDoc, data))
if (!matchedBefore){
return joinDoc(docs[index], data);
} else if (index === (docs.length - 1)) {
return createDoc(data);
}
}
return
}
/* eslint-enable no-await-in-loop */
}).catch( error => {
console.log("error query: " + error);
return { error: error };
})
async function matchedBefore(currentDoc, data){
let doc = await db.collection('users').doc(data.uid).get()
if ( !doc.exists ) return false;
// If user1 in matchedUsers
let matchedUsers = doc.get('matchedUsers');
if (matchedUsers === undefined) return true
let matchedBefore = matchedUsers.includes(currentDoc.user1);
console.log('matchedBefore: ' + matchedBefore);
return !matchedBefore
}

How can i use await, when it's don't recognize?

I'm trying to use await on var application = await SchedulerService().getIssues(issueId)
And it returns the error: SyntaxError: await is only valid in async function
I'm starting in node.js. What can I do to fix it?
I've tried already
Add async before initial function const SchedulerService = await function(){ at line 1
Add async on first return return async () => { where's return { at line 3
import schedulerConf from '../../config/scheduler';
import authConf from '../../config/auth';
import applicationConf from '../../config/application';
import request from 'request';
import schedule from 'node-schedule';
import dateformat from 'dateformat';
let interations = 0;
var serviceRecords = [];
var issueRecords = [];
const SchedulerService = function(){
return {
initialize: async () => {
console.log(`***** Starting Scheduler on ${dateformat(new Date(), "dd/mm/yyyy HH:MM:ss")}`);
var j = schedule.scheduleJob('*/1 * * * *', function(){
console.time('└─ Scheduler execution time');
if(interations === 0){
console.log(`Setting scheduler runtime to full time.`);
}else{
console.log(`------------------------`);
}
interations++;
console.log(`Job execution number: ${interations}.`);
SchedulerService().execute()
.then(response => {
console.log(`└─ Job ${interations} was successfully executed.`);
console.log(`└─ Execution date ${dateformat(new Date(), "dd/mm/yyyy HH:MM:ss")}`);
console.timeEnd('└─ Scheduler execution time');
}).catch(error => {
console.log(`└─ Job ${interations} returned error while executing.`);
});
});
},
execute: async () => {
return SchedulerService().getRecords2Sync()
.then(() => {
SchedulerService().sync().then(() => {
}).catch(error => {console.log({error})});
}).catch(error => {console.log({error})});
},
getRecords2Sync: async () => {
serviceRecords = [];
var options = {
url: `http://localhost:${authConf.serverPort}/application`,
method: 'GET',
headers: {
authorization: `${authConf.secret}`
}
}
return new Promise(function (resolve, reject) {
request(options, function (error, res, body) {
if (!error && res.statusCode == 200) {
const srs = JSON.parse(body);
const response = srs['response'];
for(let i =0;i < response.length;i++){
const { id, info } = response[i];
var status = "";
var issueId = "";
var statusClass = "";
for(let x = 0; x < info.length; x++){
if(info[x].key === "status"){
status = info[x].value;
statusClass = info[x].valueClass;
}
if(info[x].key === applicationConf.fields.issueId){
issueId = info[x].value;
}
}
if(statusClass === 0){
if(issueId !== null && issueId !== ""){
serviceRecords.push({id, status, issueId});
}
}
}
//console.log(serviceRecords);
resolve(serviceRecords);
} else {
//console.log(error);
reject(error);
}
});
});
},
getIssues : async (issueId) => {
issueRecords = [];
return new Promise(function(resolve, reject) {
var options = {
url: `http://localhost:${authConf.serverPort}/application2/${issueId}`,
method: 'GET',
headers: {
authorization: `${authConf.secret}`
}
}
request(options, function(error, res, body) {
if (!error && res.statusCode == 200) {
const issues = JSON.parse(body);
const { issue } = issues.response;
const { id, status, custom_fields } = issue;
issueRecords.push({id, status, custom_fields});
resolve(issueRecords);
} else {
reject(error);
}
});
});
},
sync : async () => {
return new Promise(function(resolve, reject) {
for (let i = 0; i < serviceRecords.length; i++) {
const application_id = serviceRecords[i].id;
const application_status = serviceRecords[i].status;
const application_issueId = serviceRecords[i].issueId;
//console.log('issueRecords.length: ', issueRecords);
//console.log('issueRecords.length: ', issueRecords.length);
console.log(`********** application`);
console.log(`└─ id ${application_id}`);
console.log(`└─ status ${application_status}`);
console.log(`└─ issueId ${application_issueId}`);
var application2 = await SchedulerService().getIssues(application_issueId)
.then(response => {
resolve(() => {
console.log(`i've found a record by issue_id ${application_issueId}`);
});
}).catch(error => {
reject(error);
});
}
});
}
}
}
export default new SchedulerService();
Thank you so much!

If you had getIssues resolve with issueId and issueRecords you might do something like this in sync:
sync: async () => {
// `map` over the serviceRecords and return a getIssues promise for each issueId
const promises = serviceRecords.map(({ issueId }) => SchedulerService().getIssues(issueId));
// Wait for all the promises to resolve
const data = await Promise.all(promises);
// Loop over the resolved promises and log the issueId
data.forEach((issueId, issueRecords) => console.log(issueId));
}

How to properly format nested try/catch blocks inside a function that returns a server response

I have a function that is called that must return a response to a server. Inside this function are two await function calls that are nested. To track error handling, I added try/catch blocks. Is there a way to avoid having nested try catch blocks to track all cases where the function might fail so I can send back an error server response?
Here's my function, it queries for a user's unique device id's and sends a push notification to each one. If a token becomes invalid, then I delete it from my database:
function findUserDevices(uid: string, message) {
collectionData(fb.firestore().collection('devices').where('userId', '==', uid)).pipe(
filter((userDevices) => userDevices && userDevices.length > 0),
take(1)
).subscribe( async (devices: any) => {
var userDeviceTokens: string[] = devices.map((device: any) => device.token);
if (userDeviceTokens !== undefined && userDeviceTokens.length != 0) {
try {
message['tokens'] = userDeviceTokens;
const pushResponse = await admin.messsaging().sendMulticast(message);
if (pushResponse.failureCount > 0) {
const failedTokens = [];
pushResponse.responses.forEach((resp, idx) => {
if (!resp.success) {
failedTokens.push(userDeviceTokens[idx]);
}
});
failedTokens.forEach( async (token) => {
var tokenInstanceID = token.split(':')[0];
try {
await deleteOldToken(tokenInstanceID);
console.log(`Token ${tokenInstanceID} deleted`)
} catch {
return res.status(500).send("err");
}
})
return res.status(200).send("ok");
} else {
return res.status(200).send("ok");
}
} catch {
return res.status(500).send("err");
}
} else {
return res.status(200).send("ok");
}
})
}
It just feels a bit excessive with all the returns I must have. Where can I improve?
EDIT, broke apart code into three blocks to prevent arrow coding
function findUserDevices(uid: string, message) {
collectionData(fb.firestore().collection('devices').where('userId', '==', uid)).pipe(
filter((userDevices) => userDevices && userDevices.length > 0),
take(1)
).subscribe(async (devices: any) => {
var userDeviceTokens: string[] = devices.map((device: any) => device.token);
if (userDeviceTokens !== undefined && userDeviceTokens.length != 0) {
try {
message['tokens'] = userDeviceTokens;
const response = await admin.messaging().sendMulticast(message);
const oldTokensArray = checkOldTokens(response, userDeviceTokens);
if (oldTokensArray.length > 0) {
await deleteOldTokens(oldTokensArray);
return res.status(200).send("ok");
} else {
return res.status(200).send("ok");
}
} catch (err) {
return res.status(500).send(err);
}
} else {
return res.status(200).send("ok");
}
})
}
function checkOldTokens(response, userDeviceTokens) {
if (response.failureCount > 0) {
const failedTokens = [];
response.responses.forEach((resp, idx) => {
if (!resp.success) {
failedTokens.push(userDeviceTokens[idx]);
}
});
return failedTokens;
} else {
return [];
}
}
async function deleteOldTokens(tokenArray) {
for (const token of tokenArray) {
await fb.firestore().collection('devices').doc(token).delete();
}
}

Updating many(100k+) documents in the most efficient way MongoDB

I have a function that runs periodically, that updates the item.price of some Documents in my Prices Collection. The Price Collection has 100k+ items. The function looks like this:
//Just a helper function for multiple GET requests with request.
let _request = (urls, cb) => {
let results = {}, i = urls.length, c = 0;
handler = (err, response, body) => {
let url = response.request.uri.href;
results[url] = { err, response, body };
if (++c === urls.length) {
cb(results);
}
};
while (i--) {
request(urls[i], handler);
}
};
// function to update the prices in our Prices collection.
const update = (cb) => {
Price.remove({}, (err, remove) => {
if (err) {
return logger.error(`Error removing items...`);
}
logger.info(`Removed all items... Beginning to update.`);
_request(urls, (responses) => {
let url, response, gameid;
for (url in responses) {
id = url.split('/')[5].split('?')[0];
response = responses[url];
if (response.err) {
logger.error(`Error in request to ${url}: ${err}`);
return;
}
if (response.body) {
logger.info(`Request to ${url} successful.`)
let jsonResult = {};
try {
jsonResult = JSON.parse(response.body);
} catch (e) {
logger.error(`Could not parse.`);
}
logger.info(`Response body for ${id} is ${Object.keys(jsonResult).length}.`);
let allItemsArray = Object.keys(jsonResult).map((key, index) => {
return {
itemid: id,
hash_name: key,
price: jsonResult[key]
}
});
Price.insertMany(allItemsArray).then(docs => {
logger.info(`Saved docs for ${id}`)
}, (e) => {
logger.error(`Error saving docs.`);
});
}
}
if (cb && typeof cb == 'function') {
cb();
}
})
});
}
As you can see, to avoid iterating through 100k+ Documents, and updating each and every one of them separately, I delete them all at the beginning, and just call the API that gives me these Items with prices, and use InsertMany to Insert all of them into my Prices Collection.
This updating process will happen every 30 minutes.
But I just now realised, what if some user wants to check the Prices and my Prices Collection is currently empty because it's in the middle of updating itself?
The Question
So do I have to iterate through all of them in order to not delete it? (Remember, there are MANY documents to be updated every 30 mins.) Or is there another solution?
Here's a picture of how my Prices Collection looks (there are 100k docs like these, I just want to update the price property):
Update:
I have re-written my update function a bit and now it looks like this:
const update = (cb = null) => {
Price.remove({}, (err, remove) => {
if (err) {
return logger.error(`Error removing items...`);
}
logger.info(`Removed all items... Beginning to update.`);
_request(urls, (responses) => {
let url, response, gameid;
for (url in responses) {
gameid = url.split('/')[5].split('?')[0];
response = responses[url];
if (response.err) {
logger.error(`Error in request to ${url}: ${err}`);
return;
}
if (response.body) {
logger.info(`Request to ${url} successful.`)
let jsonResult = {};
try {
jsonResult = JSON.parse(response.body);
} catch (e) {
logger.error(`Could not parse.`);
}
logger.info(`Response body for ${gameid} is ${Object.keys(jsonResult).length}.`);
let allItemsArray = Object.keys(jsonResult).map((key, index) => {
return {
game_id: gameid,
market_hash_name: key,
price: jsonResult[key]
}
});
let bulk = Price.collection.initializeUnorderedBulkOp();
allItemsArray.forEach(item => {
bulk.find({market_hash_name: item.market_hash_name})
.upsert().updateOne(item);
});
bulk.execute((err, bulkers) => {
if (err) {
return logger.error(`Error bulking: ${e}`);
}
logger.info(`Updated Items for ${gameid}`)
});
// Price.insertMany(allItemsArray).then(docs => {
// logger.info(`Saved docs for ${gameid}`)
// }, (e) => {
// logger.error(`Error saving docs.`);
// });
}
}
if (cb && typeof cb == 'function') {
cb();
}
})
});
}
Notice the bulk variable now (Thanks #Rahul) but now, the collection takes ages to update. My processor is burning up and it literally takes 3+ minutes to update 60k+ documents. I honestly feel like the previous method, while it might delete all of them and then reinserting them, it also takes 10x faster.
Anyone?

From my experience (updating millions of mongo docs on a hourly basis), here's a realistic approach to very large bulk updates:
do all your API calls separately and write results in as bson into a file
invoke mongoimport and import that bson file into a new empty collection prices_new. Javascript, let alone high-level OO wrappers, are just too slow for that
rename prices_new -> prices dropTarget=true (this will be atomic hence no downtime)
Schematically, it would look like this in JS
let fname = '/tmp/data.bson';
let apiUrls = [...];
async function doRequest(url) {
// perform a request and return an array of records
}
let responses = await Promise.all(apiUrls.map(doRequest));
// if the data too big to fit in memory, use streams instead of this:
let data = flatMap(responses, BSON.serialize).join('\n'));
await fs.writeFile(fname, data);
await child_process.exec(`mongoimport --collection prices_new --drop ${fname}`);
await db.prices_new.renameCollection('prices', true);

There's no need to clear the database and do a fresh insert. You can use the bulkWrite() method for this or use the updateMany() method to do the updates.
You can refactor the existing code to
const update = (cb) => {
_request(urls, responses => {
let bulkUpdateOps = [], gameid;
responses.forEach(url => {
let response = responses[url];
gameid = url.split('/')[5].split('?')[0];
if (response.err) {
logger.error(`Error in request to ${url}: ${response.err}`);
return;
}
if (response.body) {
logger.info(`Request to ${url} successful.`)
let jsonResult = {};
try {
jsonResult = JSON.parse(response.body);
} catch (e) {
logger.error(`Could not parse.`);
}
Object.keys(jsonResult).forEach(key => {
bulkUpdateOps.push({
"updateOne": {
"filter": { market_hash_name: key },
"update": { "$set": {
game_id: gameid,
price: jsonResult[key]
} },
"upsert": true
}
});
});
}
if (bulkUpdateOps.length === 1000) {
Price.bulkWrite(bulkUpdateOps).then(result => {
logger.info(`Updated Items`)
}).catch(e => logger.error(`Error bulking: ${e}`));
bulkUpdateOps = [];
}
});
if (bulkUpdateOps.length > 0) {
Price.bulkWrite(bulkUpdateOps).then(result => {
logger.info(`Updated Items`)
}).catch(e => logger.error(`Error bulking: ${e}`));
}
});
if (cb && typeof cb == 'function') {
cb();
}
}

I have not tested anything but you can try this, might be helpful. I am using bluebird library for concurrency.
let _request = (url) => {
return new Promise((resolve, reject) => {
request(url, (err, response, body) => {
if (err) {
reject(err);
}
resolve(body);
});
});
};
const formatRespose = async (response) => {
// do stuff
return {
query: {}, // itemid: id,
body: {}
};
}
const bulkUpsert = (allItemsArray) => {
let bulk = Price.collection.initializeUnorderedBulkOp();
return new Promise((resolve, reject) => {
allItemsArray.forEach(item => {
bulk.find(item.query).upsert().updateOne(item.body);
});
bulk.execute((err, bulkers) => {
if (err) {
return reject(err);
}
return resolve(bulkers);
});
});
}
const getAndUpdateData = async (urls) => {
const allItemsArray = urls.map((url) => {
const requestData = await _request(url); // you can make this also parallel
const formattedData = formatRespose(requestData); // return {query: {},body: {} };
return formattedData;
});
return await (bulkUpsert(allItemsArray));
};
function update() {
// split urls into as per your need 100/1000
var i, j, chunkUrls = [],
chunk = 100;
for (i = 0, j = urls.length; i < j; i += chunk) {
chunkUrls.push(getAndUpdateData(urls.slice(i, i + chunk)));
}
Bluebird.map(chunkUrls, function (chunk) {
return await chunk;
}, {
concurrency: 1 // depends on concurrent request change 1 = 100 request get and insert in db at time
}).then(function () {
console.log("done");
}).catch(function () {
console.log("error");
});
}

In Node.js, an asynchronous recursive function is resolving empty promises

I'm pulling information from an API that has the following response format:
{
items: [{}, {}, {}],
nextPage: {
startIndex: 11
}
}
so I wrote this program that checks if there's a nextPage property, and I make a subsequent request to the API with an offset = startIndex. Here's my code:
Serp.prototype.search = function(query, start, serps) {
let deferred = this.q.defer();
let url = '';
if (start === 0) {
url = `${GCS_BASE}/?key=${this.key}&cx=${this.cx}&q=${query}`;
} else {
url = `${GCS_BASE}/?key=${this.key}&cx=${this.cx}&q=${query}&start=${start}`;
}
this.https.get(url, (res) => {
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk;
});
res.on('end', () => {
let contactInfo = [];
let result = JSON.parse(rawData);
let totalResults = result.searchInformation.totalResults;
// if total results are zero, return nothing.
if (totalResults === 0) {
serps.push(contactInfo);
deferred.resolve(serps);
// there's just one page of results.
} else if (totalResults <= 10) {
contactInfo = this._extractContactInfo(result.items, query.toLowerCase());
serps.push(contactInfo);
deferred.resolve(serps);
// if there are more than 10, then page through the response.
} else if ((totalResults > 10) && (result.queries.hasOwnProperty('nextPage'))) {
// recursively and asynchronously pull 100 results.
if (result.queries.nextPage[0].startIndex < 91) {
contactInfo = this._extractContactInfo(result.items, query.toLowerCase());
serps.push(contactInfo);
this.search(query, result.queries.nextPage[0].startIndex, serps)
.then(() => {
deferred.resolve();
});
} else {
contactInfo = this._extractContactInfo(result.items, query.toLowerCase());
serps.push(contactInfo);
let res = this.flatten(serps);
deferred.resolve(res);
}
}
});
});
return deferred.promise;
};
That part of the code works just fine, the problem arises when I'm trying to call that search function I wrote this way:
let promises = keywords.map((keyword) => {
return Serps.search(keyword, startIndex, serps);
});
q.allSettled(promises)
.then((results) => {
console.log(results); // [ { state: 'fulfilled', value: undefined } ]
}
My problem is that the promises are being fulfilled, but the value is undefined.
So what am I doing wrong, and how can I fix it?

I solved the problem by not returning an empty promise here:
this.search(query, result.queries.nextPage[0].startIndex, serps)
.then(() => {
deferred.resolve(serps);
});
I still need to flatten the results, so maybe there's a smarter solution, so far that works perfectly.