Develop Reference

Async/Await function failing

I'm trying to build a nodeJS script that pulls records from an Airtable base, bumps a UPC list up against the [UPC Item DB API][1], writes the product description ("Title") and product image array from the API response to an object, and then updates corresponding Airtable records with the pre-formatted using the Airtable API. I can't link directly to the Airtable API for my base, but the "Update Record" should look like this:
{
record_id: 'myRecord',
fields: {
'Product Description': 'J.R. Watkins Gel Hand Soap, Lemon, 11 oz',
'Reconstituted UPC': '818570001330',
Images: [
'https://images.thdstatic.com/productImages/b3e507dc-2d4a-48d4-a469-51a34c454959/svn/j-r-watkins-hand-soaps-23051-64_1000.jpg',
'http://pics.drugstore.com/prodimg/332476/450.jpg',
]
}
}
var Airtable = require('airtable');
var base = new Airtable({apiKey: 'myKey'}).base('myBase');
var request = require('request');
// Function to slow the code down for easier console watching
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
// Function to slow the code down for easier console watching
async function init(x) {
console.log(1);
await sleep(x*1000);
console.log(2);
}
// Big nasty async function
async function imagesToAirtable() {
///Run through the airtable list
/// create the UPC_list that will be updated and pushed to Airtable to update records
const upc_list = [];
/// Pull from Airtable and assign array to an object
const airtable_records = await base('BRAND')
.select( { maxRecords : 3 })
.all()
/// Troubleshooting console.logs
console.log(airtable_records.length);
console.log("Entering the FOR loop")
/// Loop through the list, append req'd fields to the UPC object, and call the UPCItemDB API
for (var i = 0 ; i< airtable_records.length ; i++) {
/// Push req'd fields to the UPC object
await upc_list.push(
{ record_id : airtable_records[i].get("Record ID"),
fields: {
"Product Description" : "",
"Reconstituted UPC": airtable_records[i].get("Reconstituted UPC"),
"Images": []
}
}
);
/// Troubleshooting console.logs
console.log(upc_list)
console.log("Break");
/// call API
await request.post({
uri: 'https://api.upcitemdb.com/prod/trial/lookup',
headers: {
"Content-Type": "application/json",
"key_type": "3scale"
},
gzip: true,
body: "{ \"upc\": \""+airtable_records[i].get("Reconstituted UPC")+"\" }",
}, /// appending values to upc_list object
function (err, resp, body) {
console.log("This is loop "+ i)
upc_list[i].fields["Images"] = JSON.parse(body).items[0].images
upc_list[i].fields["Product Description"] = JSON.parse(body).items[0].title
console.log(upc_list[i]);
}
)}
};
imagesToAirtable();
I haven't gotten to writing the Airtable "Update Record" piece yet because I can't get the API response written to the upc_list array.
I get an error message on the last run of the FOR loop. In this case, the first and second time through the loop work fine and update the upc_list object, but the third time, I get this error:
upc_list[i].fields["Images"] = JSON.parse(body).items[0].images
^
TypeError: Cannot read property 'fields' of undefined
I know this has to do with async/await, but I'm just not experienced enough at this point to understand what I need to do.
I also know that this big nasty async/await function should be written into individual functions and then called in one single main() function but I can't figure out how to make everything chain together properly with async/await. Tips on that would be welcome as well :)
I have tried separating FOR loop into two FOR loops. The first for the initial append of the upc_list item, and the second for the API call and append with the parsed response.

I was going to skip by this question until I saw this:
I also know that this big nasty async/await function should be written
into individual functions
You are so right about that. Let's do it!
// get records from any base, up to limit
async function getRecords(base, limit) {
return base(base)
.select( { maxRecords : limit })
.all();
}
// return a new UPC object from an airtable brand record
// note - nothing async is being done here
function upcFromBrandRecord(brand) {
return {
record_id: brand.get("Record ID"),
fields: {
"Product Description": "",
"Reconstituted UPC": brand.get("Reconstituted UPC"),
"Images": []
}
};
}
The request module you're using doesn't use promises. There's a promise-using variant, I believe, but without installing anything new, we can "promise-ify" the post method you're using.
async function requestPost(uri, headers, body) {
return new Promise((resolve, reject) => {
request.post({ uri: uri, headers, gzip: true, body },
(err, resp, body) => {
err ? reject(err) : resolve(body)
}
)}
});
}
Now we can write a particular one for your usage...
async function upcLookup(brand) {
const uri = 'https://api.upcitemdb.com/prod/trial/lookup';
const headers = {
"Content-Type": "application/json",
// probably need an api key in here
"key_type": "3scale"
};
const body = JSON.stringify({ upc: brand.get("Reconstituted UPC") });
const responseBody = await requestPost(uri, headers, body);
// not sure if you must parse, but copying the OP
return JSON.parse(responseBody);
}
For a given brand record, build a complete upc record by creating the structure and calling the upc api...
async function brandToUPC(brand) {
const result = upcFromBrandRecord(brand);
const upcData = await upcLookup(brand);
result.fields["Images"] = upcData.items[0].images;
result.fields["Product Description"] = upcData.items[0].title;
return result;
}
Now we have all the tools needed to write the OP function simply...
// big and nasty no more!
async function imagesToAirtable() {
try {
const airtable_records = await getRecords('BRAND', 3);
const promises = airtable_records.map(brandToUPC);
const upc_list = await Promise.all(promises); //edit: forgot await
console.log(upc_list);
} catch (err) {
// handle error here
}
}
That's it. Caveat. I haven't parsed this code, and I know little or nothing about the services you're using, or whether there was a bug hidden underneath the one you've been encountering. So it seems unlikely that this will run out of the box. What I hope I've done is demonstrate the value of decomposition for making nastiness disappear.

What is going wrong with my express call? I need an array of ID's but its returning an empty array

Im guessing this problem is because I don't know how to use async await effectively. I still dont get it and I've been trying to understand for ages. sigh.
Anyway, heres my function:
app.post("/declineTrades", async (request, response) => {
//---------------------------------------------
const batch = db.batch();
const listingID = request.body.listingID;
const tradeOfferQuery = db
//---------------------------------------------
//Get trade offers that contain the item that just sold
//(therefore it cannot be traded anymore, I need to cancel all existing trade offers that contain the item because this item isn't available anymore)
//---------------------------------------------
.collection("tradeOffers")
.where("status", "==", "pending")
.where("itemIds", "array-contains", listingID);
//---------------------------------------------
//Function that gets all trade offers that contain the ID of the item.
async function getIdsToDecline() {
let tempArray = [];
tradeOfferQuery.get().then((querySnapshot) => {
querySnapshot.forEach((doc) => {
//For each trade offer found
let offerRef = db.collection("tradeOffers").doc(doc.id);
//Change the status to declined
batch.update(offerRef, { status: "declined" });
//Get the data from the trade offer because I want to send an email
//to the who just got their trade offer declined.
const offerGet = offerRef.get().then((offer) => {
const offerData = offer.data();
//Check the items that the receiving person had in this trade offer
const receiverItemIds = Array.from(
offerData.receiversItems
.reduce((set, { itemID }) => set.add(itemID), new Set())
.values()
);
//if the receiver item id's array includes this item that just sold, I know that
//I can get the sender ID (users can be sender or receiver, so i need to check which person is which)
if (receiverItemIds.includes(listingID)) {
tempArray.push(offerData.senderID);
}
});
});
});
//With the ID's now pushed, return the tempArray
return tempArray;
}
//---------------------------------------------
//Call the above function to get the ID's of people that got declined
//due to the item no longer being available
const peopleToDeclineArray = await getIdsToDecline();
//Update the trade offer objects to declined
const result = await batch.commit();
//END
response.status(201).send({
success: true,
result: result,
idArray: peopleToDeclineArray,
});
});
Im guessing that my return tempArray is in the wrong place? But I have tried putting it in other places and it still returns an empty array. Is my logic correct here? I need to run the forEach loop and add to the array before the batch.commit happens and before the response is sent.
TIA Guys!

As #jabaa pointed out in their comment, there are problems with an incorrectly chained Promise in your getIdsToDecline function.
Currently the function initializes an array called tempArray, starts executing the trade offer query and then returns the array (which is currently still empty) because the query hasn't finished yet.
While you could throw in await before tradeOfferQuery.get(), this won't solve your problem as it will only wait for the tradeOfferQuery to execute and the batch to be filled with entries, while still not waiting for any of the offerRef.get() calls to be completed to fill the tempArray.
To fix this, we need to make sure that all of the offerRef.get() calls finish first. To get all of these documents, you would use the following code to fetch each document, wait for all of them to complete and then pull out the snapshots:
const itemsToFetch = [ /* ... */ ];
const getAllItemsPromise = Promise.all(
itemsToFetch.map(item => item.get())
);
const fetchedItemSnapshots = await getAllItemsPromise;
For documents based on a query, you'd tweak this to be:
const querySnapshot = /* ... */;
const getSenderDocPromises = [];
querySnapshot.forEach((doc) => {
const senderID = doc.get("senderID");
const senderRef = db.collection("users").doc(senderID);
getSenderDocPromises.push(senderRef.get());
}
const getAllSenderDocPromise = Promise.all(getSenderDocPromises);
const fetchedSenderDataSnapshots = await getAllSenderDocPromise;
However neither of these approaches are necessary, as the document you are requesting using these offerRef.get() calls are already returned in your query so we don't even need to use get() here!
(doc) => {
let offerRef = db.collection("tradeOffers").doc(doc.id);
//Change the status to declined
batch.update(offerRef, { status: "declined" });
//Get the data from the trade offer because I want to send an email
//to the who just got their trade offer declined.
const offerGet = offerRef.get().then((offer) => {
const offerData = offer.data();
//Check the items that the receiving person had in this trade offer
const receiverItemIds = Array.from(
offerData.receiversItems
.reduce((set, { itemID }) => set.add(itemID), new Set())
.values()
);
//if the receiver item id's array includes this item that just sold, I know that
//I can get the sender ID (users can be sender or receiver, so i need to check which person is which)
if (receiverItemIds.includes(listingID)) {
tempArray.push(offerData.senderID);
}
});
}
could be replaced with just
(doc) => {
// Change the status to declined
batch.update(doc.ref, { status: "declined" });
// Fetch the IDs of items that the receiving person had in this trade offer
const receiverItemIds = Array.from(
doc.get("receiversItems") // <-- this is the efficient form of doc.data().receiversItems
.reduce((set, { itemID }) => set.add(itemID), new Set())
.values()
);
// If the received item IDs includes the listed item, add the
// sender's ID to the array
if (receiverItemIds.includes(listingID)) {
tempArray.push(doc.get("senderID"));
}
}
which could be simplified to just
(doc) => {
//Change the status to declined
batch.update(doc.ref, { status: "declined" });
// Check if any items that the receiving person had in this trade offer
// include the listing ID.
const receiversItemsHasListingID = doc.get("receiversItems")
.some(item => item.itemID === listingID);
// If the listing ID was found, add the sender's ID to the array
if (receiversItemsHasListingID) {
tempArray.push(doc.get("senderID"));
}
}
Based on this, getIdsToDecline actually queues declining the invalid trades and returns the IDs of those senders affected. Instead of using the batch and tradeOfferQuery objects that are outside of the function that make this even more unclear, you should roll them into the function and pull it out of the express handler. I'll also rename it to declineInvalidTradesAndReturnAffectedSenders.
async function declineInvalidTradesAndReturnAffectedSenders(listingID) {
const tradeOfferQuery = db
.collection("tradeOffers")
.where("status", "==", "pending")
.where("itemIds", "array-contains", listingID);
const batch = db.batch();
const affectedSenderIDs = [];
const querySnapshot = await tradeOfferQuery.get();
querySnapshot.forEach((offerDoc) => {
batch.update(offerDoc.ref, { status: "declined" });
const receiversItemsHasListingID = offerDoc.get("receiversItems")
.some(item => item.itemID === listingID);
if (receiversItemsHasListingID) {
affectedSenderIDs.push(offerDoc.get("senderID"));
}
}
await batch.commit(); // generally, the return value of this isn't useful
return affectedSenderIDs;
}
This then would change your route handler to:
app.post("/declineTrades", async (request, response) => {
const listingID = request.body.listingID;
const peopleToDeclineArray = await declineInvalidTradesAndReturnAffectedSenders(listingID);
response.status(201).send({
success: true,
result: result,
idArray: peopleToDeclineArray,
});
});
Then adding the appropriate error handling, swapping out the incorrect use of HTTP 201 Created for HTTP 200 OK, and using json() instead of send(); you now get:
app.post("/declineTrades", async (request, response) => {
try {
const listingID = request.body.listingID;
const affectedSenderIDs = await declineInvalidTradesAndReturnAffectedSenders(listingID);
response.status(200).json({
success: true,
idArray: affectedSenderIDs, // consider renaming to affectedSenderIDs
});
} catch (error) {
console.error(`Failed to decline invalid trades for listing ${listingID}`, error);
if (!response.headersSent) {
response.status(500).json({
success: false,
errorCode: error.code || "unknown"
});
} else {
response.end(); // forcefully end corrupt response
}
}
});
Note: Even after all these changes, you are still missing any form of authentication. Consider swapping the HTTPS Event Function out for a Callable Function where this is handled for you but requires using a Firebase Client SDK.

Can anyone tell me why it is showing array is empty?

I want to send notification to multiple devices and for that am getting the token via querying the docs and saving the token to array but it shows that array is empty. Most probably error is because am not able to add elements in the array.
My code is:-
var registrationTokens=[];
const indexOfSender=usersList.indexOf(chatItem.senderUsername);
let removedUsername=usersList.splice(indexOfSender,1); //to remove the senders name from list
usersList.forEach(async(element)=>{
const query = admin.firestore().collection('users').where("username","==",element);
const querySnapshot = await query.get();
if (querySnapshot.docs.length > 0) {
const doc = querySnapshot.docs[0];
const data = doc.data();
registrationTokens.push(data.androidNotificationToken); //adding token over here
}
else {
console.log("Unable to get token for the username ", element);
}
});
const message =
{
notification: {
title:'Message',
body: body,
imageUrl: url,
},
tokens: registrationTokens,
data: { recipient: senderUserId },
};
admin.messaging().sendMulticast(message)
.then(response =>
{
if (response.failureCount > 0) {
const failedTokens = [];
response.responses.forEach((resp, idx) => {
if (!resp.success) {
failedTokens.push(registrationTokens[idx]);
}
});
console.log('List of tokens that caused failures: ' + failedTokens);
}
else
{
console.log('Successfully sent messages ', response);
}
});
Error
Error: tokens must be a non-empty array
at FirebaseMessagingError.FirebaseError [as constructor] (/workspace/node_modules/firebase-admin/lib/utils/error.js:42:28)
at FirebaseMessagingError.PrefixedFirebaseError [as constructor] (/workspace/node_modules/firebase-admin/lib/utils/error.js:88:28)
at new FirebaseMessagingError (/workspace/node_modules/firebase-admin/lib/utils/error.js:254:16)
at Messaging.sendMulticast (/workspace/node_modules/firebase-admin/lib/messaging/messaging.js:294:19)
at sendNotificationForGroupChat (/workspace/index.js:238:35)
at exports.onCreateMessage.functions.region.firestore.document.onCreate (/workspace/index.js:116:9)
at process._tickCallback (internal/process/next_tick.js:68:7)

async inside forEach does not work the way you expect. If you add some logging, you will see that the loop ends before any of its async work is complete, leaving your tokens array empty before you pass it to FCM. Each iteration through the loop simply generates a promise that is not resolved. You will need to rewrite the code to actually wait for all those promises before calling FCM.
Read more about that:
Using async/await with a forEach loop

for-of loop will work just fine with asynchronous calls :)
Cheers

Asynchronous verification within the .map function

I am developing the backend of an application using Node JS, Sequelize and Postgres database.
When the course is registered, the user must inform which organizations, companies and teachers will be linked to it.
The organization IDs are passed through an array to the backend, I am trying to do a check to make sure that the passed IDs exist.
What I've done so far is this:
const { organizations } = req.body;
const organizationsArray = organizations.map(async (organization) => {
const organizationExists = await Organization.findByPk(organization);
if (!organizationExists) {
return res
.status(400)
.json({ error: `Organization ${organization} does not exists!` });
}
return {
course_id: id,
organization_id: organization,
};
});
await CoursesOrganizations.bulkCreate(organizationsArray);
This link has the complete controller code, I believe it will facilitate understanding.
When !OrganizationExists is true, I am getting the return that the organization does not exist. The problem is when the organization exists, I am getting the following message error.

The Array.map() is returning an array of promises that you can resolve to an array using Promise.all(). Inside the map you should use throw new Error() to break out of the map - this error will be raised by Promise.all() and you can then catch it and return an error to the client (or swallow it, etc).
This is a corrected version of your pattern, resolving the Promise results.
const { organizations } = req.body;
try {
// use Promise.all to resolve the promises returned by the async callback function
const organizationsArray = await Promise.all(
// this will return an array of promises
organizations.map(async (organization) => {
const organizationExists = await Organization.findByPk(organization, {
attributes: ['id'], // we only need the ID
raw: true, // don't need Instances
});
if (!organizationExists) {
// don't send response inside the map, throw an Error to break out
throw new Error(`Organization ${organization} does not exists!`);
}
// it does exist so return/resolve the value for the promise
return {
course_id: id,
organization_id: organization,
};
})
);
// if we get here there were no errors, create the records
await CoursesOrganizations.bulkCreate(organizationsArray);
// return a success to the client
return res.json({ success: true });
} catch (err) {
// there was an error, return it to the client
return res.status(400).json({ error: err.message });
}
This is a refactored version that will be a bit faster by fetching all the Organizations in one query and then doing the checks/creating the Course inserts.
const { Op } = Sequelize;
const { organizations } = req.body;
try {
// get all Organization matches for the IDs
const organizationsArray = await Organization.findAll({
attributes: ['id'], // we only need the ID
where: {
id: {
[Op.in]: organizations, // WHERE id IN (organizations)
}
},
raw: true, // no need to create Instances
});
// create an array of the IDs we found
const foundIds = organizationsArray.map((org) => org.id);
// check to see if any of the IDs are missing from the results
if (foundIds.length !== organizations.length) {
// Use Array.reduce() to figure out which IDs are missing from the results
const missingIds = organizations.reduce((missingIds, orgId) => {
if (!foundIds.includes(orgId)){
missingIds.push(orgId);
}
return missingIds;
}, []); // initialized to empty array
throw new Error(`Unable to find Organization for: ${missingIds.join(', ')}`);
}
// now create an array of courses to create using the foundIds
const courses = foundIds.map((orgId) => {
return {
course_id: id,
organization_id: orgId,
};
});
// if we get here there were no errors, create the records
await CoursesOrganizations.bulkCreate(courses);
// return a success to the client
return res.json({ success: true });
} catch (err) {
// there was an error, return it to the client
return res.status(400).json({ error: err.message });
}

If you have an array of Ids and you want to check if they exist you should you use the (in) operator, this makes it so that you are hitting the DB only once and getting all the records in one hit (instead of getting them one by one in a loop), after you get these records you can check their lengths to determine if they all exist or not.
const { Op } = require("sequelize");
let foundOrgs = await Organization.findAll({
where: {
id: {
[Op.in]: organizationsArray,
}
}
});

Redis ReJSON returns me two errors systematically: missing key at non-terminal path and new object must be created at root

The second error is relatively straighforward to understand.
The first is a bit more challenging.
I have tried different combination to overcome this error but none improvement occurs.
My console returns me:
// console log > first promise { ReplyError: ERR new objects must be
created at the root
at parseError (node_modules/redis-parser/lib/parser.js:193:12)
at parseType (node_modules/redis-parser/lib/parser.js:303:14) command: 'JSON.SET', args: [ 'jsonTest7',
'.user.here.now',
'{".nestedValue": "I am a nested value"}' ], code: 'ERR' } // console log > second promise // console log > jsonTest7 response:
null
Here my snippet.js:
const redis=require("redis");
rejson = require('redis-rejson');
const {promisify} = require('util');
rejson(redis); /* important - this must come BEFORE creating the client */
let client= redis.createClient({
port:6380,
host:'localhost',
});
const setAsync = promisify(client.json_set).bind(client);
const getAsync = promisify(client.json_get).bind(client);
const existsAsync= promisify(client.exists).bind(client);
client.exists('jsonTest2', function(err, reply) {
if (reply === 1) {
return true
} else {
return false
}
});
async function isExist(object){
var isExist= await existsAsync(object).then(data=> data)
.catch((err) => console.error(err));
console.log("isExist: ", typeof isExist)
if(isExist !== 1) {
console.log("creating object...")
await setAsync(object, '.', '{"here":"something"}');
console.log("object created: " + object)
}
}
async function myFunc(object, rootKey) {
console.log("then 1")
await isExist(object)
await setAsync(object, ".user.here.now", '{".nestedValue": "I am a nested value"}')
.catch((err) => console.error(err));
console.log("then 2")
const res = await getAsync(object, '.user.here.now')
.catch((err) => console.error(err));
console.log(object + " response: ", res)
}
myFunc("jsonTest7")
Any hint would be great,
thanks

The first error means that you're trying to create a new document - i.e. the RedisJSON key does not exist - but are only supplying a subdocument (i.e. '.user.here.now'). New keys must be set to the root ('.') level like your current code sample shows.