I'm trying to build a nodeJS script that pulls records from an Airtable base, bumps a UPC list up against the [UPC Item DB API][1], writes the product description ("Title") and product image array from the API response to an object, and then updates corresponding Airtable records with the pre-formatted using the Airtable API. I can't link directly to the Airtable API for my base, but the "Update Record" should look like this:
{
record_id: 'myRecord',
fields: {
'Product Description': 'J.R. Watkins Gel Hand Soap, Lemon, 11 oz',
'Reconstituted UPC': '818570001330',
Images: [
'https://images.thdstatic.com/productImages/b3e507dc-2d4a-48d4-a469-51a34c454959/svn/j-r-watkins-hand-soaps-23051-64_1000.jpg',
'http://pics.drugstore.com/prodimg/332476/450.jpg',
]
}
}
var Airtable = require('airtable');
var base = new Airtable({apiKey: 'myKey'}).base('myBase');
var request = require('request');
// Function to slow the code down for easier console watching
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
// Function to slow the code down for easier console watching
async function init(x) {
console.log(1);
await sleep(x*1000);
console.log(2);
}
// Big nasty async function
async function imagesToAirtable() {
///Run through the airtable list
/// create the UPC_list that will be updated and pushed to Airtable to update records
const upc_list = [];
/// Pull from Airtable and assign array to an object
const airtable_records = await base('BRAND')
.select( { maxRecords : 3 })
.all()
/// Troubleshooting console.logs
console.log(airtable_records.length);
console.log("Entering the FOR loop")
/// Loop through the list, append req'd fields to the UPC object, and call the UPCItemDB API
for (var i = 0 ; i< airtable_records.length ; i++) {
/// Push req'd fields to the UPC object
await upc_list.push(
{ record_id : airtable_records[i].get("Record ID"),
fields: {
"Product Description" : "",
"Reconstituted UPC": airtable_records[i].get("Reconstituted UPC"),
"Images": []
}
}
);
/// Troubleshooting console.logs
console.log(upc_list)
console.log("Break");
/// call API
await request.post({
uri: 'https://api.upcitemdb.com/prod/trial/lookup',
headers: {
"Content-Type": "application/json",
"key_type": "3scale"
},
gzip: true,
body: "{ \"upc\": \""+airtable_records[i].get("Reconstituted UPC")+"\" }",
}, /// appending values to upc_list object
function (err, resp, body) {
console.log("This is loop "+ i)
upc_list[i].fields["Images"] = JSON.parse(body).items[0].images
upc_list[i].fields["Product Description"] = JSON.parse(body).items[0].title
console.log(upc_list[i]);
}
)}
};
imagesToAirtable();
I haven't gotten to writing the Airtable "Update Record" piece yet because I can't get the API response written to the upc_list array.
I get an error message on the last run of the FOR loop. In this case, the first and second time through the loop work fine and update the upc_list object, but the third time, I get this error:
upc_list[i].fields["Images"] = JSON.parse(body).items[0].images
^
TypeError: Cannot read property 'fields' of undefined
I know this has to do with async/await, but I'm just not experienced enough at this point to understand what I need to do.
I also know that this big nasty async/await function should be written into individual functions and then called in one single main() function but I can't figure out how to make everything chain together properly with async/await. Tips on that would be welcome as well :)
I have tried separating FOR loop into two FOR loops. The first for the initial append of the upc_list item, and the second for the API call and append with the parsed response.
I was going to skip by this question until I saw this:
I also know that this big nasty async/await function should be written
into individual functions
You are so right about that. Let's do it!
// get records from any base, up to limit
async function getRecords(base, limit) {
return base(base)
.select( { maxRecords : limit })
.all();
}
// return a new UPC object from an airtable brand record
// note - nothing async is being done here
function upcFromBrandRecord(brand) {
return {
record_id: brand.get("Record ID"),
fields: {
"Product Description": "",
"Reconstituted UPC": brand.get("Reconstituted UPC"),
"Images": []
}
};
}
The request module you're using doesn't use promises. There's a promise-using variant, I believe, but without installing anything new, we can "promise-ify" the post method you're using.
async function requestPost(uri, headers, body) {
return new Promise((resolve, reject) => {
request.post({ uri: uri, headers, gzip: true, body },
(err, resp, body) => {
err ? reject(err) : resolve(body)
}
)}
});
}
Now we can write a particular one for your usage...
async function upcLookup(brand) {
const uri = 'https://api.upcitemdb.com/prod/trial/lookup';
const headers = {
"Content-Type": "application/json",
// probably need an api key in here
"key_type": "3scale"
};
const body = JSON.stringify({ upc: brand.get("Reconstituted UPC") });
const responseBody = await requestPost(uri, headers, body);
// not sure if you must parse, but copying the OP
return JSON.parse(responseBody);
}
For a given brand record, build a complete upc record by creating the structure and calling the upc api...
async function brandToUPC(brand) {
const result = upcFromBrandRecord(brand);
const upcData = await upcLookup(brand);
result.fields["Images"] = upcData.items[0].images;
result.fields["Product Description"] = upcData.items[0].title;
return result;
}
Now we have all the tools needed to write the OP function simply...
// big and nasty no more!
async function imagesToAirtable() {
try {
const airtable_records = await getRecords('BRAND', 3);
const promises = airtable_records.map(brandToUPC);
const upc_list = await Promise.all(promises); //edit: forgot await
console.log(upc_list);
} catch (err) {
// handle error here
}
}
That's it. Caveat. I haven't parsed this code, and I know little or nothing about the services you're using, or whether there was a bug hidden underneath the one you've been encountering. So it seems unlikely that this will run out of the box. What I hope I've done is demonstrate the value of decomposition for making nastiness disappear.
Im guessing this problem is because I don't know how to use async await effectively. I still dont get it and I've been trying to understand for ages. sigh.
Anyway, heres my function:
app.post("/declineTrades", async (request, response) => {
//---------------------------------------------
const batch = db.batch();
const listingID = request.body.listingID;
const tradeOfferQuery = db
//---------------------------------------------
//Get trade offers that contain the item that just sold
//(therefore it cannot be traded anymore, I need to cancel all existing trade offers that contain the item because this item isn't available anymore)
//---------------------------------------------
.collection("tradeOffers")
.where("status", "==", "pending")
.where("itemIds", "array-contains", listingID);
//---------------------------------------------
//Function that gets all trade offers that contain the ID of the item.
async function getIdsToDecline() {
let tempArray = [];
tradeOfferQuery.get().then((querySnapshot) => {
querySnapshot.forEach((doc) => {
//For each trade offer found
let offerRef = db.collection("tradeOffers").doc(doc.id);
//Change the status to declined
batch.update(offerRef, { status: "declined" });
//Get the data from the trade offer because I want to send an email
//to the who just got their trade offer declined.
const offerGet = offerRef.get().then((offer) => {
const offerData = offer.data();
//Check the items that the receiving person had in this trade offer
const receiverItemIds = Array.from(
offerData.receiversItems
.reduce((set, { itemID }) => set.add(itemID), new Set())
.values()
);
//if the receiver item id's array includes this item that just sold, I know that
//I can get the sender ID (users can be sender or receiver, so i need to check which person is which)
if (receiverItemIds.includes(listingID)) {
tempArray.push(offerData.senderID);
}
});
});
});
//With the ID's now pushed, return the tempArray
return tempArray;
}
//---------------------------------------------
//Call the above function to get the ID's of people that got declined
//due to the item no longer being available
const peopleToDeclineArray = await getIdsToDecline();
//Update the trade offer objects to declined
const result = await batch.commit();
//END
response.status(201).send({
success: true,
result: result,
idArray: peopleToDeclineArray,
});
});
Im guessing that my return tempArray is in the wrong place? But I have tried putting it in other places and it still returns an empty array. Is my logic correct here? I need to run the forEach loop and add to the array before the batch.commit happens and before the response is sent.
TIA Guys!
As #jabaa pointed out in their comment, there are problems with an incorrectly chained Promise in your getIdsToDecline function.
Currently the function initializes an array called tempArray, starts executing the trade offer query and then returns the array (which is currently still empty) because the query hasn't finished yet.
While you could throw in await before tradeOfferQuery.get(), this won't solve your problem as it will only wait for the tradeOfferQuery to execute and the batch to be filled with entries, while still not waiting for any of the offerRef.get() calls to be completed to fill the tempArray.
To fix this, we need to make sure that all of the offerRef.get() calls finish first. To get all of these documents, you would use the following code to fetch each document, wait for all of them to complete and then pull out the snapshots:
const itemsToFetch = [ /* ... */ ];
const getAllItemsPromise = Promise.all(
itemsToFetch.map(item => item.get())
);
const fetchedItemSnapshots = await getAllItemsPromise;
For documents based on a query, you'd tweak this to be:
const querySnapshot = /* ... */;
const getSenderDocPromises = [];
querySnapshot.forEach((doc) => {
const senderID = doc.get("senderID");
const senderRef = db.collection("users").doc(senderID);
getSenderDocPromises.push(senderRef.get());
}
const getAllSenderDocPromise = Promise.all(getSenderDocPromises);
const fetchedSenderDataSnapshots = await getAllSenderDocPromise;
However neither of these approaches are necessary, as the document you are requesting using these offerRef.get() calls are already returned in your query so we don't even need to use get() here!
(doc) => {
let offerRef = db.collection("tradeOffers").doc(doc.id);
//Change the status to declined
batch.update(offerRef, { status: "declined" });
//Get the data from the trade offer because I want to send an email
//to the who just got their trade offer declined.
const offerGet = offerRef.get().then((offer) => {
const offerData = offer.data();
//Check the items that the receiving person had in this trade offer
const receiverItemIds = Array.from(
offerData.receiversItems
.reduce((set, { itemID }) => set.add(itemID), new Set())
.values()
);
//if the receiver item id's array includes this item that just sold, I know that
//I can get the sender ID (users can be sender or receiver, so i need to check which person is which)
if (receiverItemIds.includes(listingID)) {
tempArray.push(offerData.senderID);
}
});
}
could be replaced with just
(doc) => {
// Change the status to declined
batch.update(doc.ref, { status: "declined" });
// Fetch the IDs of items that the receiving person had in this trade offer
const receiverItemIds = Array.from(
doc.get("receiversItems") // <-- this is the efficient form of doc.data().receiversItems
.reduce((set, { itemID }) => set.add(itemID), new Set())
.values()
);
// If the received item IDs includes the listed item, add the
// sender's ID to the array
if (receiverItemIds.includes(listingID)) {
tempArray.push(doc.get("senderID"));
}
}
which could be simplified to just
(doc) => {
//Change the status to declined
batch.update(doc.ref, { status: "declined" });
// Check if any items that the receiving person had in this trade offer
// include the listing ID.
const receiversItemsHasListingID = doc.get("receiversItems")
.some(item => item.itemID === listingID);
// If the listing ID was found, add the sender's ID to the array
if (receiversItemsHasListingID) {
tempArray.push(doc.get("senderID"));
}
}
Based on this, getIdsToDecline actually queues declining the invalid trades and returns the IDs of those senders affected. Instead of using the batch and tradeOfferQuery objects that are outside of the function that make this even more unclear, you should roll them into the function and pull it out of the express handler. I'll also rename it to declineInvalidTradesAndReturnAffectedSenders.
async function declineInvalidTradesAndReturnAffectedSenders(listingID) {
const tradeOfferQuery = db
.collection("tradeOffers")
.where("status", "==", "pending")
.where("itemIds", "array-contains", listingID);
const batch = db.batch();
const affectedSenderIDs = [];
const querySnapshot = await tradeOfferQuery.get();
querySnapshot.forEach((offerDoc) => {
batch.update(offerDoc.ref, { status: "declined" });
const receiversItemsHasListingID = offerDoc.get("receiversItems")
.some(item => item.itemID === listingID);
if (receiversItemsHasListingID) {
affectedSenderIDs.push(offerDoc.get("senderID"));
}
}
await batch.commit(); // generally, the return value of this isn't useful
return affectedSenderIDs;
}
This then would change your route handler to:
app.post("/declineTrades", async (request, response) => {
const listingID = request.body.listingID;
const peopleToDeclineArray = await declineInvalidTradesAndReturnAffectedSenders(listingID);
response.status(201).send({
success: true,
result: result,
idArray: peopleToDeclineArray,
});
});
Then adding the appropriate error handling, swapping out the incorrect use of HTTP 201 Created for HTTP 200 OK, and using json() instead of send(); you now get:
app.post("/declineTrades", async (request, response) => {
try {
const listingID = request.body.listingID;
const affectedSenderIDs = await declineInvalidTradesAndReturnAffectedSenders(listingID);
response.status(200).json({
success: true,
idArray: affectedSenderIDs, // consider renaming to affectedSenderIDs
});
} catch (error) {
console.error(`Failed to decline invalid trades for listing ${listingID}`, error);
if (!response.headersSent) {
response.status(500).json({
success: false,
errorCode: error.code || "unknown"
});
} else {
response.end(); // forcefully end corrupt response
}
}
});
Note: Even after all these changes, you are still missing any form of authentication. Consider swapping the HTTPS Event Function out for a Callable Function where this is handled for you but requires using a Firebase Client SDK.
I need to display some data in my component, unfortunately the first call to my API returns just part of the information I want to display, plus some IDs. I need another call on those IDs to retrieve other meaningful data. The first call is wrapped in a useEffect() React.js function:
useEffect(() => {
const getData = async () => {
try {
const { data } = await fetchContext.authAxios.get(
'/myapi/' + auth.authState.id
);
setData(data);
} catch (err) {
console.log(err);
}
};
getData();
}, [fetchContext]);
And returns an array of objects, each object representing an appointment for a given Employee, as follows:
[
{
"appointmentID": 1,
"employeeID": 1,
"customerID": 1,
"appointmentTime": "11:30",
"confirmed": true
},
... many more appointments
]
Now I would like to retrieve information about the customer as well, like name, telephone number etc. I tried setting up another method like getData() that would return the piece of information I needed as I looped through the various appointment to display them as rows of a table, but I learned the hard way that functions called in the render methods should not have any side-effects. What is the best approach to make another API call, replacing each "customerID" with an object that stores the ID of the customer + other data?
[Below the approach I've tried, returns an [Object Promise]]
const AppointmentElements = () => {
//Loop through each Appointment to create a single row
var output = Object.values(data).map((i) =>
<Appointment
key={i['appointmentID'].toString()}
employee={i["employeeID"]} //returned a [Object premise]
customer={getEmployeeData((i['doctorID']))} //return a [Object Promise]
time={index['appointmentTime']}
confirmed = {i['confirmed']}
/>
);
return output;
};
As you yourself mentioned functions called in the render methods should not have any side-effects, you shouldn't be calling the getEmployeeData function inside render.
What you can do is, inside the same useEffect and same getData where you are calling the first api, call the second api as well, nested within the first api call and put the complete data in a state variable. Then inside the render method, loop through this complete data instead of the data just from the first api.
Let me know if you need help in calling the second api in getData, I would help you with the code.
Update (added the code)
Your useEffect should look something like:
useEffect(() => {
const getData = async () => {
try {
const { data } = await fetchContext.authAxios.get('/myapi/' + auth.authState.id);
const updatedData = data.map(value => {
const { data } = await fetchContext.authAxios.get('/mySecondApi/?customerId=' + value.customerID);
// please make necessary changes to the api call
return {
...value, // de-structuring
customerID: data
// as you asked customer data should replace the customerID field
}
}
);
setData(updatedData); // this data would contain the other details of customer in it's customerID field, along with all other fields returned by your first api call
} catch (err) {
console.log(err);
}
};
getData();
}, [fetchContext]);
This is assuming that you have an api which accepts only one customer ID at a time.
If you have a better api which accepts a list of customer IDs, then the above code can be modified to:
useEffect(() => {
const getData = async () => {
try {
const { data } = await fetchContext.authAxios.get('/myapi/' + auth.authState.id);
const customerIdList = data.map(value => value.customerID);
// this fetches list of all customer details in one go
const customersDetails = (await fetchContext.authAxios.post('/mySecondApi/', {customerIdList})).data;
// please make necessary changes to the api call
const updatedData = data.map(value => {
// filtering the particular customer's detail and updating the data from first api call
const customerDetails = customersDetails.filter(c => c.customerID === value.customerID)[0];
return {
...value, // de-structuring
customerID: customerDetails
// as you asked customer data should replace the customerID field
}
}
);
setData(updatedData); // this data would contain the other details of customer in it's customerID field, along with all other fields returned by your first api call
} catch (err) {
console.log(err);
}
};
getData();
}, [fetchContext]);
This will reduce the number of network calls and generally preferred way, if your api supports this.
I'm using yahoo weather api for get weather feed of single city, now I want to get weather feed for multiple cities in single request, how can I do using yahoo api. I also want to know, is ther any api yahoo provides to get the list of city of any country.
My weather.js
import OAuth from 'oauth';
const header = {
"X-Yahoo-App-Id": "myappid"
};
const request = new OAuth.OAuth(
null,
null,
'myconsumerkey',
'myconsumersecret',
'1.0',
null,
'HMAC-SHA1',
null,
header
);
request.get('https://weather-ydn-yql.media.yahoo.com/forecastrss?w=713169&format=json', null,null, function (err, data, result) {
if (err) {
console.log(err);
} else {
console.log(data)
}
});
Using this code i can be able to get weather details for only one city i want to fetch weather details for multiple cities at once.
thanx in advance!!
So reading the documentation it doesn't seem possible to send a batch of locations to the Yahoo Weather API. But what you can do is .map() over an array of locations and make multiple requests.
https://developer.yahoo.com/weather/documentation.html#params
Since OAuth 1.0 is a callback, I've wrapped that with a new Promise(), which will give us an array of unfulfilled promises. Then finally, Promise.all() method returns a single Promise that fulfills when all of the promises passed as an iterable have been fulfilled.
const OAuth = require('oauth')
const header = {
'X-Yahoo-App-Id': 'your-app-id',
}
const request = new OAuth.OAuth(null, null, 'your-consumer-key', 'your-consumer-secret', '1.0', null, 'HMAC-SHA1', null, header)
const locations = ['pittsburgh,pa', 'london']
const getWeatherData = () =>
Promise.all(
locations.map(
location =>
new Promise((resolve, reject) =>
request.get(`https://weather-ydn-yql.media.yahoo.com/forecastrss?location=${location}&format=json`, null, null, (err, data) => {
if (err) return reject(err)
return resolve(data)
})
)
)
)
const main = async () => {
const test = await getWeatherData()
console.log(test)
}
main()
I have tested this with the API and here is an example response for the code above.
[
'{"location":{"city":"Pittsburgh","region":" PA","woeid":2473224,"country":"United States","lat":40.431301,"long":-79.980698,"timezone_id":"America/New_York"},"current_observation":{"wind":{"chill":32,"direction":280,"speed":5.59},"atmosphere":{"humidity":70,"visibility":10.0,"pressure":29.03,"rising":0},"astronomy":{"sunrise":"6:42 am","sunset":"7:59 pm"},"condition":{"text":"Partly Cloudy","code":30,"temperature":37},"pubDate":1586862000},"forecasts":[{"day":"Tue","date":1586836800,"low":38,"high":45,"text":"Mostly Cloudy","code":28},{"day":"Wed","date":1586923200,"low":32,"high":47,"text":"Partly Cloudy","code":30},{"day":"Thu","date":1587009600,"low":31,"high":45,"text":"Partly Cloudy","code":30},{"day":"Fri","date":1587096000,"low":35,"high":42,"text":"Rain And Snow","code":5},{"day":"Sat","date":1587182400,"low":35,"high":51,"text":"Scattered Showers","code":39},{"day":"Sun","date":1587268800,"low":42,"high":59,"text":"Rain","code":12},{"day":"Mon","date":1587355200,"low":43,"high":55,"text":"Mostly Cloudy","code":28},{"day":"Tue","date":1587441600,"low":37,"high":58,"text":"Partly Cloudy","code":30},{"day":"Wed","date":1587528000,"low":44,"high":61,"text":"Partly Cloudy","code":30},{"day":"Thu","date":1587614400,"low":50,"high":59,"text":"Mostly Cloudy","code":28}]}',
'{"location":{"city":"London","region":" England","woeid":44418,"country":"United Kingdom","lat":51.506401,"long":-0.12721,"timezone_id":"Europe/London"},"current_observation":{"wind":{"chill":46,"direction":70,"speed":6.84},"atmosphere":{"humidity":50,"visibility":10.0,"pressure":30.27,"rising":0},"astronomy":{"sunrise":"6:04 am","sunset":"7:58 pm"},"condition":{"text":"Mostly Sunny","code":34,"temperature":49},"pubDate":1586862000},"forecasts":[{"day":"Tue","date":1586818800,"low":38,"high":54,"text":"Partly Cloudy","code":30},{"day":"Wed","date":1586905200,"low":34,"high":62,"text":"Mostly Sunny","code":34},{"day":"Thu","date":1586991600,"low":38,"high":68,"text":"Partly Cloudy","code":30},{"day":"Fri","date":1587078000,"low":45,"high":62,"text":"Rain","code":12},{"day":"Sat","date":1587164400,"low":45,"high":60,"text":"Rain","code":12},{"day":"Sun","date":1587250800,"low":42,"high":63,"text":"Partly Cloudy","code":30},{"day":"Mon","date":1587337200,"low":44,"high":64,"text":"Scattered Showers","code":39},{"day":"Tue","date":1587423600,"low":44,"high":66,"text":"Partly Cloudy","code":30},{"day":"Wed","date":1587510000,"low":45,"high":67,"text":"Mostly Cloudy","code":28},{"day":"Thu","date":1587596400,"low":44,"high":65,"text":"Mostly Cloudy","code":28}]}',
]
I'm currently trying to modify my Cloud Functions and move in under https.onRequest so that i can call use it to schedule a cron job. How it i'm getting the following error in the logs.
TypeError: admin.database.ref is not a function
at exports.scheduleSendNotificationMessageJob.functions.https.onRequest (/user_code/index.js:30:20)
at cloudFunction (/user_code/node_modules/firebase-functions/lib/providers/https.js:57:9)
exports.scheduleSendNotificationMessageJob = functions.https.onRequest((req, res) => {
admin.database.ref('/notifications/{studentId}/notifications/{notificationCode}')
.onCreate((dataSnapshot, context) => {
const dbPath = '/notifications/' + context.params.pHumanId + '/fcmCode';
const promise = admin.database().ref(dbPath).once('value').then(function(tokenSnapshot) {
const theToken = tokenSnapshot.val();
res.status(200).send(theToken);
const notificationCode = context.params.pNotificationCode;
const messageData = {notificationCode: notificationCode};
const theMessage = { data: messageData,
notification: { title: 'You have a new job reminder' }
};
const options = { contentAvailable: true,
collapseKey: notificationCode };
const notificationPath = '/notifications/' + context.params.pHumanId + '/notifications/' + notificationCode;
admin.database().ref(notificationPath).remove();
return admin.messaging().sendToDevice(theToken, theMessage, options);
});
return null;
});
});
You cannot use the definition of an onCreate() Realtime Database trigger within the definition of an HTTP Cloud Function.
If you switch to an HTTP Cloud Function "so that (you) can call use it to schedule a cron job" it means the trigger will be the call to the HTTP Cloud Function. In other words you will not be anymore able to trigger an action (or the Cloud Function) when new data is created in the Realtime Database.
What you can very well do is to read the data of the Realtime Database, as follows, for example (simplified scenario of sending a notification):
exports.scheduleSendNotificationMessageJob = functions.https.onRequest((req, res) => {
//get the desired values from the request
const studentId = req.body.studentId;
const notificationCode = req.body.notificationCode;
//Read data with the once() method
admin.database.ref('/notifications/' + studentId + '/notifications/' + notificationCode)
.once('value')
.then(snapshot => {
//Here just an example on how you would get the desired values
//for your notification
const theToken = snapshot.val();
const theMessage = ....
//......
// return the promise returned by the sendToDevice() asynchronous task
return admin.messaging().sendToDevice(theToken, theMessage, options)
})
.then(() => {
//And then send back the result (see video referred to below)
res.send("{ result : 'message sent'}") ;
})
.catch(err => {
//........
});
});
You may watch the following official Firebase video about HTTP Cloud Functions: https://www.youtube.com/watch?v=7IkUgCLr5oA&t=1s&list=PLl-K7zZEsYLkPZHe41m4jfAxUi0JjLgSM&index=3. It shows how to read data from Firestore but the concept of reading and sending back the response (or an error) is the same for the Realtime Database. Together with the 2 other videos of the series (https://firebase.google.com/docs/functions/video-series/?authuser=0), it also explains how it is important to correctly chain promises and to indicate to the platform that the work of the Cloud Function is finished.
For me, this error happened when writing admin.database instead of admin.database().