I'm very exited to be working on a nodeJS REST API project that provides top sales data about the largest ecommerce giant in Latin America (MercadoLibre) by webscraping the heck out of it with Cheerio which is like jQuery for the backend
GitHub repo
Project on VS Code Online
like many others they use a non specific number of nested categories to organize their products, and im trying to map the hole category tree automatically to get back every category's "most sold" list to then save it and provide it as an API.
My problem is that the recuring functions don't seem await fetching with Axios but let me explain how it works first so you can follow along
The first thing I did was build a sort of kickstart function, that would map the main category tree to then call the recursive functions to iterate over every (n)*sub-categories.... like the one down below
const categoriasURL = 'https://www.mercadolibre.com.ar/categorias#nav-header'
async function checkCategories() {
try {
let response = await axios(categoriasURL);
let html = response.data
let $ = cheerio.load(html)
let categories = {}
let subcategories = {}
$('.categories__container', html).each( async function () {
let categoryName = lodash.kebabCase(lodash.deburr($(this).find('.categories__title a').text()))
$(this).children().find('.categories__item').each(async function(){
let subcategory = lodash.kebabCase(lodash.deburr($(this).find('a').text()))
let subcatList
try {
let subcategoryFetch = await axios($(this).find('a').attr('href'));
console.log('CAT ' + $(this).find('a').attr('href'))
subcatList = await checkSubcategories (subcategoryFetch)
} catch (error) {
console.log("error en checkCategories")
}
})
})
} catch (error) {
console.log(error)
}
}
after maping the main categories the function above calls checkSubcategories (subcategoryFetch) to check if there are more subcategories to be checked and if so it will call fetchSubcat(url) to fetch them and call checkSubcategories (subcategoryFetch) function again and so on and so forth
checkSubcategories (fetched):
async function checkSubcategories (fetched){
let html = fetched.data
let subcatList = {}
$ = cheerio.load(html)
if ($('.ui-search-filter-dl .ui-search-filter-dt-title:contains("Categorías")').length) {
$('.ui-search-filter-dl .ui-search-filter-dt-title:contains("Categorías")').parent().children().find('.ui-search-filter-container').each( async function(){
let subcategoryName = lodash.kebabCase(lodash.deburr($(this).find('.ui-search-filter-name').text()))
try {
let url = await $(this).find('a').attr('href')
let fetchedData = await fetchSubcat(url)
//
} catch (error) {
console.log("error en checkSubcategories")
}
})
} else {
return {moreSubcat: false, url:"url"}
}
}
fetchSubcat(url):
async function fetchSubcat(url){
try {
let subcategoryFetch = await axios(url);
let subcatList
subcatList = await checkSubcategories (subcategoryFetch)
return subcatList
} catch (error) {
console.log('error en fetchSubcat')
}
}
Related
I am developping an app to order food online. As backend service I am using firestore to store the data and files. The user can order dishes and there are limited stocks. So every time a user order a dish and create a basket I update the stock of the corresponding ordered dishes. I am using a firebase function in order to perform this action. To be honest it is the first I am creating firebase function.
Into the Basket object, there is a list of ordered Dishes with the corresponding database DishID. When the basket is created, I go through the DishID list and I update the Quantity in the firestore database. On my local emulator it works perfectly and very fast. But online it takes minutes to perform the first update. I can deal with some seconds. Even if it takes a few seconds (like for cold restart) it's okay. But sometimes it can take 3 minutes and someone else can order a dish during this time.
Here is my code:
//Update the dishes quantities when a basket is created
exports.updateDishesQuantity = functions.firestore.document('/Baskets/{documentId}').onCreate(async (snap, context) => {
try{
//Get the created basket
const originalBasket = snap.data();
originalBasket.OrderedDishes.forEach(async dish => {
const doc = await db.collection('Dishes').doc(dish.DishID);
console.log('Doc created');
return docRef = doc.get()
.then((result) =>{
console.log('DocRef created');
if(result.exists){
console.log('Result exists');
const dishAvailableOnDataBase = result.data().Available;
console.log('Data created');
const newQuantity = { Available: Math.max(dishAvailableOnDataBase - dish.Quantity, 0)};
console.log('Online doc updated');
return result.ref.set(newQuantity, { merge: true });
}else{
console.log("doc doesnt exist");
}
})
.catch(error =>{
console.log(error);
return null;
});
});
}catch(error){
console.log(error);
}
});
I have a couple of logs output to debug the outputs on the server. It's the doc.get() function that takes 2 minutes to execute as you can see on the logger below:
Firebase logger
Thanks for your help,
Thansk for your help. I just edited a little bit your code to make it work. I post my edited code. Thanks a lot, now it takes just 4 seconds to update the quantities.
Kid regards
//Update the dishes quantities when a basket is created
exports.updateDishesQuantity = functions.firestore.document('/Baskets/{documentId}').onCreate(async (snap, context) => {
try {
//Get the created basket
const originalBasket = snap.data();
const promises = [];
const quantities = [];
originalBasket.OrderedDishes.forEach(dish => {
promises.push(db.collection('Dishes').doc(dish.DishID).get());
quantities.push(dish.Quantity);
});
const docSnapshotsArray = await Promise.all(promises);
console.log("Promises", promises);
const promises1 = [];
var i = 0;
docSnapshotsArray.forEach(result => {
if (result.exists) {
const dishAvailableOnDataBase = result.data().Available;
const newQuantity = { Available: Math.max(dishAvailableOnDataBase - quantities[i], 0) };
promises1.push(result.ref.set(newQuantity, { merge: true }));
}
i++;
})
return Promise.all(promises1)
} catch (error) {
console.log(error);
return null;
}
});
You should not use async/await within a forEach() loop, see "JavaScript: async/await with forEach()" and "Using async/await with a forEach loop".
And since your code executes, in parallel, a variable number of calls to the asynchronous Firebase get() and set() methods, you should use Promise.all().
You should refactor your Cloud Function along the following lines:
//Update the dishes quantities when a basket is created
exports.updateDishesQuantity = functions.firestore.document('/Baskets/{documentId}').onCreate(async (snap, context) => {
try {
//Get the created basket
const originalBasket = snap.data();
const promises = [];
originalBasket.OrderedDishes.forEach(dish => {
promises.push(db.collection('Dishes').doc(dish.DishID).get());
});
const docSnapshotsArray = await Promise.all(promises);
const promises1 = [];
docSnapshotsArray.forEach(snap => {
if (result.exists) {
const dishAvailableOnDataBase = result.data().Available;
const newQuantity = { Available: Math.max(dishAvailableOnDataBase - dish.Quantity, 0) };
promises1.push(result.ref.set(newQuantity, { merge: true }));
}
})
return Promise.all(promises1)
} catch (error) {
console.log(error);
return null;
}
});
Note that instead of looping and calling push() you could use the map() method for a much concise code. However, for SO answers, I like the clarity brought by creating an empty array, populating it with a forEach() loop and passing it to Promise.all()...
Also note that since you are updating quantities in a basket you may need to use a Transaction.
I am pulling data from a Google Sheet and after some help, I have reached the point where the discord bot is responding to the inquiry, with the correct data, but it responds with each row of the map array in separate responses rather than gathering the data and sending it in one message. So I believe I need to use an async function and the await feature. I just can't seem to figure out where or how to put it in. I've been stuck on this for a few hours now and can't seem to get it to work?
Here is my code:
const { google } = require('googleapis');
const { sheets } = require('googleapis/build/src/apis/sheets');
const keys = require('../Data/client_secret.json');
const { Team, DiscordAPIError } = require('discord.js');
const Command = require("../Structures/Command");
const gclient = new google.auth.JWT(
keys.client_email,
null,
keys.private_key,
['https://www.googleapis.com/auth/spreadsheets']
);
module.exports = new Command({
name: "freeagents",
description: "Potential free agents for next season.",
async run(message, args, client) {
gclient.authorize(function(err,tokens){
if(err){
console.log(err);
return;
} else {
console.log("Connected!");
gsrun(gclient);
}
});
async function gsrun(cl){
const gsapi = google.sheets({version:'v4', auth: cl });
gsapi.spreadsheets.values.get({
spreadsheetId: "11e5nFk50pDztDLngwTSmossJaNXNAGOaLqaGDEwrbQM",
range: 'Keepers!C1:F',
}, (err, res) => {
if (err) return console.log('The API returned an error: ' + err);
const rows = res.data.values;
if (rows.length) {
const cells = rows.filter(cell => cell[3])
cells.map((cell) => {
console.log(`${cell[0]}, ${cell[1]}`);
console.log(`${cells}`);
return message.reply(`${cell[0]}, ${cell[1]}`);
});
} else {
console.log('No data found.');
}
})
};
}
});
Any help would be greatly appreciated!
You're receiving multiple messages because you're sending the message inside cells.map((cell) => {}). That method will call a function on each member of the array. You should instead iterate over the cells array to produce a single string that you can then send as a message, for example:
const strings = [];
for (let cell of cells) {
strings.push(`${cell[0]}, ${cell[1]}`);
}
return message.reply(strings.join("\n"));
This is my first time using Cloud Functions. I'm trying to make a simple call to access all the businesses stored in my Firestore collection, but when I try to log the results, I always get an empty array.
All things w/ Firebase/store are set up properly, collection name is listed properly, and have confirmed access to the database by logging db. Is there something obviously wrong with my code here? Thanks!
// The Cloud Functions for Firebase SDK to create Cloud Functions and setup triggers.
const functions = require('firebase-functions');
// The Firebase Admin SDK to access Firestore.
const admin = require('firebase-admin');
admin.initializeApp();
exports.updateBusinessData = functions.https.onRequest((request, response) => {
const db = admin.firestore()
const businessesReference = db.collection('businesses')
var businesses = []
const getBusinesses = async () => {
const businessData = await businessesReference.get()
businesses = [...businessData.docs.map(doc => ({...doc.data()}))]
for (let business in businesses) {
console.log(business)
}
response.send("Businesses Updated")
}
getBusinesses()
});
I tweaked the way you were processing the docs and I'm getting proper data from firestore.
exports.updateBusinessData = functions.https.onRequest((request, response) => {
const db = admin.firestore();
const businessesReference = db.collection("businesses");
const businesses = [];
const getBusinesses = async () => {
const businessData = await businessesReference.get();
businessData.forEach((item)=> {
businesses.push({
id: item.id,
...item.data(),
});
});
// used for of instead of in
for (const business of businesses) {
console.log(business);
}
response.send(businesses);
};
getBusinesses();
});
Your getBusinesses function is async: you then need to call it with await. Then, since you use await in the Cloud Function you need to declare it async.
The following should do the trick (untested):
exports.updateBusinessData = functions.https.onRequest(async (request, response) => {
try {
const db = admin.firestore()
const businessesReference = db.collection('businesses')
var businesses = []
const getBusinesses = async () => {
const businessData = await businessesReference.get()
businesses = businessData.docs.map(doc => doc.data());
for (let business in businesses) {
console.log(business)
}
}
await getBusinesses();
// Send back the response only when all the asynchronous
// work is done => This is why we use await above
response.send("Businesses Updated")
} catch (error) {
response.status(500).send(error);
}
});
You are probably going to update docs in the for (let business in businesses) loop to use await. Change it to a for … of loop instead as follows:
for (const business of businesses) {
await db.collection('businesses').doc(...business...).update(...);
}
Update following the comments
Can you try with this one and share what you get from the console.logs?
exports.updateBusinessData = functions.https.onRequest(async (request, response) => {
try {
console.log("Function started");
const db = admin.firestore()
const businessesReference = db.collection('businesses');
const businessData = await businessesReference.get();
console.log("Snapshot size = " + businessData.size);
const businesses = businessData.docs.map(doc => doc.data());
for (const business of businesses) {
console.log(business);
}
response.send("Businesses Updated")
} catch (error) {
console.log(error);
response.status(500).send(error);
}
});
I have a function which checks url address and returns true if that address is valid or no.
I need to write tests to that using Jest. To complete this task I wrote testcase which located in the bottom of the document. At first it threw regenerator-runtime error, but I fixed that with according import. But then it started to throw that error. To fix it, I tried to import fetch library, but error didn't solved. Despite that error function in my app working normally. How to fix that error?
ReferenceError: fetch is not defined
const fetch = require('node-fetch');
test("Testing valid product", async ()=>{
const result = await valid('#products/1');
expect(result).toBe(true);
});
My function
//FUNCTION WITH VALIDATION
export async function valid(path){
//GETTING GROUP OF THE PRODUCTS
let group = path.substr(path.indexOf('#')+1,path.indexOf('/')-1);
//GET ID OF ITEM OF THE GROUP
let url = path.substr(path.indexOf('/')+1);
if(group=='products'){
//CHECKING IF ITEM WITH THAT ID EXISTS
let items = await fetch('https://my-json-server.typicode.com/ValeryDrozd/Valerydrozd.github.io/products').then(res => res.json());
for(let i=0;i<items.length;i++){
if(String(items[i]['id'])==url)return true;
}
return false;
}
if(group=='promos'){
let items = await fetch('https://my-json-server.typicode.com/ValeryDrozd/Valerydrozd.github.io/promos').then(res => res.json());
for(let i=0;i<items.length;i++){
if(String(items[i]['id'])==url)return true;
}
return false;
}
if(group=='order'){
let orders = JSON.parse(localStorage.getItem('orders'));
if(orders==null)return false;
if(orders['orderids'].indexOf(url)==-1)return false;
return true;
}
return false;
}
My jest.config.js file
module.exports = {
collectCoverage: true,
transform: { '\\.js$': 'babel-jest', },
};
I think you could resolve this issue by either using jest-fetch-mock or create a mock fetch on your own before your test as following:
// Mock at global level
global.fetch = jest.fn(() =>
Promise.resolve({
json: () => Promise.resolve({/* whatever you want to ressolve */}),
})
);
test("Testing valid product", async ()=>{
const result = await valid('#products/1');
expect(result).toBe(true);
});
This code searches for a company then searches for all of the websites listed in an array on that company, then searches for all the conversations on that website, then searches for all the messages for each conversation, then sends those arrays of message ObjectIDs to the helper function which then returns an array of JSON data for each message. Fhew.. that was a mouthful.
I need to somehow wait for all of this to complete before cleaning it up a bit then res.send'ing it. All of the code works and the console.log(messagesWithData) posts a few arrays of messages (since it is sent a few in this scenario).
All help is appreciated :)
Company.findOne({ 'roles.admins': userId }, function (err, doc) {
if (!err) {
for (const item of doc.websites) {
Website.findById(item, function (err, doc) {
for (const item of doc.conversations) {
Conversation.findById(item, function (err, doc) {
async function findMessageData() {
var messagesWithData = await helper.takeMessageArray(
doc.messages
);
await sendMessages(messagesWithData);
}
findMessageData();
async function sendMessages(messagesWithData) {
// not sure what to put here!
console.log(messagesWithData)
}
});
}
});
}
} else {
res.send(err);
}
});
Code above can be simplified a bit with async/await
const company = await Company.findOne({ 'roles.admins': userId });
let allMessages = []
for (const websiteId of company.websites) {
const website = await Website.findById(websiteId);
for (const conversationId of website.conversations) {
const conversation = await Conversation.findById(conversationId);
const messagesWithData = await helper.takeMessageArray(
conversation.messages
);
allMessages = [...allMessages, ...messagesWithData]
}
}
// Everything completed, messages stored in one place...
console.log(allMessages)