This code searches for a company then searches for all of the websites listed in an array on that company, then searches for all the conversations on that website, then searches for all the messages for each conversation, then sends those arrays of message ObjectIDs to the helper function which then returns an array of JSON data for each message. Fhew.. that was a mouthful.
I need to somehow wait for all of this to complete before cleaning it up a bit then res.send'ing it. All of the code works and the console.log(messagesWithData) posts a few arrays of messages (since it is sent a few in this scenario).
All help is appreciated :)
Company.findOne({ 'roles.admins': userId }, function (err, doc) {
if (!err) {
for (const item of doc.websites) {
Website.findById(item, function (err, doc) {
for (const item of doc.conversations) {
Conversation.findById(item, function (err, doc) {
async function findMessageData() {
var messagesWithData = await helper.takeMessageArray(
doc.messages
);
await sendMessages(messagesWithData);
}
findMessageData();
async function sendMessages(messagesWithData) {
// not sure what to put here!
console.log(messagesWithData)
}
});
}
});
}
} else {
res.send(err);
}
});
Code above can be simplified a bit with async/await
const company = await Company.findOne({ 'roles.admins': userId });
let allMessages = []
for (const websiteId of company.websites) {
const website = await Website.findById(websiteId);
for (const conversationId of website.conversations) {
const conversation = await Conversation.findById(conversationId);
const messagesWithData = await helper.takeMessageArray(
conversation.messages
);
allMessages = [...allMessages, ...messagesWithData]
}
}
// Everything completed, messages stored in one place...
console.log(allMessages)
Related
I'm very exited to be working on a nodeJS REST API project that provides top sales data about the largest ecommerce giant in Latin America (MercadoLibre) by webscraping the heck out of it with Cheerio which is like jQuery for the backend
GitHub repo
Project on VS Code Online
like many others they use a non specific number of nested categories to organize their products, and im trying to map the hole category tree automatically to get back every category's "most sold" list to then save it and provide it as an API.
My problem is that the recuring functions don't seem await fetching with Axios but let me explain how it works first so you can follow along
The first thing I did was build a sort of kickstart function, that would map the main category tree to then call the recursive functions to iterate over every (n)*sub-categories.... like the one down below
const categoriasURL = 'https://www.mercadolibre.com.ar/categorias#nav-header'
async function checkCategories() {
try {
let response = await axios(categoriasURL);
let html = response.data
let $ = cheerio.load(html)
let categories = {}
let subcategories = {}
$('.categories__container', html).each( async function () {
let categoryName = lodash.kebabCase(lodash.deburr($(this).find('.categories__title a').text()))
$(this).children().find('.categories__item').each(async function(){
let subcategory = lodash.kebabCase(lodash.deburr($(this).find('a').text()))
let subcatList
try {
let subcategoryFetch = await axios($(this).find('a').attr('href'));
console.log('CAT ' + $(this).find('a').attr('href'))
subcatList = await checkSubcategories (subcategoryFetch)
} catch (error) {
console.log("error en checkCategories")
}
})
})
} catch (error) {
console.log(error)
}
}
after maping the main categories the function above calls checkSubcategories (subcategoryFetch) to check if there are more subcategories to be checked and if so it will call fetchSubcat(url) to fetch them and call checkSubcategories (subcategoryFetch) function again and so on and so forth
checkSubcategories (fetched):
async function checkSubcategories (fetched){
let html = fetched.data
let subcatList = {}
$ = cheerio.load(html)
if ($('.ui-search-filter-dl .ui-search-filter-dt-title:contains("CategorÃas")').length) {
$('.ui-search-filter-dl .ui-search-filter-dt-title:contains("CategorÃas")').parent().children().find('.ui-search-filter-container').each( async function(){
let subcategoryName = lodash.kebabCase(lodash.deburr($(this).find('.ui-search-filter-name').text()))
try {
let url = await $(this).find('a').attr('href')
let fetchedData = await fetchSubcat(url)
//
} catch (error) {
console.log("error en checkSubcategories")
}
})
} else {
return {moreSubcat: false, url:"url"}
}
}
fetchSubcat(url):
async function fetchSubcat(url){
try {
let subcategoryFetch = await axios(url);
let subcatList
subcatList = await checkSubcategories (subcategoryFetch)
return subcatList
} catch (error) {
console.log('error en fetchSubcat')
}
}
I am pulling data from a Google Sheet and after some help, I have reached the point where the discord bot is responding to the inquiry, with the correct data, but it responds with each row of the map array in separate responses rather than gathering the data and sending it in one message. So I believe I need to use an async function and the await feature. I just can't seem to figure out where or how to put it in. I've been stuck on this for a few hours now and can't seem to get it to work?
Here is my code:
const { google } = require('googleapis');
const { sheets } = require('googleapis/build/src/apis/sheets');
const keys = require('../Data/client_secret.json');
const { Team, DiscordAPIError } = require('discord.js');
const Command = require("../Structures/Command");
const gclient = new google.auth.JWT(
keys.client_email,
null,
keys.private_key,
['https://www.googleapis.com/auth/spreadsheets']
);
module.exports = new Command({
name: "freeagents",
description: "Potential free agents for next season.",
async run(message, args, client) {
gclient.authorize(function(err,tokens){
if(err){
console.log(err);
return;
} else {
console.log("Connected!");
gsrun(gclient);
}
});
async function gsrun(cl){
const gsapi = google.sheets({version:'v4', auth: cl });
gsapi.spreadsheets.values.get({
spreadsheetId: "11e5nFk50pDztDLngwTSmossJaNXNAGOaLqaGDEwrbQM",
range: 'Keepers!C1:F',
}, (err, res) => {
if (err) return console.log('The API returned an error: ' + err);
const rows = res.data.values;
if (rows.length) {
const cells = rows.filter(cell => cell[3])
cells.map((cell) => {
console.log(`${cell[0]}, ${cell[1]}`);
console.log(`${cells}`);
return message.reply(`${cell[0]}, ${cell[1]}`);
});
} else {
console.log('No data found.');
}
})
};
}
});
Any help would be greatly appreciated!
You're receiving multiple messages because you're sending the message inside cells.map((cell) => {}). That method will call a function on each member of the array. You should instead iterate over the cells array to produce a single string that you can then send as a message, for example:
const strings = [];
for (let cell of cells) {
strings.push(`${cell[0]}, ${cell[1]}`);
}
return message.reply(strings.join("\n"));
I am trying to retrieve all the documents from a MongoDB cluster. I have followed code I've seen online, however I am facing a small problem.
const MongoClient = require('mongodb');
const uri = "mongodb+srv://<user>:<password>#cluster0-10soy.mongodb.net/test?retryWrites=true&w=majority";
var questionsArray = [];
MongoClient.connect(uri, function (err, client) {
const database = client.db("WhatSportWereYouMadeFor");
database.collection("Questions").find({}, (error, cursor) =>{
cursor.each(function(error, item){
if (item == null){
console.log(error);
}
questionsArray.push(item);
});
})
});
module.exports = { questionsArray };
I connect fine to the database, however I've set a breakpoint at the stop variable and that gets hit before any of the documents retrieved from the database get pushed to the questions array.
I've also tried wrapping the code inside an async function and then awaiting it before the stop variable, but still that breakpoint gets hit first and only after the documents get pushed to the array.
What I would do, this wrap the whole thing into a promise, and the export that.
const MyExport = () => {
return new Promise((resolve, reject) => {
var questionsArray = [];
MongoClient.connect(uri, function (err, client) {
const database = client.db("WhatSportWereYouMadeFor");
database.collection("Questions").find({}, (error, cursor) =>{
cursor.each(function(error, item){
if (item == null){
console.log(error);
}
questionsArray.push(item);
});
resolve(questionsArray)
})
});
})
}
module.exports.questionsArray = MyExport
But then when you import it, you need to run and await it
cosnt questionsArrayFunc = require("path/to/this/file").questionsArray
const questionsArray = await questionsArrayFunc()
I hope this is what you looking for. There might be some other way, but I think this works.
So I have two for loops, and one is nested inside another but the results they return seem to be running the first loop and returning its results than the nested loop. How could I make it run in a synchronous behavior?
For example, all the topicData gets printed in a row instead of printing one topicData and moving on to the nested for loop.
I'm not sure if this is the proper way to implement the async await. Any pointers would be appreciated. Thanks
exports.create = (event, context, callback) => {
var topicTimestamp = "";
var endpoint = "";
sns.listTopics(async function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
console.log(data);
for (var topic in data.Topics){ //first loop
//var topicData = "";
//retrieve each topic and append to topicList if it is lakeview topic
var topicData = await data.Topics[topic].TopicArn;
topicTimestamp = topicData.slice(22, 34); //get only the topic createdAt
var params = {
TopicArn: topicData //topicData
};
console.log("SUBS per" + params.TopicArn);
//retrieve subscriptions attached to each topic
sns.listSubscriptionsByTopic(params, async function(err, subscriptionData){
console.log(subscriptionData);
//console.log("SUBS per" + params.TopicArn);
if (err) {
console.log(err, err.stack); // an error occurred
} else {
var endpointList = [];
for (var sub in subscriptionData.Subscriptions) { //nested loop
endpoint = await subscriptionData.Subscriptions[sub].Endpoint;
console.log("ENDPOINT:: " + endpoint);
endpointList.push(endpoint);
}
} // end of else listSub
//put topic info into table
var topicsParams = {
TableName: tableName,
Item: {
id: uuidv4(),
createdAt: timestamp,
topicCreatedAt: topicTimestamp,
topic: topicData,
phoneNumbers: endpointList
},
};
endpointList = []; //reset to empty array
dynamoDb.put(topicsParams, (error) => {...}
There are couple of issues here
You are trying to do callback style code in loops while you have promise methods available.
You could also do things in parallel using promise.all
Because of callback style the code is very complicated
You are awaiting where it is not required. For example in the callback
You can try to use this way
exports.create = async (event, context, callback) => {
try {
let topicTimestamp = "";
let endpoint = "";
const data = await sns.listTopics().promise();
// eslint-disable-next-line guard-for-in
for (const topic in data.Topics) { // first loop
// var topicData = "";
// retrieve each topic and append to topicList if it is lakeview topic
const topicData = data.Topics[topic].TopicArn;
topicTimestamp = topicData.slice(22, 34); // get only the topic createdAt
const params = {
"TopicArn": topicData // topicData
};
console.log(`SUBS per${ params.TopicArn}`);
const subscriptionData = await sns.listSubscriptionsByTopic(params).promise();
const endpointList = [];
// eslint-disable-next-line guard-for-in
for (const sub in subscriptionData.Subscriptions) { // nested loop
endpoint = subscriptionData.Subscriptions[sub].Endpoint;
console.log(`ENDPOINT:: ${ endpoint}`);
endpointList.push(endpoint);
}
// put topic info into table
const topicsParams = {
"TableName": tableName,
"Item": {
"id": uuidv4(),
"createdAt": timestamp,
"topicCreatedAt": topicTimestamp,
"topic": topicData,
"phoneNumbers": endpointList
}
};
// Similarly use dynamodb .promise functions here
}
} catch (Err) {
console.log(Err);
}
};
aws-sdk by default supports callback style. To convert them to promise you need to add .promise() at end.
At the moment this example is using for loop but you could do the same thing using Promise.all as well.
Hope this helps.
I am trying to create a simple back end blog api with user authentication and authorization. It is built with mongoose and express. In my userSchema, I have a property that is an array called "subscribedTo". Here, users can subscribe to different users to get their blogs. The subscribedTo array stores objectIDs of the users that wished to be subscribed too.
Here is my code:
router.get('/blogs', auth, async (req, res) => {
//auth middleware attaches user to the request obj
try {
let blogs = []
req.user.subscribedTo.forEach(async (id) => {
let ownersBlogs = await Blog.find({owner:id})
blogs = [...blogs, ...ownersBlogs]
console.log(blogs)//consoles desired output of users blogs
})
console.log(blogs)//runs first and returns []
res.send(blogs)
}catch(e){
res.status(500).send(e)
}
})
When I use postman for this route it returns [] which is understandable. I can't seem to res.send(blogs) even though the blogs variable returns correctly in the forEach function.
Is there a better way to do this?
You can use without loop like as bellow
Blog.find({ owner: { $in: req.user.subscribedTo } }, function (err, blogResult) {
if (err) {
response.send(err);
} else {
response.send(blogResult);
}
});
OR
send response after loop completed like as bellow
router.get('/blogs', auth, async (req, res) => {
//auth middleware attaches user to the request obj
try {
let blogs = []
let len = req.user.subscribedTo.length;
let i = 0;
if (len > 0) {
req.user.subscribedTo.forEach(async (id) => {
let ownersBlogs = await Blog.find({ owner: id })
blogs = [...blogs, ...ownersBlogs]
console.log(blogs)//consoles desired output of users blogs
i++;
if (i === len) {
//send response when loop reached at the end
res.send(blogs)
}
})
} else {
res.send(blogs);
}
} catch (e) {
res.status(500).send(e)
}
});
You can find all the documents without a foreach loop, use $in
Blog.find({owner:{$in:[array of ids]}});