Hope for your help.
I have collection tasks with document like this schema.
Task = {
title:'taskName',
performers:[ {userId:1,price:230}, {userId:2,price:260} ]
}
Profiles = { id:1, name: 'Alex', surname: 'Robinson', etc.. }
Finally, I shoul collect all data, and in response return Array of profiles objects. Problem is that for-loop end before finished all .findOne() for every elements, and it return empty Array.
This is code form get.
CODE HERE:
apiRoutes.get('/performers/:id', function(req,res,next){
var profArr = [];
Task.findOne({'_id':req.params.id},function(err, doc){
for(var i = 0; i<doc.performers.length; i++){
var profile = {
price: 0,
name: '',
surname: ''
};
profile.price = doc.performers[i].price;
Profile.findOne({'_id':doc.performers[i].userId},function(err,doc){
if (err) throw err;
profile.name = doc.name;
profile.surname = doc.surname;
profArr.push(profile);
});
}
return res.json({success:true,
message:'Performers data collected',
data:profArr});
});
The problem is you need to return response inside mongoose query. You can't use any values assigned inside the query outside. For example :
var sampleArr = [];
Users.find({}, function(err, users) {
users.forEach(function(user) {
Students.find({'userid' : user.id}, function(err, student) {
sampleArr.push({
'student' : student
});
})
console.log(sampleArr);
// It will only return empty array[];
})
})
So, your task should be like this:
apiRoutes.get('/performers/:id', function(req,res,next){
var profArr = [];
// Get a task by ID
Task.findById(req.params.id, function (err, task) {
// Get all profiles
Profile.find({}, function (err, profiles) {
task.performers.forEach(function(taskPerformer) {
profiles.forEach(function(profile) {
// Check the performer ID is the same with userID or not
if (profile._id == taskPerformer.userId) {
profArr.push({
price: taskPerformer.price,
name: profile.name,
surname: profile.surname
});
}
})
});
return res.json({
success:true,
message:'Performers data collected',
data:profArr
});
});
})
A simple idea would be to introduce a countdown-counter before you start your for-loop like this:
var countdown = doc.performers.length;
Decrement the countdown in the callback function of the findOne-Calls. Check if you have reached 0 and call a function outside to send the result.
But still your code doesn't look very efficient. There are a lot of calls to the db. Maybe you could rethink your datamodel in order to minimize the calls to the db.
Your "for" loop will be finished before findOne will be finished.
Related
So I have two for loops, and one is nested inside another but the results they return seem to be running the first loop and returning its results than the nested loop. How could I make it run in a synchronous behavior?
For example, all the topicData gets printed in a row instead of printing one topicData and moving on to the nested for loop.
I'm not sure if this is the proper way to implement the async await. Any pointers would be appreciated. Thanks
exports.create = (event, context, callback) => {
var topicTimestamp = "";
var endpoint = "";
sns.listTopics(async function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
console.log(data);
for (var topic in data.Topics){ //first loop
//var topicData = "";
//retrieve each topic and append to topicList if it is lakeview topic
var topicData = await data.Topics[topic].TopicArn;
topicTimestamp = topicData.slice(22, 34); //get only the topic createdAt
var params = {
TopicArn: topicData //topicData
};
console.log("SUBS per" + params.TopicArn);
//retrieve subscriptions attached to each topic
sns.listSubscriptionsByTopic(params, async function(err, subscriptionData){
console.log(subscriptionData);
//console.log("SUBS per" + params.TopicArn);
if (err) {
console.log(err, err.stack); // an error occurred
} else {
var endpointList = [];
for (var sub in subscriptionData.Subscriptions) { //nested loop
endpoint = await subscriptionData.Subscriptions[sub].Endpoint;
console.log("ENDPOINT:: " + endpoint);
endpointList.push(endpoint);
}
} // end of else listSub
//put topic info into table
var topicsParams = {
TableName: tableName,
Item: {
id: uuidv4(),
createdAt: timestamp,
topicCreatedAt: topicTimestamp,
topic: topicData,
phoneNumbers: endpointList
},
};
endpointList = []; //reset to empty array
dynamoDb.put(topicsParams, (error) => {...}
There are couple of issues here
You are trying to do callback style code in loops while you have promise methods available.
You could also do things in parallel using promise.all
Because of callback style the code is very complicated
You are awaiting where it is not required. For example in the callback
You can try to use this way
exports.create = async (event, context, callback) => {
try {
let topicTimestamp = "";
let endpoint = "";
const data = await sns.listTopics().promise();
// eslint-disable-next-line guard-for-in
for (const topic in data.Topics) { // first loop
// var topicData = "";
// retrieve each topic and append to topicList if it is lakeview topic
const topicData = data.Topics[topic].TopicArn;
topicTimestamp = topicData.slice(22, 34); // get only the topic createdAt
const params = {
"TopicArn": topicData // topicData
};
console.log(`SUBS per${ params.TopicArn}`);
const subscriptionData = await sns.listSubscriptionsByTopic(params).promise();
const endpointList = [];
// eslint-disable-next-line guard-for-in
for (const sub in subscriptionData.Subscriptions) { // nested loop
endpoint = subscriptionData.Subscriptions[sub].Endpoint;
console.log(`ENDPOINT:: ${ endpoint}`);
endpointList.push(endpoint);
}
// put topic info into table
const topicsParams = {
"TableName": tableName,
"Item": {
"id": uuidv4(),
"createdAt": timestamp,
"topicCreatedAt": topicTimestamp,
"topic": topicData,
"phoneNumbers": endpointList
}
};
// Similarly use dynamodb .promise functions here
}
} catch (Err) {
console.log(Err);
}
};
aws-sdk by default supports callback style. To convert them to promise you need to add .promise() at end.
At the moment this example is using for loop but you could do the same thing using Promise.all as well.
Hope this helps.
I'm trying to push a value inside a const but its in a .then and it's not working do you know how can I do that ?
I get a value in my console.log(newResult) in my if but the data is not pushed in my const newResult in the return
res.status(200).json(newResult);
.then(function (friends) {
if (friends) {
const newResult = [];
friends.forEach((r) => {
if (r.UserID == userFound.id) {
models.User.findOne({
where: {
id: r.idFriend
}
})
.then(function(userFound) {
newResult.push({
id: r.id,
user: {
id: r.User.id,
email: userFound.email,
username: userFound.username
}
});
console.log(newResult)
})
} else
newResult.push({
id: r.id,
user: {
id: r.User.id,
email: r.User.email,
username: r.User.username
}
});
console.log(newResult)
});
res.status(200).json(newResult);
}
}
every test realised return an empty tab when i go in my if condition
It will never work because, you are doing async calls
models.User.findOne inside forEach.
You'll get results on console.log when async call to database for fetching user is complete.
But before this all happens the forEach is done executing and code hits the line res.status(200).json(newResult); and you see no results from your if condition.
Instead of using this approach go for mongoose populate and populate userObject based userID while finding friends this way you won't have to do async call inside the forEach.
Read about mongoose populate at: http://mongoosejs.com/docs/populate.html
app.get('/calculatePrice', function(req, res) {
let toyIds = req.query.id,
qty = req.query.qty,
totalPrice = 0,
subtotal = 0,
result = {"items": [], "totalPrice": totalPrice},
item = {};
for (let i = 0; i < toyIds.length; i++) {
Toy.findOne({id: toyIds[0]}, { id: 1, price: 1, _id: 0 }, function(err, toy) {
if (toy) {
item[item] = toy.id; //doesnt work
item[qty] = qty[0]; //doesnt work
item[subtotal] = (toy.price * Number(qty[0])); //doesnt work
totalPrice += item[subtotal]; //doesnt work
}
});
result.items.push(item);
}
res.json(result);
});
I am trying to find specific toys inside mongodb by their respective ToyIds provided by the url "/calculateprice?id[0]=1234&qty[0]=2&id[1]=1235&qty[1]=1&id[2]=1236&qty[2]=5".
However, after it finds one, I cant seem to use the data to add values to my "item" object properties.
I have done some research and I am guessing it has to do with Toy.findOne being async but I really don't understand and can't seem to figure out a solution myself. Help and explaination to a noob learning node/mongodb would be much appreciated!
You can replace loop + findOne with find and then you need to move res.json(result) inside a callback to make it run when your async process is done:
app.get('/calculatePrice', function(req, res) {
let toyIds = req.query.id;
let result = { items: [] };
Toy.find({id: {$in: toyIds}}, function(err, toys) {
if(err) {
res.status(500).json({error: true});
}
if (toys) {
// process the result
// push toys data to result.items
}
res.json(result);
});
});
The data size in telemetry table is HUGE. So, I get "JavaScript heap out of memory" error.
How do I overcome that error?
const aloUrl = `mongodb://${userName}:${pwd}#${host}:${port}/${dbName}`;
MongoClient.connect(aloUrl, function(err, client) {
if (err) {
return console.log('ERROR:: ', err);
}
console.log("INFO:: OK");
const db = client.db(dbName);
var arr = db.collection('endpoint').find({provider:"KMR"}).map(e => e._id).toArray((err, result) => {
if (err){
console.log("ERROR", err)
}
var son = db.collection('telemetry').find({endpoint: {$in: result}}).toArray().then(arr =>{
console.log("Let's start to party")
for (let i = 0; i < 10; i++) {
console.log("\t" + arr[i]._id)
}
}).catch(e => {
console.log(`ERROR::${e}`)
})
})
});
From the mongodb docs,
The toArray() method returns an array that contains all the documents
from a cursor. The method iterates completely the cursor, loading all
the documents into RAM and exhausting the cursor.
Thus instead of calling toArray, you should use the next or forEach (or some other method which doesn't load everything at once to RAM), to iterate through the elements one by one.
For example, to print all the documents in your telemetry collection ONE BY ONE, you can do this,
db.collection('telemetry')
.find({
endpoint: {
$in: result
}
})
.forEach((document) => {
console.log(document)
});
I would suggest you to use forEach instead of toArrayin order to fetch and load w/o exhaustion.
For huge data it's always advised to stream (it's achieved by cursor in mongo).
also
$lookup is new in MongoDB 3.2. It performs a left outer join to an unsharded collection in the same database to filter in documents from the “joined” collection for processing.
You can have a look at aggregation pipeline for mongo
updating your code with aggregate.
var MongoClient = require('mongodb').MongoClient;
// Connection URL
const aloUrl = `mongodb://${userName}:${pwd}#${host}:${port}/${dbName}`;
MongoClient.connect(aloUrl, function (err, client) {
console.log("INFO:: OK");
const db = client.db(dbName);
const col = db.collection('endpoint');
var cursor = col.aggregate([
{
$match: {provider: "KMR"}
},
{
$lookup:
{
from : "telemetry",
localField : "_id",
foreignField: "endpoint",
as : "telemetry"
}
}
]);
console.log("Let's start to party")
cursor.on('data', function (data) {
console.log("\t" + data._id)
});
cursor.on('end', function () {
console.log("Done ");
});
});
In below my code, I try to append the populate in mainQuery object. If I pass only types, then I get the expected result and query also build. If I pass both types and sub category, then query was broken and object is not added as per expected query bellow. Kindly give solution for this ?
mainQuery = Category.find();
if(req.param('types')) {
searchKey = "types";
typesObj.where = {
id: 1
};
mainQuery.populate(searchKey, typesObj);
}
if(req.param('subcat')) {
searchkeySubCat = "subcategory";
typesSubcat.where = {
id: 1
};
mainQuery.populate(searchkeySubCat, typesSubcat);
}
mainQuery.exec(function (err, category) {
});
Expected Query as below
Category.find().populate('types', {where: {id:1}}).populate('subcategory', {where: {id:1}}).exec(function (err, res) {
console.log(res)
})
Try as below:
function _getWhereClause(key, value) {
return {where: {key: value}};
}
if (req.param('types')) {
populateKey = 'types';
mainQuery.populate(populateKey, _getWhereClause('id', 1));
}
// Do the same for any number of populate you want
At last execute the query:
mainQuery.exec(function (err, category) {
});
Check the documentation for more understanding- http://sailsjs.org/documentation/reference/waterline-orm/queries/populate
What are you tryin to is basically method chaining with mainQuery object . So basically you are forming one query and later on use that query . I will suggest that you make this query as a string .
for eg.
mainQuery = "Category.find().";
mainQuery+= "populate("+searchKey+","+ typesObj+").";
mainQuery +="populate("+searchkeySubCat+","+ typesSubcat+").";
mainQuery +="exec(function (err, res) { console.log(res)});";
So when you will access mainQuery you will have :
Category.find().populate('types', {where: {id:1}}).populate('subcategory', {where: {id:1}}).exec(function (err, res) {
console.log(res)
});