This question already has answers here:
Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference
(7 answers)
Closed 7 years ago.
I'm working on a Node / Mongoose / Express app and I'm having problem getting an array to update while using forEach. I'm not 100% sure what I'm missing. Does this have something to do with synchronous vs asynchronous? I may also just have the return wrong.
router.route('/:slug/episodes')
.get(function(req, res) {
var episodeArray = [];
var episodeDetails = null;
Show.findOne({ 'slug': req.params.slug }, function(err, show) {
if (err) {
res.send(err);
}
var episodes = show.episodes
episodes.forEach(function(episodeID, index) {
Episode.findById(episodeID, function(err, episode) {
if (err) {
res.send(err);
}
episodeArray.push(episode);
});
});
res.send(episodeArray)
});
});
episodeArray isn't getting the episode added in and ends up just being full of null values.
Your code is suffering from a misunderstanding of how async works. First and foremost you should read the Felix Kling link that #elclanrs posted here. SO contributors tend to get tired of answering the same async question over and over. I haven't quite gotten that tired yet, so I'll bite for the purposes of explaining async, but I'll also suggest another option that might be a much better way to solve your problem.
The async solution: There are many ways to await an array of asynchronous operations to complete. async.queue is a popular choice. How it works is you push a set of pending operations into a queue and then tell that queue to wait until all results have been received, at which point you perform your res.send(). The code would look something like this:
var async = require('async');
Show.findOne({
'slug': req.params.slug
}, function(err, show) {
if (err) {
res.send(err);
}
var episodeArray = [];
var queue = async.queue(function(episodeID, callback) {
Episode.findById(episodeID, function(err, episode) {
if (err) {
throw err;
}
episodeArray.push(episode);
callback();
});
});
// Note that forEach is synchronous.
// The tasks will be pushed to the queue before drain()
episodes.forEach(function(episodeID, index) {
queue.push(episodeId);
});
queue.drain = function() {
res.send(episodeArray);
};
});
This is not the best way to solve your problem, this is only for the purpose of demonstrating how you could fix your existing code.
As long as your episodes array is not obscenely huge, then a far better way to query your episodes might be to use mongoDB's $in operator, like so:
Show.findOne({
'slug': req.params.slug
}, function(err, show) {
if (err) {
res.send(err);
}
Episode.find({
_id: {
$in: show.episodes
}
}, function(err, episodes) {
if (err) {
throw err;
}
res.send(episodes);
});
});
Edit:
If you want to go a bit deeper, I should also mention that mongoose supports promises, which you can use to make your code less nested and repetitive, like this:
Show.findOne({
slug: req.params.slug
})
.then(function(show) {
return Episode.find({
_id: {
$in: show.episodes
}
});
})
.then(function(episodes) {
res.send(episodes);
})
// Any errors returned above will short circuit to the middleware next() here
.error(next);
You're not waiting for the asynchronous operations to finish before sending an array back to the client.
Try something like this:
var togo = episodes.length;
var error;
episodes.forEach(function(episodeID, index) {
Episode.findById(episodeID, function(err, episode) {
if (err) {
if (!error)
error = err;
} else
episodeArray.push(episode);
if (--togo === 0)
res.send(error ? error : episodeArray);
});
});
Additionally you should really add a return; if you encounter an error during an async operation, in order to prevent the rest of the code from being executed.
Related
I'm very new to javascript in general and I'm doing a school project that builds a simple lambda function to save some data to DynamoDB from an HTTP request.
First, I had this version:
exports.handler = async (event) => {
var params = {
TableName: 'graph',
ReturnConsumedCapacity: "TOTAL",
Item: null
};
for (let input of event.inputs) {
params.Item = {
'key' : input.key,
'value' : input.value
};
await DynamoDB.DocumentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data);
}
});
}
};
To my shallow understanding, an async function like the handler should wait for any await statement to finish executing, but I got nothing in the database. But even for some reasons if the lambda function execution didn't wait for DynamoDB.DocumentClient.put() to finish, wouldn't the DynamoDB.DocumentClient.put() finish on its own after the lambda has returned?
Then I followed some other people's examples and added promise() to the end:
exports.handler = async (event) => {
var params = {
TableName: 'graph',
ReturnConsumedCapacity: "TOTAL",
Item: null
};
for (let input of event.inputs) {
params.Item = {
'key' : input.key,
'value' : input.value
};
await DynamoDB.DocumentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data);
}
}).promise();
}
};
Now even though it successfully put data into the database, the log shows 6 'Success' messages where I only put 3 pairs of key-value pairs. After some playing around, I realize the 'Success' messages appear once in the first callback, twice in the second, and three times in the third. I have no clues when it behaves like this at all. Can someone shed some light on this, please? Thank you
You're mixing callbacks and promises. Callbacks were the preferred way to deal with async code (and still is preferred by some i guess), until promises were introduced.
It's not recommended to mix these two for the same purpose.
This would be more correct as you're using the built-in 'then' method of the Promise class:
await DynamoDB.DocumentClient.put(params)
.promise()
.then((data) => {
console.log(data);
})
.catch((err) => {
console.log(err);
});
It doesn't break anything using the callback as well though. Here is a really good answer that explains promises:
Aren't promises just callbacks?
When it comes to it logging 6 times, that doesn't make any sense if the array passed to the loop only has 3 items. If you could provide the output in your original post, I'll see if i can make sense of it and give you an updated answer.
This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 4 years ago.
I'm creating a Node project cause i want to knwo how to use mongoose but i'm facing a problem right now. I'm getting an undefined variable while returing a collection from my MongoDB database. This is my code:
app.js
...
case 'index-user':
let indexUserResponse = userController.index();
console.log(indexUserResponse); // undefined
break;
...
user.js (controller)
...
const index = () => {
User.find({}, function(err, res) {
if (err) throw err;
return res;
});
}
...
I know that there is a user on the database and i can see it if add a console.log(user) just before the return res; so... What is happening here? I don't get it.
The issue is that userController.index() is an calling an async function.
You should use a callback or a promise in order to fetch the data from you db.
Promise example:
Here we are returning a promise from the the executing the query on the Users collection.
const index = () => User.find({}).exec();
And in app.js:
case 'index-user':
userController.index().then(result => {
console.log(result); // as you use find there will be a list of user documents
});
break;
Or, you can use good old callbacks as so:
const index = (callback) => {
User.find({}, function(err, res) {
if (err) throw err;
return callback(res);
});
}
And in app.js:
case 'index-user':
let indexUserResponse = userController.index((result) => {
console.log(result);
});
break;
When you execute it the way you do now, the console.log() runs before the index() method can execute the find() callback.
Hope it's clear enough and helps
I'm making on app for camps where user can come and create their camping experience and comments over it. I try to remove first if any camps there in mongodb, after that to make 3 dummy camps data and then associate comments on it. but it seems always all 3 camps creating first and then comments because of that comments can't be associated with them.
Campground.remove({}, function (err) {
if (err) {
console.log('some error in campground');
}
campdata.forEach(function (seed) {
Campground.create(seed, function (err, createdData) {
if (err) {
console.log('camps not created');
} else {
// create comments
Comment.create({
description: 'this is the best place but wish if there is internet',
author: 'satty'
}, function (err, commentdata) {
if (err) {
console.log(err);
} else {
createdData.comments.push(commentdata);
createdData.save();
console.log(commentdata);
}
});
console.log(createdData);
} //else completed
}); // campground create completed
}); // for each
console.log('removed campgrounds');
}); // campground remove
Remember that Node is asynchronous. forEach runs synchronously, but the functions within are asynchronous — meaning that they are still executing after the forEach loop completes. This is a problem for you because the iterator on forEach has already reached the last element in the array long before the asynchronous comment-adding function executes.
One way to solve this is to use async:
(Removed superfluous code for brevity)
let async = require('async')
Campground.remove({}, function(err) {
async.each(campdata, function(seed, callback) {
Campground.create(seed, function(err, createdData) {
let comment = {
description: 'this is the best place but wish if there is internet',
author: 'satty'
}
Comment.create(comment, function(err, commentdata) {
createdData.comments.push(commentdata)
createdData.save()
callback(err)
})
})
}, function(err) {
// all done!
})
})
I'm currently writing a simple api where you post an array (length = 200) and since each element in the array needs to do 1-2 look up requests, I'm using the async library to control the flow of things. I'm using node 0.12.5 & Express.
router.post('/data', function(req, res, next) {
var cloudDB = db.cloudant.use('events');
var tempStorage = {"docs": []};
// This each loop is to make sure all events get iterated through before submitting response
async.each(req.body, function(singleEvent, loopCallback) {
// this should be async waterfall or something better to organize it
async.waterfall(
[
function(callback) { // get user data from db
db.getUserInfo(singleEvent.email, function (error, dbResponse) {
if(error) { // Houston, we have a problem
return callback(error);
}
return callback(null, dbResponse);
})
},
function(dbResponse, callback) { // decide what to do about results
if(!dbResponse) { // we were unable to get the user from DB
db.searchForUser(singleEvent.email, function (err, searchResponse) {
if(err)
return callback(err);
else
return callback(null, JSON.parse(searchResponse));
})
}
else {
return callback(null,JSON.parse(dbResponse));
}
},
function(userInfo, callback) { // combine data into proper logic
callback(null,combineEventAndUserData(singleEvent,userInfo));
}
],
function (err, result) {
// User event has been processed, so if there are no errors, lets add it to the queue
if(err) {
console.log(err);
}
else {
tempStorage.docs.push(result);
}
loopCallback(); // We're done with this singleEvent
}
)
}, function(err) { // function gets called when all singleEvents have been looped through
console.log("Finished each");
if(err) {
res.status(500).send(err);
}
else {
cloudDB.bulk(tempStorage, function(err, body) {
if(!err) {
res.status(200).send(body);
}
else {
res.status(500).send(err);
}
})
}
});
});
So, this code works! However... (sniff sniff), I seem to have created a memory leak. I have taken a look at both memwatch-next and heapdump, and all I've been able to tell was that 'arrays' keep growing when I look at the heap dump.
I don't know why, but I have a suspicion that this might have something to do with closures and how I'm storing the items generated from each of the waterfalls and perhaps the tempStorage.docs is not being released? Am I storing the tempStorage in the correct way? Or should I change how I do that?
Ive been playing around with mongodb in node.js. I have made a basic collection with some data (i know its there ive checked). When I try to run a find() on the collection it returns undefined. I dont know why this is. The code is below:
function get_accounts(){
var MongoClient = mongodb.MongoClient;
var url = "url";
MongoClient.connect(url, function (err, db) {
if (err) {
console.log('Unable to connect to the mongoDB server. Error:', err);
} else {
//HURRAY!! We are connected. :)
console.log('Connection established to database');
var collection = db.collection('accounts');
collection.find().toArray(function(err, docs) {
console.log("Printing docs from Array")
docs.forEach(function(doc) {
console.log("Doc from Array ");
console.dir(doc);
});
});
console.log("mission complete");
}
db.close();
}
);
}
If you know why this is happening i would like to hear your thoughts. thanks! The database is a mongolab hosted database if that makes any difference.
You are getting an undefined value because of the asynchronous nature of node.js, nowhere in your code exists logic that tells the console.log statement to wait until the find() statement finishes before it prints out the documents. You have to understand the concept of callbacks in Node.js. There are a few problems here, though, that you could fix. A lot of people getting started with node have the tendency to nest lots of anonymous functions, creating the dreaded "pyramid of doom" or callback hell. By breaking out some functions and naming them, you can make it a lot cleaner and easier to follow:
var MongoClient = require("mongodb").MongoClient
// move connecting to mongo logic into a function to avoid the "pyramid of doom"
function getConnection(cb) {
MongoClient.connect("your-mongo-url", function(err, db) {
if (err) return cb(err);
var accounts = db.collection("accounts");
cb(null, accounts);
})
}
// list all of the documents by passing an empty selector.
// This returns a 'cursor' which allows you to walk through the documents
function readAll(collection, cb) {
collection.find({}, cb);
}
function printAccount(account) {
// make sure you found your account!
if (!account) {
console.log("Couldn't find the account you asked for!");
}
console.log("Account from Array "+ account);
}
// the each method allows you to walk through the result set,
// notice the callback, as every time the callback
// is called, there is another chance of an error
function printAccounts(accounts, cb) {
accounts.each(function(err, account) {
if (err) return cb(err);
printAccount(account);
});
}
function get_accounts(cb) {
getConnection(function(err, collection) {
if (err) return cb(err);
// need to make sure to close the database, otherwise the process
// won't stop
function processAccounts(err, accounts) {
if (err) return cb(err);
// the callback to each is called for every result,
// once it returns a null, you know
// the result set is done
accounts.each(function(err, account) {
if (err) return cb(err)
if (hero) {
printAccount(account);
} else {
collection.db.close();
cb();
}
})
}
readAll(collection, processAccounts);
})
}
// Call the get_accounts function
get_accounts(function(err) {
if (err) {
console.log("had an error!", err);
process.exit(1);
}
});
You might have to add an empty JSON object inside the find.
collection.find({})
Documentation can be found here.
You must enter this code in an async function and you will be fine here data is the your desired value and you must use promises to not make your code look messy.
var accountCollection = db.collection('accounts);
let data = await accountCollection.find().toArray.then(data=>data).catch(err=>err);