I am trying to use the lodash forEach method with a nested function that calls a mongo database.
var jobs = [];
_.forEach(ids, function(id) {
JobRequest.findByJobId(id, function(err, result) {
if(err) callback(err);
jobs.push(result);
});
});
callback(null, jobs);
I am having problems because the forEach and callbacks will run through before the inner function is ever called. How can I resolve this?
I want the callback to be called after the for each and inner function have completed.
One more approach is to wrap everything into promises, in this case job results will be pushed into array in correct order:
var promises = ids.map(function(id) {
return new Promise(function(resolve, reject) {
JobRequest.findByJobId(id, function (err, result) {
if (err) reject(err);
resolve(result);
});
});
});
Promise.all(promises).then(function(jobs) {
callback(null, jobs);
}, callback);
// or shorter: Promise.all(promises).then(callback.bind(null, null), callback);
Note, that you also need to handle potential situation when JobRequest.findByJobId request fails, with promises it's very easy: just pass callback as error callback to Promise.all.
JobRequest.findByJobId is an asynchronous operation. You cannot block asynchronous operations in JavaScript, so you'll need to manually synchronize by counting. Example (error handling omitted for the sake of brevity):
var results = [];
var pendingJobCount = ids.length;
_.forEach(ids, function(id) {
JobRequest.findByJobId(id, function(err, result) {
results.push(result);
if (--pendingJobCount === 0) callback(null, results);
});
});
There are, of course, wrapper constructs for doing stuff like this, but I prefer to explain how it actually works. Check out dfsq's answer for more details on one of those wrappers, called promises.
Also note that asynchronous operations may complete out of order. The order in the results array will not necessarily match the order of the ids array. If you need that information connected, you'll need to track it yourself, for example by collecting the results in a map instead of an array:
var results = {};
var pendingJobCount = ids.length;
_.forEach(ids, function(id) {
JobRequest.findByJobId(id, function(err, result) {
results[id] = result;
if (--pendingJobCount === 0) callback(null, results);
});
});
This example assumes that there are no duplicates in your ids array. Results for duplicate keys would be overridden.
Error handling would work similarly, by inserting additional information into your result. Another example:
results.push({id: id, error: null, value: result});
Related
I am listing all files from all directories in /home/myComputer/Desktop/Research, and then filtering them with an if statement to only get the .txt files that I would like to read and store into arrays. All works fine, but pushing the data into the arrays is not functioning. When I console log them, they return no value [].
I tried promise as well as call back function, but they didn't work for me because I didn't know how to implement them properly.
app.get('/jsonData', function(req, res) {
/* Define Arrays */
var theFile = [];
var theCategory = [];
var theContent = [];
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
};
//walk(process.env.HOME, function(err, results) {
walk("/home/myComputer/Desktop/Research", function(err, results) {
if (err) throw err;
//console.log(results);
results.map(function(val) {
//Get the filename
var fileName = val.match(/[^\/]+$/).join();
//Get the category
var category = val.substr(48).match(/[^\/]+/);
if (fileName == 'written-speech.txt') {
console.log('FOUND!: ' + fileName + ' Category: ' + category) //this works
fs.readFile(val, 'utf8', function(err, contents) {
console.log(contents); // this works
theFile.push(fileName);
theCategory.push(category);
theContent.push(contents);
});
}
})
});
console.log(theFile); // The problem: This returns an empty Array []
console.log(theCategory); // The problem: This returns an empty Array []
console.log(theContent); // The problem: This returns an empty Array []
});
I expect console.log(theFile); console.log(theCategory); and console.log(theContent); to return the data pushed in them.
The reason for this is that many callbacks in Javascript are asynchronous, which means both fs.readdir and fs.readFile are asynchronous and their callbacks are not called immediately but slightly later (please read about Event Loop in javascript). So at the moment, when you log your arrays they are empty and data to them will be pushed later, e.g. in future. To avoid this you can either use synchronous methods (fs.readdirSync and fs.readFileSync) which is ugly and can cause performance issues if the app has a lot of other asynchronous operations. If in your case it is just a simple script to read some data, it might be fine.
And the other, preferred way is to use promises or some library for managing callbacks, e.g. async. Please read some articles regarding managing async code if these concepts are fully unfamiliar for you, e.g. https://dev.to/mrm8488/from-callbacks-to-fspromises-to-handle-the-file-system-in-nodejs-56p2 to get a basic understanding and see some use case examples.
Regarding your current version, there is no easy way to make it work without a lot of changes. It is better to rewrite it to use the concepts I described earlier.
walk is an asynchronous function because fs.readdir is an asynchronous method and the console.log statements are running (in a synchronous manner) before the callback of fs.readdir getting invoked.
You can console the values of these variables at the end inside the callback of walk.
I have a for loop and I want to call an async function on each iteration. But I am getting a JS stack trace error. Below is the prototype of my code. I have also used IIFE pattern but it's not working.
for(let i = 0; i<99999;i++){
getData(i, function(err, result){
if(err) return err;
else{
console.log(result);
}
});
}
function getData(number, callback){
request('http://someapiurl'+number, function(err, response){
if(err) callback(err, null);
else{
callback(null, response)
}
})
}
You re trying to make 99999 at the same time, this just won t work on most browsers.
Use promises instead, push them in an array, then use Promise.all to let the browser handle the request, simply handle the final resolved response.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
var p1 = Promise.resolve(3);
var p2 = 1337;
var p3 = new Promise((resolve, reject) => {
setTimeout(resolve, 100, 'foo');
});
Promise.all([p1, p2, p3]).then(values => {
console.log(values); // [3, 1337, "foo"]
});
First of all I highly suggest that you debug your code to see exactly what happens with the for loop.
Its very likely that the loop runs completely before its contents (the getData function) executes, asynchronous behavior of via a fetching data request. If you tried using the IIFE pattern you'd have at least wrapped the contents inside your for loop in a way that guarantees its contents to execute. An example of an implementation looks like so:
for(let i = 0; i<99999;i++){
(function(){ // <-- IIFE used to represent a closure.
getData(i, function(err, result){
if(err) return err;
else{
console.log(result);
}
});
})(i);
}
To let #mika sit on the subject of promises for you, which you can read on their answer; just keep in mind the promise anti-pattern and you'll be fine. It is especially tricky if you are not used to promises and want to use them in a for loop.
I'm new to js.
I am using express for node js, and mongoose as a mongo orm.
function direct_tags_search_in_db(tags){
var final_results = [];
for (var i=0; i<tags.length; ++i) {
var tag = tags[i];
Question.find({tags: tag}).exec(function(err, questions) {
final_results.push(questions);
if (i == tags.length -1 ){
return final_results;
}
});
}
};
I get empty results, because of the asynchronously of the find. But I don't know what the best approach for this.
Appriciate a little help, thanks.
You will often find that methods such as Question.find().exec that accept a function as an argument are async. It is especially common for methods that perform network requests or file system operations. These are most commonly referred to as a callback. That being the case, if you would like something to occur when the async task(s) complete, you need to also implement a callback.
Also, it is possible that your reference to tag is being changed in a way that is likely undesired. There are a number of solutions, here is a simple one.
function direct_tags_search_in_db(tags, callback){
var final_results = [];
// Array.forEach is able to retain the appropriate `tag` reference
tags.forEach(function(tag){
Question.find({tags: tag}).exec(function(err, questions) {
// We should be making sure to handle errors
if (err) {
// Return errors to the requester
callback(err);
} else {
final_results.push(questions);
if (i == tags.length -1 ){
// All done, return the results
callback(null, final_results);
}
}
});
});
};
You will notice that when we implement our own callback, that we follow the same common pattern as the callback for Question.find().exec(function(err, result){}); -- first argument a potential error, second argument the result. That is why when we return the results, we provide null as the first argument callback(null, final_results);
Quick example of calling this function:
direct_tags_search_in_db([1, 2, 3], function(err, results){
if (err) {
console.error('Error!');
console.error(err);
} else {
console.log('Final results');
console.log(results);
}
});
Another option for solving various async goals is the async module, promises, or otherwise.
I have frustrating problem with learning to work with callback style of programming in Node.js. I have a query to a MongoDB database. If I pass in a function to execute on the result it works but I'd rather flatten it out and have it return the value. Any help or direction on how to do this correctly is appreciated. Here's my code:
var getLots = function(response){
db.open(function(err, db){
db.collection('lots', function(err, collection){
collection.find(function(err, cursor){
cursor.toArray(function(err, items){
response(items);
})
})
})
})
}
I want something more like this:
lots = function(){
console.log("Getting lots")
return db.open(openCollection(err, db));
}
openCollection = function(err, db){
console.log("Connected to lots");
return (db.collection('lots',findLots(err, collection))
);
}
findLots = function(err, collection){
console.log("querying 2");
return collection.find(getLots(err, cursor));
}
getLots = function(err, cursor) {
console.log("Getting lots");
return cursor.toArray();
}
Where the final set of data would bubble back up through the function calls.
The problem is that I get an error from Node.js saying that err is not defined or that the collection is not defined. For some reason when I nest the callbacks the correct object is getting passed down. When I try going to this flattened style it complains that things are not defined. I don't know how to get it to pass the necessary objects.
What you need is one of the many control flow libraries available for node via npm and catalogued on the Node.js wiki. My specific recommendation is caolan/async, and you would use the async.waterfall function to accomplish this type of flow where each async operation must be executed in order and each requires the results from the previous operation.
Pseudocode example:
function getLots(db, callback) {
db.collection("lots", callback);
}
function findLots(collection, callback) {
collection.find(callback);
}
function toArray(cursor, callback) {
cursor.toArray(callback);
}
async.waterfall([db.open, getLots, find, toArray], function (err, items) {
//items is the array of results
//Do whatever you need here
response(items);
});
async is a good flow control library. Frame.js offers some specific advantages like better debugging, and better arrangement for synchronous function execution. (though it is not currently in npm like async is)
Here is what it would look like in Frame:
Frame(function(next){
db.open(next);
});
Frame(function(next, err, db){
db.collection('lots', next);
});
Frame(function(next, err, collection){
collection.find(next);
});
Frame(function(next, err, cursor){
cursor.toArray(next);
});
Frame(function(next, err, items){
response(items);
next();
});
Frame.init();
I have an interesting case where I need to do a few queries in MongoDB using Mongoose, but the response is returning before I can complete all of them.
I have two document types, list and item. In one particular call, I need to get all of the lists for a particular user, then iterate over each of them and fetch all of the items and append them to the appropriate list before returning.
List.find({'user_id': req.params.user_id}, function(err, docs){
if (!err) {
if (docs) {
var results = [];
_und.each(docs, function(value, key) {
var list = value.toObject();
list.items = [];
Item.find({'list_id': value._id}, function(err, docs) {
if (!err) {
_und.each(docs, function(value, key) { list.items.push(value.toObject()); });
results.push(list);
}
else {
console.log(err);
}
});
});
res.send(results);
(_und is how I've imported underscore.js)
Obviously the issue are the callbacks, and since there's multiple loops I can't return within a callback.
Perhaps this is a case where I would need to get the count in advance and check it on every iteration to decide when to return the results. This doesn't seem elegant though.
Code solution
First of all the issue is with the code. Your sending the results before the Item.find queries finish. You can fix this quite easily
var count = docs.length + 1;
next()
_und.each(docs, function(value, key) {
var list = value.toObject();
list.items = [];
Item.find({
'list_id': value._id
}, function(err, docs) {
if (!err) {
_und.each(docs, function(value, key) {
list.items.push(value.toObject());
});
// push asynchronous
results.push(list);
next()
}
else {
console.log(err);
}
});
});
function next() {
--count === 0 && finish()
}
function finish() {
res.send(results)
}
The easiest way is reference counting, you default count to the number of documents. Then every time your finished getting an item you call next and decrement the count by one.
Once your finished getting all items your count should be zero. Note that we do .length + 1 and call next immediately. This gaurds against the the case where there are no documents, which would otherwise do nothing.
Database solution
The best solution is to use mongo correctly. You should not be doing what is effectively a join in your code, it's slow and inefficient as hell. You should have a nested document and denormalize your list.
so list.items = [Item, Item, ...]
As a further aside, avoid mongoose, it's inefficient, use the native mongo driver.
I use with this module:
https://github.com/caolan/async