Question: Why won't a var things return a value from outside the walk() function? And how do I fix it?
Hypothosis: this is async and the console.log is happening too early. Which would lead me to how can I make this a Promise (i'm using node 4.1.1)
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
};
function traverseDirectories() {
var things = walk('src/', function(err, results){
if(err) throw err;
console.log(results) // ['dir/thing.png', 'dir/thing2.png']
return results;
});
console.log(things) // undefined
};
traverseDirectories();
Q. Why won't a var things return a value from outside the walk() function?
R. Because walk doesn't return anything (take a look and you''ll see that it's a void function).
Even if you make it a Promise, you won't be able to use it like:
var things = walk(...);
console.log(things);
Because Promises are thenable, and are still async, so it will be:
walk(...).then(function(things) {
// do something with things here
});
To do what you want, you would need something that doesn't exist in current Javascript yet.
There is an ES7 proposal of native async/await that will be a callback heaven, but atm, you can use:
Async/Await library (It's an amazing library, but very far from native, and performance isn't cool)
ES7 transpiler - you can write the ES7 code today, and it will transpile for you to ES5 (e.g Babel)
But, if you're already using the newest version of NodeJS (4.0.0 as the time of writing) - and if you're not, you really should - the best way of achieving what you want is to use generators.
Combined with a small library named co, it will help you to achieve almost what the ES7 async/await proposes, and it will mostly use native code, so both readability and performance are really good:
var co = require('co');
var traverseDirectories = co(function *traverseDirectories() {
var things = yield walk('src/');
console.log(things) // there we go!
});
function walk(dir, results) {
return new Promise(function(resolve, reject) {
fs.readdir(dir, function(err, list) {
if (err)
reject(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) resolve(results);
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file).then(function(res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
});
}
You can read more about this subject in this awesome Thomas Hunter's blog post.
Related
How can i write this code in a better way.
var fs = require('fs');
var file = '/test.txt';
fs.readFile(file, 'utf8', function (err, txt) {
if (err) return console.log(err);
txt = txt + '\nAppended something!';
fs.writeFile(myFile, txt, function (err) {
if(err) return console.log(err);
console.log('Appended text!');
});
});
Suppose i have multiple callback then how can we prevent this callback of callback and so on....
getData(function(a){
getMoreData(a, function(b){
getMoreData(b, function(c){
getMoreData(c, function(d){
getMoreData(d, function(e){
...
});
});
});
});
});
I really like bluebird for this:
First you have to 'promisify' fs. n the example below they directly promisify the readFile method:
var readFile = Promise.promisify(require("fs").readFile);
readFile("myfile.js", "utf8").then(function(contents) {
return eval(contents);
}).then(function(result) {
console.log("The result of evaluating myfile.js", result);
}).catch(SyntaxError, function(e) {
console.log("File had syntax error", e);
//Catch any other error
}).catch(function(e) {
console.log("Error reading file", e);
});
or:
var fs = Promise.promisifyAll(require("fs"));
// note now you have to put 'async' after the methods like so:
fs.readFileAsync("myfile.js", "utf8").then(function(contents) {
console.log(contents);
}).catch(function(e) {
console.error(e.stack);
});
I suggest async waterfall
Your first snippet would look like following:
var txt;
async.waterfall([
function(callback) {
fs.readFile(file, 'utf8', callback);
},
function(txt, callback) {
txt = txt + '\nAppended something!';
fs.writeFile(myFile, txt, callback);
},
function(callback) {
console.log('Appended text!');
callback();
}
], function (err, result) {
console.log(err)
});
What you're describing is callback hell and there's a couple of smart ways to get around it. I don't claim to be the know it all but there's a whole website called callbackhell.com that you might want to check out.
But for the short answer, you can do these things
1. keep your code shallow, or name your functions
Instead of writing your fs.readFile with an anonymous function, name it and call it like so
fs.readFile(file, 'utf8', function readFileCb(err, txt) {
if (err) throw new Error(err);
txt = txt + '\nAppended something!';
fs.writeFile(myFile, txt, function (err) {
// no need to return a console.log, just throw Error should suffice
if(err) throw new Error(err);
console.log('Appended text!');
});
});
2. Modularize your code. Have named functions or libraries that do exactly one thing
function writeFile(file, txt, cb){
fs.writeFile(file, txt, cb)
}
function writeFileCb(err){
if(err) throw new Error(err);
console.log('Appended Text!');
}
fs.readFile(file, 'utf8', function readFileCb(err, txt) {
if (err) throw new Error(err);
txt = txt + '\nAppended something!';
writeFile(myFile, txt, writeFileCb);
});
3. Ensure that all errors are caught. You seem to be doing that well, so kudos!
You can also use Promises, libraries like Async waterfall, but callbacks are an essential parts of JavaScript and going through callback hell is just a matter of having good sense in writing your code.
I've been pointed towards using the async module, but I'm not quite sure how to use waterfall to solve my problem.
My original code had problems with asynchronicity.
var Image = require('./models/image');
var User = require('./models/user');
var query = Image.find({});
query.limit(10);
query.sort('-date')
query.exec(function (err, collected) {
if (err) return console.error(err);
var i = 0;
var authors = [];
while (i < 8) {
var search = User.find({'twitter.id' : collected[i].author});
search.exec(function (err, user){
if (err) return console.error(err);
var result = (user[0].twitter.username);
authors.push(result);
});
i = i + 1;
}
}
console.log(authors);
I want the authors array to hold all the found usernames. However when that last console.log() call returns '[]'
So, you want to wait for all of the searches to complete first. You should put all your async calls into an array, and then use an async library to chain them together (waterfall) or execute simultaneously (parallel). Parallel tends to execute "faster":
var searches = [];
while (i < 8) {
var search = User.find({'twitter.id' : collected[i].author});
searches.push(function(cb) {
search.exec(function (err, user){
if (err) cb(err, null);
else cb(null, user[0].twitter.username);
});
});
i++;
}
async.parallel(searches, function( err, authors ) {
if ( err ) return console.error( err );
console.log(authors);
// Authors only defined here.
// TODO: More code
});
// Authors not defined here.
I need to resize some images. Problem is the library I'm using do one resize per callback.
I want to resize several images so I put it in a loop:
exports.resizeImages = function (req, res) {
var images = fs.readdirSync('uploads/');
for (var n = 0; n < files.length; n++) {
var tgt = 'uploads/resized/' + images[n];
gm(tgt).resize(150).write(tgt, function (err) {
if (err) {
console.log('resizing failed');
res.status(400).send('failed to resize');
return;
}
if (n == images.length) {
res.status(200).send();
}
});
}
}
I'm aware I can't do it like this. I need make the loop wait until the callback responds somehow. I've seen some examples but I can't get it to work.
Any ideas?
You could also use the node async module
Something like this:
var async = require('async');
exports.resizeImages = function (req, res) {
var images = fs.readdirSync('uploads/');
async.each(images, function(file, callback) {
var tgt = 'uploads/resized/' + file;
gm(tgt).resize(150).write(tgt, callback);
}, function(err) {
if(err) {
console.log('resizing failed');
return res.status(400).send('failed to resize');
} else {
//no error
return res.status(200).send();
}
});
}
You need to write a for loop using promises. Pick your favorite promise library. The details are covered in this question or this question. Or in this blog post:
var Promise = require('bluebird');
var promiseWhile = function(condition, action) {
var resolver = Promise.defer();
var loop = function() {
if (!condition()) return resolver.resolve();
return Promise.cast(action())
.then(loop)
.catch(resolver.reject);
};
process.nextTick(loop);
return resolver.promise;
};
Promises are great for this, indeed. There are other libraries that wrap callbacks into this kind of abstractions, but since Promises are standard it's better to learn just one thing.
If you want to keep it bare, you can use an external counter:
var images = fs.readdirSync('uploads/');
var processed = 0;
for (var n = 0; n < files.length; n++) {
var tgt = 'uploads/resized/' + images[n];
gm(tgt).resize(150).write(tgt, function (err) {
if (err) {
console.log('resizing failed');
res.status(400).send('failed to resize');
return;
}
processed++;
if (processed == images.length) {
res.status(200).send();
}
});
}
That is assuming that you only send 200 OK if all images have been correctly resized.
Like others have mentioned you could use promises, but if you don't want to start using promises in your project async.js is perfect for this kind of thing.
Your code rewritten using async.js:
exports.resizeImages = function (req, res) {
async.waterfall([
function readImages_step(done) {
readdir('uploads/', done);
},
function uploadImages_step(images, done) {
async.each(images, function(image, cb) {
var target = 'uploads/resized/' + image;
gm(target).resize(150).write(target, cb);
}, done);
}
], function (err) {
if (err) {
console.log('resizing failed');
return res.status(400).send('failed to resize');
}
return res.status(200).send();
}
};
I changed your readdirsync call to be asynchronous. Async.each will run each upload in parallel.
I have this chunk of code
User.find({}, function(err, users) {
for (var i = 0; i < users.length; i++) {
pseudocode
Friend.find({
'user': curUser._id
}, function(err, friends) * * ANOTHER CALLBACK * * {
for (var i = 0; i < friends.length; i++) {
pseudocode
}
console.log("HERE I'm CHECKING " + curUser);
if (curUser.websiteaccount != "None") {
request.post({
url: 'blah',
formData: blah
}, function(err, httpResponse, body) { * * ANOTHER CALLBACK * *
pseudocode
sendMail(friendResults, curUser);
});
} else {
pseudocode
sendMail(friendResults, curUser);
}
});
console.log("finished friend");
console.log(friendResults);
sleep.sleep(15);
console.log("finished waiting");
console.log(friendResults);
}
});
There's a couple asynchronous things happening here. For each user, I want to find their relevant friends and concat them to a variable. I then want to check if that user has a website account, and if so, make a post request and grab some information there. Only thing is, that everything is happening out of order since the code isn't waiting for the callbacks to finish. I've been using a sleep but that doesn't solve the problem either since it's still jumbled.
I've looked into async, but these functions are intertwined and not really separate, so I wasn't sure how it'd work with async either.
Any suggestions to get this code to run sequentially?
Thanks!
I prefer the promise module to q https://www.npmjs.com/package/promise because of its simplicity
var Promises = require('promise');
var promise = new Promises(function (resolve, reject) {
// do some async stuff
if (success) {
resolve(data);
} else {
reject(reason);
}
});
promise.then(function (data) {
// function called when first promise returned
return new Promises(function (resolve, reject) {
// second async stuff
if (success) {
resolve(data);
} else {
reject(reason);
}
});
}, function (reason) {
// error handler
}).then(function (data) {
// second success handler
}, function (reason) {
// second error handler
}).then(function (data) {
// third success handler
}, function (reason) {
// third error handler
});
As you can see, you can continue like this forever. You can also return simple values instead of promises from the async handlers and then these will simply be passed to the then callback.
I rewrote your code so it was a bit easier to read. You have a few choices of what to do if you want to guarantee synchronous execution:
Use the async library. It provides some helper functions that run your code in series, particularly, this: https://github.com/caolan/async#seriestasks-callback
Use promises to avoid making callbacks, and simplify your code APIs. Promises are a new feature in Javascript, although, in my opinion, you might not want to do this right now. There is still poor library support for promises, and it's not possible to use them with a lot of popular libraries :(
Now -- in regards to your program -- there's actually nothing wrong with your code at all right now (assuming you don't have async code in the pseucode blocks). Your code right now will work just fine, and will execute as expected.
I'd recommend using async for your sequential needs at the moment, as it works both server and client side, is essentially guaranteed to work with all popular libraries, and is well used / tested.
Cleaned up code below
User.find({}, function(err, users) {
for (var i = 0; i < users.length; i++) {
Friend.find({'user':curUser._id}, function(err, friends) {
for (var i = 0; i < friends.length; i++) {
// pseudocode
}
console.log("HERE I'm CHECKING " + curUser);
if (curUser.websiteaccount != "None") {
request.post({ url: 'blah', formData: 'blah' }, function(err, httpResponse, body) {
// pseudocode
sendMail(friendResults, curUser);
});
} else {
// pseudocode
sendMail(friendResults, curUser);
}
});
console.log("finished friend");
console.log(friendResults);
sleep.sleep(15);
console.log("finished waiting");
console.log(friendResults);
}
});
First lets go a bit more functional
var users = User.find({});
users.forEach(function (user) {
var friends = Friend.find({
user: user._id
});
friends.forEach(function (friend) {
if (user.websiteaccount !== 'None') {
post(friend, user);
}
sendMail(friend, user);
});
});
Then lets async that
async.waterfall([
async.apply(Users.find, {}),
function (users, cb) {
async.each(users, function (user, cb) {
async.waterfall([
async.apply(Friends.find, { user, user.id}),
function (friends, cb) {
if (user.websiteAccount !== 'None') {
post(friend, user, function (err, data) {
if (err) {
cb(err);
} else {
sendMail(friend, user, cb);
}
});
} else {
sendMail(friend, user, cb);
}
}
], cb);
});
}
], function (err) {
if (err) {
// all the errors in one spot
throw err;
}
console.log('all done');
});
Also, this is you doing a join, SQL is really good at those.
You'll want to look into something called promises. They'll allow you to chain events and run them in order. Here's a nice tutorial on what they are and how to use them http://strongloop.com/strongblog/promises-in-node-js-with-q-an-alternative-to-callbacks/
You can also take a look at the Async JavaScript library: Async It provides utility functions for ordering the execution of asynchronous functions in JavaScript.
Note: I think the number of queries you are doing within a handler is a code smell. This problem is probably better solved at the query level. That said, let's proceed!
It's hard to know exactly what you want, because your psuedocode could use a cleanup IMHO, but I'm going to what you want to do is this:
Get all users, and for each user
a. get all the user's friends and for each friend:
send a post request if the user has a website account
send an email
Do something after the process has finished
You can do this many different ways. Vanilla callbacks or async work great; I'm going to advocate for promises because they are the future, and library support is quite good. I'll use rsvp, because it is light, but any Promise/A+ compliant library will do the trick.
// helpers to simulate async calls
var User = {}, Friend = {}, request = {};
var asyncTask = User.find = Friend.find = request.post = function (cb) {
setTimeout(function () {
var result = [1, 2, 3];
cb(null, result);
}, 10);
};
User.find(function (err, usersResults) {
// we reduce over the results, creating a "chain" of promises
// that we can .then off of
var userTask = usersResults.reduce(function (outerChain, outerResult) {
return outerChain.then(function (outerValue) {
// since we do not care about the return value or order
// of the asynchronous calls here, we just nest them
// and resolve our promise when they are done
return new RSVP.Promise(function (resolveFriend, reject){
Friend.find(function (err, friendResults) {
friendResults.forEach(function (result) {
request.post(function(err, finalResult) {
resolveFriend(outerValue + '\n finished user' + outerResult);
}, true);
});
});
});
});
}, RSVP.Promise.resolve(''));
// handle success
userTask.then(function (res) {
document.body.textContent = res;
});
// handle errors
userTask.catch(function (err) {
console.log(error);
});
});
jsbin
I have the following code:
// Retrieve
var MongoClient = require("mongodb").MongoClient;
var accounts = null;
var characters = null;
// Connect to the db
MongoClient.connect("mongodb://localhost:27017/bq", function(err, db) {
if(err) { return console.dir(err); }
db.createCollection('accounts', function(err, collection) {
if(err) { return console.dir(err); }
else { accounts = collection; }
createAccount("bob","bob");
createAccount("bob","bob");
createAccount("bob","bob");
createAccount("bob","bob");
});
});
function createAccount(email, password)
{
accounts.findOne({"email":email}, function(err, item) {
if(err) { console.dir(err); }
else {
if(item === null) {
accounts.insert({"email":email, "password":password}, function(err, result) {
if(err) { console.dir(err); }
else { console.dir("Account " + email + " created."); }
});
}
else {
console.dir("Account already exists.")
}
}
});
}
When I run the script the first time, I end up with 4 accounts for bob. When I run it the second time, I get 4 messages that the account already exists.
I'm pretty sure I know why this is, and the solution I have come up with is to use some kind queue for processing each read/write of the database in order one at a time. What I am wanting to know, is whether that is the proper way to go about it, and what would the general best practice for this be?
Some languages provide a special language construct to deal with this problem. For example, C# has async/await keywords that let you write the code as if you were calling synchronous APIs.
JavaScript does not and you have to chain the createAccount calls with callbacks.
Some people have developed libraries that may help you organize this code. For example async, step, node-promise and Q
You can also use the fibers library, a native library that extends the JavaScript runtime with fibers / coroutines.
And some people have extended the language with constructs that are similar to async/await: streamline.js, IcedCoffeeScript or wind.js. For example, streamline.js (I'm the author so I'm obviously biased) uses _ as a special callback placeholder and lets you write your example as:
var db = MongoClient.connect("mongodb://localhost:27017/bq", _):
var accounts = db.createCollection('accounts', _);
createAccount("bob","bob", _);
createAccount("bob","bob", _);
createAccount("bob","bob", _);
createAccount("bob","bob", _);
function createAccount(email, password, _) {
var item = accounts.findOne({"email":email}, _);
if (item === null) {
accounts.insert({"email":email, "password":password}, _);
console.log("Account " + email + " created."); }
} else {
console.log("Account already exists.")
}
}
And, last but not least, new language features such as generators and deferred functions are being discussed for future versions of JavaScript (generators are very likely to land in ES6, deferred functions seem to be a bit stalled).
So you have many options:
stick to callbacks
use a helper library
use the fibers runtime extension
use a language extension
wait for ES6
Add a unique constraint on email and you will not have to check if user exists anymore!
JavaScript is asynchronous. accounts.findOne returns immediately, so basically all your 4 statements are getting executed together.
What accounts.findOne does is, it says find one {"email":email} and when you find it, run the function that is in the second argument. Then it returns the function and continues to next CreateAccount statement. In the meanwhile when the results are returned from the harddrive (which takes a lot longer than executing these statements), it goes into the function, and since there is no user, it adds one. Makes sense?
UPDATE This is the right way of doing this in JavaScript.
MongoClient.connect("mongodb://localhost:27017/bq", function(err, db) {
if(err) { return console.dir(err); }
db.createCollection('accounts', function(err, collection) {
if(err) { return console.dir(err); }
else { accounts = collection; }
createAccount("bob","bob", function() {
createAccount("bob","bob", function() {
createAccount("bob","bob", function() {
createAccount("bob","bob", function() {
});
});
});
});
});
});
function createAccount(email, password, fn)
{
accounts.findOne({"email":email}, function(err, item) {
if(err) { console.dir(err); }
else {
if(item === null) {
accounts.insert({"email":email, "password":password}, function(err, result) {
if(err) { console.dir(err); }
else { console.dir("Account " + email + " created."); }
fn();
});
}
else {
console.dir("Account already exists.")
fn();
}
}
});
}