Develop Reference

async waterfall push coding

I am new to Node.JS programming. Any thoughts of my following problem, would be of great help.
I have the following async.waterfall code with sequence of three WasterfallTasks
calling dependent REST API calls in these three tasks.
router.get('/:inputId', function(req, res, next) {
var inputId = req.params.inputId;
var waterfallTasks = [];
waterfallTasks.push(function(callback) {
request(req, "/admin/id/" + inputId, function(error, results) {
var result = {};
result.grpInfo = results;
//result.params = results.params;
callback(error, result);
});
});
waterfallTasks.push(function(result1, callback)
var userID = result1.grpInfo.x.userID[1];
request(req, userID, "/con/metrix/", function(error,results){
result1.error=error;
result1.metrix=results;
callback(null,result1);
waterfallTasks.push(function(result2,callback)
request(req, "/xx/xy/", function(error, results){
callback(error,result3);
}); });
async.waterfall(waterfallTasks, function(error, results){
if(!error) {
res.render('TestDetails',{
'properties' : results.properties,
// 'params': "result.params,
'error': results.error }); });
My questions are:
Can I bring any parameters from the first waterfallTask function
call to the final async.waterfall call for rendering(like the way I
commented with //). Is it possible?
If I have to additional properties(JSON as well) from a different API
and add to this async function, what is the best way to do it? like
can I call that API in the "result2" function and pass it?
Thank you

Node JS synchronous Database Calls

i am pretty new to java script and nodejs.
i am trying to write a code that checks if an email is already exists in the database, and if not send an error.
the problem is that before the database function is ending, my code going to the next line, resulting in undefined variable(emailExists)
this is my code for the sign Up:
app.post('/signUpWeb', function (req, res) {
var reqBody = req.body;
var email= reqBody.email;
var password= reqBody.password;
var fullName= reqBody.fullName;
var webDbInsertion = {email: email, password: password, fullName: fullName};
var emailExists= DButils.checkIfPKexists(connection, "webusersMail", "email", webDbInsertion.email);
if(emailExists == false){
DButils.insertInfoToDB(connection, "webusersMail" ,webDbInsertion);
console.log("successfull signup");
res.send("successfull signup");
}else{
console.log("signup failed, email: " + email + " allready exits");
res.send("signup failed");
}
res.end();
});
and this is for my database call
exports.checkIfPKexists= function(dbConnection, tableName, PK, newPK){
var query = dbConnection.query('select count(*) as mailPkCount from ' +tableName+ ' where ' +PK+ ' = ?', newPK, function (err, row, result) {
if (err) {
console.error(err);
return;
}
var count= row[0].mailPkCount;
var bool = (count > 0);
return bool;
});
};

Well you see, node.JS is designed to be async, while what you're asking isn't impossible, it would shut down the main thread that runs the event loop for that instance of your server.
What I suggest doing is adding a callback function to your "checkIfPKexists" function, and return with the result as a parameter, much like you do in the DBConnection.query. You would then move all the code that isn't getting initialized into your callback.
Edit: Heres a quick code example, polish it up to your liking http://pastebin.com/bEHp4bi2

Using JS Promises to Execute Multiple Async Queries to Build an Object

After recently discovering JS promises, I have been studying them so that I might build a certain functionality that allows me to execute 4 async queries, use the result of each to build an object that I can finally send as a response to a request directed at my node app.
The final object is made up of 3 array properties containing the resulting rows of each query.
It seems that I've done something wrong handling the promises, though, because ultimately, game is not being built. It is sent as an empty object. Here's a JSFiddle.
What is my mistake?
Here's what I have so far:
function sendGame(req, res, sales, settings, categories) {
var game = new Object();
game.sales = sales;
game.settings = settings;
game.categories = categories;
JSONgame = JSON.stringify(game);
res.writeHead(200, {
'Access-Control-Allow-Origin': 'http://localhost',
'Content-Length': JSONgame.length,
'Content-Type': 'application/json'
});
res.write(JSONgame);
res.end();
console.log('Game: ' + JSON.stringify(game, null, 4));
console.log('--------------------------------------');
console.log('User ' + req.body.username + ' successfully retrieved game!');
}
function retrieveSales(req, connection, timeFrame) {
console.log('User ' + req.body.username + ' retrieving sales...');
connection.query('select * from sales_entries where date BETWEEN ? AND ?', timeFrame,
function (err, rows, fields) {
if (err) {
callback(new Error('Failed to connect'), null);
} else {
sales = [];
for (x = 0; x < rows.length; x++) {
sales.push(rows[x]);
}
//console.log('Sales: ' + JSON.stringify(sales, null, 4));
return sales;
}
});
}
retrieveCategories() and retrieveSettings() omitted for readability; they are the same as retrieveSales() mostly.
function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment().days(0).format("YYYY-MM-DD HH:mm:ss"), moment().days(6).format("YYYY-MM-DD HH:mm:ss")];
var connection = createConnection();
connection.connect(function (err) {
if (err) return callback(new Error('Failed to connect'), null);
console.log('Connection with the Officeball MySQL database openned for game retrieval...');
var sales = retrieveSales(req, connection, timeFrame);
var settings = retrieveSettings(req, connection);
var categories = retrieveCategories(req, connection);
var all = q.all([sales, settings, categories]);
all.done(function () {
sendGame(req, res, sales, settings, categories);
});
});
}

Your problem is that you're not using promises. All your APIs use callbacks.
A promise is like a closed box:
A promise also has a method that opens the box, works on the value and returns another box on the value (also opening any additional boxes along the way). That method is .then:
In boxes, it does:
=>( . => ) =>
That is, it adds a handler that gets an open box and returns a box. Everything else just combines stuff. All .all does is wait for a list of promises to resolve, it is exactly like .then in the fact it waits for a result. Because promises are boxes, you can pass them around and return them which is very cool.
Generally:
Whenever you return from a promise handler (not a rejection), you are fullfilling it indicating normal flow continuation.
Whenever you throw at a promise handler, you are rejecting indication exceptional flow.
So basically in node speak:
Whenever you returned a null error and a response, you resolve the promise.
Whenever you returned an error and no response, you reject the promise.
So:
function myFunc(callback){
nodeBack(function(err,data){
if(err!== null){
callback(new Error(err),null);
}
callback(data+"some processing");
})
});
Becomes:
function myFunc(){
return nodeBack().then(function(data){ return data+"some processing"; });
}
Which I think is a lot clearer. Errors are propagated across the promise chain just like in synchronous code - it's very common to find synchronous analogs to promise code.
Q.all takes a list of promises and waits for them to complete, instead you want Q.nfcall to transform a callback based API to a promise one and then use Q.all on that.
That is:
var sales = Q.nfcall(retrieveSales,req, connection, timeFrame);
var settings = Q.nfcall(retrieveSettings,req, connection);
var categories = Q.nfcall(retrieveCategories, req, connection);
Q.nfcall takes a nodeback in the err,data convention and converts it to a promise API.
Also, when you do
return sales;
You are not really returning anything, since it returns synchronously. You need to use callback like in your error case or promisify it altogether. If you don't mind, I'll do it with Bluebird since it comes with much better facilities for dealing with these interop cases and does so much much faster, if you'd like you can switch promisifyAll for a bunch of Q.nfcall calls.
// somewhere, on top of file
connection = Promise.promisifyAll(connection);
// note I'm passing just the username - passing the request breaks separation of concerns.
var retrieveSales = Promise.method(username, connection, timeFrame) {
console.log('User ' + username + ' retrieving sales...');
var q = 'select * from sales_entries where date BETWEEN ? AND ?';
return connection.queryAsync(q, timeFrame).then(function(rows, fields){
return rows;
});
}
Note that suddenly you don't need a lot of boilerplate for making a query, you can use queryAsync directly instead if you'd like.
Now the code that wraps it becomes:
var gameSucceed = Promise.method(function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment()....];
var connection = Promise.promisifyAll(createConnection());
return conn.connectAsync().then(function () {
console.log('Connection with the ...');
//sending req, but should really be what they use.
return Promise.all([retrieveSales(req,conn,timeFrame),
retrieveSettings(req,conn),
retrieveCategories(req,conn)]);
});
});
Now you can call sendGame(req, res, sales, settings, categories); outside of gameSucceed which doesn't hide what it does as much -
gameSucceed(req,res).spread(function(sales,settings,cats){
return sendGame(req,res,sales,settings,cats);
});

Having trouble with promises in nodejs

I'm trying to use promises with nodejs (I'm trying with node-promise package); however, without any success. See the code below:
var express = require('express'),
request = require('request'),
promise = require('node-promise');
app.get('/promise', function(req, res) {
var length = -1;
new promise.Promise(request(
{uri: "http://www.bing.com"},
function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log("An error occurred when connected to the web site");
return;
}
console.log("I'll return: " + body.length);
length = body.length;
}
)).then(function(result) {
console.log("This is what I got: " + length);
console.log("Done!");
});
res.end();
});
The output of the above code is I'll return: 35857 only and it doesn't go to the then part.
I change the code then to be:
app.get('/promise', function(req, res) {
var length = -1;
promise.when(
request(
{uri: "http://www.bing.com"},
function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log("An error occurred when connected to the web site");
return;
}
console.log("I'll return: " + body.length);
length = body.length;
}
),
function(result) {
console.log("This is what I got: " + length);
console.log("Done!");
},
function(error) {
console.log(error);
}
);
res.end();
});
This time the output is This is what I got: -1 then Done!... looks like the "promise" was not called this time.
So:
What's needed to be done to fix the code above? Obviously I'm not doing it right :)
Is node-promise "the way to go" when I'm doing promises, or is there a better way/package? i.e. simpler and more production-ready.
Thanks.

Try jquery-deferred-for-node.
I'm not an expert but understand that this lib tends to be favoured by programmers who work both server-side and client-side.
Even if you don't already know jQuery's Deferreds, the advantages of going this route are that :
the documentation is excellent (it comprises links to the jQuery docs), though you may struggle to find examples specific to Node.
methods are chainable.
jQuery Callbacks are also included.
when one day you need to do asynchronous stuff client-side, then there's virtually nothing to relearn - the concepts are identical and the syntax very nearly so. See the "Correspondances" section in the github page hyperlinked above.
EDIT
I'm not a node.js person so I'm guessing here but based on your code above, you might want to consider something along the following lines with jquery-deferred-for-node :
var express = require('express'),
request = require('request'),
Deferred = require('JQDeferred');
function fetch(uri, goodCodes) {
goodCodes = (!goodCodes) ? [200] : goodCodes;
var dfrd = Deferred(); // A Deferred to be resolved/rejected in response to the `request()`.
request(uri, function(error, response, body) {
if (!error) {
var isGood = false;
// Loop to test response.statusCode against `goodCodes`.
for (var i = 0; i < goodCodes.length; i++) {
if (response.statusCode == goodCodes[i]) {
isGood = true;
break;
}
}
if (isGood) {
dfrd.resolve(response.statusCode, body);
} else {
dfrd.reject(response.statusCode, "An invalid response was received from " + uri);
}
} else {
dfrd.reject(response.statusCode, "An error occurred attempting to connect to " + uri);
}
});
// Make promise derived from dfrd available to "consumer".
return dfrd.promise();
};
//...
app.get('/promise', function(req, resp) {
fetch("http://www.bing.com").done(function(statusCode, result) {
console.log("Done! This is what I got: " + result.length);
}).fail(function(statusCode, message) {
console.log("Error (" + statusCode + "): " + message);
});
resp.end();
};
Here, I have tried to write a generalized utility for fetching a resource in such a way that the asynchronous response (or error) can be handled externally. I think this is broadly along the lines of what you were trying to achieve.
Out of interest, where do console.log() messages end up with node.js?
EDIT 2
Above, I have given Deferred an initial capital, as is conventional for Constructors
With jQuery Deferreds, there must be any number of ways to fetch() consecutively. The approach below leaves fetch() as it was, and introduces fetch_() to act as its front-end. There may be simpler ways but this allows fetch() to remain a general utility, functionally equivalent to the client-side jQuery.ajax().
function fetch_(uri){
return function(){
return fetch(uri, [200]).then(function(statusCode, result){
console.log("Done! This is what I got: " + result.length);
},function(statusCode, message){
console.log("Error (" + statusCode + "): " + message);
});
};
}
Note that function fetch() returns a function. It has to be like this because where fetch() is called, we want an unexecuted function, not (yet) the result of that function.
Now let's assume an array of uris is available. This can be hard-coded or built dynamically - whatever the application demands.
var uris = [
'http://xxx.example.com',
'http://yyy.example.com',
'http://zzz.example.com'
];
And now, a variety of ways in which fetch_() might be called :
//v1. To call `resp.end()` when the fetching process starts.
app.get('/promise', function(req, resp) {
fetch_(uris[0])().then(fetch_(uris[1])).then(fetch_(uris[2]));
resp.end();
});
//v2. To call `resp.end()` when the fetching process has finished.
app.get('/promise', function(req, resp){
fetch_(uris[0])().then(fetch_(uris[1])).then(fetch_(uris[2])).always(resp.end);
});
//v3. As v2 but building a `.then()` chain of any (unknown) length.
app.get('/promise', function(req, resp){
var dfrd = Deferred().resolve();//
$.each(uris, function(i, uri){
dfrd = dfrd.then(fetch_(uri));
});
dfrd = dfrd.always(resp.end);
});
untested
I have more confidence in v1 and v2. v3 may work.
v2 and v3 should both give exactly the same behaviour but v3 is generalized for any number of uris.
Everything may need debugging.

I would recommend using Q: https://github.com/kriskowal/q. I believe that it's used internally by other frameworks (like jQuery deferred implementation).
I believe that the documentation is "fine"; the syntax is consistent with other promise implementations... and it has a node adapter.
So your deferred style approach:
var deferred = Q.defer();
FS.readFile("foo.txt", "utf-8", function (err, res) {
if (!err) {
deferred.resolve(res);
} else {
deferred.reject(err);
}
});
return deferred.promise;
Can be written more concisely as:
var deferred = Q.defer();
FS.readFile("foo.txt", "utf-8", deferred.makeNodeResolver());
return deferred.promise;

Node.js Variable scope

I have a http server setup which basically needs to look up stuff in the database.
Here is the code snippet :
var sys = require('sys');
var Client = require('mysql').Client;
var client = new Client();
client.host = '_';
client.user = '_';
client.password = '_';
client.database = '_';
var http = require('http');
http.createServer(function(req, res) {
req.on('end', function() {
client.connect(function(error, results) {
if (error) {
console.log('Connection Error:');
return;
}
ClientConnectionReady(client);
});
ClientConnectionReady = function(client) {
var final = '';
client.query('select * from table', function selectCb(error, result, fields) {
if (error) {
console.log('ERROR');
client.end();
return;
}
final += "{" + JSON.stringify(result);
});
client.query("SELECT COUNT(*) from table", function selectCb(error, result, fields) {
if (error) {
console.log('ERROR');
client.end();
return;
}
final += "," + JSON.stringify(result) + "}";
});
res.writeHead(200, {
'Content-Type': 'text/plain'
});
res.write(final);
res.end();
client.end();
};
});
}).listen(8007, "127.0.0.1");
  
If I print the values of the variable 'final' at the places where I assign them, I see valid values, but at the lines when I do 'res.write(final)', final is still blank.
How do I make this work and why is this failing?? Thanks for the help, I am new to node.js

The Node.js environment is asynchronous. Those statements that modify "final" are inside callbacks that are executed only when the database operations finish. The code immediately after the initiation of the database operations, where you write the result, are executed long before those callbacks run.
You've almost stumbled upon the answer to the problem already: you must not write the result until the operations are finished, which you know will be the case inside the callbacks. If you must wait for both to finish (seems like you do), then you can do something like keep a counter in the outer scope. Each callback can increment the counter, and call the same result-writer function only when the counter indicates that both callbacks are complete. (I have the idea that the Node runtime has a fancier way of doing that sort of thing, but I'm not that familiar with it. In a simple case like this, keeping something like a counter is easy enough to do.)
Also, an unrelated note: that "ClientConnectionReady" variable should probably either be written as a function definition:
function ClientConnectionReady(client) {
// ...
}
or else it should be declared with var. (I'm a little surprised in fact that it's not throwing an error, but again I'm not that familiar with Node.js.)

By the looks of it, you are trying to write final before it is ever assigned a value.
I'm assuming that client.query is asynchronous. Given that, the callback function is most likely being called after the res.writeHead and res.write lines. What you need to do is put other calls and the client.write* lines within the first callback.
This should give you an idea (didn't check if it compiles)
ClientConnectionReady = function(client)
{
var final = '';
//Get the rows
client.query('select * from table',
function selectCb(error, result, fields)
{
if (error)
{
console.log('ERROR');
client.end();
return;
}
final+="{"+JSON.stringify(result);
//Get the count query
client.query("SELECT COUNT(*) from table",
function selectCb(error, result, fields)
{
if (error)
{
console.log('ERROR');
client.end();
return;
}
final+=","+JSON.stringify(result)+"}";
//Return the final results to the client now
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write(final);
res.end();
client.end();
});
});
};
What this does is first gets the rows. In that callback, it then gets the count. Finally, when that works, it sends the data to the client within the count callback.