I am new from Node js , just i am trying implementing this functionality last few days but i am unable to fix
exports.get_exercises_for_trainer = function(req, res)
{
connection.query('SELECT * FROM ag_exercise', function(err, exercise)
{
console.log('------------------------------before add fields ----------------------------------');
console.log(exercise);
for (var i in exercise)
{
fields(exercise[i].wt_id, function(result1) {
exercise[i].wt_fields = result1; //here i am adding result set of other query but i am not geting this fields data
console.log(result1) //but i printed here working fine but i need this result1 data out side query
});
}
console.log('------------------------------after add fields ----------------------------------');
console.log(exercise);
res.render('pages/trainer_home.ejs',{page_title:"Exercise Create",exercise:exercise});
});
}
function fields(wt_id,callback)
{
connection.query('SELECT * FROM ag_workout_type_fields WHERE wt_id = "'+wt_id+'"', function( err1, result1){
callback(result1);
});
}
I have one more query ? in node js : If table having users , users having different relation tables like orders , profiles , address
How to implement this
First i am getting users
User loop
getting every user profiles ,address,orders
end user loop
but above scenario i am unable to implement in node js but in php very simple like this
$get_users = ... //users data
foreach($getusers as $usr)
{
$usr->orders = //orders data
.... like etc
}
There are three main questions here, I will adress each seperately.
Question 1: When making an async function, how do I then access my data outside that function?
All data from async calls are accessed via callback, event listeners or promises (a fancy callback and event listener handler). For the most part, you are going to just be using callbacks. So, instead of :
get_user = function(user_id){
//Do some stuff to get the user
return the_user;
};
var user = get_user('1234');
//do whatever you want with user
You will see :
get_user = function(user_id,callback){
//Do some stuff to get the user
callback(null,the_user);
}
get_user('1234',function(err,user){
//do whatever you want with user
});
When we get to Question 3, you will see the more complicated use case you were speaking of.
Question 2: How do I loop through my data, perform a subsiquent query on each row, and append that data to my current data?
There are a couple of issues here.
Every time you query the database, you are performing an asynchronous function, so you need to manage all of those callbacks accordingly. Fortunately there are some great tools to do that for you, and we will be using async.
Every time you call an asynchronous function in a for loop, the for loop continues, thus your iterator is overwritten, but your asynchronous function is not done with it yet, so you will get all sorts of unexpected results like vanishing variables, or missmapped results. You can handle this with JavaScript closures, or, you can rely again on libraries like async which handle it all for you.
Instead of running a for loop over your queries results, we're going to pass it to async.forEachOf, which we will use to modify the existing array and append the results of the subsequent queries to the primary query's rows. It is important to note that forEachOf will run the subsequent queries in parallel, so you should not be using a single database connection, but a pool. If you MUST use a single database connection, use forEachOfSeries instead.
async = require('async');
exports.get_exercises_for_trainer = function(req, res){
connection.query('SELECT * FROM ag_exercise', function(err, exercises)
{
console.log('------------------------------before add fields ----------------------------------');
console.log(exercises);
async.forEachOf(exercises,function(exercise,index,callback){
connection.query('SELECT * FROM ag_workout_type_fields WHERE wt_id = "' + exercise.wt_id + '"', function( err, result1){
if(err)return callback(err1);
exercises[index].wt_fields = result1; //Modify original array
return callback();
});
},function(err){
if(err){return;} //do some error handling
console.log('------------------------------after add fields ----------------------------------');
console.log(exercises);
res.render('pages/trainer_home.ejs',{page_title:"Exercise Create",exercise:exercises});
});
});
};
Question 3: How do I perform many related but different queries so that I can populate information about my object?
This is another great usage of the async library. In this case since the queries are all different, we'll use parallel instead of forEachOf.
async = require('async');
populate_user = function(user,_callback){
async.paralell({
profile: function(callback){
var sql = "SELECT * FROM profiles WHERE user_id = " + user.id + " LIMIT 1 ";
var connection.query(sql,function(err,rows,fields){
if(err)return callback(err);
if(rows.length === 1)return callback(null,rows[0]);
return callback(null,[]);
});
},
address: function(callback){
var sql = "SELECT * FROM addresses WHERE user_id = " + user.id + " LIMIT 1 ";
var connection.query(sql,function(err,rows,fields){
if(err)return callback(err);
if(rows.length === 1)return callback(null,rows[0]);
return callback(null,[]);
});
},
orders: function(callback){
var sql = "SELECT * FROM orders WHERE user_id = " + user.id;
var connection.query(sql,function(err,rows,fields){
if(err)return callback(err);
if(rows.length > 0)return callback(null,rows); //notice how this one could have multiple results so we're returning them all
return callback(null,[]);
});
},
},
function(err,result){
if(err)return _callback(err);
for(var att in result){user[att] = result[att];}
callback(null,user);
}
}
user = {id:1234};
populate_user(user,function(err,populated_user)){
console.log(user); //wow notice how it's populated too!
console.log(populated_user); //this is really just a new handle for the same object
});
I want to note that NONE of this was tested, not even for syntax, so it may take a little reworking.
Related
I am trying to figure out why one of my queries won't return the value from a query...my code looks like this:
var client = new pg.Client(conString);
client.connect();
var query = client.query("SELECT count(*) as count FROM sat_scores")
// Don't use demo key in production. Get a key from https://api.nasa.gov/index.html#apply-for-an-api-key
function getNEO(callback) {
var data = '';
query.on('rows', function(rows) {
console.log("Row count is: %s", rows[0].count)
data += rows[0].count;
});
query.on('end', function() {
callback(data);
});
}
with that, getNEO returns a blank...but if I set var data = '4', then getNEO returns 4....the query should return 128 but it just returns a blank...
First of all, getNEO() doesn't return anything - I'm operating on the assumption that you call getNEO() exactly once for your query, and pass in a callback to handle the data, and that callback is what's not getting the appropriate data?
My typical recommendation for troubleshooting things like this is to simplify your code, and try and get really close to any example code given (for instance):
var client = new pg.Client(conString);
// define your callback here, in theory
client.connect(function (err) {
if (err) throw err;
var query = client.query("SELECT count(*) as count FROM sat_scores"),
function(err, result) {
if (err) throw err;
console.log(result.rows.length);
}
);
});
... I'm doing a couple things here you'll want to note:
It looks like the client.connect() method is asynchronous - you can't just connect and then go run your query, you have to wait until the connection is completed, hence the callback. Looking through the code, it looks like it may emit a connect event when it's ready to send queries, so you don't have to use a callback on the connect() method directly.
I don't see a data event in the documentation for the query object nor do I see one in the code. You could use the row event, or you could use a callback directly on the query as in the example on the main page - that's what I've done here in the interest of simplicity.
I don't see the count property you're using, and row[0] is only going to be the first result - I think you want the length property on the whole rows array if you're looking for the number of rows returned.
I don't know if you have a good reason to use the getNEO() function as opposed to putting the code directly in procedurally, but I think you can get a closer approximation of what you're after like this:
var client = new pg.Client(conString);
// define your callback here, in theory
client.connect();
function getNEO(callback) {
client.on('connect', function () {
var query = client.query("SELECT count(*) as count FROM sat_scores"));
query.on('end', function(result) {
callback(result.rowCount);
});
});
}
... so, you can call your getNEO() function whenever you like, it'll appropriately wait for the connection to be completed, and then you can skip tracking each row as it comes; the end event receives the result object which will give you all the rows and the row count to do with what you wish.
so here is how I was able to resolve the issue....I moved the var query inside of the function
function getNEO(state, callback) {
var conString = "postgres://alexa:al#alexadb2.cgh3p2.us-east-1.redshift.amazonaws.com:5439/alexa";
var client = new pg.Client(conString);
client.connect();
var data = '';
var query = client.query("SELECT avg(Math) as math, avg(Reading) as reading FROM sat_scores WHERE State = '" + state + "'");
console.log("query is: %s", query);
query.on('row', function(row) {
console.log("Row cnt is: %s", row.math);
console.log("row is: " + row)
data += row;
});
console.log("made it");
query.on('end', function() {
callback(data);
});
}
I would like to 'functionalize' my queries by putting them into functions which have apt names for the task.
I want to avoid putting everything in the req, res functions (my controllers), and instead put them in 'models' of sorts, that is, another JavaScript file that will be imported and used to run the functions that execute queries and return the results on behalf of the controller.
Assuming that I have the following setup for the queries:
UserController.js
exports.userAccount = function(req, res, next) {
var queryText = "\
SELECT *\
FROM users\
WHERE id = $1\
";
var queryValues = [168];
pg.connect(secrets.DATABASE_URL, function(err, client, done) {
client.query(queryText, queryValues, function(err, result) {
res.render('pathToSome/page', {
queryResult: result.rows
});
});
});
}
Here, while I'm in the query, I essentially redirect and render a page with the data. That works fine. But I want to take out all that pg.connect and client.query code and move it to a separate file to be imported as a model. I've come up with the following:
UserModel.js
exports.findUser = function(id) {
// The user to be returned from the query
// Local scope to 'findUser' function?
var user = {};
var queryText = "\
SELECT *\
FROM users\
WHERE id = $1\
";
var queryValues = [id];
pg.connect(secrets.DATABASE_URL, function(err, client, done) {
client.query(queryText, queryValues, function(err, result) {
// There is only ever 1 row returned, so get the first one in the array
// Apparently this is local scope to 'client.query'?
// I want this to overwrite the user variable declared at the top of the function
user = result.rows;
// Console output correct; I have my one user
console.log("User data: " + JSON.stringify(user));
});
});
// I expect this to be correct. User is empty, because it was not really
// assigned in the user = result.rows call above.
console.log("User outside of 'pg.connect': " + JSON.stringify(user));
// I would like to return the user here, but it's empty!
return user;
};
and I'm calling my model function as so:
var user = UserModel.findUser(req.user.id);
The query executes perfectly fine in this fashion - except that the user object is not being assigned correctly (I'm assuming a scope issue), and I can't figure it out.
The goal is to be able to call a function (like the one above) from the controller, have the model execute the query and return the result to the controller.
Am I missing something blatantly obvious here?
pgconnect is an asynchronous call. Instead of waiting for data to return from the database before proceeding with the next line, it goes ahead with the rest of the program before Postgres answers. So in the code above, findUser returns a variable that has not yet been populated.
In order to make it work correctly, you have to add a callback to the findUser function. (I told you wrong in a previous edit: The done parameter in pg.connect is called in order to release the connection back to the connection pool.) The final result should look something like this:
exports.findUser = function(id, callback) {
var user = {};
var queryText = "SELECT FROM users WHERE id = $1";
var queryValues = [id];
pg.connect(secrets.DATABASE_URL, function(err, client, done) {
client.query(queryText, queryValues, function(err, result) {
user = result.rows;
done(); // Releases the connection back to the connection pool
callback(err, user);
});
});
return user;
};
And you'd use it, not like this:
var user = myModule.findUser(id);
But like this:
myModule.findUser(id, function(err, user){
// do something with the user.
});
If you have several steps to perform, each of them dependent on data from a previous asynchronous call, you'll wind up with confusing, Inception-style nested callbacks. Several asynchronous libraries exist to help you with making such code more readable, but the most popular is npm's async module.
I'm working with node.js and redis. I've got a redis database with a bunch of keys. Something like this:
user/chris/potion
user/pete/potion
user/chris/race
user/pete/race
user/chris/weapon
user/pete/weapon
I want to do a redis call which retrieves all user stats, puts the stats into a JS object, then passes it to the client for displaying character stats in the browser. Using javascript I inject the username chris at u into the redis call like this:
KEYS user/u/*
which returns:
1) "user/chris/weapon"
2) "user/chris/race"
3) "user/chris/potion"
Now I can iterate through those results, get the value of each key with GET, and make a javascript object. Seems super simple so I write the code. I quickly run into problems using forEach:
var redis = require('redis');
var client = redis.createClient();
exports.getUserObject = function(requesteduser, callback) {
var userstats = {}; // the object to hold the user stats once retrieved from the db
client.KEYS('user/' + requesteduser + '/*', function(err, replies) {
replies.forEach(function (reply, i) {
client.GET(reply, function(err, value) {
// get the key name so we can populate a js object
var n = reply.lastIndexOf('/');
var key = reply.substring(n + 1, reply.length);
userstats[key] = value;
console.dir(userstats);
callback(null, userstats); // parent expects (err, userstats)
});
});
});
}
When ran, output is like this:
{ weapon: 'twosword' }
{ weapon: 'twosword', race: 'elf' }
{ weapon: 'twosword', race: 'elf', potion: 'mana' }
callback(null, userstats) is called three times. Calling callback more than once will be problematic, since eventually callback will trigger data being sent data to the client.
I think I know what is happening. client.GET is ran three times because I asked it to. The replies array has three elements, so each time the result comes in for client.GET, the callback is called.
What I need to happen is for the forEach loop to finish iterating through all the redis keys, and once that's done, call the callback once. I tried solving this problem first with promises, then with async, because async has async.each. I got stuck solving the problem with both. I'm back to square one now, I'm convinced I have to do something different with forEach to make progress.
How can I accomplish this?
Since you're iterating over replies, you can check when you've reached the last element and only call callback in that instance.
client.KEYS('user/' + requesteduser + '/*', function(err, replies) {
replies.forEach(function (reply, i) {
client.GET(reply, function(err, value) {
// get the key name so we can populate a js object
var n = reply.lastIndexOf('/');
var key = reply.substring(n + 1, reply.length);
userstats[key] = value;
console.dir(userstats);
if (i == replies.length-1) callback(null, userstats); // parent expects (err, userstats)
});
});
});
I know it's a little late to help #Grimtech but I'll leave my opinion for other people that arrives here eventually:
First of all I rather to model #Grimtech's problem differently. Something like this:
client.hmset("user:chris", "weapon", "some weapon", "race", "some race", "potion", "some potion");
client.hmset("user:pete", "weapon", "same weapon", "race", "other race", "potion", "other potion");
Then I would have a method in Node returning a json like the following, assuming that I can have more than one key starting with "user:chris":
router.get('/users/:user_id', function(req, res, next) {
var response = [];
var client = redis.createClient();
client.keys("user:" + req.params.user_id + "*", function (err, users) {
users.forEach(function (key, pos) {
client.hgetall(key, function (err, user) {
response.push(user);
if (pos === users.length - 1) {
res.json(response);
client.quit();
}
});
});
});
});
The if (pos === users.length - 1) solves the async issue.
The json returned will have all the "user:chris" attributes so yo can do whatever you want in the client browser.
After recently discovering JS promises, I have been studying them so that I might build a certain functionality that allows me to execute 4 async queries, use the result of each to build an object that I can finally send as a response to a request directed at my node app.
The final object is made up of 3 array properties containing the resulting rows of each query.
It seems that I've done something wrong handling the promises, though, because ultimately, game is not being built. It is sent as an empty object. Here's a JSFiddle.
What is my mistake?
Here's what I have so far:
function sendGame(req, res, sales, settings, categories) {
var game = new Object();
game.sales = sales;
game.settings = settings;
game.categories = categories;
JSONgame = JSON.stringify(game);
res.writeHead(200, {
'Access-Control-Allow-Origin': 'http://localhost',
'Content-Length': JSONgame.length,
'Content-Type': 'application/json'
});
res.write(JSONgame);
res.end();
console.log('Game: ' + JSON.stringify(game, null, 4));
console.log('--------------------------------------');
console.log('User ' + req.body.username + ' successfully retrieved game!');
}
function retrieveSales(req, connection, timeFrame) {
console.log('User ' + req.body.username + ' retrieving sales...');
connection.query('select * from sales_entries where date BETWEEN ? AND ?', timeFrame,
function (err, rows, fields) {
if (err) {
callback(new Error('Failed to connect'), null);
} else {
sales = [];
for (x = 0; x < rows.length; x++) {
sales.push(rows[x]);
}
//console.log('Sales: ' + JSON.stringify(sales, null, 4));
return sales;
}
});
}
retrieveCategories() and retrieveSettings() omitted for readability; they are the same as retrieveSales() mostly.
function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment().days(0).format("YYYY-MM-DD HH:mm:ss"), moment().days(6).format("YYYY-MM-DD HH:mm:ss")];
var connection = createConnection();
connection.connect(function (err) {
if (err) return callback(new Error('Failed to connect'), null);
console.log('Connection with the Officeball MySQL database openned for game retrieval...');
var sales = retrieveSales(req, connection, timeFrame);
var settings = retrieveSettings(req, connection);
var categories = retrieveCategories(req, connection);
var all = q.all([sales, settings, categories]);
all.done(function () {
sendGame(req, res, sales, settings, categories);
});
});
}
Your problem is that you're not using promises. All your APIs use callbacks.
A promise is like a closed box:
A promise also has a method that opens the box, works on the value and returns another box on the value (also opening any additional boxes along the way). That method is .then:
In boxes, it does:
=>( . => ) =>
That is, it adds a handler that gets an open box and returns a box. Everything else just combines stuff. All .all does is wait for a list of promises to resolve, it is exactly like .then in the fact it waits for a result. Because promises are boxes, you can pass them around and return them which is very cool.
Generally:
Whenever you return from a promise handler (not a rejection), you are fullfilling it indicating normal flow continuation.
Whenever you throw at a promise handler, you are rejecting indication exceptional flow.
So basically in node speak:
Whenever you returned a null error and a response, you resolve the promise.
Whenever you returned an error and no response, you reject the promise.
So:
function myFunc(callback){
nodeBack(function(err,data){
if(err!== null){
callback(new Error(err),null);
}
callback(data+"some processing");
})
});
Becomes:
function myFunc(){
return nodeBack().then(function(data){ return data+"some processing"; });
}
Which I think is a lot clearer. Errors are propagated across the promise chain just like in synchronous code - it's very common to find synchronous analogs to promise code.
Q.all takes a list of promises and waits for them to complete, instead you want Q.nfcall to transform a callback based API to a promise one and then use Q.all on that.
That is:
var sales = Q.nfcall(retrieveSales,req, connection, timeFrame);
var settings = Q.nfcall(retrieveSettings,req, connection);
var categories = Q.nfcall(retrieveCategories, req, connection);
Q.nfcall takes a nodeback in the err,data convention and converts it to a promise API.
Also, when you do
return sales;
You are not really returning anything, since it returns synchronously. You need to use callback like in your error case or promisify it altogether. If you don't mind, I'll do it with Bluebird since it comes with much better facilities for dealing with these interop cases and does so much much faster, if you'd like you can switch promisifyAll for a bunch of Q.nfcall calls.
// somewhere, on top of file
connection = Promise.promisifyAll(connection);
// note I'm passing just the username - passing the request breaks separation of concerns.
var retrieveSales = Promise.method(username, connection, timeFrame) {
console.log('User ' + username + ' retrieving sales...');
var q = 'select * from sales_entries where date BETWEEN ? AND ?';
return connection.queryAsync(q, timeFrame).then(function(rows, fields){
return rows;
});
}
Note that suddenly you don't need a lot of boilerplate for making a query, you can use queryAsync directly instead if you'd like.
Now the code that wraps it becomes:
var gameSucceed = Promise.method(function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment()....];
var connection = Promise.promisifyAll(createConnection());
return conn.connectAsync().then(function () {
console.log('Connection with the ...');
//sending req, but should really be what they use.
return Promise.all([retrieveSales(req,conn,timeFrame),
retrieveSettings(req,conn),
retrieveCategories(req,conn)]);
});
});
Now you can call sendGame(req, res, sales, settings, categories); outside of gameSucceed which doesn't hide what it does as much -
gameSucceed(req,res).spread(function(sales,settings,cats){
return sendGame(req,res,sales,settings,cats);
});
I have a webserver running in node.js and Express which retrieves data from mongodb . In mongodb collections are getting created dynamically and the name of newly created collection will be stored in one metadata collection “project” . My requirement is to firstly iterate to metadata collection to get the collection name and then get inside the each collection to do multiple query based on some condition . Because my collection metadata is dynamic I have tried to do using for loop .
But it is giving wrong data . It is not executing sequent . Before finishing the loop execution it is returning the value .How to perform sequential execution in node.js using node core modules only (Not other library like async..);
exports.projectCount = function (req, res) {
var mongo = require("mongodb"),
Server = mongo.Server,
Db = mongo.Db;
var server = new Server("localhost", 27017, {
auto_reconnect: true
});
var db = new Db("test", server);
// global JSON object to store manipulated data
var projectDetail = {
projectCount: 0,
projectPercent: 0
};
var totalProject = 0;
db.open(function (err, collection) {
//metadata collection
collection = db.collection("project");
collection.find().toArray(function (err, result) {
// Length of metadata collection
projectDetail.projectCount = result.length;
var count = 0;
//iterate through each of the array which is the name of collection
result.forEach(function (item) {
//change collection object to new collection
collection = db.collection(item.keyParameter.wbsName);
// Perform first query based on some condition
collection.find({
$where: "this.status == 'Created'"
}).toArray(function (err, result) {
// based on result of query one increment the value of count
count += result.lenght;
// Perform second query based on some condition
collection.find({
$where: "this.status=='Completed'"
}).toArray(function (err, result) {
count += result.length;
});
});
});
// it is returning the value without finishing the above manipulation
// not waiting for above callback and value of count is coming zero .
res.render('index', {
projectDetail: projectDetail.projectCount,
count: count
});
});
});
};
When you want to call multiple asynchronous functions in order, you should call the first one, call the next one in it's callback and so on. The code would look like:
asyncFunction1(args, function () {
asyncFunction2(args, function () {
asyncFunction3(args, function () {
// ...
})
})
});
Using this approach, you may end up with an ugly hard-to-maintain piece of code.
There are various ways to achieve the same functionality without nesting callbacks, like using async.js or node-fibers.
Here is how you can do it using node.js EventEmitter:
var events = require('events');
var EventEmitter = events.EventEmitter;
var flowController = new EventEmitter();
flowController.on('start', function (start_args) {
asyncFunction1(args, function () {
flowController.emit('2', next_function_args);
});
});
flowController.on('2', function (args_coming_from_1) {
asyncFunction2(args, function () {
flowController.emit('3', next_function_args);
});
});
flowController.on('3', function (args_coming_from_2) {
asyncFunction3(args, function () {
// ...
});
});
flowController.emit('start', start_args);
For loop simulation example:
var events = require('events');
var EventEmitter = events.EventEmitter;
var flowController = new EventEmitter();
var items = ['1', '2', '3'];
flowController.on('doWork', function (i) {
if (i >= items.length) {
flowController.emit('finished');
return;
}
asyncFunction(item[i], function () {
flowController.emit('doWork', i + 1);
});
});
flowController.on('finished', function () {
console.log('finished');
});
flowController.emit('doWork', 0);
Use callbacks or promises or a flow control library. You cannot program servers in node without understanding at the very least one of these approaches, and honestly all halfway decent node programmers thoroughly understand all three of them (including a handful of different flow control libraries).
This is not a something you are going to just get an answer coded for you by someone else on stackoverflow and then move on. This is a fundamental thing you have to go and study and learn generically as it is only going to come up over and over again on a daily basis.
http://howtonode.org/control-flow
http://callbackhell.com/
Per the resources in the answer above me, nesting the callback when you iterate and only calling it if you are on the last iteration will solve you problem.