I have a script that I want to run on a scheduled basis in node. The script is not terminating and exiting. I suspect that this is because my database client is still open.
var client = new pg.Client(conString);
client.connect();
function registerBundle (innerHash, outterHash) {
// some stuff here
}
var query = client.query("SELECT id, chain FROM mytable where \
state_ready = true and transaction_id='' ");
query.on('row', function(row) {
var chain = row['chain'];
var pg_record = row['id'];
console.log(pg_record);
var innerHash = "something";
var outerHash = "something else";
var registrar = registerBundle(innerHash, outerHash);
var update = client.query('UPDATE mytable SET transaction_id = $1::text \
where id=$2::int', [transactionHash, pg_record]);
console.log(chain);
});
if I include the following, the client connection closes before the updates have a time to fire.
query.on('end', function() {
client.end();
});
I cannot use setTimeout or any other such mechanism because I don't know how long to wait for the registerBundle function to complete. Also I think query.on('end' will fire when the update is completed. Not sure how to test this.
My question, I need things to fire in order.
Query DB
Process each row (query.on
Update each row with value returned from registerBundle
Close db client/connection when all rows have been processed.
Terminate script and exit node
Seems pretty straightforward from a python/php world but falls apart in my javascript world.
A promise-based interface like pg-promise is the way to go:
var bluebird = require('bluebird');
var pgp = require('pg-promise')({
promiseLib: bluebird
});
var db = pgp(/*connection details*/);
db.tx(t => {
// BEGIN executed
return t.map('SELECT id, chain FROM mytable where state_ready = $1 and transaction_id = $2', [true, 123], a => {
var chain = data.chain;
var pg_record = data.id;
return t.none('UPDATE mytable SET transaction_id = $1::text where id=$2::int', [transactionHash, pg_record]);
}).then(t.batch); // settling all internal queries
})
.then(data => {
// success, COMMIT executed
})
.catch(error => {
// error, ROLLBACK executed
})
.finally(pgp.end); // shuts down the connection pool
The example above does exactly what you asked for, plus it uses a transaction. But in reality you're gonna want to do it all in one query, for performance reasons ;)
See more examples.
Related
My problem is that the code does not seem to be running in order, as seen below.
This code is for my discord.js bot that I am creating.
var Discord = require("discord.js");
var bot = new Discord.Client();
var yt = require("C:/Users/username/Documents/Coding/Discord/youtubetest.js");
var youtubetest = new yt();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var prefix = "!";
var vidid;
var commands = {
play: {
name: "!play ",
fnc: "Gets a Youtube video matching given tags.",
process: function(msg, query) {
youtubetest.respond(query, msg);
var vidid = youtubetest.vidid;
console.log(typeof(vidid) + " + " + vidid);
console.log("3");
}
}
};
bot.on('ready', () => {
console.log('I am ready!');
});
bot.on("message", msg => {
if(!msg.content.startsWith(prefix) || msg.author.bot || (msg.author.id === bot.user.id)) return;
var cmdraw = msg.content.split(" ")[0].substring(1).toLowerCase();
var query = msg.content.split("!")[1];
var cmd = commands[cmdraw];
if (cmd) {
var res = cmd.process(msg, query, bot);
if (res) {
msg.channel.sendMessage(res);
}
} else {
let msgs = [];
msgs.push(msg.content + " is not a valid command.");
msgs.push(" ");
msgs.push("Available commands:");
msgs.push(" ");
msg.channel.sendMessage(msgs);
msg.channel.sendMessage(commands.help.process(msg));
}
});
bot.on('error', e => { console.error(e); });
bot.login("mytoken");
The youtubetest.js file:
var youtube_node = require('youtube-node');
var ConfigFile = require("C:/Users/username/Documents/Coding/Discord/json_config.json");
var mybot = require("C:/Users/username/Documents/Coding/Discord/mybot.js");
function myyt () {
this.youtube = new youtube_node();
this.youtube.setKey(ConfigFile.youtube_api_key);
this.vidid = "";
}
myyt.prototype.respond = function(query, msg) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
msg.channel.sendMessage("There was an error finding requested video.");
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
}
});
console.log("2");
};
module.exports = myyt;
As the code shows, i have an object for the commands that the bot will be able to process, and I have a function to run said commands when a message is received.
Throughout the code you can see that I have put three console.logs with 1, 2 and 3 showing in which order I expect the parts of the code to run. When the code is run and a query is found the output is this:
I am ready!
string +
2
3
1
This shows that the code is running in the wrong order that I expect it to.
All help is very highly appreciated :)
*Update! Thank you all very much to understand why it isn't working. I found a solution where in the main file at vidid = youtubetest.respond(query, msg) when it does that the variable is not assigned until the function is done so it goes onto the rest of my code without the variable. To fix I simply put an if statement checking if the variable if undefined and waiting until it is defined.*
Like is mentioned before, a lot of stuff in javascript runs in async, hence the callback handlers. The reason it runs in async, is to avoid the rest of your code being "blocked" by remote calls. To avoid ending up in callback hell, most of us Javascript developers are moving more and more over to Promises. So your code could then look more like this:
myyt.prototype.respond = function(query, msg) {
return new Promise(function(resolve, reject) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
reject("There was an error finding requested video."); // passed down to the ".catch" statement below
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
resolve(2); // Resolve marks the promises as successfully completed, and passes along to the ".then" method
}
});
}).then(function(two) {
// video is now the same as myyt.vidid as above.
console.log(two);
}).catch(function(err) {
// err contains the error object from above
msg.channel.sendMessage(err);
})
};
This would naturally require a change in anything that uses this process, but creating your own prototypes seems.. odd.
This promise returns the vidid, so you'd then set vidid = youtubetest.response(query, msg);, and whenever that function gets called, you do:
vidid.then(function(id) {
// id is now the vidid.
});
Javascript runs async by design, and trying to hack your way around that leads you to dark places fast. As far as I can tell, you're also targetting nodeJS, which means that once you start running something synchronously, you'll kill off performance for other users, as everyone has to wait for that sync call to finish.
Some suggested reading:
http://callbackhell.com/
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://stackoverflow.com/a/11233849/3646975
I'd also suggest looking up ES6 syntax, as it shortens your code and makes life a hellofalot easier (native promises were only introduced in ES6, which NodeJS 4 and above supports (more or less))
In javascript, please remember that any callback function you pass to some other function is called asynchronously. I.e. the calls to callback function may not happen "in order". "In order" in this case means the order they appear on the source file.
The callback function is simply called on certain event:
When there is data to be processed
on error
in your case for example when the youtube search results are ready,
'ready' event is received or 'message' is received.
etc.
I am trying to figure out why one of my queries won't return the value from a query...my code looks like this:
var client = new pg.Client(conString);
client.connect();
var query = client.query("SELECT count(*) as count FROM sat_scores")
// Don't use demo key in production. Get a key from https://api.nasa.gov/index.html#apply-for-an-api-key
function getNEO(callback) {
var data = '';
query.on('rows', function(rows) {
console.log("Row count is: %s", rows[0].count)
data += rows[0].count;
});
query.on('end', function() {
callback(data);
});
}
with that, getNEO returns a blank...but if I set var data = '4', then getNEO returns 4....the query should return 128 but it just returns a blank...
First of all, getNEO() doesn't return anything - I'm operating on the assumption that you call getNEO() exactly once for your query, and pass in a callback to handle the data, and that callback is what's not getting the appropriate data?
My typical recommendation for troubleshooting things like this is to simplify your code, and try and get really close to any example code given (for instance):
var client = new pg.Client(conString);
// define your callback here, in theory
client.connect(function (err) {
if (err) throw err;
var query = client.query("SELECT count(*) as count FROM sat_scores"),
function(err, result) {
if (err) throw err;
console.log(result.rows.length);
}
);
});
... I'm doing a couple things here you'll want to note:
It looks like the client.connect() method is asynchronous - you can't just connect and then go run your query, you have to wait until the connection is completed, hence the callback. Looking through the code, it looks like it may emit a connect event when it's ready to send queries, so you don't have to use a callback on the connect() method directly.
I don't see a data event in the documentation for the query object nor do I see one in the code. You could use the row event, or you could use a callback directly on the query as in the example on the main page - that's what I've done here in the interest of simplicity.
I don't see the count property you're using, and row[0] is only going to be the first result - I think you want the length property on the whole rows array if you're looking for the number of rows returned.
I don't know if you have a good reason to use the getNEO() function as opposed to putting the code directly in procedurally, but I think you can get a closer approximation of what you're after like this:
var client = new pg.Client(conString);
// define your callback here, in theory
client.connect();
function getNEO(callback) {
client.on('connect', function () {
var query = client.query("SELECT count(*) as count FROM sat_scores"));
query.on('end', function(result) {
callback(result.rowCount);
});
});
}
... so, you can call your getNEO() function whenever you like, it'll appropriately wait for the connection to be completed, and then you can skip tracking each row as it comes; the end event receives the result object which will give you all the rows and the row count to do with what you wish.
so here is how I was able to resolve the issue....I moved the var query inside of the function
function getNEO(state, callback) {
var conString = "postgres://alexa:al#alexadb2.cgh3p2.us-east-1.redshift.amazonaws.com:5439/alexa";
var client = new pg.Client(conString);
client.connect();
var data = '';
var query = client.query("SELECT avg(Math) as math, avg(Reading) as reading FROM sat_scores WHERE State = '" + state + "'");
console.log("query is: %s", query);
query.on('row', function(row) {
console.log("Row cnt is: %s", row.math);
console.log("row is: " + row)
data += row;
});
console.log("made it");
query.on('end', function() {
callback(data);
});
}
I would like to 'functionalize' my queries by putting them into functions which have apt names for the task.
I want to avoid putting everything in the req, res functions (my controllers), and instead put them in 'models' of sorts, that is, another JavaScript file that will be imported and used to run the functions that execute queries and return the results on behalf of the controller.
Assuming that I have the following setup for the queries:
UserController.js
exports.userAccount = function(req, res, next) {
var queryText = "\
SELECT *\
FROM users\
WHERE id = $1\
";
var queryValues = [168];
pg.connect(secrets.DATABASE_URL, function(err, client, done) {
client.query(queryText, queryValues, function(err, result) {
res.render('pathToSome/page', {
queryResult: result.rows
});
});
});
}
Here, while I'm in the query, I essentially redirect and render a page with the data. That works fine. But I want to take out all that pg.connect and client.query code and move it to a separate file to be imported as a model. I've come up with the following:
UserModel.js
exports.findUser = function(id) {
// The user to be returned from the query
// Local scope to 'findUser' function?
var user = {};
var queryText = "\
SELECT *\
FROM users\
WHERE id = $1\
";
var queryValues = [id];
pg.connect(secrets.DATABASE_URL, function(err, client, done) {
client.query(queryText, queryValues, function(err, result) {
// There is only ever 1 row returned, so get the first one in the array
// Apparently this is local scope to 'client.query'?
// I want this to overwrite the user variable declared at the top of the function
user = result.rows;
// Console output correct; I have my one user
console.log("User data: " + JSON.stringify(user));
});
});
// I expect this to be correct. User is empty, because it was not really
// assigned in the user = result.rows call above.
console.log("User outside of 'pg.connect': " + JSON.stringify(user));
// I would like to return the user here, but it's empty!
return user;
};
and I'm calling my model function as so:
var user = UserModel.findUser(req.user.id);
The query executes perfectly fine in this fashion - except that the user object is not being assigned correctly (I'm assuming a scope issue), and I can't figure it out.
The goal is to be able to call a function (like the one above) from the controller, have the model execute the query and return the result to the controller.
Am I missing something blatantly obvious here?
pgconnect is an asynchronous call. Instead of waiting for data to return from the database before proceeding with the next line, it goes ahead with the rest of the program before Postgres answers. So in the code above, findUser returns a variable that has not yet been populated.
In order to make it work correctly, you have to add a callback to the findUser function. (I told you wrong in a previous edit: The done parameter in pg.connect is called in order to release the connection back to the connection pool.) The final result should look something like this:
exports.findUser = function(id, callback) {
var user = {};
var queryText = "SELECT FROM users WHERE id = $1";
var queryValues = [id];
pg.connect(secrets.DATABASE_URL, function(err, client, done) {
client.query(queryText, queryValues, function(err, result) {
user = result.rows;
done(); // Releases the connection back to the connection pool
callback(err, user);
});
});
return user;
};
And you'd use it, not like this:
var user = myModule.findUser(id);
But like this:
myModule.findUser(id, function(err, user){
// do something with the user.
});
If you have several steps to perform, each of them dependent on data from a previous asynchronous call, you'll wind up with confusing, Inception-style nested callbacks. Several asynchronous libraries exist to help you with making such code more readable, but the most popular is npm's async module.
I have a webserver running in node.js and Express which retrieves data from mongodb . In mongodb collections are getting created dynamically and the name of newly created collection will be stored in one metadata collection “project” . My requirement is to firstly iterate to metadata collection to get the collection name and then get inside the each collection to do multiple query based on some condition . Because my collection metadata is dynamic I have tried to do using for loop .
But it is giving wrong data . It is not executing sequent . Before finishing the loop execution it is returning the value .How to perform sequential execution in node.js using node core modules only (Not other library like async..);
exports.projectCount = function (req, res) {
var mongo = require("mongodb"),
Server = mongo.Server,
Db = mongo.Db;
var server = new Server("localhost", 27017, {
auto_reconnect: true
});
var db = new Db("test", server);
// global JSON object to store manipulated data
var projectDetail = {
projectCount: 0,
projectPercent: 0
};
var totalProject = 0;
db.open(function (err, collection) {
//metadata collection
collection = db.collection("project");
collection.find().toArray(function (err, result) {
// Length of metadata collection
projectDetail.projectCount = result.length;
var count = 0;
//iterate through each of the array which is the name of collection
result.forEach(function (item) {
//change collection object to new collection
collection = db.collection(item.keyParameter.wbsName);
// Perform first query based on some condition
collection.find({
$where: "this.status == 'Created'"
}).toArray(function (err, result) {
// based on result of query one increment the value of count
count += result.lenght;
// Perform second query based on some condition
collection.find({
$where: "this.status=='Completed'"
}).toArray(function (err, result) {
count += result.length;
});
});
});
// it is returning the value without finishing the above manipulation
// not waiting for above callback and value of count is coming zero .
res.render('index', {
projectDetail: projectDetail.projectCount,
count: count
});
});
});
};
When you want to call multiple asynchronous functions in order, you should call the first one, call the next one in it's callback and so on. The code would look like:
asyncFunction1(args, function () {
asyncFunction2(args, function () {
asyncFunction3(args, function () {
// ...
})
})
});
Using this approach, you may end up with an ugly hard-to-maintain piece of code.
There are various ways to achieve the same functionality without nesting callbacks, like using async.js or node-fibers.
Here is how you can do it using node.js EventEmitter:
var events = require('events');
var EventEmitter = events.EventEmitter;
var flowController = new EventEmitter();
flowController.on('start', function (start_args) {
asyncFunction1(args, function () {
flowController.emit('2', next_function_args);
});
});
flowController.on('2', function (args_coming_from_1) {
asyncFunction2(args, function () {
flowController.emit('3', next_function_args);
});
});
flowController.on('3', function (args_coming_from_2) {
asyncFunction3(args, function () {
// ...
});
});
flowController.emit('start', start_args);
For loop simulation example:
var events = require('events');
var EventEmitter = events.EventEmitter;
var flowController = new EventEmitter();
var items = ['1', '2', '3'];
flowController.on('doWork', function (i) {
if (i >= items.length) {
flowController.emit('finished');
return;
}
asyncFunction(item[i], function () {
flowController.emit('doWork', i + 1);
});
});
flowController.on('finished', function () {
console.log('finished');
});
flowController.emit('doWork', 0);
Use callbacks or promises or a flow control library. You cannot program servers in node without understanding at the very least one of these approaches, and honestly all halfway decent node programmers thoroughly understand all three of them (including a handful of different flow control libraries).
This is not a something you are going to just get an answer coded for you by someone else on stackoverflow and then move on. This is a fundamental thing you have to go and study and learn generically as it is only going to come up over and over again on a daily basis.
http://howtonode.org/control-flow
http://callbackhell.com/
Per the resources in the answer above me, nesting the callback when you iterate and only calling it if you are on the last iteration will solve you problem.
i have a recursive query like this (note: this is just an example):
var user = function(data)
{
this.minions = [];
this.loadMinions = function()
{
_user = this;
database.query('select * from users where owner='+data.id,function(err,result,fields)
{
for(var m in result)
{
_user.minions[result[m].id] = new user(result[m]);
_user.minions[result[m].id].loadMinions();
}
}
console.log("loaded all minions");
}
}
currentUser = new user(ID);
for (var m in currentUser.minions)
{
console.log("minion found!");
}
this don't work because the timmings are all wrong, the code don't wait for the query.
i've tried to do this:
var MyQuery = function(QueryString){
var Data;
var Done = false;
database.query(QueryString, function(err, result, fields) {
Data = result;
Done = true;
});
while(Done != true){};
return Data;
}
var user = function(data)
{
this.minions = [];
this.loadMinions = function()
{
_user = this;
result= MyQuery('select * from users where owner='+data.id);
for(var m in result)
{
_user.minions[result[m].id] = new user(result[m]);
_user.minions[result[m].id].loadMinions();
}
console.log("loaded all minions");
}
}
currentUser = new user(ID);
for (var m in currentUser.minions)
{
console.log("minion found!");
}
but he just freezes on the while, am i missing something?
The first hurdle to solving your problem is understanding that I/O in Node.js is asynchronous. Once you know how this applies to your problem the recursive part will be much easier (especially if you use a flow control library like Async or Step).
Here is an example that does some of what you're trying to do (minus the recursion). Personally, I would avoid recursively loading a possibly unknown number/depth of records like that; Instead load them on demand, like in this example:
var User = function(data) {
this.data = data
this.minions;
};
User.prototype.getMinions = function(primaryCallback) {
var that = this; // scope handle
if(this.minions) { // bypass the db query if results cached
return primaryCallback(null, this.minions);
}
// Callback invoked by database.query when it has the records
var aCallback = function(error, results, fields) {
if(error) {
return primaryCallback(error);
}
// This is where you would put your recursive minion initialization
// The problem you are going to have is callback counting, using a library
// like async or step would make this party much much easier
that.minions = results; // bypass the db query after this
primaryCallback(null, results);
}
database.query('SELECT * FROM users WHERE owner = ' + data.id, aCallback);
};
var user = new User(someData);
user.getMinions(function(error, minions) {
if(error) {
throw error;
}
// Inside the function invoked by primaryCallback(...)
minions.forEach(function(minion) {
console.log('found this minion:', minion);
});
});
The biggest thing to note in this example are the callbacks. The database.query(...) is asynchronous and you don't want to tie up the event loop waiting for it to finish. This is solved by providing a callback, aCallback, to the query, which is executed when the results are ready. Once that callback fires and after you perform whatever processing you want to do on the records you can fire the primaryCallback with the final results.
Each Node.js process is single-threaded, so the line
while(Done != true){};
takes over the thread, and the callback that would have set Done to true never gets run because the thead is blocked on an infinite loop.
You need to refactor your program so that code that depends on the results of the query is included within the callback itself. For example, make MyQuery take a callback argument:
MyQuery = function(QueryString, callback){
Then call the callback at the end of your database.query callback -- or even supply it as the database.query callback.
The freezing is unfortunately correct behaviour, as Node is single-threaded.
You need a scheduler package to fix this. Personally, I have been using Fibers-promise for this kind of issue. You might want to look at this or another promise library or at async