Problem with Promised Connections
I recently converted my Node app from running on my local machine to utilizing an Amazon EC2 for the Node app and a VPN for the file-serving and MySQL.
I learned just enough about Promises to write the following connection snippet (which runs 3 queries before responding to the client), utilizing Bluebird. The connections worked on my machine, but with the VPN hosted MySQL settings, the connections crashed every time, about 30 seconds after the app started, which I realized was probably because I'd forgotten to close them.
EDIT: Based on the comments, it appears the issue is not in the connection closures.
So I modified my script in the best way I knew to close the connections, but with Promises, this is confusing. This version of the connection doesn't work. It doesn't fail or cause any errors. It just returns no results on the server side. I think my problem is in the way I've closed the connections.
What's causing the issue?
Is it the connection closures?
If so, how would I close them properly?
My (Simplified) MySQL Connection Attempt with Bluebird Promises
var mysql = require('mysql');
var Promise = require('bluebird');
var moment = require('moment');
function createConnection() {
var connection = mysql.createConnection({
dateStrings : true,
host : 'hostname',
user : 'username',
password : 'password',
database : 'database'
});
connection = Promise.promisifyAll(connection);
return connection;
}
function sendGame(req, res, sales, settings, categories, players) {
var game = new Object();
game.sales = sales;
game.players = players;
game.settings = settings;
game.categories = categories;
var JSONgame = JSON.stringify(game);
console.log("Game: " + JSON.stringify(game, undefined, 4));
}
var retrieveSales = Promise.method(function (username, connection, timeFrame) {
console.log('User ' + username + ' retrieving sales...');
var q = 'select * from sales_entries where date BETWEEN ? AND ?';
return connection.queryAsync(q, timeFrame).then(function (results) {
return results[0];
});
});
var retrieveSettings = Promise.method(function (username, connection) {
console.log('User ' + username + ' retrieving settings...');
var q = 'select * from sales_settings';
return connection.queryAsync(q).then(function (results) {
return results[0];
});
});
var retrieveCategories = Promise.method(function (username, connection) {
console.log('User ' + username + ' retrieving categories...');
var q = 'select * from sales_categories';
return connection.queryAsync(q).then(function (results) {
return results[0];
});
});
var retrievePlayers = Promise.method(function (username, connection) {
console.log('User ' + username + ' retrieving players...');
var q = 'select * from users';
return connection.queryAsync(q).then(function (results) {
return results[0];
});
});
var gameSucceed = Promise.method(function gameSucceed(req, res) {
var username = req.body.username;
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment().days(0).hour(0).minute(0).second(0).format("YYYY-MM-DD HH:mm:ss"), moment().days(6).hour(0).minute(0).second(0).format("YYYY-MM-DD HH:mm:ss")];
//var connection = Promise.promisifyAll(createConnection());
return connection.connectAsync().then(function () {
console.log('Connection with the MySQL database openned for Game retrieval...');
return Promise.all([retrieveSales(username, connection, timeFrame), retrieveSettings(username, connection), retrieveCategories(username, connection), retrievePlayers(username, connection)]);
}).then(function () {
connection.end(),
console.log("...Connection with the MySQL database for Game retrieval ended")
});
});
function getGameData(req, res) {
gameSucceed(req, res).spread(function (sales, settings, categories, players) {
return sendGame(req, res, sales, settings, categories, players);
});
};
var req = new Object();
var res = new Object();
req.body = {
"username" : "user123",
"password" : "password"
}
getGameData(req, res);
Console Result
User user123 retrieving game...
Connection with the MySQL database openned for Game retrieval...
User user123 retrieving sales...
User user123 retrieving settings...
User user123 retrieving categories...
User user123 retrieving players...
...Connection with the MySQL database for Game retrieval ended
Game: {}
var gameSucceed = function gameSucceed(req, res) {
…
var connection = createConnection());
return connection.connectAsync().then(function () {
return Promise.all([…]);
}).then(function () {
connection.end();
});
};
The promise that is ultimately returned from this method does not have a resolution value. It is created by that then call from whose callback you do not return - which will lead to undefined. To fix this, just route the result through:
.then(function(results) {
connection.end();
return results;
});
However, if you do it like that the connection won't be closed in case of an error. The best solution is to use the finally() method, which just works like a finally clause in synchronous code. It's callback will be invoked both for resolutions and rejections, and the resulting promise will automatically carry on the value.
.finally(function() {
connection.end();
})
// .then(function(results) { })
Your code has a particular resource management problem like Bergi put it. You have to keep remembering when to close the collection and when not to.
The optimal solution would be to use Promise.using however, that's only available in the v2 branch of Bluebird so you're going to have to wait a while.
Until then, you can create your own wrapper method that does more basic scoped resource management:
function connect(fn,timeout){
timeout = (timeout === undefined) ? 8000 : timeout; // connection timeout
return createConnection().then(function(connection){
// run the function, when it resolves - close the connection
// set a 7 second timeout on the connection
return fn(connection).timeout(timeout).finally(function(){
connection.end();
});
});
}
Which would let you do:
connect(function(connection){
return gameSucceed(req,resp,connection); // connection is injected to that fn now
}).then(function(val){
// gameSucceed resolution value here
});
Now, when the gameSucceed is done, the connection will close itself automatically. This would make gameSucceed itself look like:
var gameSucceed = Promise.method(function gameSucceed(req, res,connection) {
var username = req.body.username;
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment().days(0).hour(0).minute(0).second(0).format("YYYY-MM-DD HH:mm:ss"), moment().days(6).hour(0).minute(0).second(0).format("YYYY-MM-DD HH:mm:ss")];
return connection.connectAsync().then(function () {
console.log('Connection with the MySQL database openned for Game retrieval...');
return Promise.all([retrieveSales(username, connection, timeFrame), retrieveSettings(username, connection), retrieveCategories(username, connection), retrievePlayers(username, connection)]);
}); // no longer its responsibility to handle the connection
});
Generally, you might also want to consider a more OOPish style of coding for your code.
Good luck, and happy coding.
Related
I am unable to execute the sql, when using the global database connection in node.js.
I have followed the steps as in Azure documentation: https://learn.microsoft.com/en-us/azure/mysql/connect-nodejs and able to display the output on the console. But, I want to put all my Azure SQL database connection in a separate file, but the select query is not printing the output on the console.
DatabaseManager.js
var Connection = require('tedious').Connection;
var Request = require('tedious').Request;
var sqlConnection = function sqlConnection() {
// Create connection to database
var config =
{
userName: 'uname',
password: 'password',
server: 'dbserver.database.windows.net',
options:
{
database: 'mydatabase',
encrypt: true
}
}
var connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on('connect', function(err) {
if (err)
{
console.log(err)
}
else
{
console.log('CONNECTED TO DATABASE');
}
}
);
}
module.exports = sqlConnection;
app.js
var restify = require('restify');
var builder = require('botbuilder');
var botbuilder_azure = require("botbuilder-azure");
var azure = require('azure-storage');
var dbconnection = require('./DatabaseManager');
bot.dialog('profileDialog',
(session) => {
session.send('You reached the profile intent. You said \'%s\'.', session.message.text);
console.log('Reading rows from the Table...');
dbconnection("select FNAME from StudentProfile where ID=1"),
function (err, result, fields) {
if (err) throw err;
console.log(result);
}
session.endDialog();
}
Console Output:
Reading rows from the Table...
CONNECTED TO DATABASE
I was expecting the output of FNAME, but nothing is printing on the console. Is there anything, I am missing?
Thank you.
There's a couple of problems here. First off, you should only ever import a module once per file. This is just a performance consideration and won't actually break your code.
Next, pay attention to what you're exporting from your DatabaseManager module. Right now, you're exporting a function that creates the connection and then doesn't do anything with it. We can fix this by using a pattern called a "callback" which lets us provide a function that will then be called with the connection as an argument.
I added a ton of comments to the code explaining things. This code won't run as-is - there's a couple places where I have "do this or this". You'll have to choose one.
var Tedious = require('tedious'); // Only require a library once per file
var Connection = Tedious.Connection;
var Request = Tedious.Request;
// Or using the object spread operator
var { Connection, Request } = require('tedious');
// You called this `sqlConnection`. I'm going to use a verb since it's a
// function and not a variable containing the connection. I'm also going
// to change the declaration syntax to be clearer.
function connect(cb) { // cb is short for callback. It should be a function.
var config = {
userName: 'uname',
password: 'password',
server: 'dbserver.database.windows.net',
options: {
database: 'mydatabase',
encrypt: true
}
}; // Put a semi-colon on your variable assignments
var connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on('connect', function(err) {
if (err) {
console.log(err);
return; // Stop executing the function if it failed
}
// We don't need an "else" because of the return statement above
console.log('CONNECTED TO DATABASE');
// We have a connection, now let's do something with it. Call the
// callback and pass it the connection.
cb(connection);
});
}
module.exports = connect; // This exports a function that creates the connection
Then back in your main file, you can use it like so.
var restify = require('restify');
var builder = require('botbuilder');
var botbuilder_azure = require('botbuilder-azure');
var azure = require('azure-storage');
var connect = require('./DatabaseManager'); // renamed to be a verb since it's a function.
bot.dialog('profileDialog', (session) => { // Hey, this is a callback too!
session.send('You reached the profile intent. You said \'%s\'.', session.message.text);
console.log('Creating a connection');
connect((connection) => {
// or with the traditional function notation
connect(function(connection) {
console.log('Reading rows from the Table...');
// Execute your queries here using your connection. This code is
// taken from
// https://github.com/tediousjs/tedious/blob/master/examples/minimal.js
request = new Request("select FNAME from StudentProfile where ID=1", function(err, rowCount) { // Look another callback!
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
connection.close();
});
request.on('row', function(columns) { // Iterate through the rows using a callback
columns.forEach(function(column) {
if (column.value === null) {
console.log('NULL');
} else {
console.log(column.value);
}
});
});
connection.execSql(request);
});
I currently have an issue in Node.js whereby I make a call to a mysql database using the promise-mysql and bluebird packages. I followed the tutorials on the webpage and it seems to work. I keep ending up with a timeout error as once the database has been queried the process doesn't break (if that's the right term) and I either have to terminate the process using ctrl + c or when testing on Alexa when the skill replies with there was a problem. Is there a way to end the promise and db connection once I have retrieved the value of my query? Below is an example of the code.
index.js
'use strict';
const Alexa = require('alexa-sdk');
const mysql = require('promise-mysql');
const querydb = require('./sqlQuery.js');
const insertdb = require('./sqlInsert.js');
var testSQL = 'SELECT weight, height from users where pin=1100';
var valuesPromise = querydb.getItem(testSQL);
valuesPromise.then((result)=>{
rows = result.height;
row2 = result.weight;
console.log(rows);
console.log (row2);
return true;
}).catch(function(error){
console.log (error);
});
databaseConnection.js
var mysql = require('promise-mysql');
pool = mysql.createPool({
host: "hostURL",
database: "dbName",
user: "dbUser",
password: "dbPassword",
connectionLimit: 1
});
function getSqlConnection() {
return pool.getConnection().disposer(function(connection) {
pool.releaseConnection(connection);
});
}
module.exports = getSqlConnection;
sqlQuery.js
var Promise = require("bluebird");
var getSqlConnection = require('./databaseConnection');
function getItem(sql){
return Promise.using(getSqlConnection(), function(connection) {
return connection.query(sql).then(function(rows) {
return rows[0];
}).catch(function(error) {
return (error);
});
})
};
module.exports.getItem = getItem;
Since you are in node process you can just exit from it with process.exitCode = 1 and then process.exit() or process.exit(1).
Here is the documentation.
That however is not the best practice. The easiest way would be to just throw an error:
throw "Exit!" or some error you would like to catch and re-throw to exit (in the scenario where you have a catch statement)
I am using Node.js with MySQL and restify.
I have the following code which is run as part of a REST API. It works fine.
server.get('/test', function (req, res, next) {
var query_string =
"SELECT DATE(date_transacted) AS transaction_date, " +
" MonthReports.tb AS MonthReports__tb " +
" FROM monthly_reports MonthReports " +
" WHERE ( date_transacted >= \'2015-01-00\' AND date_transacted <= \'2015-09-00\' ) ";
connection.query(
query_string
, function (err, rows, fields) {
if (err) throw err;
res.send(rows);
});
});
If I deliberately turn off the MySQL database and makes a REST API call which will run the query, I will get the error
Cannot enqueue Query after fatal error.
At this point, I turn on the MySQL database. The node.js process is unable to recover and the same error keeps appearing when I make a REST API call. The REST API server is dead.
What can be done to make the Node.js REST API server code recoverable?
I am assuming you are connecting globally inside your script.
One simple way would be to create a connection per request:
server.get('/test', function (req, res, next) {
var query_string =
"SELECT DATE(date_transacted) AS transaction_date, " +
" MonthReports.tb AS MonthReports__tb " +
" FROM monthly_reports MonthReports " +
" WHERE ( date_transacted >= \'2015-01-00\' AND date_transacted <= \'2015-09-00\' ) ";
var connection = getConnection(function connected(err) {
if (err) {
// error connecting to mysql! alert user
} else {
connection.query(
query_string
, function (err, rows, fields) {
if (err) throw err;
res.send(rows);
});
}
});
});
The above code is psuedo code as i'm not familiar with the node mysql library. This will allow each request to see if mysql is able to be connected to, at the expense of having a connection per web request.
Another strategy could be to check err when you issue a query, and if there is an error try to reestablish the global connection
server.get('/test', function (req, res, next) {
var query_string =
"SELECT DATE(date_transacted) AS transaction_date, " +
" MonthReports.tb AS MonthReports__tb " +
" FROM monthly_reports MonthReports " +
" WHERE ( date_transacted >= \'2015-01-00\' AND date_transacted <= \'2015-09-00\' ) ";
connection.query(
query_string
, function (err, rows, fields) {
if (err) {
// Try to reconnect here instead of throwing error and stopping node process, and reissue query
}
res.send(rows);
});
});
This website gives a complete answer. Credit goes to the writer of this article, not me.
https://www.exratione.com/2013/01/nodejs-connections-will-end-close-and-otherwise-blow-up/
/**
* #fileOverview A simple example module that exposes a getClient function.
*
* The client is replaced if it is disconnected.
*/
var mysql = require("mysql");
var client = mysql.createConnection({
host: "127.0.0.1",
database: "mydb",
user: "username",
password: "password"
});
/**
* Setup a client to automatically replace itself if it is disconnected.
*
* #param {Connection} client
* A MySQL connection instance.
*/
function replaceClientOnDisconnect(client) {
client.on("error", function (err) {
if (!err.fatal) {
return;
}
if (err.code !== "PROTOCOL_CONNECTION_LOST") {
throw err;
}
// client.config is actually a ConnectionConfig instance, not the original
// configuration. For most situations this is fine, but if you are doing
// something more advanced with your connection configuration, then
// you should check carefully as to whether this is actually going to do
// what you think it should do.
client = mysql.createConnection(client.config);
replaceClientOnDisconnect(client);
client.connect(function (error) {
if (error) {
// Well, we tried. The database has probably fallen over.
// That's fairly fatal for most applications, so we might as
// call it a day and go home.
//
// For a real application something more sophisticated is
// probably required here.
process.exit(1);
}
});
});
}
// And run this on every connection as soon as it is created.
replaceClientOnDisconnect(client);
/**
* Every operation requiring a client should call this function, and not
* hold on to the resulting client reference.
*
* #return {Connection}
*/
exports.getClient = function () {
return client;
};
This answer was extracted from another link nodejs mysql Error: Connection lost The server closed the connection
The extracted code;
var db_config = {
host: 'localhost',
user: 'root',
password: '',
database: 'example'
};
var connection;
function handleDisconnect() {
connection = mysql.createConnection(db_config); // Recreate the connection, since
// the old one cannot be reused.
connection.connect(function(err) { // The server is either down
if(err) { // or restarting (takes a while sometimes).
console.log('error when connecting to db:', err);
setTimeout(handleDisconnect, 2000); // We introduce a delay before attempting to reconnect,
} // to avoid a hot loop, and to allow our node script to
}); // process asynchronous requests in the meantime.
// If you're also serving http, display a 503 error.
connection.on('error', function(err) {
console.log('db error', err);
if(err.code === 'PROTOCOL_CONNECTION_LOST') { // Connection to the MySQL server is usually
handleDisconnect(); // lost due to either server restart, or a
} else { // connnection idle timeout (the wait_timeout
throw err; // server variable configures this)
}
});
}
handleDisconnect();
I am looking at some node.js code that does push notification on a MySQL database.
http://www.gianlucaguarini.com/blog/push-notification-server-streaming-on-a-mysql-database/
There is a polling function.
var pollingLoop = function() {
// Doing the database query
var query = connection.query('SELECT * FROM users'),
users = []; // this array will contain the result of our db query
// setting the query listeners
query
.on('error', function(err) {
// Handle error, and 'end' event will be emitted after this as well
console.log(err);
updateSockets(err);
})
.on('result', function(user) {
// it fills our array looping on each user row inside the db
users.push(user);
})
.on('end', function() {
// loop on itself only if there are sockets still connected
if (connectionsArray.length) {
pollingTimer = setTimeout(pollingLoop, POLLING_INTERVAL);
updateSockets({
users: users
});
} else {
console.log('The server timer was stopped because there are no more socket connections on the app')
}
});
};
The particular code segment above that puzzles me is this;
updateSockets({
users: users
});
Why is the argument users: users?
The code for updateSockets() is here;
var updateSockets = function(data) {
// adding the time of the last update
data.time = new Date();
console.log('Pushing new data to the clients connected ( connections amount = %s ) - %s', connectionsArray.length , data.time);
// sending new data to all the sockets connected
connectionsArray.forEach(function(tmpSocket) {
tmpSocket.volatile.emit('notification', data);
});
};
{
users : users
}
This code is just a plain objet. The first users is the name of the object property and the second users is just a variable.
You can write like this if you want :
var myUsers = users;
updateSockets({
users: myUsers
});
It's an additional information stored in data
When this code performs emit(data), it sends the packet with parameters user and time (added in updateSockets)
It's the message you want to send
After recently discovering JS promises, I have been studying them so that I might build a certain functionality that allows me to execute 4 async queries, use the result of each to build an object that I can finally send as a response to a request directed at my node app.
The final object is made up of 3 array properties containing the resulting rows of each query.
It seems that I've done something wrong handling the promises, though, because ultimately, game is not being built. It is sent as an empty object. Here's a JSFiddle.
What is my mistake?
Here's what I have so far:
function sendGame(req, res, sales, settings, categories) {
var game = new Object();
game.sales = sales;
game.settings = settings;
game.categories = categories;
JSONgame = JSON.stringify(game);
res.writeHead(200, {
'Access-Control-Allow-Origin': 'http://localhost',
'Content-Length': JSONgame.length,
'Content-Type': 'application/json'
});
res.write(JSONgame);
res.end();
console.log('Game: ' + JSON.stringify(game, null, 4));
console.log('--------------------------------------');
console.log('User ' + req.body.username + ' successfully retrieved game!');
}
function retrieveSales(req, connection, timeFrame) {
console.log('User ' + req.body.username + ' retrieving sales...');
connection.query('select * from sales_entries where date BETWEEN ? AND ?', timeFrame,
function (err, rows, fields) {
if (err) {
callback(new Error('Failed to connect'), null);
} else {
sales = [];
for (x = 0; x < rows.length; x++) {
sales.push(rows[x]);
}
//console.log('Sales: ' + JSON.stringify(sales, null, 4));
return sales;
}
});
}
retrieveCategories() and retrieveSettings() omitted for readability; they are the same as retrieveSales() mostly.
function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment().days(0).format("YYYY-MM-DD HH:mm:ss"), moment().days(6).format("YYYY-MM-DD HH:mm:ss")];
var connection = createConnection();
connection.connect(function (err) {
if (err) return callback(new Error('Failed to connect'), null);
console.log('Connection with the Officeball MySQL database openned for game retrieval...');
var sales = retrieveSales(req, connection, timeFrame);
var settings = retrieveSettings(req, connection);
var categories = retrieveCategories(req, connection);
var all = q.all([sales, settings, categories]);
all.done(function () {
sendGame(req, res, sales, settings, categories);
});
});
}
Your problem is that you're not using promises. All your APIs use callbacks.
A promise is like a closed box:
A promise also has a method that opens the box, works on the value and returns another box on the value (also opening any additional boxes along the way). That method is .then:
In boxes, it does:
=>( . => ) =>
That is, it adds a handler that gets an open box and returns a box. Everything else just combines stuff. All .all does is wait for a list of promises to resolve, it is exactly like .then in the fact it waits for a result. Because promises are boxes, you can pass them around and return them which is very cool.
Generally:
Whenever you return from a promise handler (not a rejection), you are fullfilling it indicating normal flow continuation.
Whenever you throw at a promise handler, you are rejecting indication exceptional flow.
So basically in node speak:
Whenever you returned a null error and a response, you resolve the promise.
Whenever you returned an error and no response, you reject the promise.
So:
function myFunc(callback){
nodeBack(function(err,data){
if(err!== null){
callback(new Error(err),null);
}
callback(data+"some processing");
})
});
Becomes:
function myFunc(){
return nodeBack().then(function(data){ return data+"some processing"; });
}
Which I think is a lot clearer. Errors are propagated across the promise chain just like in synchronous code - it's very common to find synchronous analogs to promise code.
Q.all takes a list of promises and waits for them to complete, instead you want Q.nfcall to transform a callback based API to a promise one and then use Q.all on that.
That is:
var sales = Q.nfcall(retrieveSales,req, connection, timeFrame);
var settings = Q.nfcall(retrieveSettings,req, connection);
var categories = Q.nfcall(retrieveCategories, req, connection);
Q.nfcall takes a nodeback in the err,data convention and converts it to a promise API.
Also, when you do
return sales;
You are not really returning anything, since it returns synchronously. You need to use callback like in your error case or promisify it altogether. If you don't mind, I'll do it with Bluebird since it comes with much better facilities for dealing with these interop cases and does so much much faster, if you'd like you can switch promisifyAll for a bunch of Q.nfcall calls.
// somewhere, on top of file
connection = Promise.promisifyAll(connection);
// note I'm passing just the username - passing the request breaks separation of concerns.
var retrieveSales = Promise.method(username, connection, timeFrame) {
console.log('User ' + username + ' retrieving sales...');
var q = 'select * from sales_entries where date BETWEEN ? AND ?';
return connection.queryAsync(q, timeFrame).then(function(rows, fields){
return rows;
});
}
Note that suddenly you don't need a lot of boilerplate for making a query, you can use queryAsync directly instead if you'd like.
Now the code that wraps it becomes:
var gameSucceed = Promise.method(function gameSucceed(req, res) {
console.log('User ' + req.body.username + ' retrieving game...');
var timeFrame = [moment()....];
var connection = Promise.promisifyAll(createConnection());
return conn.connectAsync().then(function () {
console.log('Connection with the ...');
//sending req, but should really be what they use.
return Promise.all([retrieveSales(req,conn,timeFrame),
retrieveSettings(req,conn),
retrieveCategories(req,conn)]);
});
});
Now you can call sendGame(req, res, sales, settings, categories); outside of gameSucceed which doesn't hide what it does as much -
gameSucceed(req,res).spread(function(sales,settings,cats){
return sendGame(req,res,sales,settings,cats);
});