I'm using mysqljs to access MySQL with javascript.
I would like to point out that this process seems to work fine if a single piece of data.
I am feeding into my code a large set of data, to be processed line by line as a batch.
I create my connection like this:
var connection = mysql.createConnection({
//debug: ['ComQueryPacket'],
host : dataSource.host,
user : dataSource.user,
password: dataSource.password,
database: dataSource.database
});
I have three functions that make database queries.
The function containing the SELECT query is built like this:
dbSearch(data){
var sql = "SELECT * from table where field =? and otherfield=?";
return new Promise((resolve, reject) => {
connection.query(sql, [data[0], data[1], (error, results, fields) => {
if (error){
console.log(error);
reject("Database connection error: " + error);
} else {
resolve(results);
}
});
});
}
The code executes in another function:
if (dataItem){
dbSearch(dataItem)
.then((row) => {
processingfunction(row);
});
If I leave out connection.end() the code hangs and the stream of data is held up at the first item being processed.
If I put connection.end() inside the function, i get this error:
Database connection error: Error: Cannot enqueue Query after invoking quit.
I put connection.end() as the last line of the code, everything works fine
The problem though is for the update and insert functions:
updateRecord(data){
var sql = "UPDATE table set field=? where id=?";
return new Promise((resolve, reject) => {
connection.query(sql, [data[0], data[1], (error, results, fields) => {
if (error){
console.log(error);
reject("Database connection error: " + error);
} else {
resolve(results);
}
});
});
}
inputRecord(data){
var sql = "INSERT INTO table (field1, field2, field3) VALUES(?,?,?)";
return new Promise((resolve, reject) => {
connection.query(sql, [data[0], data[1], data[2]], (error, results, fields) => {
if (error){
console.log(error);
reject("Database connection error: " + error);
} else {
resolve(results);
}
});
});
}
With connection.end() in the function I get this error.
Database connection error: Error: Cannot enqueue Query after invoking quit.
(node:40700) UnhandledPromiseRejectionWarning: Database connection error: Error: Cannot enqueue Query after invoking quit.
(node:40700) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:40700) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
Based on the documentation, I have no clarity on how to properly handle closing the connection so that the code can process properly.
Not sure what I am doing wrong. Could use some mentorinng from someone experienced with using the connections to process chunks of data and how to properly handle closing the connections?
NOTE:
A similar problem happens when I try connection pooling, so that was not a workable solution.
You using connection pool, you don't have to close the conn manually. There is a .query() convenient method. Since you're already using .query(), don't close it at the end of the function.
This is a shortcut for the pool.getConnection() -> connection.query() -> connection.release() code flow.
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 10,
host : 'example.org',
user : 'bob',
password : 'secret',
database : 'my_db'
});
pool.query('SELECT 1 + 1 AS solution', function (error, results, fields) {
if (error) throw error;
console.log('The solution is: ', results[0].solution);
});
You are using an implicit connection. Meaning, you are attempting to query the database without explicitly connecting first. This means that each time you run this line:
reject("Database connection error: " + error);
You can't be certain that it was a connection error. It may have been a query error. I believe it is always best to explicitly create/destroy your connections.
inputRecord(data){
var sql = "INSERT INTO table (field1, field2, field3) VALUES(?,?,?)";
return new Promise((resolve, reject) => {
connection.connect(function(err){
if(err) reject("Database connection error: " + error);
connection.query(sql, [data[0], data[1], data[2]], (error, results, fields) => {
connection.end(); // Now you are certain a connection was made, and can be terminated
if (error){
console.log(error);
reject("Database connection error: " + error);
} else {
resolve(results);
}
});
});
});
}
Because this is a pattern that will be used over and over, I would suggest moving it to it's own function:
executeQuery(query, params){
return new Promise((resolve, reject) => {
connection.connect(function(err){
if(err) reject("Database connection error: " + error); // Connection error with certainty
connection.query(query, params, (error, results, fields) => {
connection.end(); // Now you are certain a connection was made, and can be terminated
if (error){
console.log(error);
reject("Database query error: " + error); // Query error
} else {
resolve(results);
}
});
});
});
}
And then simply call the function:
inputRecord(data){
var sql = "INSERT INTO table (field1, field2, field3) VALUES(?,?,?)";
return executeQuery(sql,[data[0],data[1],data[2]]);
}
updateRecord(data){
var sql = "UPDATE table set field=? where id=?";
return executeQuery(sql,[data[0],data[1]]);
}
dbSearch(data){
var sql = "SELECT * from table where field =? and otherfield=?";
return executeQuery(sql,[data[0],data[1]]);
}
To use connection pooling, executeQuery would become:
executeQuery(query, params){
return new Promise((resolve, reject) => {
pool.query(query, params,function(err,res){ // shortcut for connect, query,end - no need to terminate the connection
if(err) reject(err);
else resolve(res);
});
});
}
This should solve your problem, but if not, it should at least help narrow down the possibilities by breaking out the error conditions and bringing all query related code into the same function.
Checking for connection first
getConnection(){
return new Promise((resolve, reject) => {
if(connection.state === 'connected') resolve(connection);
else {
connection.connect(function(err) => {
if(err) reject ("Connection error: " + error);
else resolve(connection);
});
}
});
}
executeQuery(query, params){
return new Promise((resolve, reject) => {
getConnection().then(connection =>{
connection.query(query, params, (error, results, fields) => {
connection.end(); // Now you are certain a connection was made, and can be terminated
if (error){
console.log(error);
reject("Database query error: " + error); // Query error
} else {
resolve(results);
}
});
});
});
}
Related
I have the following function that gets a hexcode from the database
function getColour(username, roomCount)
{
connection.query('SELECT hexcode FROM colours WHERE precedence = ?', [roomCount], function(err, result)
{
if (err) throw err;
return result[0].hexcode;
});
}
My problem is that I am returning the result in the callback function but the getColour function doesn't return anything. I want the getColour function to return the value of result[0].hexcode.
At the moment when I called getColour it doesn't return anything
I've tried doing something like
function getColour(username, roomCount)
{
var colour = '';
connection.query('SELECT hexcode FROM colours WHERE precedence = ?', [roomCount], function(err, result)
{
if (err) throw err;
colour = result[0].hexcode;
});
return colour;
}
but of course the SELECT query has finished by the time return the value in colour
You have to do the processing on the results from the db query on a callback only. Just like.
function getColour(username, roomCount, callback)
{
connection.query('SELECT hexcode FROM colours WHERE precedence = ?', [roomCount], function(err, result)
{
if (err)
callback(err,null);
else
callback(null,result[0].hexcode);
});
}
//call Fn for db query with callback
getColour("yourname",4, function(err,data){
if (err) {
// error handling code goes here
console.log("ERROR : ",err);
} else {
// code to execute on data retrieval
console.log("result from db is : ",data);
}
});
If you want to use promises to avoid the so-called "callback hell" there are various approaches.
Here's an example using native promises and the standard MySQL package.
const mysql = require("mysql");
//left here for testing purposes, although there is only one colour in DB
const connection = mysql.createConnection({
host: "remotemysql.com",
user: "aKlLAqAfXH",
password: "PZKuFVGRQD",
database: "aKlLAqAfXH"
});
(async () => {
connection.connect();
const result = await getColour("username", 2);
console.log(result);
connection.end();
})();
function getColour(username, roomCount) {
return new Promise((resolve, reject) => {
connection.query(
"SELECT hexcode FROM colours WHERE precedence = ?",
[roomCount],
(err, result) => {
return err ? reject(err) : resolve(result[0].hexcode);
}
);
});
}
In async functions, you are able to use the await expression which will pause the function execution until a Promise is resolved or rejected. This way the getColour function will return a promise with the MySQL query which will pause the main function execution until the result is returned or a query error is thrown.
A similar but maybe more flexible approach might be using a promise wrapper package of the MySQL library or even a promise-based ORM.
I am using express and mysql packages to just test out queries.
I had this code:
connection.connect();
connection.query('SELECT 1 + 1 AS solution', (error, results) => {
if (error) {
connection.end();
console.log('error');
throw error;
}
connection.end();
console.log(results);
return results;
});
And it works just fine. But when I move it to a different file and try to export it like this:
query.js
const connection = require('../config/connection');
connection.connect();
module.exports = async (query) => connection.query(query, (error, results) => {
console.log(query);
if (error) {
// connection.end();
console.log('error');
throw error;
}
// connection.end();
console.log(results);
return results;
});
app.js
try {
const queryResult = await query('SELECT 1 + 1 AS solution');
console.log(queryResult, 2);
res.send(queryResult);
} catch (err) {
console.log(err);
}
I get this error:
TypeError: Converting circular structure to JSON
at JSON.stringify (<anonymous>)
at stringify (C:\Users\aironsid\Documents\tatooify\server\node_modules\express\lib\response.js:1119:12)
at ServerResponse.json (C:\Users\aironsid\Documents\tatooify\server\node_modules\express\lib\response.js:260:14)
at ServerResponse.send (C:\Users\aironsid\Documents\tatooify\server\node_modules\express\lib\response.js:158:21)
at app.get (C:\Users\aironsid\Documents\tatooify\server\app.js:44:9)
at process._tickCallback (internal/process/next_tick.js:68:7)
Also in console logs I get, a big object followed by number 2, then I get the errthe ``or, then I get the query value, and then the result. Like so:
Big object 2
error
SELECT 1 + 1 AS solution
[ RowDataPacket { solution: 2 } ]
So what this tells me is that my code runs in async. Removing async/await from both files doesn't change anything. So my code first runs the app.js code, then the query.js code, which takes some time. But why? And how can I make it wait? connection.query() does not return a promise.
Problem is that your async function does not wait for query to finish.
It should return a promise that resolves when the query completes, but instead returns promise that immediately resolves into return value of function connection.query().
You should change that to:
module.exports = (query) => {
return new Promise(resolve =>
connection.query(query, (error, results) => {
console.log(query);
if (error) {
// connection.end();
console.log('error');
throw error;
}
// connection.end();
console.log(results);
resolve(results);
})
);
}
With omited commented lines and shorter syntax:
module.exports = (query) => new Promise((resolve, reject) =>
connection.query(query, (error, results) => {
if (error) return reject(error);
resolve(results);
})
);
And even shorter:
module.exports = (query) =>
new Promise((resolve, reject) =>
connection.query(query, (error, results) =>
error ? reject(error) : resolve(results)
)
);
To clarify: your async keyword did not have any effect, because what it does is allow you to use await in its function and wraps whatever you return into a promise.
You did not use any awaits and immediately returned result via lambda function.
Note that your return statement returned from callback passed to connection.query function and not from the main function.
I created query function to connect to mssql database like below:
query(query) {
return sql.connect(config.connection.sqlserver).then(() => sql.query(query)).then((result) => {
sql.close();
return result.recordset;
}).catch((err) => console.log(err));
},
When I'm starting nodejs server everything works fine. When I'm doing refresh many times I'm getting result from database e.g.
But when I'm sending request from client side to server side I'm getting an error like below:
(node:19724) UnhandledPromiseRejectionWarning: Error: Global connection already
exists. Call sql.close() first.
at Object.connect (C:\repos\mtg-app\node_modules\mssql\lib\base.js:1723:31)
at query (C:\repos\mtg-app\server\query.js:6:16)
at asyncCall (C:\repos\mtg-app\server\routes\index.js:19:11)
at router.get (C:\repos\mtg-app\server\routes\index.js:29:3)
at Layer.handle [as handle_request] (C:\repos\mtg-app\node_modules\express\l
ib\router\layer.js:95:5)
at next (C:\repos\mtg-app\node_modules\express\lib\router\route.js:137:13)
at Route.dispatch (C:\repos\mtg-app\node_modules\express\lib\router\route.js:112:3)
at Layer.handle [as handle_request] (C:\repos\mtg-app\node_modules\express\lib\router\layer.js:95:5)
at C:\repos\mtg-app\node_modules\express\lib\router\index.js:281:22
at Function.process_params (C:\repos\mtg-app\node_modules\express\lib\router\index.js:335:12)
(node:19724) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:19724) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
I don't understand first of all why it's working only on server and secondly why it's not working despite sql.close() method?
Please explain me this issue very well.
In regards to my comment, something along these lines.
async function query(q) {
return new Promise((resolve, reject) => {
new sql.ConnectionPool({/* Config */}).connect().then((pool) => {
return pool.request().query(q)
}).then((result) => {
// Resolve result
resolve(result);
// Close sql.close();
sql.close();
}).catch((error) => {
// Reject error
reject(error);
// Close sql.close();
sql.close();
});
});
}
query(`SELECT * FROM`)
.then((response) => {
// Success
console.log(response);
})
.catch((error) => {
// Error
console.log(error);
})
.finally(() => {
// Clean-up
});
Or another way, with mysqlijs all from the documentation here.
const mysql = require('mysql');
const pool = mysql.createPool({/* Config */});
pool.getConnection(function(error, connection) {
if (error) {
throw error;
}
// Use the connection
connection.query(`SELECT * FROM`, function (error, results, fields) {
// When done with the connection, release it.
connection.release();
// Handle error after the release.
if (error) throw error;
// Don't use the connection here, it has been returned to the pool.
});
});
pool.on('acquire', function (connection) {
console.log('Connection %d acquired', connection.threadId);
});
pool.on('enqueue', function () {
console.log('Waiting for available connection slot');
});
pool.on('release', function (connection) {
console.log('Connection %d released', connection.threadId);
});
pool.end(function (err) {
// Ending all connection the the MySQL pool.
});
I did it in the simpler way than below. But thanks Raymond because I realized that I should use ConnectionPool.
My query module:
/* eslint-disable import/no-unresolved */
const mssql = require('mssql');
const database = require('./config/connection');
const pool = new mssql.ConnectionPool(database.config).connect();
async function query(sql) {
try {
const connection = await pool;
const result = await connection.query(sql);
return result.recordset;
} catch (err) {
console.log('SQL error', err);
}
return [];
}
module.exports.query = query;
Now when I want to use it e.g. in router module:
router.get('/get-users', (req, res, next) => {
const usersStandings = [];
sql.query('select u.name, s.points from dbo.[user] u join dbo.standings s on s.id = u.standings_id join dbo.category c on c.id = s.category_id where c.type = \'Tournament\' order by points desc').then((rows) => {
rows.forEach((element) => {
usersStandings.push(element);
});
res.send(usersStandings);
});
});
Now I don't have problems with Global connection etc.
I'm building a node.js app which in production will act as a SSH client to many servers, some of which may be inaccessible at any given time. I'm trying to write a function which attempts to run a SSH command with each client in its config upon startup, and I'm not able to handle both successful sessions and those which end in error. I wrapped a ssh2 client in a promise. If I remove the third (trash) server and only successes result, this works fine! See the output:
STDOUT: Hello World
STDOUT: Hello World
Session closed
Session closed
Successful session: Hello World,Hello World
But if one of the connections times out, even though I handle the error, I don't get to keep any of my data. It looks like the error message overwrites all of the resolved promises
Successful session: Error: Timed out while waiting for handshake,Error:
Timed out while waiting for handshake,Error: Timed out while waiting
for handshake
Here's my code, forgive me if this is a bit scattered, as I've combined a few of my modules for the sake of this question. My goal is to keep the data from the successful session and gracefully handle the failure.
var Client = require('ssh2').Client;
const labs = {
"ny1": "192.168.1.2",
"ny2": "192.168.1.3",
"ny3": "1.1.1.1"
};
function checkLabs() {
let numLabs = Object.keys(labs).length;
let promises = [];
for(i=0;i<numLabs;i++){
let labName = Object.keys(labs)[i];
promises.push(asyncSSH("echo 'Hello World'", labs[labName]));
}
Promise.all(promises.map(p => p.catch(e => e)))
.then(results => console.log("Successful session: " + results))
.catch(e => console.log("Error! " + e));
}
var sendSSH = function (command, dest, callback) {
var conn = new Client();
conn.on('ready', function() {
return conn.exec(command, function(err, stream) {
if (err) throw err;
stream.on('data', function(data) {
callback(null, data);
console.log('STDOUT: ' + data);
}).stderr.on('data', function(data){
callback(err);
console.log('STDERR: ' + data);
}).on('close', function(err) {
if(err) {
console.log('Session closed due to error');
} else {
console.log('Session closed');
}
});
stream.end('ls -l\nexit\n');
});
}).on('error', function(err){
callback(err);
}).connect({
host: dest,
port: 22,
username: 'root',
readyTimeout: 10000,
privateKey: require('fs').readFileSync('link-to-my-key')
});
};
function asyncSSH(command, dest) {
return new Promise(function(resolve, reject){
sendSSH(command, dest, function(err,data) {
if (!err) {
resolve(data);
} else {
reject(err);
}
});
});
}
checklabs();
How can I better use this promise wrapper to handle whatever errors come from the ssh2 client? Any tips are appreciated.
To get best use from each connection, you can (and arguably should) promisify separately :
the instatiation of each Client() instance
each instance's conn.exec() method (and any other asynchronous methods as required)
This will allow each instance of Client() to be reused with different commands (though not required in this example).
You should also be sure to disconnect each socket when its job is done, by calling client_.end(). For this, a "disposer pattern" is recommended.
With those points in mind and with a few assumptions, here's what I ended up with :
var Client = require('ssh2').Client;
const labs = {
"ny1": "192.168.1.2",
"ny2": "192.168.1.3",
"ny3": "1.1.1.1"
};
function checkLabs() {
let promises = Object.keys(labs).map((key) => {
return withConn(labs[key], (conn) => {
return conn.execAsync("echo 'Hello World'")
.catch((e) => "Error: " + e.message); // catch in order to immunise the whole process against any single failure.
// and inject an error message into the success path.
});
});
Promise.all(promises)
.then(results => console.log("Successful session: " + results))
.catch(e => console.log("Error! " + e.message)); // with individual errors caught above, you should not end up here.
}
// disposer pattern, based on https://stackoverflow.com/a/28915678/3478010
function withConn(dest, work) {
var conn_;
return getConnection(dest).then((conn) => {
conn_ = conn;
return work(conn);
}).then(() => {
if(conn_) {
conn_.end(); // on success, disconnect the socket (ie dispose of conn_).
}
}, () => {
if(conn_) {
conn_.end(); // on error, disconnect the socket (ie dispose of conn_).
}
});
// Note: with Bluebird promises, simplify .then(fn,fn) to .finally(fn).
}
function getConnection(dest) {
return new Promise((resolve, reject) => {
let conn = promisifyConnection(new Client());
conn.on('ready', () => {
resolve(conn);
})
.on('error', reject)
.connect({
host: dest,
port: 22,
username: 'root',
readyTimeout: 10000,
privateKey: require('fs').readFileSync('link-to-my-key')
});
});
}
function promisifyConnection(conn) {
conn.execAsync = (command) => { // promisify conn.exec()
return new Promise((resolve, reject) => {
conn.exec(command, (err, stream) => {
if(err) {
reject(err);
} else {
let streamSegments = []; // array in which to accumulate streamed data
stream.on('close', (err) => {
if(err) {
reject(err);
} else {
resolve(streamSegments.join('')); // or whatever is necessary to combine the accumulated stream segments
}
}).on('data', (data) => {
streamSegments.push(data);
}).stderr.on('data', function(data) {
reject(new Error(data)); // assuming `data` to be String
});
stream.end('ls -l\nexit\n'); // not sure what this does?
}
});
});
};
// ... promisify any further Client methods here ...
return conn;
}
NOTES:
the promisification of conn.exec() includes an assumption that data may be received in a series of segments (eg packets). If this assumption is not valid, then the need for the streamSegments array disappears.
getConnection() and promisifyConnection() could be written as one function but with separate function it's easier to see what's going on.
getConnection() and promisifyConnection() keep all the messy stuff well away from the application code.
I use promisifyAll to the following module since I want to use it with promises and I got error "TypeError: Cannot read property 'then' of undefined"
const DBWrapper = Promise.promisifyAll(require("node-dbi").DBWrapper);
var dbWrapper = new DBWrapper('pg', dbConnectionConfig);
dbWrapper.connect();
dbWrapper.insert('USERS', data, function (err, data) {
if (err) {
console.log("error to insert data: " + err);
} else {
console.log("test" + data);
}
}).then(() => {
//read data
dbWrapper.fetchAll("SELECT * FROM USERS", null, function (err, result) {
if (!err) {
console.log("Data came back from the DB.", result);
} else {
console.log("DB returned an error: %s", err);
}
dbWrapper.close(function (close_err) {
if (close_err) {
console.log("Error while disconnecting: %s", close_err);
}
});
});
})
You have two things going on here that are incorrect from what I can tell.
You're not properly implementing the promises in the above code. Error first callbacks aren't passed in when invoking Promise returning methods, instead they pass the result value to the nearest .then(). If an error occurs they will pass that to the nearest .catch(). If there is no .catch() and an error occurs, an unhandledRejection error will be thrown.
By default, promisifyAll() appends the suffix Async to any promisified methods, so you will need to modify any method calls on dbWrapper accordingly.
Assuming node-dbi can be promisifed and that your DB calls are correct the following code should work as you were initially anticipating
const Promise = require('bluebird');
const DBWrapper = require("node-dbi").DBWrapper;
const dbWrapper = Promise.promisifyAll(new DBWrapper('pg', dbConnectionConfig));
return dbWrapper.insertAsync('USERS', data)
.then((data) => {
console.log("test" + data);
//read data
return dbWrapper.fetchAllAsync("SELECT * FROM USERS", null)
})
.then((result) => {
console.log('Data came back from DB.', result);
return dbWrapper.closeAsync();
})
.catch((err) => {
console.log('An error occurred:', err);
});