I'm aware of the best practice of MongoDB connection pooling in NodeJS of the singleton DB connection type like this
var db = null;
var connection = function getDBConnection(callback) {
if(db) { callback(null, db) } else { MongoClient.connect( .... ) }
}
module.exports = getDBConnection;
However, what I cannot get my head around at the moment is how to handle this in a one-shot script that, say, does some pre-initialization on the documents of a certain db collection:
getDBConnection(function (err, database) {
var collection = database.collection("objects");
var allObjectsArray = collection.find( /* ... */
).toArray(function (err, objects) {
if(err != null) console.log(err);
assert.equal(null, err);
_.each(objects, function (item) {
collection.update(
{ id: item.id},
{ $set: { /* ... */ }},
function (err, result) {
if(err != null) console.log(err);
assert.equal(null, err);
}
);
});
// database.close(); <-- this fails with "MongoError: Connection Closed By Application", thrown by the update callback
});
// database.close(); <-- this fails too, thrown by the toArray callback
});
If I call the script like that, it never terminates, due to the still open connection. If I close the connection at the bottom, it fails because of, well, a closed connection.
Considering that opening a new connection for every update is not really an option, what am I missing? Keeping the connection open may be fine for webapps, but for a one-shot script called from a shell script this really doesn't work out, does it?
Sorry if this question has arisen before, I've given it some research but have not quite been able to come up with a working answer for me...
Thanks!
Julian
As a "pooled connection" there is code running to keep the connection alive and establish more connections in the pool if required under the driver connection. So much like various "server code" methods, event loop handlers have been invoked and the process does not exit at the end of your code until these are de-registered.
Therefore your two choices to call after all your code has executed are either:
Call db.close() or in your code context specifically database.close() once all is done.
Call process.exit() which is a generic call in node.js applications which will shut the whole process down and therefore stop any other current event loop code. This actually gives you an option to throw an error on exit if you want your code to be "shell integrated" somewhere and look for the exit status.
Or call both. db.close() will allow execution to the next line of code and whatever you put there will also run.
But you have to wait until everything is called, so you can cannot use synchronous loops with asynchronous code in the middle:
async.each(objects,function(item,callback) {
collection.update(
{ "_id": item._id },
{
// updates
},
callback
);
},function(err) {
if (err) throw err;
database.close();
});
Related
I am creating a RESTful API that uses Node.js and Express. My application uses an Oracle database so I installed the node-oracledb module from npm. I've looked through the documentation and viewed some of the samples presented in the module's github page; however, I don't see any examples where a connection pool is used. Correct me if I'm wrong, but for applications that will make many calls to the database the recommendation is to use connection pools as opposed to using standalone connections. Below is a sample of the code I wrote:
createPool = function(poolAttrs, fetchPool){
oracledb.createPool(poolAttrs, function(error, pool){
if(error){
console.error(`Could not create pool using specified attributes: `, error.message);
}
else{
console.log(`Pool created successfully using poolAlias: ${poolAttrs.poolAlias}`);
fetchPool(pool);
}
});
};
createConnection = function(poolAlias, connection){
oracledb.getConnection(poolAlias, function(error, conn){
if(error){
console.error(`Could not get connection: `, error.message);
} else {
console.log(`New connection obtained: ${conn}`);
connection(conn);
}
});
};
executeQuery = function(queryString, poolAlias){
console.log(queryString);
var conn = createConnection(poolAlias, function connection(conn){
conn.execute(queryString, function(error, result){
if(error){
console.error(`Could not execute query: `, error.message);
} else {
console.log(result);
}
conn.close(function(error){
if(error){
console.error(`Could not close connection.`);
} else {
console.log(`Connection successfully closed.`);
}
})
});
});
}
closePool = function(pool){
pool.close(60, function(error){
if(error){
console.error(`Could not close connection pool ${pool.poolAlias}`, error.message);
} else {
console.log(`Pool closed successfully.`);
}
});
};
createPool(attrs, function fetchPool(pool){
var poolAlias = pool.poolAlias;
console.log(poolAlias);
});
executeQuery(queryString, attrs.poolAlias);
When I run this code I get the following error:
Could not get connection: NJS-047: poolAlias "amDBpool" not found in the connection pool cache
I know why the error is happening. As my understanding of callbacks go, I know that calling the asynchronous createPool() function with the fetchPool(pool) callback registers this callback (fetchPool) in the callback stack. All synchronous code will execute before it. So when I call executeQuery and the function reaches the line of execution where it calls createConnection(poolAlias...) the poolAlias variable is null since the createPool function is still waiting in the callback stack to execute. Hence, no pool with the alias "poolAlias" exists and the call fails. I know that I could stick the call to createPool inside the executeQuery method, but wouldn't that try to create a new pool every time I execute a query? My question is, is there a way to test inside the executeQuery method that the pool exists, and if it does then not try to recreate it. Other than this, the only other way to do this would be with an Promises or Async/Await right?
The node-oracledb examples using a connection pool are the connectionpool.js and examples/webap.js files. Yes, a connection pool would be a good idea in a web service. Also I recommend using the async/await style of programming.
You can see the examples creates a connection pool before doing anything else:
await oracledb.createPool({
user: dbConfig.user,
password: dbConfig.password,
connectString: dbConfig.connectString
});
The returned pool from the oracledb.createPool() call is ignored because the pool is later accessed via the pool cache since no credentials are passed to getConnection():
let connection = await oracledb.getConnection();
This uses the default pool alias (which happens to be the string "default"), so no alias is specified when the pool is created or used.
Documentation and tips on using node-oracledb connection pools is in Connection Pooling.
You may be interested in Creating a REST API with Node.js and Oracle Database,
A node-oracledb Web Service in Docker and Demo: GraphQL with Oracle Database and node-oracledb
Right now i am creating a very large application in Node JS. I am trying to make my code clean and short (Just like most of the developer). I've create my own js file to handle connection to mysql. Please see code below.
var mysql = require('mysql');
var config = {
'default' : {
connectionLimit : process.env.DB_CONN_LIMIT,
host : process.env.DB_HOST,
user : process.env.DB_USER,
password : process.env.DB_PASS,
database : process.env.DB_NAME,
debug : false,
socketPath : process.env.DB_SOCKET
}
};
function connectionFunc(query,parameters,callback,configName) {
configName = configName || "default";
callback = callback || null;
parameters = parameters;
if(typeof parameters == 'function'){
callback = parameters;
parameters = [];
}
//console.log("Server is starting to connect to "+configName+" configuration");
var dbConnection = mysql.createConnection(config[configName]);
dbConnection.connect();
dbConnection.query(query,parameters, function(err, rows, fields) {
//if (!err)
callback(err,rows,fields);
//else
//console.log('Error while performing Query.');
});
dbConnection.end();
}
module.exports.query = connectionFunc;
I am using the above file in my models, like below :
var database = require('../../config/database.js');
module.exports.getData = function(successCallBack){
database.query('SAMPLE QUERY GOES HERE', function(err, result){
if(err) {console.log(err)}
//My statements here
});
}
Using this coding style, everything works fine but when i am trying to create a function that will loop my model's method for some reason. Please see sample below :
for (i = 0; i < 10000; i++) {
myModel.getData(param, function(result){
return res.json({data : result });
});
}
It gives me an ER_CON_COUNT_ERROR : Too Many Conenction. The question is why i still get an error like these when my connection always been ended by this dbConnection.end();? I'm still not sure if i am missing something. I am still stuck on this.
My connection limit is 100 and i think adding more connection is a bad idea.
Because query data form the database is async.
In your loop the myModel.getData (or more precisely the underling query) will not halt/paus your code until the query is finished, but send the query to the database server and as soon as the database response the callback will be called.
The calling end on dbConnection will not close the connection immediately, it will just mark the connection to be closed as soon as all queries that where created with that connection are finished.
mysql: Terminating connections
Terminating a connection gracefully is done by calling the end() method. This will make sure all previously enqueued queries are still before sending a COM_QUIT packet to the MySQL server.
An alternative way to end the connection is to call the destroy() method. This will cause an immediate termination of the underlying socket. Additionally destroy() guarantees that no more events or callbacks will be triggered for the connection.
But with destroy the library will not wait for the result so the results are lost, destroy is rarely useful.
So with your given code you try to create 10000 connections at one time.
You should only use on connection by task, e.g. if a user requests data using the browser, then you should use one connection for this given request. The same is for timed task, if you have some task that is done in certain intervals.
Here an example code:
var database = require('./config/database.js');
function someTask( callback ) {
var conn = database.getConnection();
myModel.getData(conn, paramsA, dataReceivedA)
function dataReceivedA(err, data) {
myModel.getData(conn, paramsB, dataReceivedB)
}
function dataReceivedB(err, data) {
conn.end()
callback();
}
}
If you want to entirely hide your database connection in your model code. Then you would need to doe something like that:
var conn = myModel.connect();
conn.getData(params, function(err, data) {
conn.end();
})
How to actually solve this depends only many factors so it is only possible to give you hints here.
I'm using Node.js to write system scripts that run on a server. Due to Node's asynchronous nature, my script is exiting before the database calls have a chance to complete and nothing is ever written to the database.
I'm using Mongoose as an ORM and talking to a MongoDB, if that makes any difference. Node.js offers SYNCHRONOUS method calls for this very reason, for example: https://nodejs.org/api/child_process.html
I guess my questions are:
1) Does mongoose offer a way to block so my scripting process can wait for the database call to return?
2) If not, is there another method I should consider other than something like:
(function wait () {
if (!SOME_EXIT_CONDITION) setTimeout(wait, 1000);
})();
3) Is node not the best tool for the job for writing scripts? I love node for web app development, and can write nested callbacks or work with promises all day long. But what about as a scripting language?
EDIT -----------------------------------------------
Below is an quick example of the script to provide more clarity of the situation:
#!/usr/bin/env node
# Please note the above that this is a bash script
var schema = mongoose.Schema({
// ... attributes ...
});
var model = new (mongoose.model('ModelObject'))();
model['attribute'] = 42;
console.log('This gets printed first');
model.save(function(err) {
console.log('Nothing in the callback gets printed because callback is never called');
if(err) { // Can't check for errors because this is never reached
console.log('This never gets printed to the screen');
console.log('And consequently nothing is ever saved to mongo');
} else {
console.log('This never gets printed either');
}
});
console.log('This gets printed second');
If your model does not get saved, there is a Mongo error. Following MongoDB conventions you have to check for errors:
model.save(function(error, savedItem) {
if(error) {
// nothing is saved
}
});
Otherwise, have you considered using Promises? It useful for chaining events and simpler error handling.
Promise = require('bluebird');
Promise.promisifyAll(mongoose.Query.base);
model.saveAsync().then(function(savedItem) {
// saved
})
.catch(function(error) {
// handle error
});
I think you are looking for this, check below if this help you.
var mongoose = require('mongoose'),
model1 = mongoose.model('model1'),
model2 = mongoose.model('model2');
model1.findOne({"type" : 'Active'}, function err(err, catConfig) {
if(!err.error){
//This will execute once above DB call is done!
model2.findOne(condition).remove(function(err, gAnalysis) {
//Lines of code that you want to execute after second DB call
});
}
});
I don't see you opening a connection to the database so presumably saving a model instance does nothing, not even call the callback with an error...
I've tested the below example:
test.js:
var mongoose = require('mongoose');
var kittySchema = mongoose.Schema({
name: String
});
var Kitten = mongoose.model('Kitten', kittySchema);
mongoose.connect('mongodb://localhost:27017/test', function (err) {
if (err) throw err;
var silence = new Kitten({ name: 'Silence' });
silence.save(function (err, saved) {
if (err) throw err;
console.log('Kitty Silence is saved!');
mongoose.disconnect(function (err) {
if (err) throw err;
console.log('done...');
});
});
});
Running node test.js prints this to the console:
Kitty Silence is saved!
done...
and examining my local test database shows that Silence is indeed saved.
I have a small problem with nested callbacks in javascript. Apparently im doing something wrong, but i did my research and tried to follow the tutorials avaialble throughout the web. I know that my code works, since query returns proper data, but i have no idea why my code doesnt "wait" within executeQuery method till the res is fetched from database, it just goes straight to "oh noes" section.
DatabaseConnection.prototype.executeQuery = function(query, executeQueryDone){
var activeConnection;
console.log("YEAAA, executing Query: " + query);
this.pool.getConnection(function (err, connection){
console.log("Got Connection, we are ready to go!");
if (err){
console.log("Error, DAMMNIT! " + err);
executeQueryDone(err);
}
activeConnection = connection;
activeConnection.connect();
activeConnection.query(query, function(error, res){
console.log("Connection from pool is executing Query");
if(error){
console.log("Error during executing query");
executeQueryDone(error);
}
else {
console.log(" OK now release connection (dont be selfish)! ");
activeConnection.release();
executeQueryDone(null, res);
}
});
});
console.log("oh noes! IM AFTER CONNECTION, why dude? WHY???? ");
};
I'd be grateful for any hints since im struggling with that since yesterday.
=====================
PROBLEM SOLVED:
generally all was OK, the "issue" was mistakenly written test:
i made it like that:
describe('testDB2', function () {
it('should return proper STUFF', function (done) {
assert.equal(1, someService.getStuff(function(err, result){
if (err === null){
console.log("err is null, as it should be!");
}
console.log(" result from DB " + result[1].NUMBERS);
}));
});
});
while is should be like that:
describe('testDB2', function () {
it('should return proper STUFF', function (done) {
someService.getStuff(function(err, result){
assert.equal(err, null);
assert.equal(result[1].NUMBERS, 43637654);
done();
});
});
});
as a result (in the incorrect case), i didnt fetch the result the way i wanted as assert couldnt "catch up"
thanks to all for the enlightment ;)
Your console.log call isn't part of a callback, so it will be called as soon as the getConnection call is made. If you want it to be called only after your callback to getConnection fires, you either need to call it at the end of that call, or you need to use some form of promises.
Javascript is single threaded, but it does use a task queue. When the database connection is instantiated and the pool connected to, the response to that is placed into the task queue to be executed or actioned when complete.
Directly after placing that in the task queue the next piece of execution is the log that does the "oh noes" - nice messaging btw lol.
So essentially what happens is the db call gets placed in the task queue for later execution, and then the log occurs, and then the task queue executes at a later time with the db response.
I'm writing my first (non tutorial) node application and am at a point where I'm writing a function that should take the username and password as parameters and query them against the user table of my database to return either true or false. The database is setup, and the app is connecting to it successfully.
However, I haven't worked with SQL very much, nor node, and I'm unsure how to proceed with this function (and short surrounding script). Here it is:
console.log('validator module initialized');
var login = require("./db_connect");
function validate(username, password){
connection.connect();
console.log('Connection with the officeball MySQL database openned...');
connection.query(' //SQL query ', function(err, rows, fields) {
//code to execute
});
connection.end();
console.log('...Connection with the officeball MySQL database closed.');
if(){ //not exactly sure how this should be set up
return true;
}
else{ //not exactly sure how this should be set up
return false;
}
}
exports.validate = validate;
This is using node-mysql. I'm looking for a basic example of how I might set the query and validation up.
I think you'll want to rethink your app into a more node-like way (i.e. one that recognizes that many/most things happen asynchronously, so you're not usually "returning" from a function like this, but doing a callback from it. Not sure what you plan to get from node-mysql, but I would probably just use the plain mysql module. The following code is still most likely not entirely what you want, but will hopefully get you thinking about it correctly.
Note that the use of 'return' below is not actually returning a result (the callback itself should not return anything, and thus its like returning undefined. The return statements are there so you exit the function, which saves a lot of tedious if/else blocks.
Hope this helps, but I'd suggest looking at various node projects on github to get a better feel for the asynchronous nature of writing for node.
function validate(username, password, callback){
var connection = mysql.createConnection({ user:'foo',
password: 'bar',
database: 'test',
host:'127.0.0.1'});
connection.connect(function (err){
if (err) return callback(new Error('Failed to connect'), null);
// if no error, you can do things now.
connection.query('select username,password from usertable where username=?',
username,
function(err,rows,fields) {
// we are done with the connection at this point), so can close it
connection.end();
// here is where you process results
if (err)
return callback(new Error ('Error while performing query'), null);
if (rows.length !== 1)
return callback(new Error ('Failed to find exactly one user'), null);
// test the password you provided against the one in the DB.
// note this is terrible practice - you should not store in the
// passwords in the clear, obviously. You should store a hash,
// but this is trying to get you on the right general path
if (rows[0].password === password) {
// you would probably want a more useful callback result than
// just returning the username, but again - an example
return callback(null, rows[0].username);
} else {
return callback(new Error ('Bad Password'), null);
}
});
});
};