I'm very new to express/mongo stack, so the question is pretty simple, couldn't find anything on stackoverflow which would resolve my problem, so here it is:
I have a index.js file which looks more or less like this:
var mongoose = require('mongoose');
// Connections
var developmentDb = 'mongodb://localhost/admin';
var usedDb;
// If we're in development...
if (process.env.NODE_ENV === 'development') {
// set our database to the development one
usedDb = developmentDb;
// connect to it via mongoose
mongoose.connect(usedDb);
}
// get an instance of our connection to our database
var db = mongoose.connection;
db.once('open', function callback () {
console.log('Databsae Connection Successfully Opened at ' + usedDb);
});
module.exports = db;
Then I'm requireing it in my express route like this:
var express = require('express');
var router = express.Router();
var db = require('../../database');
/* GET users listing. */
router.get('/', function(req, res) {
var users = db.collection('users');
var test = users.find();
res.send(test);
});
module.exports = router;
I'm sending request and the result I'm getting is 'undefined' so nothing is returned from the back end.
Db connection is 100% correct and works.
I'm not entirely sure, do I have to have a schema definition on express side, or is it possible to query any data, without knowing the schema?
What you are missing here is a bit of the "mongoose magic" that is actually happening "behind the scenes " as it were. It's also a base concept of most operations in node.js that the operations ( particularly where IO is concerned ) are largely asynchronous in nature, so you are generally working with callbacks that fire when the operation is complete.
Take this part of your listing:
// get an instance of our connection to our database
var db = mongoose.connection;
db.once('open', function callback () {
console.log('Databsae Connection Successfully Opened at ' + usedDb);
});
module.exports = db;
So while you may have coded that in sequence, the actual order of events is not as you might think. While you can call the db object from mongoose.connection ( and actually this is a connection object and not theDb` object as implemented by the underlying driver ) there is no guarantee that the database is actually connected at this time. In fact, in all likelihood it is not.
The sort point here is that your database connection actually happens after you export the variable from the module, and not before. It does not wait for the preceding line to complete, nor can you make it do so.
Mongoose itself rather has a concept of "models" to represent the collections in your database. So the general approach is to define these model objects and use them for accessing the data:
var Model = mongoose.model( 'Model', modelSchema, 'optionalCollectionName' );
Model.find({}, function(err,data) {
// do stuff in the callback
});
Part of the reason for this ( aside from the schema definitions that are attached ) is that there is actually something else going on here related to the connection. These objects in fact have internal logic that only processes the actions on the tied "collection" object only when the connection to the database is available. So there is an "internal callback" function that is happening here where an internal connection object is actually being used.
Here is some simple code that "overrides" the usage of the internal methods to just try and get the underlying collection object from the driver. It will fail:
var mongoose = require('mongoose'),
Schema = mongoose.Schema
mongoose.connect('mongodb://localhost/test');
var modelSchema = new Schema({},{ strict: false });
var Model = mongoose.model( 'Model', modelSchema, 'optionalCollectionName' );
Model.collection.find({}, function(err,data) {
// do stuff in the callback
});
Since this asked for the collection object as implemented in the underlying driver to be returned and the native .find() method from that object to be used, the problem occurs that the database is not actually connected yet. So in order to make this work, you would need to wrap the call inside an event handler that fired only when the database was truly connected. Or otherwise make sure that a connection had been made before this was called:
mongoose.connection.on('open',function(err,conn) {
// Now we know we are connected.
Model.collection.find({}, function(err,data) {
// do stuff in the callback
});
});
So the model objects are actually doing this for you, and are providing their own implementations of the standard methods such as "find", "update" etc.
If you don't want to do this kind of wrapping, and the definition of models seems like too much work, and even using the { strict: false } modifier here, which relaxes the constraints on the schema to effectively allow any data, then you are probably better off using the base driver rather than mongoose.
But of course you want something smarter than just wrapping all the code inside a callback to the connection. Here is one approach to define an object that you can use to "fetch" the database connection for you, and make sure that nothing is executed until that connection is made:
var async = require('async'),
mongodb = require('mongodb'),
MongoClient = mongodb.MongoClient;
var Model = (function() {
var _db,
conlock;
return {
getDb: function(callback) {
var err = null;
if ( _db == null && conlock == null ) {
conlock = 1;
MongoClient.connect('mongodb://localhost/test',function(err,db) {
_db = db;
conlock == null;
if (!err) {
console.log("Connected")
}
callback(err,_db);
});
} else if ( conlock != null ) {
var count = 0;
async.whilst(
function() { return ( _db == null ) && (count < 5) },
function(callback) {
count++
setTimeout(callback,500);
},
function(err) {
if ( count == 5 )
err = new Error("connect wait exceeded");
callback(err,_db);
}
);
} else {
callback(err,_db);
}
}
};
})();
async.parallel(
[
function(callback) {
console.log("call model");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined");
callback();
});
},
function(callback) {
console.log("call model again");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined here as well");
callback();
});
}
],
function(err) {
Model.getDb(function(err,db) {
db.close();
});
}
);
That basically wraps up an object which we called "Model" here, with a single method to .getDb(). That method simply accepts a callback, which is there to be your actual piece of code that you want to access the database with, which in turn exposes the Db object from the connection.
The Db object is stored internally in that object, so it is basically a singleton which only ever connects to the database once. But as your logic is passed in a a callback function, then it will either just pass in the existing stored object, or wait until the connection is made before passing in the code.
Output from the sample usage should be:
call model
call model again
Connected
db is defined
db is defined here as well
And that shows the order of events to how they actually happen.
So there are different ways to handle this. Mongoose models "abstract" a lot of that for you. You can of course either take a base approach with the base driver as given in the example, or take that further and implement your own connection system including overridden methods that does much of the same thing that mongoose is doing underneath. There are also other wrapper libraries that already do this without the schema concepts that is generally inherent to mongoose as well.
Basically though, every higher level library above the base driver is doing much the same as described, where there are wrappers around the methods to make sure the connection is there without needing to embed all of your code in an event listener that is checking for that.
Related
Im writting a node app to log some informations in a mongo database.
Below is the snippet code that called each time i need to store log in the mongo database.
const mongo = {}
const mongo_cli = require('mongodb').MongoClient
module.exports = {
log (l) {
mongo_cli.connect(the_mongo_url, (error, client) => {
if (error) throw error;
mongo.cli = client;
mongo.db = client.db(the_database);
//insert and update operations
});
}
}
The code above work for now. I mean, I can insert and update logs already inserted at the price of one (or more) connection (s) that I never close due to my lack of control of callback functions.
So, how can i structure it better so that i can just have only one mongo_cli call to not consume too many ressources ?
Right now i am creating a very large application in Node JS. I am trying to make my code clean and short (Just like most of the developer). I've create my own js file to handle connection to mysql. Please see code below.
var mysql = require('mysql');
var config = {
'default' : {
connectionLimit : process.env.DB_CONN_LIMIT,
host : process.env.DB_HOST,
user : process.env.DB_USER,
password : process.env.DB_PASS,
database : process.env.DB_NAME,
debug : false,
socketPath : process.env.DB_SOCKET
}
};
function connectionFunc(query,parameters,callback,configName) {
configName = configName || "default";
callback = callback || null;
parameters = parameters;
if(typeof parameters == 'function'){
callback = parameters;
parameters = [];
}
//console.log("Server is starting to connect to "+configName+" configuration");
var dbConnection = mysql.createConnection(config[configName]);
dbConnection.connect();
dbConnection.query(query,parameters, function(err, rows, fields) {
//if (!err)
callback(err,rows,fields);
//else
//console.log('Error while performing Query.');
});
dbConnection.end();
}
module.exports.query = connectionFunc;
I am using the above file in my models, like below :
var database = require('../../config/database.js');
module.exports.getData = function(successCallBack){
database.query('SAMPLE QUERY GOES HERE', function(err, result){
if(err) {console.log(err)}
//My statements here
});
}
Using this coding style, everything works fine but when i am trying to create a function that will loop my model's method for some reason. Please see sample below :
for (i = 0; i < 10000; i++) {
myModel.getData(param, function(result){
return res.json({data : result });
});
}
It gives me an ER_CON_COUNT_ERROR : Too Many Conenction. The question is why i still get an error like these when my connection always been ended by this dbConnection.end();? I'm still not sure if i am missing something. I am still stuck on this.
My connection limit is 100 and i think adding more connection is a bad idea.
Because query data form the database is async.
In your loop the myModel.getData (or more precisely the underling query) will not halt/paus your code until the query is finished, but send the query to the database server and as soon as the database response the callback will be called.
The calling end on dbConnection will not close the connection immediately, it will just mark the connection to be closed as soon as all queries that where created with that connection are finished.
mysql: Terminating connections
Terminating a connection gracefully is done by calling the end() method. This will make sure all previously enqueued queries are still before sending a COM_QUIT packet to the MySQL server.
An alternative way to end the connection is to call the destroy() method. This will cause an immediate termination of the underlying socket. Additionally destroy() guarantees that no more events or callbacks will be triggered for the connection.
But with destroy the library will not wait for the result so the results are lost, destroy is rarely useful.
So with your given code you try to create 10000 connections at one time.
You should only use on connection by task, e.g. if a user requests data using the browser, then you should use one connection for this given request. The same is for timed task, if you have some task that is done in certain intervals.
Here an example code:
var database = require('./config/database.js');
function someTask( callback ) {
var conn = database.getConnection();
myModel.getData(conn, paramsA, dataReceivedA)
function dataReceivedA(err, data) {
myModel.getData(conn, paramsB, dataReceivedB)
}
function dataReceivedB(err, data) {
conn.end()
callback();
}
}
If you want to entirely hide your database connection in your model code. Then you would need to doe something like that:
var conn = myModel.connect();
conn.getData(params, function(err, data) {
conn.end();
})
How to actually solve this depends only many factors so it is only possible to give you hints here.
I'm using Node.js to write system scripts that run on a server. Due to Node's asynchronous nature, my script is exiting before the database calls have a chance to complete and nothing is ever written to the database.
I'm using Mongoose as an ORM and talking to a MongoDB, if that makes any difference. Node.js offers SYNCHRONOUS method calls for this very reason, for example: https://nodejs.org/api/child_process.html
I guess my questions are:
1) Does mongoose offer a way to block so my scripting process can wait for the database call to return?
2) If not, is there another method I should consider other than something like:
(function wait () {
if (!SOME_EXIT_CONDITION) setTimeout(wait, 1000);
})();
3) Is node not the best tool for the job for writing scripts? I love node for web app development, and can write nested callbacks or work with promises all day long. But what about as a scripting language?
EDIT -----------------------------------------------
Below is an quick example of the script to provide more clarity of the situation:
#!/usr/bin/env node
# Please note the above that this is a bash script
var schema = mongoose.Schema({
// ... attributes ...
});
var model = new (mongoose.model('ModelObject'))();
model['attribute'] = 42;
console.log('This gets printed first');
model.save(function(err) {
console.log('Nothing in the callback gets printed because callback is never called');
if(err) { // Can't check for errors because this is never reached
console.log('This never gets printed to the screen');
console.log('And consequently nothing is ever saved to mongo');
} else {
console.log('This never gets printed either');
}
});
console.log('This gets printed second');
If your model does not get saved, there is a Mongo error. Following MongoDB conventions you have to check for errors:
model.save(function(error, savedItem) {
if(error) {
// nothing is saved
}
});
Otherwise, have you considered using Promises? It useful for chaining events and simpler error handling.
Promise = require('bluebird');
Promise.promisifyAll(mongoose.Query.base);
model.saveAsync().then(function(savedItem) {
// saved
})
.catch(function(error) {
// handle error
});
I think you are looking for this, check below if this help you.
var mongoose = require('mongoose'),
model1 = mongoose.model('model1'),
model2 = mongoose.model('model2');
model1.findOne({"type" : 'Active'}, function err(err, catConfig) {
if(!err.error){
//This will execute once above DB call is done!
model2.findOne(condition).remove(function(err, gAnalysis) {
//Lines of code that you want to execute after second DB call
});
}
});
I don't see you opening a connection to the database so presumably saving a model instance does nothing, not even call the callback with an error...
I've tested the below example:
test.js:
var mongoose = require('mongoose');
var kittySchema = mongoose.Schema({
name: String
});
var Kitten = mongoose.model('Kitten', kittySchema);
mongoose.connect('mongodb://localhost:27017/test', function (err) {
if (err) throw err;
var silence = new Kitten({ name: 'Silence' });
silence.save(function (err, saved) {
if (err) throw err;
console.log('Kitty Silence is saved!');
mongoose.disconnect(function (err) {
if (err) throw err;
console.log('done...');
});
});
});
Running node test.js prints this to the console:
Kitty Silence is saved!
done...
and examining my local test database shows that Silence is indeed saved.
Background Information
I'm attempting my first node.js API/application. As a learning exercise, I'm trying to create some test cases initially delete all records in a table, insert 3 specific records, and then query for those 3 records.
Code
Here's the code I have cobbled together:
http://pastebin.com/duQQu3fm
Problem
As you can see from the code, I'm trying to put the database connection logic in a dbSession.js file and pass it around.
I am able to start up the http server by doing the following:
dev#devbox:~/nimble_node$ sudo nodejs src/backend/index.js
Server started and listening on port: 8080
Database connection successful
However, when I try to run my jasmine tests, it fails with the following error:
F
Failures:
1) The API should respond to a GET request at /api/widgets/
Message:
TypeError: Object #<MongoClient> has no method 'collection'
Stacktrace:
TypeError: Object #<MongoClient> has no method 'collection'
at resetDatabase (/home/dev/nimble_node/spec/resetDatabase.js:6:29)
at /home/dev/nimble_node/spec/e2e/apiSpec.js:23:25
at /home/dev/nimble_node/node_modules/async/lib/async.js:683:13
at iterate (/home/dev/nimble_node/node_modules/async/lib/async.js:260:13)
at async.forEachOfSeries.async.eachOfSeries (/home/dev/nimble_node/node_modules/async/lib/async.js:279:9)
at _parallel (/home/dev/nimble_node/node_modules/async/lib/async.js:682:9)
at Object.async.series (/home/dev/nimble_node/node_modules/async/lib/async.js:704:9)
at null.<anonymous> (/home/dev/nimble_node/spec/e2e/apiSpec.js:19:9)
at null.<anonymous> (/home/dev/nimble_node/node_modules/jasmine-node/lib/jasmine-node/async-callback.js:45:37)
Finished in 0.01 seconds
1 test, 1 assertion, 1 failure, 0 skipped
Database connection successful
Line 6 of resetDatabase is:
var collection = dbSession.collection('widgets');
Given that after the error appears, I get the "Database connection successful" message, I think what's happening is that when the tests request the dbSession library, the database hasn't finished running the code to connect. And therefore, I can't get the collection object.
I'm currently reading through the mongodb online manual to see if I can find some hints as to how to do something like this.
Any suggestions or pointers would be appreciated.
EDIT 1
To prove that there is a collection method on the MongoClient object, I changed the dbSession.js code to look like this:
'use strict';
var DBWrapper = require('mongodb').MongoClient;
var dbWrapper = new DBWrapper;
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
var collection = dbWrapper.collection('widgets');
console.log('just created a collection...');
}
});
module.exports = dbWrapper;
And now, when I start up the http server (index.js), notice the messages:
dev#devbox:~/nimble_node$ sudo nodejs src/backend/index.js
Server started and listening on port: 8080
Database connection successful
just created a collection...
It could be an async issue.
Your code in dbSessionjs
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
}
});
module.exports = dbWrapper;
Starts the connection at dbWrapper asynchronously, but exports dbWrapper right away, which is then imported in resetDatabase. Thus yes, the connect function may have not yet returned from the async function when you call it in resetDatabase (and is what the log suggests,as the error appears before the success log).
You could add a callback after dbWrapper.connect() returns, in order to actually only be able to use dbWrapper when the connection finished.
(With sqlite, this may not happen as it accesses the DB faster on the commandline).
This may not be your problem but looks like a candidate.
EDIT: Here's a possible example for a callback, but please take note it depends on what you need to do so there are a lot of different solutions. The key is to call a callback function when you are done initializing.
Another solution could be to simply wait, and/or poll (e.g. chcke a variable 'initialized').
'use strict';
var DBWrapper = require('mongodb').MongoClient;
var dbWrapper = new DBWrapper;
function doConnect(callback) {
console.log("Initializing DB connection...");
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
var collection = dbWrapper.collection('widgets');
console.log('just created a collection...');
console.log('calling callback...');
callback(dbWrapper);
} else {
console.log("Error connectingi: " + err);
}
});
};
doConnect(function(correctDbWrapper) {
//Now you can use the wrapper
console.log("Inside callback, now consuming the dbWrapper");
dbWrapper = correctDbWrapper;
var collection = dbWrapper.collection('widgets');
});
It's interesting though I never ran into this issue, although I have generally used similar code like yours. I guess because normally I have this DB initialization right at the top, and then have to do lots of initializations on the node app, which gives the app time enough to return from the connect call....
I am new with NodeJs and Express frameworks. I have understood that Node works with only one thread on the server side. So, I have noticed this causes me some problems in order to develop correctly my application.
In my routes folder, I have a file index.js.
This file manage the navigation asked by the user from app.js.
So I decided to create a route function "test".
In this function, I had just that code
exports.test = function(req, res){
res.render('test', {});
};
So simple, so easy. That's rend the template test.jade in my views folder. Greats !
But I wanna complexify the process. In this test route function, I want load some content from my MYSQL database.
For that, I have created a folder Models in the folders node_modules
Inside, I have only 2 file, the first mysqlConnection.js which exports the variable DB in order to make queries.
var mysql = require('mysql');
var DB = mysql.createConnection(
{
host : 'localhost',
user : 'root',
password : '',
database : 'test',
}
);
DB.connect();
module.exports = DB;
In the second file, articles_class.js, I just have
var DB = require('models/mysqlConnection');
var Article = function() {
this.getArticles = function()
{
DB.query('SELECT * FROM articles;', function(err, rows, fields)
{
if (err)
throw err;
else
{
console.log(rows);
return (rows);
}
});
}
}
module.exports = Article;
Go back in my route test function :
I just want to load from table "test" all the articles. Very basic. But not easy.
Why ?
Because before the query is finished, NodeJs respond to the client with the template render, but, unfornlty, without the rows loaded. Asynchronous problem ... Mysql doesn't block the Nodejs javascript Instruction.
The code of the function :
exports.test = function(req, res){
var Article = require('models/articles_class');
a = new Article();
articles = a.getArticles();
console.log(articles); // undefined
res.render('test', {});
};
I Found others subjects in stackoverflow which speak about this problem. Make sync queries, work with callbacks ect ..
But for, here, if I try to manage this problem with callbacks, That's cannot work ... Because I need to send to the client the template with articles but I can't block the process with a sync method.
I am very lost ... I don't understand how I have to build my application. I am not able to create a good proceed in order to manage the sql queries. There is a pattern or a specific method ?
Or perhaps I have to make only ajax requests from the client. I load the template "test". And in a javascript file in the public folder, I ask to the server to load me the articles content and wait success callback function ? it's not very clean ...
Thx for your answers. The others answers I have found didn't help me to understand how manage that with NodeJs.
Pass a callback to getArticles:
exports.test = function(req, res){
var Article = require('models/articles_class');
a = new Article();
a.getArticles( function( articles ) {
console.log(articles); // undefined
res.render('test', { articles: articles });
});
};
Changes to your get articles function:
var DB = require('models/mysqlConnection');
var Article = function() {
this.getArticles = function( callback )
{
DB.query('SELECT * FROM articles;', function(err, rows, fields)
{
if (err)
throw err;
else
{
console.log(rows);
callback && callback(rows);
}
});
}
}
module.exports = Article;
Express will only return the template through the open http connection once res.render() is called. So it's just a matter of passing it as a callback through your call stack, so it should only be called after you have your database rows.
As we are working with callbacks, they don't block your application.