I am new with NodeJs and Express frameworks. I have understood that Node works with only one thread on the server side. So, I have noticed this causes me some problems in order to develop correctly my application.
In my routes folder, I have a file index.js.
This file manage the navigation asked by the user from app.js.
So I decided to create a route function "test".
In this function, I had just that code
exports.test = function(req, res){
res.render('test', {});
};
So simple, so easy. That's rend the template test.jade in my views folder. Greats !
But I wanna complexify the process. In this test route function, I want load some content from my MYSQL database.
For that, I have created a folder Models in the folders node_modules
Inside, I have only 2 file, the first mysqlConnection.js which exports the variable DB in order to make queries.
var mysql = require('mysql');
var DB = mysql.createConnection(
{
host : 'localhost',
user : 'root',
password : '',
database : 'test',
}
);
DB.connect();
module.exports = DB;
In the second file, articles_class.js, I just have
var DB = require('models/mysqlConnection');
var Article = function() {
this.getArticles = function()
{
DB.query('SELECT * FROM articles;', function(err, rows, fields)
{
if (err)
throw err;
else
{
console.log(rows);
return (rows);
}
});
}
}
module.exports = Article;
Go back in my route test function :
I just want to load from table "test" all the articles. Very basic. But not easy.
Why ?
Because before the query is finished, NodeJs respond to the client with the template render, but, unfornlty, without the rows loaded. Asynchronous problem ... Mysql doesn't block the Nodejs javascript Instruction.
The code of the function :
exports.test = function(req, res){
var Article = require('models/articles_class');
a = new Article();
articles = a.getArticles();
console.log(articles); // undefined
res.render('test', {});
};
I Found others subjects in stackoverflow which speak about this problem. Make sync queries, work with callbacks ect ..
But for, here, if I try to manage this problem with callbacks, That's cannot work ... Because I need to send to the client the template with articles but I can't block the process with a sync method.
I am very lost ... I don't understand how I have to build my application. I am not able to create a good proceed in order to manage the sql queries. There is a pattern or a specific method ?
Or perhaps I have to make only ajax requests from the client. I load the template "test". And in a javascript file in the public folder, I ask to the server to load me the articles content and wait success callback function ? it's not very clean ...
Thx for your answers. The others answers I have found didn't help me to understand how manage that with NodeJs.
Pass a callback to getArticles:
exports.test = function(req, res){
var Article = require('models/articles_class');
a = new Article();
a.getArticles( function( articles ) {
console.log(articles); // undefined
res.render('test', { articles: articles });
});
};
Changes to your get articles function:
var DB = require('models/mysqlConnection');
var Article = function() {
this.getArticles = function( callback )
{
DB.query('SELECT * FROM articles;', function(err, rows, fields)
{
if (err)
throw err;
else
{
console.log(rows);
callback && callback(rows);
}
});
}
}
module.exports = Article;
Express will only return the template through the open http connection once res.render() is called. So it's just a matter of passing it as a callback through your call stack, so it should only be called after you have your database rows.
As we are working with callbacks, they don't block your application.
Related
I'm new to javascript, node.js (or backend at all). I am trying to create a controller for the login page requests and I am confused about getting data from the MYSQL table and User Authentication and working with JWT package !
In my Controller, I first check if the user input is available in the user table (with a simple stored procedure), then I compare the database password and the user input, after this I want to create a token and with limited time. (I have watched some tutorial videos about JWT and there is no problem with it), my main problem is to figure out how to write a proper controller with this functions?
I have 2 other questions:
1.Is it the right and secure way to get data from MySQL table inside the route? Or should I create a JS class for my controller? (I'm a bit confused and doubtful here)
2.Assuming that comparePassword() returns true, how can I continue coding outside of the db.query callback function scope? Because I have to execute comparePasssword() inside db.query callback
loginController.js :
const { validationResult } = require('express-validator');
const bcrypt = require('bcrypt');
const db = require('../../sqlConnection')
let comparePassword = (dbPass, inputPass) => {
bcrypt.compare(inputPass, dbPass, function(err, result) {
console.log(result)
});
}
// for get request
exports.getController = (req, res) => {
res.send('login')
}
// for post request
exports.postController = (req, res) => {
let errors = validationResult(req)
if(!errors.isEmpty()) {
res.status(422).json({ errors: errors.array() })
}
// find data from MYSQL table
let sql = `CALL findUser(?)`
db.query(sql, [req.body.username], (err, res) => {
if(err) console.log(err)
//console.log(Object.values(JSON.parse(JSON.stringify(res[0]))))
var data = JSON.stringify(res[0])
data = JSON.parse(data).find(x => x)
data ? comparePassword(data.password, req.body.password) : res.status(400).send('cannot find
user')
})
res.send('post login')
}
login.js :
const express = require('express')
const router = express.Router()
const { check } = require('express-validator');
const loginCont = require('../api/controllers/loginController')
router.route('/')
.get(
loginCont.getController
)
.post(
[
check('username').isLength({min: 3}).notEmpty(),
check('password').isLength({min: 4}).notEmpty()
],
loginCont.postController
)
module.exports = router
In my point of view, looks like there is no easy answer for your question so I will try to give you some directions so you can figure out which are the gaps in your code.
First question: MySQL and business logic on controller
In a design pattern like MVC or ADR (please take a look in the links for the flow details) The Controllers(MVC) Or Actions(ADR) are the entry point for the call, and a good practice is to use these entry points to basically:
Instantiate a service/class/domain-class that supports the request;
Call the necessary method/function to resolve what you want;
Send out the response;
This sample project can help you on how to structure your project following a design pattern: https://riptutorial.com/node-js/example/30554/a-simple-nodejs-application-with-mvc-and-api
Second question: db and continue the process
For authentication, I strongly suggest you to take a look on the OAuth or OAuth2 authentication flow. The OAuth(2) has a process where you generate a token and with that token you can always check in your Controllers, making the service a lot easier.
Also consider that you may need to create some external resources/services to solve if the token is right and valid, but it would facilitate your job.
This sample project should give you an example about how to scope your functions in files: https://github.com/cbroberg/node-mvc-api
Summary
You may have to think in splitting your functions into scoped domains so you can work with them in separate instead of having all the logic inside the controllers, then you will get closer to classes/services like: authenticantion, user, product, etc, that could be used and reused amount your controllers.
I hope that this answer could guide you closer to your achievements.
Right now i am creating a very large application in Node JS. I am trying to make my code clean and short (Just like most of the developer). I've create my own js file to handle connection to mysql. Please see code below.
var mysql = require('mysql');
var config = {
'default' : {
connectionLimit : process.env.DB_CONN_LIMIT,
host : process.env.DB_HOST,
user : process.env.DB_USER,
password : process.env.DB_PASS,
database : process.env.DB_NAME,
debug : false,
socketPath : process.env.DB_SOCKET
}
};
function connectionFunc(query,parameters,callback,configName) {
configName = configName || "default";
callback = callback || null;
parameters = parameters;
if(typeof parameters == 'function'){
callback = parameters;
parameters = [];
}
//console.log("Server is starting to connect to "+configName+" configuration");
var dbConnection = mysql.createConnection(config[configName]);
dbConnection.connect();
dbConnection.query(query,parameters, function(err, rows, fields) {
//if (!err)
callback(err,rows,fields);
//else
//console.log('Error while performing Query.');
});
dbConnection.end();
}
module.exports.query = connectionFunc;
I am using the above file in my models, like below :
var database = require('../../config/database.js');
module.exports.getData = function(successCallBack){
database.query('SAMPLE QUERY GOES HERE', function(err, result){
if(err) {console.log(err)}
//My statements here
});
}
Using this coding style, everything works fine but when i am trying to create a function that will loop my model's method for some reason. Please see sample below :
for (i = 0; i < 10000; i++) {
myModel.getData(param, function(result){
return res.json({data : result });
});
}
It gives me an ER_CON_COUNT_ERROR : Too Many Conenction. The question is why i still get an error like these when my connection always been ended by this dbConnection.end();? I'm still not sure if i am missing something. I am still stuck on this.
My connection limit is 100 and i think adding more connection is a bad idea.
Because query data form the database is async.
In your loop the myModel.getData (or more precisely the underling query) will not halt/paus your code until the query is finished, but send the query to the database server and as soon as the database response the callback will be called.
The calling end on dbConnection will not close the connection immediately, it will just mark the connection to be closed as soon as all queries that where created with that connection are finished.
mysql: Terminating connections
Terminating a connection gracefully is done by calling the end() method. This will make sure all previously enqueued queries are still before sending a COM_QUIT packet to the MySQL server.
An alternative way to end the connection is to call the destroy() method. This will cause an immediate termination of the underlying socket. Additionally destroy() guarantees that no more events or callbacks will be triggered for the connection.
But with destroy the library will not wait for the result so the results are lost, destroy is rarely useful.
So with your given code you try to create 10000 connections at one time.
You should only use on connection by task, e.g. if a user requests data using the browser, then you should use one connection for this given request. The same is for timed task, if you have some task that is done in certain intervals.
Here an example code:
var database = require('./config/database.js');
function someTask( callback ) {
var conn = database.getConnection();
myModel.getData(conn, paramsA, dataReceivedA)
function dataReceivedA(err, data) {
myModel.getData(conn, paramsB, dataReceivedB)
}
function dataReceivedB(err, data) {
conn.end()
callback();
}
}
If you want to entirely hide your database connection in your model code. Then you would need to doe something like that:
var conn = myModel.connect();
conn.getData(params, function(err, data) {
conn.end();
})
How to actually solve this depends only many factors so it is only possible to give you hints here.
I have read all the questions on SO that I could find. They all use Express, Mongoose or they leave something out. I understand that Node.js is the server. I understand the MongoDB require is the driver the Node.js server uses to open a connection to the MongoDB. Then, on the server, I can do (from the documentation):
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var ObjectId = require('mongodb').ObjectID;
var url = 'mongodb://localhost:27017/test';
var findRestaurants = function(db, callback) {
var cursor =db.collection('restaurants').find( );
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
console.dir(doc);
} else {
callback();
}
});
};
// Connect to the db
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
findRestaurants(db, function() { //I don't want to do this as soon as the server starts
db.close();
});
});
//if I put findRestaurant here,
function findRestaurant(data){
}
How do I call it from the client?
I do not want to find data as soon as I start the server. I realize those are examples, but what I cannot find is a way where the client requests some data and where the Node.js server returns it.
I have seen close examples using jQuery, Angular on the client, and then Express, Mongoose, Meteor, , etc.
All I want to understand is how I make this request from the client's browser. I can do that with XMLhttpRequest(), so I can put that part together, I believe. But, any example is appreciated.
But what is waiting on the Node.js side of things (how do I set up my function to be called once the server is listening)?
How do I create a function on the server side, maybe "GetRestaurants" and have that return the data it gets using find()?
I cannot find this information, this simple, anywhere. Is it too complicated to do the example without a framework?
I do not wish to copy and paste from something using Express, etc. without understanding what's going on. Most explanations never say, this goes on the Node.js side. This is client. I know I am expected to do my own research, but I am not putting it together, too used to RDBMSes, IIS, Apache, PHP, and so on.
I believe I have a fundamental misunderstanding of what's going on in the paradigm.
Please. No REST API creation, no frameworks of any kind on Node.js other than using the MongoDB library (unless there is an absolute requirement), not even jQuery, Angular, Jade, or anything else for the client side, straight up JavaScript on all sides.
I have seen questions like this,
How to display data from MongoDB to the frontend via Node.js without using a framework
But they do not show what I am asking. They do it all at once, as soon as the database connects. What if I want to do a delete or insert or find? There are many SO questions like this, but I have not hit the one that shows what I am looking for.
This should give the guidance. Once you go to a browser and type http://localhost:5155 the callback function (request, response) { will be called and the request to db will be made. Make sure you get response and then start working on the client side code:
const http = require('http');
const MongoClient = require('mongodb').MongoClient;
const assert = require('assert');
const url = 'mongodb://localhost:27017/test';
const server = http.createServer(function (request, response) {
getData(function (data) {
response.end(data);
});
});
function getData(callback) {
// Connect to the db
MongoClient.connect(url, function (err, db) {
assert.equal(null, err);
findRestaurants(db, function (data) {
db.close();
callback(data);
});
});
const findRestaurants = function (db, callback) {
const cursor = db.collection('restaurants').find();
const data = [];
cursor.each(function (err, doc) {
assert.equal(err, null);
data.push(doc);
if (doc === null) {
callback(data);
}
});
};
}
server.listen(5155);
I'm very new to express/mongo stack, so the question is pretty simple, couldn't find anything on stackoverflow which would resolve my problem, so here it is:
I have a index.js file which looks more or less like this:
var mongoose = require('mongoose');
// Connections
var developmentDb = 'mongodb://localhost/admin';
var usedDb;
// If we're in development...
if (process.env.NODE_ENV === 'development') {
// set our database to the development one
usedDb = developmentDb;
// connect to it via mongoose
mongoose.connect(usedDb);
}
// get an instance of our connection to our database
var db = mongoose.connection;
db.once('open', function callback () {
console.log('Databsae Connection Successfully Opened at ' + usedDb);
});
module.exports = db;
Then I'm requireing it in my express route like this:
var express = require('express');
var router = express.Router();
var db = require('../../database');
/* GET users listing. */
router.get('/', function(req, res) {
var users = db.collection('users');
var test = users.find();
res.send(test);
});
module.exports = router;
I'm sending request and the result I'm getting is 'undefined' so nothing is returned from the back end.
Db connection is 100% correct and works.
I'm not entirely sure, do I have to have a schema definition on express side, or is it possible to query any data, without knowing the schema?
What you are missing here is a bit of the "mongoose magic" that is actually happening "behind the scenes " as it were. It's also a base concept of most operations in node.js that the operations ( particularly where IO is concerned ) are largely asynchronous in nature, so you are generally working with callbacks that fire when the operation is complete.
Take this part of your listing:
// get an instance of our connection to our database
var db = mongoose.connection;
db.once('open', function callback () {
console.log('Databsae Connection Successfully Opened at ' + usedDb);
});
module.exports = db;
So while you may have coded that in sequence, the actual order of events is not as you might think. While you can call the db object from mongoose.connection ( and actually this is a connection object and not theDb` object as implemented by the underlying driver ) there is no guarantee that the database is actually connected at this time. In fact, in all likelihood it is not.
The sort point here is that your database connection actually happens after you export the variable from the module, and not before. It does not wait for the preceding line to complete, nor can you make it do so.
Mongoose itself rather has a concept of "models" to represent the collections in your database. So the general approach is to define these model objects and use them for accessing the data:
var Model = mongoose.model( 'Model', modelSchema, 'optionalCollectionName' );
Model.find({}, function(err,data) {
// do stuff in the callback
});
Part of the reason for this ( aside from the schema definitions that are attached ) is that there is actually something else going on here related to the connection. These objects in fact have internal logic that only processes the actions on the tied "collection" object only when the connection to the database is available. So there is an "internal callback" function that is happening here where an internal connection object is actually being used.
Here is some simple code that "overrides" the usage of the internal methods to just try and get the underlying collection object from the driver. It will fail:
var mongoose = require('mongoose'),
Schema = mongoose.Schema
mongoose.connect('mongodb://localhost/test');
var modelSchema = new Schema({},{ strict: false });
var Model = mongoose.model( 'Model', modelSchema, 'optionalCollectionName' );
Model.collection.find({}, function(err,data) {
// do stuff in the callback
});
Since this asked for the collection object as implemented in the underlying driver to be returned and the native .find() method from that object to be used, the problem occurs that the database is not actually connected yet. So in order to make this work, you would need to wrap the call inside an event handler that fired only when the database was truly connected. Or otherwise make sure that a connection had been made before this was called:
mongoose.connection.on('open',function(err,conn) {
// Now we know we are connected.
Model.collection.find({}, function(err,data) {
// do stuff in the callback
});
});
So the model objects are actually doing this for you, and are providing their own implementations of the standard methods such as "find", "update" etc.
If you don't want to do this kind of wrapping, and the definition of models seems like too much work, and even using the { strict: false } modifier here, which relaxes the constraints on the schema to effectively allow any data, then you are probably better off using the base driver rather than mongoose.
But of course you want something smarter than just wrapping all the code inside a callback to the connection. Here is one approach to define an object that you can use to "fetch" the database connection for you, and make sure that nothing is executed until that connection is made:
var async = require('async'),
mongodb = require('mongodb'),
MongoClient = mongodb.MongoClient;
var Model = (function() {
var _db,
conlock;
return {
getDb: function(callback) {
var err = null;
if ( _db == null && conlock == null ) {
conlock = 1;
MongoClient.connect('mongodb://localhost/test',function(err,db) {
_db = db;
conlock == null;
if (!err) {
console.log("Connected")
}
callback(err,_db);
});
} else if ( conlock != null ) {
var count = 0;
async.whilst(
function() { return ( _db == null ) && (count < 5) },
function(callback) {
count++
setTimeout(callback,500);
},
function(err) {
if ( count == 5 )
err = new Error("connect wait exceeded");
callback(err,_db);
}
);
} else {
callback(err,_db);
}
}
};
})();
async.parallel(
[
function(callback) {
console.log("call model");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined");
callback();
});
},
function(callback) {
console.log("call model again");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined here as well");
callback();
});
}
],
function(err) {
Model.getDb(function(err,db) {
db.close();
});
}
);
That basically wraps up an object which we called "Model" here, with a single method to .getDb(). That method simply accepts a callback, which is there to be your actual piece of code that you want to access the database with, which in turn exposes the Db object from the connection.
The Db object is stored internally in that object, so it is basically a singleton which only ever connects to the database once. But as your logic is passed in a a callback function, then it will either just pass in the existing stored object, or wait until the connection is made before passing in the code.
Output from the sample usage should be:
call model
call model again
Connected
db is defined
db is defined here as well
And that shows the order of events to how they actually happen.
So there are different ways to handle this. Mongoose models "abstract" a lot of that for you. You can of course either take a base approach with the base driver as given in the example, or take that further and implement your own connection system including overridden methods that does much of the same thing that mongoose is doing underneath. There are also other wrapper libraries that already do this without the schema concepts that is generally inherent to mongoose as well.
Basically though, every higher level library above the base driver is doing much the same as described, where there are wrappers around the methods to make sure the connection is there without needing to embed all of your code in an event listener that is checking for that.
I have a database on MongoLab. It has several collections. All but one work. One collection is called "selectopts". It has two documents. I can clearly see these two documents.
In my express code I have...
var db = require('mongojs');
db.connect('mongodb://xxxx:xxxx#ds053xxx.mongolab.com:53xxx/rednecks',['selectopts']);
exports.selects = function (req, res) {
db.selectopts.find(function (err, s) {
if (err) return;
res.json(s);
});
};
It always errors at db.selectopts.find..., TypeError: Cannot call method 'find' of undefined. This exact same stupid simple code works fine for four other collections. Why is just this one collection not coming back from MongoLab?
I'm so completely stumped.
EDIT...
Tried db.collection('selectopts').find(... and got this error...
EDIT again...
Here are the two docs in the selectopts collection on MongoLab. Do you see some problem with the docs?...
EDIT x 3...
This is the correct/working mongo connection setup code...
var mongojs = require('mongojs');
var db = mongojs.connect(
'xxx:xxx#ds053xxx.mongolab.com:53xxx/rednecks',
);
See the main difference? (SMFH) :-/
Sometimes when I post at SO I don't receive the exact actual answer from any one reply, but somehow you guys always steer me toward the answer. In this case, it was programmer stupidity. My code at the top of the route js file is wrong. You can see it wrong at the top of this post. I edited and added the correct syntax, which magically got everything working.
To eliminate this type of repetition/syntax/non-DRY bungling, I moved the mongo connection lines into a separate database.js file and require it at the top of the route files. Genius huh?!? :-D
"Thank you" x 100 for all your replies! You always get me back on track :-)
Connect using the following style:
var MongoClient = require('mongodb').MongoClient;
var db;
MongoClient.connect(
'mongodb://xxxx:xxxx#ds053xxx.mongolab.com:53xxx/rednecks',
{auto_reconnect: true},
function(e, database)
{
db = database;
});
exports.selects = function (req, res) {
db.selectopts.find(function (err, s) {
if (err) return;
res.json(s);
});
};