I have connected a SQL Server database to a simple node.js server. When I run the code I get both recordsets and recordset returned to me. They both essentially contain the same data. I can work with this but it seems redundant and would be neater to just call exactly the records I need.
I was hoping to get an clear ELI5 explanation as mssql npm documentation is somewhat confusing in my opinion.
Here is the code below:
const express = require('express');
const cors = require('cors');
const sql = require('mssql');
const app = express();
const sqlServer = 'hasea\\SQLExpress'
const selectAllQuery = 'SELECT * FROM dbo.users';
const config = {
user: 'nbar',
password: 'nb',
server: sqlServer,
database: 'nirvanaBar'
}
// SQL Select function
function DBconn(query, res) {
sql.connect(config, function (err) {
if (err) console.log(err);
var request = new sql.Request();
request.query(query, function (err, row) {
if (err) console.log(err)
res.json({
data: row
})
})
})
}
app.use(cors());
app.get('/', (req, res) => {
res.send("Hello from the server")
});
app.get('/users', (req, res) => {
//query?
//var andrew = "select * from dbo.users where firstName = 'Andrew';"
var matt = "select * from dbo.users where firstName = 'Matt';"
//DBconn(selectAllQuery,res);
DBconn(matt, res);
})
app.listen(4000, () => {
console.log(`Server started on port 4000`)
})
The results:
I can see this has being brought up before but I am not understanding what is going on nor does the documentation provide much information as to why this would be the case. I can use the data, I just thought it would be neater to return both recordsets and recordset.
Thanks in advance
This is the expected behaviour of mssql have a look at the documentation here. When you execute multiple statements it allows you to have multiple recordsets
You will notice that there is in fact 2 properties
recordset (singular)
and
recordsets (plural)
recordset (singular) refers to the first statement that is executed. In your case you can just use this
recordsets (plural) is an array of recordsets, in your case there will only be one in the array as you are only running one statement.
I'm using the node-mongodb-native driver with MongoDB to write a website.
I have some questions about how to manage connections:
Is it enough using only one MongoDB connection for all requests? Are there any performance issues? If not, can I setup a global connection to use in the whole application?
If not, is it good if I open a new connection when request arrives, and close it when handled the request? Is it expensive to open and close a connection?
Should I use a global connection pool? I hear the driver has a native connection pool. Is it a good choice?
If I use a connection pool, how many connections should be used?
Are there other things I should notice?
The primary committer to node-mongodb-native says:
You open do MongoClient.connect once when your app boots up and reuse
the db object. It's not a singleton connection pool each .connect
creates a new connection pool.
So, to answer your question directly, reuse the db object that results from MongoClient.connect(). This gives you pooling, and will provide a noticeable speed increase as compared with opening/closing connections on each db action.
Open a new connection when the Node.js application starts, and reuse the existing db connection object:
/server.js
import express from 'express';
import Promise from 'bluebird';
import logger from 'winston';
import { MongoClient } from 'mongodb';
import config from './config';
import usersRestApi from './api/users';
const app = express();
app.use('/api/users', usersRestApi);
app.get('/', (req, res) => {
res.send('Hello World');
});
// Create a MongoDB connection pool and start the application
// after the database connection is ready
MongoClient.connect(config.database.url, { promiseLibrary: Promise }, (err, db) => {
if (err) {
logger.warn(`Failed to connect to the database. ${err.stack}`);
}
app.locals.db = db;
app.listen(config.port, () => {
logger.info(`Node.js app is listening at http://localhost:${config.port}`);
});
});
/api/users.js
import { Router } from 'express';
import { ObjectID } from 'mongodb';
const router = new Router();
router.get('/:id', async (req, res, next) => {
try {
const db = req.app.locals.db;
const id = new ObjectID(req.params.id);
const user = await db.collection('user').findOne({ _id: id }, {
email: 1,
firstName: 1,
lastName: 1
});
if (user) {
user.id = req.params.id;
res.send(user);
} else {
res.sendStatus(404);
}
} catch (err) {
next(err);
}
});
export default router;
Source: How to Open Database Connections in a Node.js/Express App
Here is some code that will manage your MongoDB connections.
var MongoClient = require('mongodb').MongoClient;
var url = require("../config.json")["MongoDBURL"]
var option = {
db:{
numberOfRetries : 5
},
server: {
auto_reconnect: true,
poolSize : 40,
socketOptions: {
connectTimeoutMS: 500
}
},
replSet: {},
mongos: {}
};
function MongoPool(){}
var p_db;
function initPool(cb){
MongoClient.connect(url, option, function(err, db) {
if (err) throw err;
p_db = db;
if(cb && typeof(cb) == 'function')
cb(p_db);
});
return MongoPool;
}
MongoPool.initPool = initPool;
function getInstance(cb){
if(!p_db){
initPool(cb)
}
else{
if(cb && typeof(cb) == 'function')
cb(p_db);
}
}
MongoPool.getInstance = getInstance;
module.exports = MongoPool;
When you start the server, call initPool
require("mongo-pool").initPool();
Then in any other module you can do the following:
var MongoPool = require("mongo-pool");
MongoPool.getInstance(function (db){
// Query your MongoDB database.
});
This is based on MongoDB documentation. Take a look at it.
Manage mongo connection pools in a single self contained module. This approach provides two benefits. Firstly it keeps your code modular and easier to test. Secondly your not forced to mix your database connection up in your request object which is NOT the place for a database connection object. (Given the nature of JavaScript I would consider it highly dangerous to mix in anything to an object constructed by library code). So with that you only need to Consider a module that exports two methods. connect = () => Promise and get = () => dbConnectionObject.
With such a module you can firstly connect to the database
// runs in boot.js or what ever file your application starts with
const db = require('./myAwesomeDbModule');
db.connect()
.then(() => console.log('database connected'))
.then(() => bootMyApplication())
.catch((e) => {
console.error(e);
// Always hard exit on a database connection error
process.exit(1);
});
When in flight your app can simply call get() when it needs a DB connection.
const db = require('./myAwesomeDbModule');
db.get().find(...)... // I have excluded code here to keep the example simple
If you set up your db module in the same way as the following not only will you have a way to ensure that your application will not boot unless you have a database connection you also have a global way of accessing your database connection pool that will error if you have not got a connection.
// myAwesomeDbModule.js
let connection = null;
module.exports.connect = () => new Promise((resolve, reject) => {
MongoClient.connect(url, option, function(err, db) {
if (err) { reject(err); return; };
resolve(db);
connection = db;
});
});
module.exports.get = () => {
if(!connection) {
throw new Error('Call connect first!');
}
return connection;
}
If you have Express.js, you can use express-mongo-db for caching and sharing the MongoDB connection between requests without a pool (since the accepted answer says it is the right way to share the connection).
If not - you can look at its source code and use it in another framework.
You should create a connection as service then reuse it when need.
// db.service.js
import { MongoClient } from "mongodb";
import database from "../config/database";
const dbService = {
db: undefined,
connect: callback => {
MongoClient.connect(database.uri, function(err, data) {
if (err) {
MongoClient.close();
callback(err);
}
dbService.db = data;
console.log("Connected to database");
callback(null);
});
}
};
export default dbService;
my App.js sample
// App Start
dbService.connect(err => {
if (err) {
console.log("Error: ", err);
process.exit(1);
}
server.listen(config.port, () => {
console.log(`Api runnning at ${config.port}`);
});
});
and use it wherever you want with
import dbService from "db.service.js"
const db = dbService.db
I have been using generic-pool with redis connections in my app - I highly recommend it. Its generic and I definitely know it works with mysql so I don't think you'll have any problems with it and mongo
https://github.com/coopernurse/node-pool
I have implemented below code in my project to implement connection pooling in my code so it will create a minimum connection in my project and reuse available connection
/* Mongo.js*/
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/yourdatabasename";
var assert = require('assert');
var connection=[];
// Create the database connection
establishConnection = function(callback){
MongoClient.connect(url, { poolSize: 10 },function(err, db) {
assert.equal(null, err);
connection = db
if(typeof callback === 'function' && callback())
callback(connection)
}
)
}
function getconnection(){
return connection
}
module.exports = {
establishConnection:establishConnection,
getconnection:getconnection
}
/*app.js*/
// establish one connection with all other routes will use.
var db = require('./routes/mongo')
db.establishConnection();
//you can also call with callback if you wanna create any collection at starting
/*
db.establishConnection(function(conn){
conn.createCollection("collectionName", function(err, res) {
if (err) throw err;
console.log("Collection created!");
});
};
*/
// anyother route.js
var db = require('./mongo')
router.get('/', function(req, res, next) {
var connection = db.getconnection()
res.send("Hello");
});
If using express there is another more straightforward method, which is to utilise Express's built in feature to share data between routes and modules within your app. There is an object called app.locals. We can attach properties to it and access it from inside our routes. To use it, instantiate your mongo connection in your app.js file.
var app = express();
MongoClient.connect('mongodb://localhost:27017/')
.then(client =>{
const db = client.db('your-db');
const collection = db.collection('your-collection');
app.locals.collection = collection;
});
// view engine setup
app.set('views', path.join(__dirname, 'views'));
This database connection, or indeed any other data you wish to share around the modules of you app can now be accessed within your routes with req.app.locals as below without the need for creating and requiring additional modules.
app.get('/', (req, res) => {
const collection = req.app.locals.collection;
collection.find({}).toArray()
.then(response => res.status(200).json(response))
.catch(error => console.error(error));
});
This method ensures that you have a database connection open for the duration of your app unless you choose to close it at any time. It's easily accessible with req.app.locals.your-collection and doesn't require creation of any additional modules.
Best approach to implement connection pooling is you should create one global array variable which hold db name with connection object returned by MongoClient and then reuse that connection whenever you need to contact Database.
In your Server.js define var global.dbconnections = [];
Create a Service naming connectionService.js. It will have 2 methods getConnection and createConnection.
So when user will call getConnection(), it will find detail in global connection variable and return connection details if already exists else it will call createConnection() and return connection Details.
Call this service using <db_name> and it will return connection object if it already have else it will create new connection and return it to you.
Hope it helps :)
Here is the connectionService.js code:
var mongo = require('mongoskin');
var mongodb = require('mongodb');
var Q = require('q');
var service = {};
service.getConnection = getConnection ;
module.exports = service;
function getConnection(appDB){
var deferred = Q.defer();
var connectionDetails=global.dbconnections.find(item=>item.appDB==appDB)
if(connectionDetails){deferred.resolve(connectionDetails.connection);
}else{createConnection(appDB).then(function(connectionDetails){
deferred.resolve(connectionDetails);})
}
return deferred.promise;
}
function createConnection(appDB){
var deferred = Q.defer();
mongodb.MongoClient.connect(connectionServer + appDB, (err,database)=>
{
if(err) deferred.reject(err.name + ': ' + err.message);
global.dbconnections.push({appDB: appDB, connection: database});
deferred.resolve(database);
})
return deferred.promise;
}
In case anyone wants something that works in 2021 with Typescript, here's what I'm using:
import { MongoClient, Collection } from "mongodb";
const FILE_DB_HOST = process.env.FILE_DB_HOST as string;
const FILE_DB_DATABASE = process.env.FILE_DB_DATABASE as string;
const FILES_COLLECTION = process.env.FILES_COLLECTION as string;
if (!FILE_DB_HOST || !FILE_DB_DATABASE || !FILES_COLLECTION) {
throw "Missing FILE_DB_HOST, FILE_DB_DATABASE, or FILES_COLLECTION environment variables.";
}
const client = new MongoClient(FILE_DB_HOST, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
class Mongoose {
static FilesCollection: Collection;
static async init() {
const connection = await client.connect();
const FileDB = connection.db(FILE_DB_DATABASE);
Mongoose.FilesCollection = FileDB.collection(FILES_COLLECTION);
}
}
Mongoose.init();
export default Mongoose;
I believe if a request occurs too soon (before Mongo.init() has time to finish), an error will be thrown, since Mongoose.FilesCollection will be undefined.
import { Request, Response, NextFunction } from "express";
import Mongoose from "../../mongoose";
export default async function GetFile(req: Request, res: Response, next: NextFunction) {
const files = Mongoose.FilesCollection;
const file = await files.findOne({ fileName: "hello" });
res.send(file);
}
For example, if you call files.findOne({ ... }) and Mongoose.FilesCollection is undefined, then you will get an error.
npm i express mongoose
mongodb.js
const express = require('express');
const mongoose =require('mongoose')
const app = express();
mongoose.set('strictQuery', true);
mongoose.connect('mongodb://localhost:27017/db_name', {
useNewUrlParser: true,
useUnifiedTopology: true
})
.then(() => console.log('MongoDB Connected...'))
.catch((err) => console.log(err))
app.listen(3000,()=>{ console.log("Started on port 3000 !!!") })
node mongodb.js
Using below method you can easily manage as many as possible connection
var mongoose = require('mongoose');
//Set up default mongoose connection
const bankDB = ()=>{
return mongoose.createConnection('mongodb+srv://<username>:<passwprd>#mydemo.jk4nr.mongodb.net/<database>?retryWrites=true&w=majority',options);
}
bankDB().then(()=>console.log('Connected to mongoDB-Atlas bankApp...'))
.catch((err)=>console.error('Could not connected to mongoDB',err));
//Set up second mongoose connection
const myDB = ()=>{
return mongoose.createConnection('mongodb+srv://<username>:<password>#mydemo.jk4nr.mongodb.net/<database>?retryWrites=true&w=majority',options);
}
myDB().then(()=>console.log('Connected to mongoDB-Atlas connection 2...'))
.catch((err)=>console.error('Could not connected to mongoDB',err));
module.exports = { bankDB(), myDB() };
The code is set up this way:
var express = require('express');
var router = express.Router();
var mongo = require('mongodb').MongoClient;
function getData(){
db.collection("collection_name").find({}).toArray(function (err, docs) {
if (err) throw err;
//doing stuff here
}
var dataset = [
{//doing more stuff here
}
];
});
}
router.get("/renderChart", function(req, res) {
mongo.connect(url_monitor, function (err, db) {
assert.equal(null, err);
getData(res);
});
});
When I run the code and trying to get to /renderChart when running, I get the "ReferenceError: db is not defined". I came across a similar case, and think it may be a similar problem caused because mongodb.connect() is called asynchronously, but I couldn't get it to work:
Express js,mongodb: "ReferenceError: db is not defined" when db is mentioned outside post function
The problem here is you don't pass the db to the function, so it's undefined.
A solution:
function getData(db, res){
db.collection("collection_name").find({}).toArray(function (err, docs) {
if (err) throw err;
//doing stuff here
}
var dataset = [
{//doing more stuff here
}
];
});
}
router.get("/renderChart", function(req, res) {
mongo.connect(url_monitor, function (err, db) {
assert.equal(null, err);
getData(db, res);
});
});
You'll probably need to pass the req at some point too, or make specific db queries. And you'll probably want to use promises or async/await to better deal with all asynchronous calls.
Its Simple Javascript.
You are using a variable db in your file, which is not defined, so it will throw an error.
You need to do something like this .
var findDocuments = function(db, callback) {
// Get the documents collection
var collection = db.collection('documents');
// Find some documents
collection.find({}).toArray(function(err, docs) {
assert.equal(err, null);
assert.equal(2, docs.length);
console.log("Found the following records");
console.dir(docs);
callback(docs);
});
}
I have the same problem before, instead of passing db to routing function, My solution is to make db variable global like
var mongojs = require('mongojs')
global.db = mongojs(<mongodb url>);
then db variable can be used in any part of your code
If you're using express, put that in your app.js file and you will never have to worry about db variable anyore.
PS: some people think that using global is not a good practices, but I argue that since global is a node.js features and especially since it works, why not
node.js global variables?
You don't have tell the codes, that which database you want to use.
how to get databases list https://stackoverflow.com/a/71895254/17576982
here is the sample code to find the movie with name 'Back to the Future' in database sample_mflix > collection movies:
const { MongoClient } = require("mongodb");
// Replace the uri string with your MongoDB deployment's connection string.
const uri =
"mongodb+srv://<user>:<password>#<cluster-url>?retryWrites=true&writeConcern=majority";
const client = new MongoClient(uri);
async function run() {
try {
await client.connect();
const database = client.db('sample_mflix');
const movies = database.collection('movies');
// Query for a movie that has the title 'Back to the Future'
const query = { title: 'Back to the Future' };
const movie = await movies.findOne(query);
console.log(movie);
} finally {
// Ensures that the client will close when you finish/error
await client.close();
}
}
run().catch(console.dir);
to get list of database, put await client.db().admin().listDatabases() on fun function. e.g.
async function run() {
try {
await client.connect();
var databasesList = await client.db().admin().listDatabases();
console.log("Databases:");
databasesList.databases.forEach(db => console.log(` - ${db.name}`));
learn MongoDB more from official docs: https://www.mongodb.com/docs
I have a simple file model.js like follows:
var mongo = require('mongodb');
var mongoUri = process.env.MONGOLAB_URI ||
process.env.MONGOHQ_URL ||
'mongodb://localhost/mydb';
exports.connect = mongo.Db.connect(mongoUri, function(err, db) {
console.log("Connect to the database successfully")
});
and in my web.js I load the model using model = require('./model.js'). One werid thing is that although I did not call model.connect(), the message "Connect to the database successfully" still got logged to my console. Why is this happening and is there a way to avoid it?
EDIT:Never mind I have found a workaround:
exports.connect = function(){
mongo.Db.connect(mongoUri, function(err, db) {
console.log("Connect to the database successfully")
});
}
exports.connect = mongo.Db.connect(mongoUri, function(err, db) {
console.log("Connect to the database successfully")
});
You just called mongo.Db.connect() and assigned its result to exports.connect.
That code runs as soon as you require() the module.
Instead, you need to create a function:
exports.connect = function() { ... };
I've built a series of database queries in my express app that reside in a /models/index.js file which I can access from app.js via var express = require('express');. I am trying to populate req.session.user with a userid that is returned by a findByEmail(); function in /models/index.js.
The findByEmail(); function works fine, however I can't figure out how to store its return value in req.session. I've tried including req.session.id = result.rows[0].id; in the 'findByEmail();function, but this returns areq is not defined` error.
Am I overlooking a simple require statement in my /models/index.js file or is there another trick to accessing req.session in a module?
I've included the relevant code from /models.index.js below:
/models.index.js:
var pg = require('pg');
function findByEmail(email){
pg.connect(function(err, client, done) {
if(err) {
console.log('pg.connect error');
throw err;
}
client.query('BEGIN', function(err) {
if(err) {
console.log('client.query BEGIN error');
return rollback(client, done);
}
process.nextTick(function() {
var text = "SELECT * FROM users WHERE email = $1";
client.query(text, [email], function(err, result) {
if(err) {
console.log(err);
return rollback(client, done);
}
console.log(result);
console.log(result.rows);
console.log('id: ', result.rows[0].id);
req.session.id = result.rows[0].id;
done();
});
});
});
});
}
module.exports.pg = pg;
exports.findByEmail = findByEmail;
As far as /models/index.js knows, req is not defined, same thing with rollback. A module is a closure and you don't have access to variables defined outside of it.
If you want to do that you must pass them as parameters but it's not very good design, as #gustavohenke said: Separation of concerns.
You might want to have a callback and call it with success/error and set the session id there so you don't have to pass in into the module:
function findByEmail(email,callback){
pg.connect(function(err, client, done) {
if(err) {
console.log('pg.connect error');
throw err;
}
// Do all the async work and when you are done ...
// An error is usually passed as the first parameter of the callback
callback(err,result)
});
}
exports.findByEmail = findByEmail;
You would then call it like this:
var models = require('./models');
models.findByEmail('thedude#lebowski.com',function(err,results) {
// set session id here where you probably have access to the req object...
})