I am fairly new to coding and I encountered something I couldn't solve nor find on google.
this is my code:
let express = require('express');
let app = express();
const {
Pool,
Client
} = require("pg");
app.use(express.static(__dirname + '/public')); //__dir and not _dir
let port = 2000; // you can use any port
app.listen(port);
console.log('serving files on' + ' ' + port);
const pool = new Pool({
user: 'postgres',
host: 'localhost',
database: 'myFirstDb',
password: '.....',
port: 3010,
max: 20,
connectionTimeoutMillis: 0,
idleTimeoutMillis: 0
})
let results = {};
app.get("/all", async (req, res) => {
const fromDate = new Date();
//return all rows
results = await pool.query("select name, income, spendings from income")
// console.table(results.rows)
// this line here doesn't work because I found out you could not use variables like this because
// node does not have a DOM element.
document.getElementById('yep').innerHTML = results
console.log(new Date())
const toDate = new Date();
const elapsed = toDate.getTime() - fromDate.getTime();
//send it to the wire
res.send({
rows: results.rows,
elapsed: elapsed,
method: "pool"
})
const timestamps = await pool.query('insert into timestamps values ($1)', [fromDate])
})
app.listen(2015, () => console.log("Listening on port 2015"))
// SECOND PART //
const client = new Client({
user: 'postgres',
host: 'localhost',
database: 'myFirstDb',
password: 'tenCores10',
port: 3010,
})
async function insert() {
// things that should happen
try {
// wait till this is finished before doing anything else
await client.connect()
// then...
console.log('Connected succesfully')
const inserting = await client.query('insert into income values ($1, $2, $3)', [8, 1, 'jay'])
// deleting it right away, just to keep it working
const deleting = await client.query('delete from income where income = 8')
const results = await client.query('select * from income')
console.table(results.rows)
//send it to the wire
// catching errors
} catch (error) {
console.log(`That went wrong ${error}`)
} // finish it
finally {
await client.end()
console.log('client disconnected succesfully')
}
}
insert();
I am writing a node.js file called 'pool.js'. I have two different connections in there, one on localhost:2000 and the other on localhost:2015/all. the first one is with a client and the second one is with a pool.
The thing is that I would like to show the results I get from my postgres database with a html file on the localhost. But I can't seem to get the constant 'results' in a way to a html file. If I now check localhost:2015/all it does show the database data but it's all cropped up and I can't adjust it how it is located in the page.
So my question is: Is there a way how I could get the database data displayed on a webpage via a html file?
This is how my folder looks like:
I hope I added enough info, but if I am missing something please let me know. And sorry for the code which looks a bit messed up, but I can't seem to get the spaces in the code sample right.
Thanks in advance,
I use mysql (without sequelize) and generally create the connection in a module.export function which I then require from other files like so:
var db;
module.exports={
getConnection = function (){return new Promise(function(resolve, reject){
// db = mysql connection stuff
resolve(db)
});},
//other database related operations similarly
}
Now, I want to separate everything for example, I want to have a userModel that will have a few database operations related to users which will just import the db connection and so on;
But what is the optimal way of achieving this? Thanks.
Since it's used so broadly and doesn't present a high risk of variable naming conflict, I prefer to add it to the global object. Then you never have to make sure it's being included, passed, or imported from a module.
Consider the following example:
// in your application initialization file such as app.js
// require items …
const mysql = require('mysql');
const connection = mysql.createPool({
connectionLimit: 10,
host: process.env.DB_HOST || '127.0.0.1',
user: process.env.DB_USER || 'local_user',
password: process.env.DB_PASSWORD || 'local_password',
database: process.env.DB_NAME || 'local_database',
multipleStatements: true,
charset: 'utf8mb4' // necessary if you might need support for emoji characters
});
connection.on('connection', function (connection) {
// handy for testing
console.log('Pool id %d connected', connection.threadId);
});
connection.on('enqueue', function () {
// handy for testing
console.log('Waiting for available connection slot');
});
// make the connection global via the the variable db
global.db = connection;
// everywhere else in your app, use the now global db variable when running queries
db.query(
'INSERT INTO users SET ?', [{name: "John Smith"}], function(error, results, fields) {
if (error) throw error;
console.log(results);
});
}
I'm using the node-mongodb-native driver with MongoDB to write a website.
I have some questions about how to manage connections:
Is it enough using only one MongoDB connection for all requests? Are there any performance issues? If not, can I setup a global connection to use in the whole application?
If not, is it good if I open a new connection when request arrives, and close it when handled the request? Is it expensive to open and close a connection?
Should I use a global connection pool? I hear the driver has a native connection pool. Is it a good choice?
If I use a connection pool, how many connections should be used?
Are there other things I should notice?
The primary committer to node-mongodb-native says:
You open do MongoClient.connect once when your app boots up and reuse
the db object. It's not a singleton connection pool each .connect
creates a new connection pool.
So, to answer your question directly, reuse the db object that results from MongoClient.connect(). This gives you pooling, and will provide a noticeable speed increase as compared with opening/closing connections on each db action.
Open a new connection when the Node.js application starts, and reuse the existing db connection object:
/server.js
import express from 'express';
import Promise from 'bluebird';
import logger from 'winston';
import { MongoClient } from 'mongodb';
import config from './config';
import usersRestApi from './api/users';
const app = express();
app.use('/api/users', usersRestApi);
app.get('/', (req, res) => {
res.send('Hello World');
});
// Create a MongoDB connection pool and start the application
// after the database connection is ready
MongoClient.connect(config.database.url, { promiseLibrary: Promise }, (err, db) => {
if (err) {
logger.warn(`Failed to connect to the database. ${err.stack}`);
}
app.locals.db = db;
app.listen(config.port, () => {
logger.info(`Node.js app is listening at http://localhost:${config.port}`);
});
});
/api/users.js
import { Router } from 'express';
import { ObjectID } from 'mongodb';
const router = new Router();
router.get('/:id', async (req, res, next) => {
try {
const db = req.app.locals.db;
const id = new ObjectID(req.params.id);
const user = await db.collection('user').findOne({ _id: id }, {
email: 1,
firstName: 1,
lastName: 1
});
if (user) {
user.id = req.params.id;
res.send(user);
} else {
res.sendStatus(404);
}
} catch (err) {
next(err);
}
});
export default router;
Source: How to Open Database Connections in a Node.js/Express App
Here is some code that will manage your MongoDB connections.
var MongoClient = require('mongodb').MongoClient;
var url = require("../config.json")["MongoDBURL"]
var option = {
db:{
numberOfRetries : 5
},
server: {
auto_reconnect: true,
poolSize : 40,
socketOptions: {
connectTimeoutMS: 500
}
},
replSet: {},
mongos: {}
};
function MongoPool(){}
var p_db;
function initPool(cb){
MongoClient.connect(url, option, function(err, db) {
if (err) throw err;
p_db = db;
if(cb && typeof(cb) == 'function')
cb(p_db);
});
return MongoPool;
}
MongoPool.initPool = initPool;
function getInstance(cb){
if(!p_db){
initPool(cb)
}
else{
if(cb && typeof(cb) == 'function')
cb(p_db);
}
}
MongoPool.getInstance = getInstance;
module.exports = MongoPool;
When you start the server, call initPool
require("mongo-pool").initPool();
Then in any other module you can do the following:
var MongoPool = require("mongo-pool");
MongoPool.getInstance(function (db){
// Query your MongoDB database.
});
This is based on MongoDB documentation. Take a look at it.
Manage mongo connection pools in a single self contained module. This approach provides two benefits. Firstly it keeps your code modular and easier to test. Secondly your not forced to mix your database connection up in your request object which is NOT the place for a database connection object. (Given the nature of JavaScript I would consider it highly dangerous to mix in anything to an object constructed by library code). So with that you only need to Consider a module that exports two methods. connect = () => Promise and get = () => dbConnectionObject.
With such a module you can firstly connect to the database
// runs in boot.js or what ever file your application starts with
const db = require('./myAwesomeDbModule');
db.connect()
.then(() => console.log('database connected'))
.then(() => bootMyApplication())
.catch((e) => {
console.error(e);
// Always hard exit on a database connection error
process.exit(1);
});
When in flight your app can simply call get() when it needs a DB connection.
const db = require('./myAwesomeDbModule');
db.get().find(...)... // I have excluded code here to keep the example simple
If you set up your db module in the same way as the following not only will you have a way to ensure that your application will not boot unless you have a database connection you also have a global way of accessing your database connection pool that will error if you have not got a connection.
// myAwesomeDbModule.js
let connection = null;
module.exports.connect = () => new Promise((resolve, reject) => {
MongoClient.connect(url, option, function(err, db) {
if (err) { reject(err); return; };
resolve(db);
connection = db;
});
});
module.exports.get = () => {
if(!connection) {
throw new Error('Call connect first!');
}
return connection;
}
If you have Express.js, you can use express-mongo-db for caching and sharing the MongoDB connection between requests without a pool (since the accepted answer says it is the right way to share the connection).
If not - you can look at its source code and use it in another framework.
You should create a connection as service then reuse it when need.
// db.service.js
import { MongoClient } from "mongodb";
import database from "../config/database";
const dbService = {
db: undefined,
connect: callback => {
MongoClient.connect(database.uri, function(err, data) {
if (err) {
MongoClient.close();
callback(err);
}
dbService.db = data;
console.log("Connected to database");
callback(null);
});
}
};
export default dbService;
my App.js sample
// App Start
dbService.connect(err => {
if (err) {
console.log("Error: ", err);
process.exit(1);
}
server.listen(config.port, () => {
console.log(`Api runnning at ${config.port}`);
});
});
and use it wherever you want with
import dbService from "db.service.js"
const db = dbService.db
I have been using generic-pool with redis connections in my app - I highly recommend it. Its generic and I definitely know it works with mysql so I don't think you'll have any problems with it and mongo
https://github.com/coopernurse/node-pool
I have implemented below code in my project to implement connection pooling in my code so it will create a minimum connection in my project and reuse available connection
/* Mongo.js*/
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/yourdatabasename";
var assert = require('assert');
var connection=[];
// Create the database connection
establishConnection = function(callback){
MongoClient.connect(url, { poolSize: 10 },function(err, db) {
assert.equal(null, err);
connection = db
if(typeof callback === 'function' && callback())
callback(connection)
}
)
}
function getconnection(){
return connection
}
module.exports = {
establishConnection:establishConnection,
getconnection:getconnection
}
/*app.js*/
// establish one connection with all other routes will use.
var db = require('./routes/mongo')
db.establishConnection();
//you can also call with callback if you wanna create any collection at starting
/*
db.establishConnection(function(conn){
conn.createCollection("collectionName", function(err, res) {
if (err) throw err;
console.log("Collection created!");
});
};
*/
// anyother route.js
var db = require('./mongo')
router.get('/', function(req, res, next) {
var connection = db.getconnection()
res.send("Hello");
});
If using express there is another more straightforward method, which is to utilise Express's built in feature to share data between routes and modules within your app. There is an object called app.locals. We can attach properties to it and access it from inside our routes. To use it, instantiate your mongo connection in your app.js file.
var app = express();
MongoClient.connect('mongodb://localhost:27017/')
.then(client =>{
const db = client.db('your-db');
const collection = db.collection('your-collection');
app.locals.collection = collection;
});
// view engine setup
app.set('views', path.join(__dirname, 'views'));
This database connection, or indeed any other data you wish to share around the modules of you app can now be accessed within your routes with req.app.locals as below without the need for creating and requiring additional modules.
app.get('/', (req, res) => {
const collection = req.app.locals.collection;
collection.find({}).toArray()
.then(response => res.status(200).json(response))
.catch(error => console.error(error));
});
This method ensures that you have a database connection open for the duration of your app unless you choose to close it at any time. It's easily accessible with req.app.locals.your-collection and doesn't require creation of any additional modules.
Best approach to implement connection pooling is you should create one global array variable which hold db name with connection object returned by MongoClient and then reuse that connection whenever you need to contact Database.
In your Server.js define var global.dbconnections = [];
Create a Service naming connectionService.js. It will have 2 methods getConnection and createConnection.
So when user will call getConnection(), it will find detail in global connection variable and return connection details if already exists else it will call createConnection() and return connection Details.
Call this service using <db_name> and it will return connection object if it already have else it will create new connection and return it to you.
Hope it helps :)
Here is the connectionService.js code:
var mongo = require('mongoskin');
var mongodb = require('mongodb');
var Q = require('q');
var service = {};
service.getConnection = getConnection ;
module.exports = service;
function getConnection(appDB){
var deferred = Q.defer();
var connectionDetails=global.dbconnections.find(item=>item.appDB==appDB)
if(connectionDetails){deferred.resolve(connectionDetails.connection);
}else{createConnection(appDB).then(function(connectionDetails){
deferred.resolve(connectionDetails);})
}
return deferred.promise;
}
function createConnection(appDB){
var deferred = Q.defer();
mongodb.MongoClient.connect(connectionServer + appDB, (err,database)=>
{
if(err) deferred.reject(err.name + ': ' + err.message);
global.dbconnections.push({appDB: appDB, connection: database});
deferred.resolve(database);
})
return deferred.promise;
}
In case anyone wants something that works in 2021 with Typescript, here's what I'm using:
import { MongoClient, Collection } from "mongodb";
const FILE_DB_HOST = process.env.FILE_DB_HOST as string;
const FILE_DB_DATABASE = process.env.FILE_DB_DATABASE as string;
const FILES_COLLECTION = process.env.FILES_COLLECTION as string;
if (!FILE_DB_HOST || !FILE_DB_DATABASE || !FILES_COLLECTION) {
throw "Missing FILE_DB_HOST, FILE_DB_DATABASE, or FILES_COLLECTION environment variables.";
}
const client = new MongoClient(FILE_DB_HOST, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
class Mongoose {
static FilesCollection: Collection;
static async init() {
const connection = await client.connect();
const FileDB = connection.db(FILE_DB_DATABASE);
Mongoose.FilesCollection = FileDB.collection(FILES_COLLECTION);
}
}
Mongoose.init();
export default Mongoose;
I believe if a request occurs too soon (before Mongo.init() has time to finish), an error will be thrown, since Mongoose.FilesCollection will be undefined.
import { Request, Response, NextFunction } from "express";
import Mongoose from "../../mongoose";
export default async function GetFile(req: Request, res: Response, next: NextFunction) {
const files = Mongoose.FilesCollection;
const file = await files.findOne({ fileName: "hello" });
res.send(file);
}
For example, if you call files.findOne({ ... }) and Mongoose.FilesCollection is undefined, then you will get an error.
npm i express mongoose
mongodb.js
const express = require('express');
const mongoose =require('mongoose')
const app = express();
mongoose.set('strictQuery', true);
mongoose.connect('mongodb://localhost:27017/db_name', {
useNewUrlParser: true,
useUnifiedTopology: true
})
.then(() => console.log('MongoDB Connected...'))
.catch((err) => console.log(err))
app.listen(3000,()=>{ console.log("Started on port 3000 !!!") })
node mongodb.js
Using below method you can easily manage as many as possible connection
var mongoose = require('mongoose');
//Set up default mongoose connection
const bankDB = ()=>{
return mongoose.createConnection('mongodb+srv://<username>:<passwprd>#mydemo.jk4nr.mongodb.net/<database>?retryWrites=true&w=majority',options);
}
bankDB().then(()=>console.log('Connected to mongoDB-Atlas bankApp...'))
.catch((err)=>console.error('Could not connected to mongoDB',err));
//Set up second mongoose connection
const myDB = ()=>{
return mongoose.createConnection('mongodb+srv://<username>:<password>#mydemo.jk4nr.mongodb.net/<database>?retryWrites=true&w=majority',options);
}
myDB().then(()=>console.log('Connected to mongoDB-Atlas connection 2...'))
.catch((err)=>console.error('Could not connected to mongoDB',err));
module.exports = { bankDB(), myDB() };
When I run the server attempting to create two databases (db1 and db2), the system kicks backs this error:
Possibly unhandled SequelizeBaseError: database "db2" does not exist
As a reference, there is similar stackoverflow question on this topic here, however, the author of the solution does not cover the solution to how the server.js file is setup. You will notice I have structured my index.js file similar to their answer.
My models/index.js file and server run and executes scripts properly, yet the second database does not work at all and does not even get initialized.
Can someone provide a solution for the server.js file to accurately initialize two databases in this one server?
The following is the partial code from the models/index.js and server.js files. In the server.js file, I am using .sync to initialize the databases.
server.js
[...]
//sync's sequelize DB and tables
db['db1'].sequelize.sync(function(err){});
db['db2'].sequelize.sync(function(err){});
models/index.js
var databasesArray = ['db1', 'db2']
var databasesObj = {
database: {
db1: {
DBName: 'db1',
User: user,
Password: password,
Config: config,
},
db2: {
DBName: 'db2',
User: user,
Password: password,
Config: config,
}
}
} // EOF databaseObj
for(var i = 0; i < databasesArray.length; ++i) {
var databasePointerToOBJ = databasesArray[i];
var database = databasesObj.database[databasePointerToOBJ]
if(database.DBName == 'db1'){
var sq = new Sequelize(database.DBName, user, password, config)
db['db1'] = {
Sequelize: Sequelize,
sequelize: sq,
Table1: sq.import(__dirname + '/...')
}
}else if(database.DBName == 'db2'){
var sq = new Sequelize(database.DBName, user, password, config)
db['db2'] = {
Sequelize: Sequelize,
sequelize: sq,
Table1: sq.import(__dirname + '/...')
}
}
}
module.exports = db;
--- EDIT ---
The author of the solution was correct. In order for a new database to be created, it must be created prior to being sync'd. As piotrbienias rightly notes, adding the code for adding a new DB in a your initialization script would be the best option. Piotrbienias solution is .js option. The code that worked for me is a .sh option and is as following:
PG_HOST=localhost
PG_PORT=5432
PG_DB=databaseName
PG_USER=ubuntu
PG_PASS='EnterPassword'
sudo -u postgres createdb -U postgres -O $PG_USER $PG_DB
You need to create the database manually before trying to access it via Sequelize - it does not create it if it does not exists. You can use a pg module inside some initialisation script to create the DB via CREATE DATABASE before doing any synchronisation and connection, or simply create it via postgres CLI
const pg = require('pg');
module.exports = function(next){
var connectionData = {
user: 'postgres',
password: 'password',
host: 'localhost'
};
var databaseName = 'db2';
var connectionUri = `postgres://${user}:${password}#${host}/postgres`;
pg.connect(connectionUri, function(err, client, done) {
client.query(`CREATE DATABASE ${databaseName}`, function(error){
// here you can perform some sequelize operations after creating the database
client.end(); // disconnect client
next(); // you can call it with some parameter like Sequelize instance etc.
});
});
};
I am currently developing a node.js backend for a mobile app with potentially many users. However it's my first time in developing node.js. I was following a tutorial on how to connect to a mysql database via mysql pools.
I am able to create a single mysql connection and do queries via my routes.
The problem arises once I establish the file structure mentioned in the tutorial:
dbConnect
-[models]
--users.js
-db.js
-server-ks
I am not getting an error message regarding the connection of the mysql database - even if I enter a wrong password.
// server.js
///////////////////////////// basic setup ///////////////////////////////////
var restify = require('restify');
var bodyParser = require('body-parser');
var mysql = require('mysql');
var db = require('./db');
var users = require('./models/users');
///////////////////////////// initilisation of the server ///////////////////////////////////
var server = restify.createServer({
name: 'testUsers',
});
server.use(restify.bodyParser({ mapParams: true }));
///////////////////////////// Säuberung der URL //////////////////////////////////////////
server.pre(restify.pre.sanitizePath());
///////////////////////////// MySQL Instanz starten //////////////////////////////////////////
db.connect(db.MODE_PRODUCTION, function (err) {
if (err) {
console.log('Unable to connect to MySQL.')
process.exit(1)
} else {
server.listen(8080, function () {
console.log('Listening on port 8080 ...')
})
}
})
///////////////////////////// implementation of the routes ///////////////////////////////////
function send(req, res, next) {
var test = users.getAll();
res.json({ test: 'Hello ' + req.params.name });
return next();
};
My DB.js file looks the following:
var mysql = require('mysql'),
sync = require('async')
var PRODUCTION_DB = 'userDB',
TEST_DB = 'userDB'
exports.MODE_TEST = 'mode_test'
exports.MODE_PRODUCTION = 'mode_production'
var state = {
pool: null,
mode: null,
}
exports.connect = function (mode, done) {
state.pool = mysql.createPool({
connectionLimit: 50,
host: 'localhost',
user: 'user',
password: 'password',
database: 'userDB' // test
//mode === exports.MODE_PRODUCTION ? PRODUCTION_DB : TEST_DB
})
state.mode = mode
done()
}
exports.get = function () {
return state.pool
}
Could it be, that the tutorial spared out an essential part in utilizing mysql pools and node.js?
Thanks in advance for at least trying to answer that question.
Are there better methods sequelize(?) available to create performant connections to a MySQL database?
It looks like creating the pool object does not actually connect to the database. A big clue is that the createPool function is not asynchronous, which is what you would expect if it was actually connecting at that moment.
You have to make use of the returned pool object to perform a query, which IS asynchronous.
From the documentation:
var mysql = require('mysql');
var pool = mysql.createPool({
connectionLimit : 10,
host : 'example.org',
user : 'bob',
password : 'secret',
database : 'my_db'
});
pool.query('SELECT 1 + 1 AS solution', function(err, rows, fields) {
if (err) throw err;
console.log('The solution is: ', rows[0].solution);
});