Hello I noticed some strange behaviour in node.js with felix geisendörfers awesome mysql module.
I have the following structure in my express app.
app.js (main)
routesA.js
routesB.js
routesC.js
The routes use the Router function of express.
Every routing file (A-C) has to access the mysql server.
But I was to lazy to write a connection and the connection options in every routing script file.
So I made another file called DBServer.js It is as follows (inspired by some hints from someone here in stackoverflow):
var mysql = require('mysql');
exports.connect = function (){
var db_config = {
host: '127.0.0.1',
user: 'my username',
password: '********',
database: 'my database'
};
var connection;
function autoConnect() {
connection = mysql.createConnection(db_config);
connection.connect(function(err) {
if(err) {
console.log('DBServer Error: cannot connect to db. Reconnect attempt in 2 seconds...\nError: ', err);
setTimeout(autoConnect, 2000);
}
else{
console.log('DBServer connected successfully...');
}
});
connection.on('error', function(err) {
if(err.code === 'PROTOCOL_CONNECTION_LOST') {
console.log('DBServer Error: lost connection. Reconnect attempt in 2 seconds...\nError: ', err);
autoConnect();
}
else {
console.log('DBServer Error: minor error\nError: ', err);
}
});
}
autoConnect();
return connection
}
In every routing file I require this DBServer file via:
var db = require('./lib/DBServer').connect();
When I start my app, the console logs 3 times
DBServer connected successfully...
DBServer connected successfully...
DBServer connected successfully...
... as intended.
Everything works perfect. I run the App with forever and every time the script losses connection to the db-server (what happens from time to time) it reconnects again... as intended.
... BUT! Except of one script. Script routesA.js stops working when it comes to a mysql query. The script freezes but does not quit. I have to stop and restart it again.
There is no difference between the invocation of DBServer.js between routesA, routesB or routesC. And it works pretty good... But it seems that if routesA looses connection it does not reconnect again... routesB and routesC still works fine.
So I changed the way how script routesA.js connects to the database. I connect now in script routesA.js not via the DBServer.js and require but the manual way
var mysql = require('mysql');
var db = mysql.createConnection({
host: '127.0.0.1',
user: 'my username',
password: '******',
database: 'ma database'
});
db.connect();
Now it works... and runs for days without problems. But the reason why this works is, because now I don't have an error handling in script routesA.js.. so forever detects a script exit and restarts... and everything works again.
But I don't want that way. I want a proper error handling like in DBServer.js. As said this works for script B and C, but not script A...
I know that it is strange and difficult to say, what might be the problem. But maybe someone has had some similar problem.
Another question here is: How do you handle the database connections with multiple script files. I there a way to share one mysql connection for all script files you have in an app?
kind regards
martin
The reason probably has to do with the fact that you're returning the initial connection object on require(), but if you get disconnected, you reassign the connection variable which the external scripts do not have a reference to (they still only have a reference to the old/original connection object).
I should also note that if you're using the mysql2 module (compatible with mysql except much faster), there is a connection.ping() method that you can use to periodically ping the server to help keep the connection alive.
Related
Ive read documentation from several pages on SO of this issue, but i havent been able to fix my issue with this particular error.
throw new Error('SASL: SCRAM-SERVER-FIRST-MESSAGE: client password must be a string')
^
Error: SASL: SCRAM-SERVER-FIRST-MESSAGE: client password must be a string
at Object.continueSession (C:\Users\CNFis\Desktop\WulfDevelopments\ThePantry\node_modules\pg\lib\sasl.js:24:11)
at Client._handleAuthSASLContinue (C:\Users\CNFis\Desktop\WulfDevelopments\ThePantry\node_modules\pg\lib\client.js:257:10)
at Connection.emit (events.js:400:28)
at C:\Users\CNFis\Desktop\WulfDevelopments\ThePantry\node_modules\pg\lib\connection.js:114:12
at Parser.parse (C:\Users\CNFis\Desktop\WulfDevelopments\ThePantry\node_modules\pg-protocol\dist\parser.js:40:17)
at Socket.<anonymous> (C:\Users\CNFis\Desktop\WulfDevelopments\ThePantry\node_modules\pg-protocol\dist\index.js:11:42)
at Socket.emit (events.js:400:28)
at addChunk (internal/streams/readable.js:290:12)
at readableAddChunk (internal/streams/readable.js:265:9)
at Socket.Readable.push (internal/streams/readable.js:204:10)
its as if in my connectDB() function its not recognizing the password to the database. I am trying to run a seeder.js script to seed the database with useful information for testing purposes, and if i run npm run server which is a script that just starts a nodemon server, itll connect to the DB just fine. but when i try to run my script to seed data, i am returning this error.
import { Sequelize } from "sequelize";
import colors from "colors";
import dotenv from "dotenv";
dotenv.config();
const user = "postgres";
const host = "localhost";
const database = "thePantry";
const port = "5432";
const connectDB = async () => {
const sequelize = new Sequelize(database, user, process.env.DBPASS, {
host,
port,
dialect: "postgres",
logging: false,
});
try {
await sequelize.authenticate();
console.log("Connection has been established successfully.".bgGreen.black);
} catch (error) {
console.error("Unable to connect to the database:".bgRed.black, error);
}
};
export default connectDB;
above is my connectDB() file, and again, it works when i run the server normally. but i receive this error only when trying to seed the database. Ill post my seeder script below:
import dotenv from "dotenv";
import colors from "colors";
import users from "./data/users.js";
import User from "./models/userModel.js";
import connectDB from "./config/db.js";
dotenv.config();
console.log(process.env.DBPASS);
connectDB();
const importData = async () => {
try {
await User.drop();
await User.sync();
await User.bulkCreate(users);
console.log("Data Imported".green.inverse);
process.exit();
} catch (e) {
console.error(`${e}`.red.inverse);
process.exit(1);
}
};
const destroyData = async () => {
try {
await User.bulkDestroy();
console.log("Data Destroyed".red.inverse);
process.exit();
} catch (e) {
console.error(`${e}`.red.inverse);
process.exit(1);
}
};
if (process.argv[2] === "-d") {
destroyData();
} else {
importData();
}
Add your .env file in your project, I think your .env file is missing in your project folder.
add like this:
So, i may have figured this out by playing around in another project with sequelize, as it turns out, the initial connection to the database in my server.js file, honestly means nothing. Unlike Mongoose where the connection is available across the whole app. its not the same for Sequelize this connection that it creates is only apparent in certain places, for example i was trying the same process in my other project as i am here, except i was trying to read data from my DB using the model that i built with sequelize and i was receiving the same type error, i went into where i defined the model and made a sequelize connection there, and i was then able to read from the database using that object model.
Long story short, to fix the error in this app i have to place a connection to the database in the seeder.js file or i have to place a connection in the User model (this is ideal since ill be using the model in various places) to be able to seed information or read information from the database.
today i have same problem like this, so if you use database with type relational. you must define password from database.
const user = "postgres";
const host = "localhost";
const database = "thePantry";
const password = "yourdatabasepassword"; if null > const password = "";
const port = "5432";
but, if you use database with type non-relational, as long as the attributes are the same, you can immediately run the program as you defined it
I also faced this issue and another solution different from the accepted solution here solved my issue, so I wanted to explain that to this lovely community, too.
Firstly, when I faced the issue, ran my project in debug mode and reached the code below.
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.database, config.username, config.password, config);
}
The problem here is actually obvious when I saw first, there is a problem in .env file as mentioned in the solutions above. In my process.env is defined as like as following line: DATABASE_URL=postgres://username:password#IP_adress:port/db_name and my config.js file is in the following format:
module.exports = {
"development": {
"url":"postgres://username:password#IP_adress:port/db_name",
"dialect": "postgres",
}, ...
}
So as a solution, I come with the following fix for the parameters that are inside Sequelize(...). My solution below is totally worked for me and I hope it also works for you too.
let sequelize;
if (config.use_env_variable) {
sequelize = new Sequelize(process.env[config.use_env_variable], config);
} else {
sequelize = new Sequelize(config.url, config);
}
Finally, the point you need to be careful about what you have written to the config file. That's the most important in this case.
Farewell y'all.
Here is my case. I have postgresql connection url in my enviroment like:
POSTGRES=postgres://postgres:test#localhost:5432/default
But my config getting like:
POSTGRES_DB_HOST=localhost
POSTGRES_DB_PORT=5432
...rest of configs
Now it has resolved.
I faced this issue because I was trying to execute nodemon from a parent folder. Once I changed my pwd, the error was resolved.
For your seeder script, i'm doing something similar but not using Sequilize, just the node-postgres package in an ExpressJS app.
To give context (so you know if this applies to your situation)
I run a separate script for testing, which uses database credentials to test batched emailing. So, I need to access my database (eventually will migrate it to an AWS lambda function).
I need to access my database and run sequential actions, since I'm not spinning up my server, all that 'under the hood' processes that would normally start your connection pool is probably not running. My guess ( I know it's an old post but this may help others).
Try passing your hardcoded password credentials. first on your seeder.js file. (i'm sure you've tried this already).
Try creating a new Pool within your seeder script and pass it your credentials (try hard coding it first to see if it works).
Pool in postgres takes a client config with the following properties (i use this to get mine to work).
const pool = new Pool({
user: '****',
database: '****',
password: '****',
port: 5432,
host: '****',
max: 5,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 5000,
})
I imagine sequilize will have a similar configuration, so try playing around with that.
Then I just connect to the pool and do everything I'd normally do.
Hope this helps with a bit of the troubleshooting. I had the EXACT same error message earlier. Ultimately I had to restructure my code to 'boot up' the Client/Connection Pool for the database. It sounds like you're not properly 'booting up' your connection so try doing it manually within your seeder script (don't pass process.env.DB_PASSWORD at first).
I saw this error when running a npx sequelize-cli db:... command
and my postgres server wasn't running or able to accept connections.
To fix it, I had to be running: postgres -D /usr/local/var/postgres in the background.
I am trying to connect to SQL Server 2014 with NodeJS, i am using the "mssql" package, i dont have answer here is my code
var sql = require('mssql');
var opciones = {
user: 'myuser',
password: 'mypass',
server: 'myserver',
database: 'mydatabase',
options: {
encrypt: true
}
};
sql.connect(opciones,function(err){
if(err){
console.log(err);
}else{
console.log("CONEXIÓN EXITOSA");
}
});
the name of that js is "cnSQL.js", when i execute on cmd "node cnSQL" I dont have answer.
Tested the code as above with my local DB instance.
The code is actually correct.
var sql = require('mssql');
var opciones = {
user: 'sa',
password: 'mypass',
server: '127.0.0.1',
database: 'mydb',
options: {
encrypt: true
}
};
sql.connect(opciones,function(err){
if(err){
console.log(err);
}else{
console.log("CONEXIÓN EXITOSA");
}
});
I have managed to get the 'Connexion Exitosa' message.
In order to further debug your issue, attempt the following:
With the username you are trying to login with (In my case 'sa')
Open SQL Server Management Studio and attempt to put in the connection information as above. Click login, does that work?
If not:
Open SQL Server Management Studio
Connect to the SQL Server Instance you are trying to connect to from NodeJS
Right click on the instance of your SQL server and click properties
Click security and ensure that the "SQL Server and Windows Authentication mode" radio button is selected.
When done click OK
On the left hand navigation expand the Node "Security"
Expand Logins
Find your user and ensure it is enabled and that the password you selected reflects the password within your NodeJS app.
If so:
Amend all the information within the code to reflect exactly the credentials used to login
Special Note: '.' will not represent localhost here. Use 127.0.0.1 or localhost
Furthermore, if the SQL server instance you are trying to connect to is not hosted locally. Ensure that the machine it is hosted on accepts connections on port 1433 (by default for SQL Server).
Hope it helps!
I'm using socket.io-client to create a socket connection to my locally-running server. See my code below:
// Working example of connecting to a local server that is not SSL protected
var io = require('socket.io-client')
var socket = io.connect('http://localhost:3000', {reconnect: true});
socket.on('connect', function(){ console.log("inside 'connect'") } );
socket.on('connection', function(){ console.log("inside 'connection'") } );
socket.on('event', function(data){ console.log("inside 'event'") } );
socket.on('disconnect', function(){ console.log("inside 'disconnect'") } );
var payload = {email: 'fake#gmail.com', password: 'tester'};
var tokens = {browserId: 'b965e554-b4d2-5d53-fd69-b2ca5483537a'};
socket.emit("publish", {logic:"user", method:"signIn"}, payload, tokens, function(err, creds) {
console.log("inside the socket client emit callback. err: " + err);
console.log("creds: " + creds);
});
Now for my problem. As I stated in the comment at the top of that code, I can connect to my local nodejs server and get the response I expect when I turn off SSL encryption on my server. As soon as I turn SSL on, I stop getting any response at all from the code above. I don't see any message in my server logs or from the command line, where I'm running the code above with node.
My goal is to be able to run the code above, with SSL turned on in my server, and get the same response that I get when SSL is turned off. I've tried a bunch of variations on the code I included above, such as:
connecting to "https://localhost:3000"
connecting to "//localhost:3000"
connecting to "https://localhost:3443" (this is the port I have to connect to when I have the nodejs server running with SSL)
changing {reconnect:true} to {reconnect:true,secure:true}
I'm truly stumped, and I've been doing a bunch of research on the web and on my node server. It's my company's code and I didn't originally implement the SSL components, so I've spent a few hours looking at our code and trying to understand how adding SSL changes everything. I'm also a student and have about 2 years of experience behind me, so I'm good but I'm no expert. Have I said anything above that indicates if my task is impossible to achieve, or if maybe I have just overlooked something? Any leads on things to check out would be appreciated :)
I'm writing a Meteor app that adds jobs to a queue that are to be processed by one or more external workers. I'm using vsivsi/meteor-job-collection (https://github.com/vsivsi/meteor-job-collection) and everything worked fine when I was (initially) processing jobs using method-based (i.e., non external) workers from within the Meteor app itself (which queues, creates and manages the jobs).
However, I want an external process, i.e., a Node.js program (not even a Meteor app) to listen for and subsequently process posted jobs instead and saw that this is indeed possible at https://github.com/vsivsi/meteor-job-collection#quick-example (third code block; "node.js Worker") - after all, this enables a more generic and Meteor-agnostic job-processing approach.
I created a separate Node.js program, successfully connected it to my running Meteor app using dpp, ddp-login and meteor-job and started the job listener in exactly the same fashion as I originally did within the Meteor app (when using method-based workers), but I get the following inexplicable error (only) when calling job.done() (other job methods seem to work flawlessly):
"Exception while invoking method 'jobs_jobDone' Error: Match error: Failed Match.OneOf or Match.Optional validation"
The code:
var DDP = require('ddp');
var DDPlogin = require('ddp-login');
var Job = require('meteor-job');
var ddp = new DDP({
host: "localhost",
port: 3000,
use_ejson: true
});
Job.setDDP(ddp);
ddp.connect(connectCB);
function connectCB(err) {
if (err)
throw err;
var options = { //Needed to be able to log in; METEOR_TOKEN is set as an environment variable and this works fine.
env: 'METEOR_TOKEN',
method: 'account',
account: null,
pass: null,
retry: 5,
plaintext: false
};
DDPlogin(ddp, options, ddpLoginCB);
}
function ddpLoginCB(err) {
if (err)
throw err;
Job.processJobs('jobs', 'dataGatherJob', {
concurrency: 8,
pollInterval: 250
}, processJobsCB);
}
function processJobsCB(job, cb) {
//This is where the actual work gets done; this callback works fine,
//is triggered when my Meteor app adds a new job, and the job object
//conforms to my expectations; just some example calls on/usages of the
//job object that work fine and are reactively updated and handled by my
//Meteor app:
//console.log(job); //Works fine.
//job.progress(50, 100); //Works fine.
//job.fail(); //Works fine.
job.done(); //Causes the Meteor-app (server) to generate an "Exception while invoking method 'jobs_jobDone' Error: Match error: Failed Match.OneOf or Match.Optional validation" error.
cb();
}
I do not understand why this is happening. The job id is identical to the one in the actual Mongo collection (the queue uses a Mongo collection to manage jobs) set by the server (I checked this) and besides that, other job operations (e.g., job.fail()) do work. I furthermore reset the "autopublish" and "insecure" packages for my Meteor app intermittently just to be sure that they are not causing the problem and this was not the case.
Any thoughts on this?
Many thanks in advance,
Danny.
Background Information
I'm attempting my first node.js API/application. As a learning exercise, I'm trying to create some test cases initially delete all records in a table, insert 3 specific records, and then query for those 3 records.
Code
Here's the code I have cobbled together:
http://pastebin.com/duQQu3fm
Problem
As you can see from the code, I'm trying to put the database connection logic in a dbSession.js file and pass it around.
I am able to start up the http server by doing the following:
dev#devbox:~/nimble_node$ sudo nodejs src/backend/index.js
Server started and listening on port: 8080
Database connection successful
However, when I try to run my jasmine tests, it fails with the following error:
F
Failures:
1) The API should respond to a GET request at /api/widgets/
Message:
TypeError: Object #<MongoClient> has no method 'collection'
Stacktrace:
TypeError: Object #<MongoClient> has no method 'collection'
at resetDatabase (/home/dev/nimble_node/spec/resetDatabase.js:6:29)
at /home/dev/nimble_node/spec/e2e/apiSpec.js:23:25
at /home/dev/nimble_node/node_modules/async/lib/async.js:683:13
at iterate (/home/dev/nimble_node/node_modules/async/lib/async.js:260:13)
at async.forEachOfSeries.async.eachOfSeries (/home/dev/nimble_node/node_modules/async/lib/async.js:279:9)
at _parallel (/home/dev/nimble_node/node_modules/async/lib/async.js:682:9)
at Object.async.series (/home/dev/nimble_node/node_modules/async/lib/async.js:704:9)
at null.<anonymous> (/home/dev/nimble_node/spec/e2e/apiSpec.js:19:9)
at null.<anonymous> (/home/dev/nimble_node/node_modules/jasmine-node/lib/jasmine-node/async-callback.js:45:37)
Finished in 0.01 seconds
1 test, 1 assertion, 1 failure, 0 skipped
Database connection successful
Line 6 of resetDatabase is:
var collection = dbSession.collection('widgets');
Given that after the error appears, I get the "Database connection successful" message, I think what's happening is that when the tests request the dbSession library, the database hasn't finished running the code to connect. And therefore, I can't get the collection object.
I'm currently reading through the mongodb online manual to see if I can find some hints as to how to do something like this.
Any suggestions or pointers would be appreciated.
EDIT 1
To prove that there is a collection method on the MongoClient object, I changed the dbSession.js code to look like this:
'use strict';
var DBWrapper = require('mongodb').MongoClient;
var dbWrapper = new DBWrapper;
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
var collection = dbWrapper.collection('widgets');
console.log('just created a collection...');
}
});
module.exports = dbWrapper;
And now, when I start up the http server (index.js), notice the messages:
dev#devbox:~/nimble_node$ sudo nodejs src/backend/index.js
Server started and listening on port: 8080
Database connection successful
just created a collection...
It could be an async issue.
Your code in dbSessionjs
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
}
});
module.exports = dbWrapper;
Starts the connection at dbWrapper asynchronously, but exports dbWrapper right away, which is then imported in resetDatabase. Thus yes, the connect function may have not yet returned from the async function when you call it in resetDatabase (and is what the log suggests,as the error appears before the success log).
You could add a callback after dbWrapper.connect() returns, in order to actually only be able to use dbWrapper when the connection finished.
(With sqlite, this may not happen as it accesses the DB faster on the commandline).
This may not be your problem but looks like a candidate.
EDIT: Here's a possible example for a callback, but please take note it depends on what you need to do so there are a lot of different solutions. The key is to call a callback function when you are done initializing.
Another solution could be to simply wait, and/or poll (e.g. chcke a variable 'initialized').
'use strict';
var DBWrapper = require('mongodb').MongoClient;
var dbWrapper = new DBWrapper;
function doConnect(callback) {
console.log("Initializing DB connection...");
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
var collection = dbWrapper.collection('widgets');
console.log('just created a collection...');
console.log('calling callback...');
callback(dbWrapper);
} else {
console.log("Error connectingi: " + err);
}
});
};
doConnect(function(correctDbWrapper) {
//Now you can use the wrapper
console.log("Inside callback, now consuming the dbWrapper");
dbWrapper = correctDbWrapper;
var collection = dbWrapper.collection('widgets');
});
It's interesting though I never ran into this issue, although I have generally used similar code like yours. I guess because normally I have this DB initialization right at the top, and then have to do lots of initializations on the node app, which gives the app time enough to return from the connect call....