Develop Reference

NodeJS Child process cannot access MySQL Pool object

My intention is upon a file upload the main Node.JS Express backend process will fork a child process which will call a java program to parse the uploaded file and then the NodeJS child will make bulk (100K+) MySQL queries. I want the child to handle the queries because I do not want the parent process to "block." Since both the parent and child processes need access to the MySQL connection I have defined in an external file containing an npm mysql pool.
mysqlConnector.js:
// Load module
var mysql = require('mysql');
// Initialize pool
var pool = mysql.createPool({
connectionLimit : 10,
host : 'localhost',
port: 3306,
user: 'bob',
password: 'my_pass',
database: 'my_db'
});
const doQuery = (query_string) => {
console.log(pool);
pool.query(query_string, function (error, results, fields) {
if (error) throw error;
console.log('in doQuery: query' );
});
};
exports.doQuery = doQuery;
exports.pool = pool
My index.js (main process) is able to access this pool and make queries with no issue. The main process will fork the child running the following code
handleprops.js:
const {execSync, fork} = require('child_process');
const {doQuery} = require('../mysqlConnector');
const exec_options = {
cwd: null,
env: null,
encoding: 'utf8',
timeout: 0,
maxBuffer: 200 * 1024,
killSignal: 'SIGTERM'
};
process.on('message', msg=>{
if(msg.filepath && msg.username){
execSync(`java -jar C:\\Users\\colli\\d2reader\\out\\artifacts\\d2reader_jar\\d2reader.jar ${msg.filepath}`, exec_options);
var filename = msg.filepath.replace(/^.*[\\\/]/, '');
filename = __dirname + "\\json\\" + filename.substring(0, filename.length-3) + "json";
try{
const fs = require('fs');
const data = fs.readFileSync(filename, 'utf8');
processSharedStashQueries(JSON.parse(data), msg.username);
} catch(err) {
console.error(err);
}
process.exit();
}
});
const processSharedStashQueries = (data, username) => {
const INSERT_USER_QUERY = `insert into sharedstashes (NumStashes, SharedGold, NumItems, UserID) values (${data.numStashes}, ${data.sharedGold}, ${data.numItems}, (select id from users where username = '${username}'))`;
doQuery(INSERT_USER_QUERY);
};
When the child calls doQuery() inside processSharedStashQueries() the pool object will be logged and I can verify it is a valid and defined object and is the same exact pool object the parent process uses. However, nothing happens when the child hits the pool.query() line of code. No error and no console logs. It's as if nothing happens.
Why can the child process "see" the pool object but not be able to use it?
Is my overall approach incorrect? I fear I might be missing some knowledge in how to use npm mysql effectively to do such a thing. Should I put the MySQL query smarts inside the external java program? Or is that bad design?
Thanks in advance!

Store morgan logs into database

I am using morgan('dev') for logging, but I want to store this object "GET /users/get 200 195.315 ms - 393" [in console log] into database.
like
- method : get
- endpoint : /users/get
- status : 200
How can I get this object?
How to explode and insert data into field(method, endpoint, status) above on database?

You can create a small middleware that will listen on 'finish' event of Response and save the data into the database:
app.use((req, res, next) => {
const method = req.method;
const endpoint = req.originalUrl;
res.on('finish', () => {
const status = res.status;
saveRequestDataToDatabase(method, endpoint, status);
next();
});
});

There are modules already out there that provide stream-compatible interfaces to known storages, such as Mongoose-Morgan which allows you to stream your Mongoose logs directly into MongoDB. However, if you can’t find a morgan-compatible module, you can simply write a function that returns a writable stream and sends the information where you need it.
Morgan NPM Logger – The Beginner’s Guide
Create a new named format:
const morgan = require('morgan')
const Writable = require("stream").Writable
morgan.token("custom", "-method: :method -endpoint: :url -status: status")
Use the new format by name:
class MyStream extends Writable {
write(line) {
//here you send the log line to wherever you need
console.log("Logger:: ", line)
}
}
let writer = new MyStream()
app.use(morgan(‘custom’, {stream: write}))

Use loopback token for authentication of socket.io

I'm working with loopback 2.0 and socket.io 1.0.6.
I'd like to use loopback authentication method for authentication of socket.io.
I found the method to authenticate users in loopback/lib/middleware/token.js. https://github.com/strongloop/loopback/blob/master/lib/middleware/token.js
Then I write like below:
var loopback = require('loopback');
var ioapp = module.exports = socketio;
function socketio(server) {
var io = require('socket.io')(server);
// auth
io.use(function(socket, next) {
loopback.token()(socket.request, null, next);
});
// listeners
...
return io;
};
But actually I won't work and causes error like this.
/Users/.../project_root/node_modules/loopback/lib/models/access-token.js:201
id = req.param(params[i]);
^
TypeError: Object #<IncomingMessage> has no method 'param'
at tokenIdForRequest (/Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/loopback/lib/models/access-token.js:201:14)
at Function.AccessToken.findForRequest (/Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/loopback/lib/models/access-token.js:123:12)
at /Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/loopback/lib/middleware/token.js:53:16
at Array.0 (/Users/ksuzuki/Projects/appsocially/repo/chat-center/server/socket.js:15:28)
at run (/Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/socket.io/lib/namespace.js:114:11)
at Namespace.run (/Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/socket.io/lib/namespace.js:126:3)
at Namespace.add (/Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/socket.io/lib/namespace.js:155:8)
at Client.connect (/Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/socket.io/lib/client.js:67:20)
at Server.onconnection (/Users/ksuzuki/Projects/appsocially/repo/chat-center/node_modules/socket.io/lib/index.js:309:10)
at Server.EventEmitter.emit (events.js:95:17)
I guess this is because I pass the wrong object type to loopback.token() method.

Well I believe Loopback token is built to be used with express request object. In the latest version(2.x) you could use it if you override AccessToken.findForRequest and implement it you yourself.
But there's another approach to this which is covered in the official documentation:
Basically it suggest using socketio-auth(Which "provides hooks to implement authentication in socket.io without using querystrings to send credentials, which is not a good security practice") and using AccessToken model directly.
I put the code here with a little bit of simplification:
On server-side:
app.io = require('socket.io')(app.start());
require('socketio-auth')(app.io, {
authenticate: function (socket, value, callback) {
var AccessToken = app.models.AccessToken;
//get credentials sent by the client
var token = AccessToken.count({
userId: value.userId,
id: value.id,
}, callback);
}
});
On client-side:
socket.on('connect', function() {
// You should have retrieved tokenId/userId by calling user.login and
// saving it in cookies or localStorage.
socket.emit('authentication', {id: tokenId, userId: userId });
});

Node.js error with SSL UNABLE_TO_VERIFY_LEAF_SIGNATURE

System: Windows 7
NodeJS version: 0.10.2
WS module: ws, last version
Error:
events.js:72
throw er; // Unhandled 'error' event
^
Error: UNABLE_TO_VERIFY_LEAF_SIGNATURE
at SecurePair. (tls.js:1283:32)
at SecurePair.EventEmitter.emit (events.js:92:17)
at SecurePair.maybeInitFinished (tls.js:896:10)
at CleartextStream.read [as _read] (tls.js:430:15)
at CleartextStream.Readable.read (_stream_readable.js:294:10)
at EncryptedStream.write [as _write] (tls.js:344:25)
at doWrite (_stream_writable.js:211:10)
at writeOrBuffer (_stream_writable.js:201:5)
at EncryptedStream.Writable.write (_stream_writable.js:172:11)
at write (_stream_readable.js:547:24)
Server:
(function(){
"use strict";
var fs = require('fs');
// you'll probably load configuration from config
var cfg = {
ssl: true,
port: 8080,
ssl_key: 'crt/server1.key',
ssl_cert: 'crt/server1.crt'
};
var httpServ = require('https')
var WebSocketServer = require('ws').Server;
var app = null;
// dummy request processing
var processRequest = function( req, res ) {
res.writeHead(200);
res.end("All glory to WebSockets!\n");
};
if ( cfg.ssl ) {
app = httpServ.createServer({
// providing server with SSL key/cert
key: fs.readFileSync( cfg.ssl_key ),
cert: fs.readFileSync( cfg.ssl_cert ),
//requestCert: true,
//rejectUnauthorized: false
}, processRequest ).listen( cfg.port );
} else {
app = httpServ.createServer( processRequest ).listen( cfg.port );
}
// passing or reference to web server so WS would knew port and SSL capabilities
var wss = new WebSocketServer( { server: app } );
wss.on( 'connection', function ( wsConnect ) {
wsConnect.on( 'message', function ( message ) {
console.log( message );
});
});
}());
Client:
var WebSocket = require('ws');
var ws = new WebSocket('wss://localhost:8080');
ws.on('open', function() {
ws.send('something');
});
The certificate confirmed.
Help> please!

I'm using a package called "superagent" and getting the same error. After trying several potential fixes, I came across this one that works for me 100% of the time:
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
There's no need to do any requires or whatever : just add this to your code before your network calls and you're good to go.

The SSL certificate used by the server in your example is probably not fully trusted by the WebSocket client so NodeJS is throwing an error in its network library on the client-side.
You need to set rejectUnauthorized to false (this is an option that most high-level network libraries will allow you to set via an option that gets passed to the lower level NodeJS networking library).
I skimmed the ws module source code and looks like you should try this:
var ws = new WebSocket('wss://localhost:8080', null, {rejectUnauthorized: false});
NOTE: rejectUnauthorized should only be false during testing/development. Production applications should always use rejectUnauthorized: true for full security.

If you do not want to disable your security. Add ca: [cert] option in http /socket client options.
Where cert is Certificate of server you are connecting to or CA of the server you are connecting to.

I've encountered a similar problem before, you may try to use
https.globalAgent.options.secureProtocol = 'SSLv3_method'
to set SSLv3 connection for https.

As per the nginx official website, they clearly mentioned certificate should be the combination of The server certificate and chained certificates.MoreInfo

Below solution is prefect and working fine for me in node js
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";

Use a library to load the certificates (pem, crt, ...) from a given folder.
https://github.com/fujifish/syswide-cas
You can export the certificates from the browser and try:
const syswidecas = require('syswide-cas');
syswidecas.addCAs('./certs');

How to access session in express, outside of the req?

I know that I can use
function(req, res) {
req.session
}
using express. However I need to access the session outside of the response function. How would I go about doing that?
I'm using socket.io to pass information for adding posts and comments. So when I receive the socket.io message on the server-side, I need to verify the person posting the information by using the session. However since this is being done via socket.io there is no req/res.

I think I have a different answer.
code:
var MongoStore = require('connect-mongo')(session);
var mongoStore = new MongoStore({
db:settings.db, //these options values may different
port:settings.port,
host:settings.host
})
app.use(session({
store : mongoStore
//here may be more options,but store must be mongoStore above defined
}));
then you should define a session key at req,just like :
code:
req.session.userEmail;
finally,you can get it this way:
code:
var cookie = require("cookie"); //it may be defined at the top of the file
io.on("connection",function(connection){
var tS = cookie.parse(connection.handshake.headers.cookie)['connect.sid'];
var sessionID = tS.split(".")[0].split(":")[1];
mongoStore.get(sessionID,function(err,session){
console.log(session.userEmail);
});
}
I had test it yesterday, it worked well.

Using socket.io, I've done this in a simple way. I assume you have an object for your application let's say MrBojangle, for mine it's called Shished:
/**
* Shished singleton.
*
* #api public
*/
function Shished() {
};
Shished.prototype.getHandshakeValue = function( socket, key, handshake ) {
if( !handshake ) {
handshake = socket.manager.handshaken[ socket.id ];
}
return handshake.shished[ key ];
};
Shished.prototype.setHandshakeValue = function( socket, key, value, handshake ) {
if( !handshake ) {
handshake = socket.manager.handshaken[ socket.id ];
}
if( !handshake.shished ) {
handshake.shished = {};
}
handshake.shished[ key ] = value;
};
Then on your authorization method, I'm using MongoDB for session storage:
io.set('authorization', function(handshake, callback) {
self.setHandshakeValue( null, 'userId', null, handshake );
if (handshake.headers.cookie) {
var cookie = connect.utils.parseCookie(handshake.headers.cookie);
self.mongoStore()
.getStore()
.get(cookie['connect.sid'], function(err, session) {
if(!err && session && session.auth && session.auth.loggedIn ) {
self.setHandshakeValue( null,
'userId',
session.auth.userId,
handshake );
}
});
}
Then before saving a record in the model, you can do:
model._author = shished.getHandshakeValue( socket, 'userId' );

I believe checking socket.handshake should get you the session:
io.sockets.on('connection', function(socket) {
console.log(socket.handshake.sessionID);
});
When the client establishes a socket connection with your socket.io server, the client sends a WebSocket handshake request. What I'm doing above is grabbing the session ID from the handshake.

Assuming your socket.io code looks kinda like this:
io.on('connection',
function(client) {
console.log(client.request)
});
The request is client.request as shown in the example above.
Edit:
As a separate thing, maybe this would help:
https://github.com/aviddiviner/Socket.IO-sessions