I have created a bare-bones HTTP proxy that performs HTTP tunnelling using HTTP CONNECT method.
const http = require('http');
const https = require('https');
const pem = require('pem');
const net = require('net');
const util = require('util');
const createHttpsServer = (callback) => {
pem.createCertificate({
days: 365,
selfSigned: true
}, (error, {serviceKey, certificate, csr}) => {
const httpsOptions = {
ca: csr,
cert: certificate,
key: serviceKey
};
const server = https.createServer(httpsOptions, (req, res) => {
// How do I know I here whats the target server port?
res.writeHead(200);
res.end('OK');
});
server.listen((error) => {
if (error) {
console.error(error);
} else {
callback(null, server.address().port);
}
});
});
};
const createProxy = (httpsServerPort) => {
const proxy = http.createServer();
proxy.on('connect', (request, requestSocket, head) => {
// Here I know whats the target server PORT.
const targetServerPort = Number(request.url.split(':')[1]);
console.log('target server port', targetServerPort);
const serverSocket = net.connect(httpsServerPort, 'localhost', () => {
requestSocket.write(
'HTTP/1.1 200 Connection established\r\n\r\n'
);
serverSocket.write(head);
serverSocket.pipe(requestSocket);
requestSocket.pipe(serverSocket);
});
});
proxy.listen(9000);
};
const main = () => {
createHttpsServer((error, httpsServerPort) => {
if (error) {
console.error(error);
} else {
createProxy(httpsServerPort);
}
});
};
main();
The server accepts a HTTPS connection and responds with "OK" message without forwarding the request further.
As you can see in the code (see // Here I know whats the target server PORT.), I can obtain the target server's port within the HTTP CONNECT event handler. However, I am unable to figure out how to pass this information to the createHttpsServer HTTP server router (see // How do I know I here whats the target server port?).
When tunnelling a TLS connection, how to pass additional information?
The above code can be tested by running:
$ node proxy.js &
$ curl --proxy http://localhost:9000 https://localhost:59194/foo.html -k
The objective is to respond with "OK localhost:59194".
You can't add anything to a TLS stream (thankfully), short of tunneling it inside another protocol—which is what the Connect method already does. But, since you have the HTTP proxy and the HTTPS server in the same codebase, you don't need to fling the TLS stream over the network another time. Instead, you want to parse the TLS stream, and then you can pass any variables to the code that handles it.
However, after parsing TLS you'll still have a raw HTTP stream, and you'll need an HTTP server to turn it into requests and to handle responses.
The quick and rather dirty way to go about it is to use Node's HTTPS server to both decode TLS and parse HTTP. But the server's API doesn't provide for dealing with sockets that are already connected, and server's code isn't cleanly separated from connection code. So you need to hijack the server's internal connection-handling logic—this is of course susceptible to breakage in case of future changes:
const http = require('http');
const https = require('https');
const pem = require('pem');
const createProxy = (httpsOptions) => {
const proxy = http.createServer();
proxy.on('connect', (request, requestSocket, head) => {
const server = https.createServer(httpsOptions, (req, res) => {
res.writeHead(200);
res.end('OK');
});
server.emit('connection', requestSocket);
requestSocket.write('HTTP/1.1 200 Connection established\r\n\r\n');
});
proxy.listen(9000);
};
const main = () => {
pem.createCertificate({
days: 365,
selfSigned: true
}, (error, {serviceKey, certificate, csr}) => {
createProxy({
ca: csr,
cert: certificate,
key: serviceKey
});
});
};
main();
To avoid creating an HTTPS server instance on every request, you can move the instance out and tack you data onto the socket object instead:
const server = https.createServer(httpsOptions, (req, res) => {
res.writeHead(200);
// here we reach to the net.Socket instance saved on the tls.TLSSocket object,
// for extra dirtiness
res.end('OK ' + req.socket._parent.marker + '\n');
});
proxy.on('connect', (request, requestSocket, head) => {
requestSocket.marker = Math.random();
server.emit('connection', requestSocket);
requestSocket.write('HTTP/1.1 200 Connection established\r\n\r\n');
});
With the above code, if you do several successive requests:
curl --proxy http://localhost:9000 https://localhost:59194/foo.html \
https://localhost:59194/foo.html https://localhost:59194/foo.html \
https://localhost:59194/foo.html https://localhost:59194/foo.html -k
then you'll also notice that they're processed on a single connection, which is nice:
OK 0.6113572936982015
OK 0.6113572936982015
OK 0.6113572936982015
OK 0.6113572936982015
OK 0.6113572936982015
I can't quite vouch that nothing will be broken by handing the socket to the HTTPS server while the proxy server already manages it. [The server has the presence of mind to not overwrite another instance on the socket object](https://github.com/nodejs/node/blob/v10.9.0/lib/_http_server.js#L331), but otherwise seems to be rather closely involved with the socket. You'll want to test it with longer-running connections.
As for the `head` argument, [which can indeed contain initial data](https://www.rfc-editor.org/rfc/rfc2817#section-5.2):
you might be able to put it back on the stream with requestSocket.unshift(head), but I'm not sure that it won't be immediately consumed by the proxy server.
Or, you might be able to chuck it over to the HTTPS server with requestSocket.emit('data', head) since the HTTP server seems to use the stream events, however TLS socket source calls read() for whatever reason, and that's mutually exclusive with the events, so I'm not sure how they even work with each other.
One solution would be to make your own wrapper for stream.Duplex that will forward all calls and events, except for read() in the case when this initial buffer exists—and then use this wrapper in place of requestSocket. But you'll then need to replicate the 'data' event also, in accordance with the logic of Node's readable streams.
Finally, you can try creating a new duplex stream, write head and pipe the socket to it, like you did initially, and use the stream in place of the socket for the HTTPS server—not sure that it will be compatible with HTTP server's rather overbearing management of the socket.
An cleaner approach is to decode the TLS stream and use a standalone parser for the resultant HTTP stream. Thankfully, Node has a tls module that is nicely isolated and turns TLS sockets into regular sockets:
proxy.on('connect', (request, requestSocket, head) => {
const httpSocket = new tls.TLSSocket(requestSocket, {
isServer: true,
// this var can be reused for all requests,
// as it's normally saved on an HTTPS server instance
secureContext: tls.createSecureContext(httpsOptions)
});
...
});
See caveats on tls.createSecureContext regarding replicating the behavior of the HTTPS server.
Alas, Node's HTTP parser isn't so usable: it's a C library, which necessitates quite a bit of legwork between the socket and the parser calls. And the API can (and does) change between versions, without warnings, with a larger surface for incompatibilities compared to the HTTP server internals used above.
There are NPM modules for parsing HTTP: e.g. one, two, but none seem too mature and maintained.
I also have doubts about the feasibility of a custom HTTP server because network sockets tend to require plenty of nurture over time due to edge cases, with hard-to-debug timeout issues and such—which should all be already accounted for in the Node's HTTP server.
P.S. One possible area of investigation is how the Cluster module handles connections: afaik the parent process in a cluster hands connection sockets over to the children, but it doesn't fork on every request—which suggests that the child processes somehow deal with connected sockets, in code that's outside of an HTTP server instance. However, since the Cluster module is now in the core, it may exploit non-public APIs.
Related
Generalizing that would be the question... how to make websockets to go through a proxy in node.js?
In my particular case I'm using pusher.com with the node.js client library they recommend. Looking inside the code I would like to know some hints on what I should change in order to make this library to work with a proxy... you can take a look in the code here
Maybe I should somehow replace or modified the websockets module that is being used by the library?
EDIT
Thanks for your answers/comments! A couple of things to take into consideration (excuse me if I'm wrong with some/all of them, just learning):
I don't want to create a proxy server. I just want to use an existent proxy server within my company in order to proxified my websockets requests (particularly pusher.com)
Just to let you know, if I use a proxifier like the one for windows Proxifier and I set up the rule to inspect for all connections to port 443 to go through the proxy server proxy-my.coporate.com:1080 (type SOCKS5) it works like a charm.
But I don't want to go this way. I want to programatically configuring this proxy server within my node js code (even if that involved to modified the pusher library I mentioned)
I know how to do this for HTTP using Request module (look for the section that mentions how to use a proxy).
I want a similarly thing for websockets.
From
https://www.npmjs.com/package/https-proxy-agent
var url = require('url');
var WebSocket = require('ws');
var HttpsProxyAgent = require('https-proxy-agent');
// HTTP/HTTPS proxy to connect to
var proxy = process.env.http_proxy || 'http://168.63.76.32:3128';
console.log('using proxy server %j', proxy);
// WebSocket endpoint for the proxy to connect to
var endpoint = process.argv[2] || 'ws://echo.websocket.org';
var parsed = url.parse(endpoint);
console.log('attempting to connect to WebSocket %j', endpoint);
// create an instance of the `HttpsProxyAgent` class with the proxy server information
var options = url.parse(proxy);
var agent = new HttpsProxyAgent(options);
// finally, initiate the WebSocket connection
var socket = new WebSocket(endpoint, { agent: agent });
socket.on('open', function () {
console.log('"open" event!');
socket.send('hello world');
});
socket.on('message', function (data, flags) {
console.log('"message" event! %j %j', data, flags);
socket.close();
});
Using a proxy for websockets should work roughly the same as for https connections; you should use the CONNECT method. At least that's what both the HTTP and HTML5 specs say. So if your proxy implements CONNECT, you're good to go.
Try node-http-proxy
It allows you to send http or websocket requests through a proxy.
var http = require('http'),
httpProxy = require('http-proxy');
//
// Create a basic proxy server in one line of code...
//
// This listens on port 8000 for incoming HTTP requests
// and proxies them to port 9000
httpProxy.createServer(9000, 'localhost').listen(8000);
//
// ...and a simple http server to show us our request back.
//
http.createServer(function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.write('request successfully proxied!' + '\n' + JSON.stringify(req.headers, true, 2));
res.end();
}).listen(9000);
Source: link
Most web proxies don't support websockets yet. The best workaround is to use encryption by specifying wss:// (websocket secure protocol):
wss://ws.pusherapp.com:[port]/app/[key]
I'm definitely a newbie with JS and node. I have telescope management software called SkyX Pro, and it has the ability to run a TCP Server on port 3040. I can connect to it using Netcat and hand it a Javascript starting with //* Javascript *// this works and allows me to startup cameras and other equipment and send commands for taking pictures etc. The issue is it needs to be run from a batch file which makes getting any information back to an HTML page tough (Like Camera, focuser and filter wheel status and temperatures).
The NC call looks like "NC localhost 3040 < Javascript-file.js
To get around the browser to local machine security issues I want to run this from node.js with maybe socket.io-client if possible, but I don't know the proper syntax for it.
I have seen plenty of client syntax sending hello's etc. but nothing send javascript and allowing for two-way connectivity that I can understand.
I have tried using:
var socket = io.connect('http://localhost');`enter code here`
socket.on('httpServer', function (data) {
console.log(data);
document.write(data + "\r\n");
socket.emit('tcp', "For TCP");
});
const net = require('net');
const client = new net.Socket();
client.connect({ port: 3040, host: process.argv[2] });
client.on('data', (data) => {
console.log(data.toString('utf-8'));
But I do not understand it well enough to troubleshoot why it's not working.
Any help would be wonderful, and please treat me like a baby that needs its step by step.
Cheer
Peter
Reading [1], We can assume socket-io isn't the perfect fit for you, because that Server you have sound like a typical tcp-socket server, not a socket.io server ( which requires special headers ) or a web-socket server.
So you only needs "net" library to do the job.
const net = require('net');
// module to send a message to TCP-socket server and wait for the response from socket-server
const sendAndReceive = async (client, message) => {
client.write(message);
let response = null
await ( new Promise( (resolve, reject) => {
client.on('data', function(data) {
response = data;
resolve()
});
}))
return response;
}
// send a single message to the socket-server and print the response
const sendJSCode = (message) => {
// create socket-client
const client = new net.Socket();
client.connect(3040, 'localhost', async function() {
console.log('Connected');
// send message and receive response
const response = await sendAndReceive(client, message)
// parse and print repsonse string
const stringifiedResponse = Buffer.from(response).toString()
console.log('from server: ', stringifiedResponse)
// clean up connection
client.destroy()
});
}
sendJSCode('var Out; \n Out="TheSky Build=" + Application.build \n\r')
This script will:
Initiate a socket client
on connection successfully, client sends a message
client receives back response from that message
client prints response to terminal
Note that TheSkyX has a limitation of 4096 bytes for each message[2], any more than that and we will need to chunk the message. So you may want to keep the js-code short and precise.
that snippet I gave is minimal, it doesn't handle errors from server. If you want, you can add client.on("error", .. ) to handle it.
Your point of connecting to the socket server directly from browser is very intriguing, unfortunately it is not allowed by modern browsers natively due to security concerns 3
[1] https://socket.io/docs/#What-Socket-IO-is-not:~:text=That%20is%20why%20a%20WebSocket%20client,to%20a%20plain%20WebSocket%20server%20either.
[2] https://www.bisque.com/wp-content/scripttheskyx/scriptOverSocket.html#MSearchField:~:text=set%20to%204096%20bytes
I need what is essentially a reverse proxy but I need it in Node.js as I will have to put in some custom functionality.
The gateway will be the only visible service, and it needs to forward traffic on to an internal network of services. A simple 302 isn't going to work here.
How can I realistically achieve this with Node.js given the asynchronous nature of it?
Are there any well known libraries used for this?
I've managed this using node-http-proxy, where http://first.test/ and http://second.test/ are the hostnames.
var http = require('http'),
httpProxy = require('http-proxy');
var proxy = httpProxy.createProxyServer({});
// reverse proxy server
http.createServer(function (req, res) {
var target = '';
if (req.headers.host.match(/first.test/)) {
target = 'http://127.0.0.1:8001';
} else if (req.headers.host.match(/second.test/)) {
target = 'http://127.0.0.1:8002';
}
console.log(req.headers.host, '->', target);
proxy.web(req, res, { target: target });
}).listen(8000);
// test server 1
http.createServer(function(req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.write('8001\n');
res.write(JSON.stringify(req.headers, true, 2));
res.end();
}).listen(8001);
// test server 2
http.createServer(function(req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.write('8002\n');
res.write(JSON.stringify(req.headers, true, 2));
res.end();
}).listen(8002);
For a simple reverse proxy that uses the reactor pattern (like node), I would check out nginx. But, you mentioned you wanted to add in some custom functionality using node, so is that a realistic goal? Absolutely! Here are some things to think about when you are designing your reverse proxy:
How will you keep track of where incoming requests need to end up? For example, if you proxy all request with /test/* to your UI, but the returned HTML has root relative URLs (/imgs/banner.jpg), how do you keep track of where the subsequent request needs to go (especially if it comes from javascript)? Are you going to tightly couple your proxy and your back end applications? Or you might consider setting a cookie to keep track.
Does this thing need to scale at all? If your answer is no, my follow up is - are you sure? If you really just need to proxy to two backend applications, I'm sure there are any number of clever ways to achieve that. If at any time you may have N back end applications, then you need a solid plan for managing (add/remove/update) them on the proxy.
Do your applications use HTTPS? If so, are you going to terminate SSL on the proxy? Can you send data in the clear between your proxy and your back end applications?
Good luck on your reverse proxy endeavors! I will update this if anything else occurs to me.
With pure core module (may be a bit ugly, but efficient):
var http = require('http');
http.createServer(function (request, response) {
if (request.headers.host === 'api.test') {
// request data from 172.17.0.1:80
} else if (request.headers.host === 'test') {
// request data from 172.17.0.2:80
} else {
// Do something else
}
}).listen(80);
If you don't like this example, you can try: https://www.npmjs.org/package/turtle.io
I want to make a Node.js daemon that runs on multiple computers and is able to exchange messages between the different daemons. Of course the communication should be encrypted, but I really don't know what kind of encryption I should use for server-to-server encryption. The protocol I'm using right now is TCP via net.createServer. How should I encrypt the communication assuming I have a already exchanged password on both devices? How do I make it secure to the most known attacks?
Edit:
Is using RSA combined with an "authentication password" secure? This password would then be submitted with every request, the whole message (including the password) would be encrypted with the RSA public key (which can be downloaded without encryption).
I think the right way to do this is to communicate via ssl, see here:
http://nodejs.org/docs/v0.4.2/api/tls.html
You could also do a quick and dirty encryption using the crypto module:
var crypto = require('crypto');
var algorithm = 'aes256'; // or any other algorithm supported by OpenSSL
exports.encryptString = function(text) {
var cipher = crypto.createCipher(algorithm, key);
return cipher.update(text, 'utf8', 'hex') + cipher.final('hex');
};
var key = "123456";
exports.decryptString = function(text) {
var decipher = crypto.createDecipher(algorithm, key);
return decipher.update(text, 'hex', 'utf8') + decipher.final('utf8');
};
Both servers need the public key.
You'll probably want to use JSON stringify and parse functions on top of the above (I had those lying around). You could do it in middleware that deciphers incoming requests and ciphers outgoing ones.
I take a different approach to this by doing the work outside of my application. Generally speaking, you don't want to reinvent wheels and secure encryption is a tough thing to get right.
I have a situation where several slave servers need to communicate to a master server to run jobs from a queue. For the server-to-server connection I actually just use Socket.IO (using the socket.io client NPM package and all transports disabled except for web sockets). This gives me a solid RPC, which works well for my needs. (I have since discovered rpc-stream which can give you RPC over arbitrary streams. This would be a bit more lightweight for server-to-server communication where Socket.IO is overkill.)
Now, for the encryption part... I just use a VPN set up between my servers. I took the lazy approach and used Hamachi for this, but you can certainly use OpenVPN or any other.
A second method you can use is to tunnel your connections through SSH.
In short, don't do any work you don't have to. Opt for speed, simplicity, and security. Use something off-the-shelf for this.
One option which might be easier to implement is to encrypt and decrypt all messages sent over a normal socket connection (net.createServer and net.connect), via pre-shared gpg keys using node-gpg. This requires that you have gpg in your $PATH on both client and server with a password-less private gpg key 'Server' on the server and a corresponding 'Client' on the client, with the respective public keys installed on the other end.
server.js:
var socketServer = net.createServer(function (c) {
// example of send to client
var output = JSON.stringify({"msg": "Stuff to send to client."});
encrypt(output, 'Client', function (error, cryptdata) {
c.write(cryptdata.toString());
});
// receive data sent from client
c.on('data', function (cryptdata) {
decrypt(cryptdata.toString(), 'Server', function (error, data) {
data = JSON.parse(data.toString());
// handle incoming data
});
});
});
socketServer.listen(port, function() {
});
client.js:
var socketClient = net.connect({"port": port}, function () {
// Send data to server
var data = JSON.stringify({"msg": "Data to server"});
encrypt(data, 'Server', function (error, cryptdata) {
socketClient.write(cryptdata.toString());
});
});
// Receive data from server
socketClient.on('data', function(cryptdata) {
decrypt(cryptdata.toString(), 'Client', function (error, data) {
data = JSON.parse(data.toString());
// handle data
});
});
And these were the functions I used in both server.js and client.js for encryption/decryption.
function encrypt(str, receiver, callback) {
gpg.encrypt(str, ['-r ' + receiver, '-a'], callback);
}
function decrypt(str, receiver, callback) {
gpg.decrypt(str, ['-u ' + receiver, '-a'], callback);
}
This eliminates any problem you may run into with self-signed SSL certificates and at least with my benchmarks it is a lot faster. Though, it might not be as secure.
Here's the deal: I'm using node.js to connect to a local CouchDB instance.
The problem: the "response" event isn't firing. So I'm assuming it's not getting a connection.
var http = require('http'),
sys = require('sys'),
url = require('url');
var server = http.createServer(function (request, response) {
var client = http.createClient(5984, '127.0.0.1'); // couchdb
var request_db = client.request('GET', '/_all_dbs');
console.log('Created a request for _all_dbs');
request_db.addListener('response', function(response_db) {
console.log('Some response from CouchDB');
});
}).listen(8124);
console.log('Server running at http://127.0.0.1:8124/');
The only only output is:
Server running at http://127.0.0.1:8124/
Created a request for _all_dbs
I'm also guessing this might be a proxy problem. In Ubuntu 10.10, I have a system-wide proxy setting, but I tell it to ignore 127.0.0.1. Doing a curl on 127.0.0.1:5984/_all_dbs gives an error on "Error Code: 502 Proxy Error.".
First, try accessing 127.0.0.1:8124/_utils as #PartlyCloudy suggests.
Second, try using a couchdb lib such as cradle to make your life easier (and minimize mistakes)
npm install cradle
Remember to stop into #node.js and ask questions! Make sure to report back with your findings.
If you are using request to make the http request set proxy as empty string.. It will take preference over the system settings (you can do it only when the target ip is current machine ip)
var options = {
agent: ...,
url: ...,
method: ...,
proxy: ''
};
var req= request(options);
.
.
.