Develop Reference

Why does client.recv(1024) return an empty byte literal in this bare-bones WebSocket Server implementation?

I need a web socket client server exchange between Python and JavaScript on an air-gapped network, so I'm limited to what I can read and type up (believe me I'd love to be able to run pip install websockets). Here's a bare-bones RFC 6455 WebSocket client-server relationship between Python and JavaScript. Below the code, I'll pinpoint a specific issue with client.recv(1024) returning an empty byte literal, causing the WebSocket Server implementation to abort the connection.
Client:
<script>
const message = {
name: "ping",
data: 0
}
const socket = new WebSocket("ws://localhost:8000")
socket.addEventListener("open", (event) => {
console.log("socket connected to server")
socket.send(JSON.stringify(message))
})
socket.addEventListener("message", (event) => {
console.log("message from socket server:", JSON.parse(event))
})
</script>
Server, found here (minimal implementation of RFC 6455):
import array
import time
import socket
import hashlib
import sys
from select import select
import re
import logging
from threading import Thread
import signal
from base64 import b64encode
class WebSocket(object):
handshake = (
"HTTP/1.1 101 Web Socket Protocol Handshake\r\n"
"Upgrade: WebSocket\r\n"
"Connection: Upgrade\r\n"
"WebSocket-Origin: %(origin)s\r\n"
"WebSocket-Location: ws://%(bind)s:%(port)s/\r\n"
"Sec-Websocket-Accept: %(accept)s\r\n"
"Sec-Websocket-Origin: %(origin)s\r\n"
"Sec-Websocket-Location: ws://%(bind)s:%(port)s/\r\n"
"\r\n"
)
def __init__(self, client, server):
self.client = client
self.server = server
self.handshaken = False
self.header = ""
self.data = ""
def feed(self, data):
if not self.handshaken:
self.header += str(data)
if self.header.find('\\r\\n\\r\\n') != -1:
parts = self.header.split('\\r\\n\\r\\n', 1)
self.header = parts[0]
if self.dohandshake(self.header, parts[1]):
logging.info("Handshake successful")
self.handshaken = True
else:
self.data += data.decode("utf-8", "ignore")
playloadData = data[6:]
mask = data[2:6]
unmasked = array.array("B", playloadData)
for i in range(len(playloadData)):
unmasked[i] = unmasked[i] ^ mask[i % 4]
self.onmessage(bytes(unmasked).decode("utf-8", "ignore"))
def dohandshake(self, header, key=None):
logging.debug("Begin handshake: %s" % header)
digitRe = re.compile(r'[^0-9]')
spacesRe = re.compile(r'\s')
part = part_1 = part_2 = origin = None
for line in header.split('\\r\\n')[1:]:
name, value = line.split(': ', 1)
if name.lower() == "sec-websocket-key1":
key_number_1 = int(digitRe.sub('', value))
spaces_1 = len(spacesRe.findall(value))
if spaces_1 == 0:
return False
if key_number_1 % spaces_1 != 0:
return False
part_1 = key_number_1 / spaces_1
elif name.lower() == "sec-websocket-key2":
key_number_2 = int(digitRe.sub('', value))
spaces_2 = len(spacesRe.findall(value))
if spaces_2 == 0:
return False
if key_number_2 % spaces_2 != 0:
return False
part_2 = key_number_2 / spaces_2
elif name.lower() == "sec-websocket-key":
part = bytes(value, 'UTF-8')
elif name.lower() == "origin":
origin = value
if part:
sha1 = hashlib.sha1()
sha1.update(part)
sha1.update("258EAFA5-E914-47DA-95CA-C5AB0DC85B11".encode('utf-8'))
accept = (b64encode(sha1.digest())).decode("utf-8", "ignore")
handshake = WebSocket.handshake % {
'accept': accept,
'origin': origin,
'port': self.server.port,
'bind': self.server.bind
}
#handshake += response
else:
logging.warning("Not using challenge + response")
handshake = WebSocket.handshake % {
'origin': origin,
'port': self.server.port,
'bind': self.server.bind
}
logging.debug("Sending handshake %s" % handshake)
self.client.send(bytes(handshake, 'UTF-8'))
return True
def onmessage(self, data):
logging.info("Got message: %s" % data)
def send(self, data):
logging.info("Sent message: %s" % data)
self.client.send("\x00%s\xff" % data)
def close(self):
self.client.close()
class WebSocketServer(object):
def __init__(self, bind, port, cls):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind((bind, port))
self.bind = bind
self.port = port
self.cls = cls
self.connections = {}
self.listeners = [self.socket]
def listen(self, backlog=5):
self.socket.listen(backlog)
logging.info("Listening on %s" % self.port)
self.running = True
while self.running:
# upon first connection rList = [784] and the other two are empty
rList, wList, xList = select(self.listeners, [], self.listeners, 1)
for ready in rList:
if ready == self.socket:
logging.debug("New client connection")
client, address = self.socket.accept()
fileno = client.fileno()
self.listeners.append(fileno)
self.connections[fileno] = self.cls(client, self)
else:
logging.debug("Client ready for reading %s" % ready)
client = self.connections[ready].client
data = client.recv(1024) # currently, this results in: b''
fileno = client.fileno()
if data: # data = b''
self.connections[fileno].feed(data)
else:
logging.debug("Closing client %s" % ready)
self.connections[fileno].close()
del self.connections[fileno]
self.listeners.remove(ready)
for failed in xList:
if failed == self.socket:
logging.error("Socket broke")
for fileno, conn in self.connections:
conn.close()
self.running = False
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG,
format="%(asctime)s - %(levelname)s - %(message)s")
server = WebSocketServer("localhost", 8000, WebSocket)
server_thread = Thread(target=server.listen, args=[5])
server_thread.start()
# Add SIGINT handler for killing the threads
def signal_handler(signal, frame):
logging.info("Caught Ctrl+C, shutting down...")
server.running = False
sys.exit()
signal.signal(signal.SIGINT, signal_handler)
while True:
time.sleep(100)
server side logs:
INFO - Hanshake successful
DEBUG - Client ready for reading 664
DEBUG - Closing client 664
and on the client side I get
WebSocket connection to 'ws://localhost:8000' failed: Unknown Reason
The problem is traced here:
if data:
self.connections[fileno].feed(data)
else: # this is being triggered on the server side
logging.debug("Closing client %s" % ready)
So researching this I found a potential problem in the Python documentation for select used to retrieve rlist, wlist, xlist
select.select(rlist, wlist, xlist[, timeout]) This is a
straightforward interface to the Unix select() system call. The first
three arguments are iterables of ‘waitable objects’: either integers
representing file descriptors or objects with a parameterless method
named fileno() returning such an integer:
rlist: wait until ready for reading
wlist: wait until ready for writing
xlist: wait for an “exceptional condition” (see the manual page for
what your system considers such a condition)
Seeing that the feature is based on the Unix system call, I realized this code might not support Windows, which is my environment. I checked the values of rlist, wlist, xlist and found they're all empty lists on the first iteration rList = [784] (or another number, such as 664) and the other two are empty, after which the connection is closed.
The documentation goes on to note:
Note: File objects on Windows are not acceptable, but sockets are. On Windows, the underlying select() function is provided by the
WinSock library, and does not handle file descriptors that don’t
originate from WinSock.
But I'm not clear on the exact meaning of this.
So in the code logic, I did some logging and traced the issue here:
rList, wList, xList = select(self.listeners, [], self.listeners, 1)
for ready in rList: # rList = [836] or some other number
# and then we check if ready (so the 836 int) == self.socket
# but if we log self.socket we get this:
# <socket.socket fd=772, family=AddressFamily.AF_INET,
# type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 8000)>
# so of course an integer isn't going to be equivalent to that
if ready == self.socket:
logging.debug("New client connection")
#so lets skip this code and see what the other condition does
else:
logging.debug("Client ready for reading %s" % ready)
client = self.connections[ready].client
data = client.recv(1024) # currently, this results in: b''
fileno = client.fileno()
if data: # data = b'', so this is handled as falsy
self.connections[fileno].feed(data)
else:
logging.debug("Closing client %s" % ready)
And as to why client.recv(1024) returns an empty binary string, I have no idea. I don't know if rList was supposed to contain more than an integer, or if the protocol is working as intended up until recv
Can anyone explain what's causing the broken .recv call here? Is the client side JavaScript WebSocket protocol not sending whatever data should be expected? Or is the WebSocket Server at fault, and what's wrong with it?

I tried running your example and it seem to be working as expected. At least server logs end with the following line:
INFO - Got message: {"name":"ping","data":0}
My environment:
OS: Arch Linux;
WebSocket client: Chromium/85.0.4183.121 running the JS-code you provided;
WebSocket server: Python/3.8.5 running the Python code you provided;
select.select docstring indeed states that
On Windows, only sockets are supported
but most likely the OS is irrelevant since the server code uses only sockets as select.select arguments.
recv returns an empty byte string when the reading end of a socket is closed. From recv(3) man:
If no messages are available to be received and the peer has performed an orderly shutdown, recv() shall return 0.
An interesting thing is a message about a successful handshake in server logs you got:
INFO - Hanshake successful
It means that in your case the connection between the client and the server has been established and some data has flown in both directions. After that the socket got closed. Looking at the server code I see no reason for the server to stop the connection. So I assume that the client you are using is to blame.
To find out exactly what is going wrong, try intercepting the network traffic using tcpdump or wireshark and running the following Python WebSocket client script that reproduces the actions my browser did when I was testing:
import socket
SERVER = ("localhost", 8000)
HANDSHAKE = (
b"GET /chat HTTP/1.1\r\n"
b"Host: server.example.com\r\n"
b"Upgrade: websocket\r\n"
b"Connection: Upgrade\r\n"
b"Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==\r\n"
b"Sec-WebSocket-Protocol: chat, superchat\r\n"
b"Sec-WebSocket-Version: 13\r\n"
b"Origin: http://example.com\r\n"
b"\r\n\r\n"
)
# a frame with `{"name":"ping","data":0}` payload
MESSAGE = b"\x81\x983\x81\xde\x04H\xa3\xb0e^\xe4\xfc>\x11\xf1\xb7jT\xa3\xf2&W\xe0\xaae\x11\xbb\xeey"
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect(SERVER)
n = s.send(HANDSHAKE)
assert n != 0
data = s.recv(1024)
print(data.decode())
n = s.send(MESSAGE)
assert n != 0

registration using sip.js and websockets in Node (wss or ws)

I'm trying to register SIP agents using sip.js and Node to start load testing to my Kamailio
Node version v0.10.32
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"
global.WebSocket = require('ws')
// SipJS Initialization
var SIP = require('./sip-0.6.0.js');
var Uaconfig = {
traceSip: true,
register: true,
userAgentString: 'sipJS Test tool',
uri: 'ramenlabs.io',
wsServers: ['wss://ramenlabs.io:5063'],
authorizationUser: 'gogasca',
password: '********'
//hackIpInContact: true
};
var userAgent = new SIP.UA(Uaconfig);
When I run I get an error:
Thu Oct 16 2014 03:40:53 GMT+0000 (UTC) | sip.transport | WebSocket connection error: Error: unexpected server response (400)
Because initial Websocket from Sip.js lacks Sec-WebSocket-Protocol: sip to indicate SIP Protocol.
If i use sample web page it works perfectly with same parameters, but not from my node command line.
How to force this?
In API js I do see the 'sip' extra header but contains the window.Websocket
try {
this.ws = new window.WebSocket(this.server.ws_uri, 'sip');
}
catch(e) {
this.logger.log('Error connecting to WebSocket ' + this.server.ws_uri + ': ' + e);
//return false;
}

You can set debug=3 in kamailio.cfg, restart and try again. Then look in syslog file (/var/log/syslog or /var/log/messages) for debug messages from Kamailio to see if you get a hint on why the connection is not accepted.

Stilts.js Unable to connect to websocket server with secure true

I am having issues with subscribing to my websocket server when I turn the secure option to true. I ran an SSL validator to make sure it was set up properly and everything passed. Also to be safe, I also checked with my hosting provider so I dont believe that is the issue. And also, everything does work when secure is set to false.
The WSS is running on port 8676, and I did make sure that port is open.
I am using the Stomp Javascript library. I downloaded the latest version from their github master branch (https://github.com/projectodd/stilts/blob/master/stomp-client-js/src/main/javascript/stomp.js).
I keep getting this error inside the _transmit() function: can't call transmit on undefined
It seems to not be setting the _transport property inside of _buildConnector() function. I added some console.logs and it always goes to the else statement in this function.
Any ideas on a fix? Or am I just missing something?
This is how I am initializing my connection:
client = new Stomp.Client('my.ip.address', 8676, true);
Here is where I am logging some of the functions:
Stomp.Client = function(host, port, secure) {
console.log('host param: ' + host);
console.log('port param: ' + port);
console.log('secure param: ' + secure);
this._host = host || Stomp.DEFAULT_HOST;
this._port = port || Stomp.DEFAULT_PORT || 8675;
this._secure = secure || Stomp.DEFAULT_SECURE_FLAG || false;
console.log('this._host: ' + this._host);
console.log('this._port: ' + this._port);
console.log('this._secure: ' + this._secure);
}
Output:
host param: my.ip.address
port param: 8676
secure param: true
this._host: my.ip.address
this._port: 8676
this._secure: true
_buildConnector: function(transports, i) {
console.log('INSIDE _buildConnector()');
var callback = this._connectCallback;
var client = this;
if ( i < transports.length ) {
console.log('IF!!!');
return function() {
var fallback = client._buildConnector( transports, i+1, callback );
try {
console.log('_buildConnector::IF::TRY');
transports[i].connect( function() {
client._transport = transports[i];
callback();
}, fallback );
} catch (err) {
console.log('_buildConnector::IF::CATCH');
fallback();
}
};
} else {
console.log('_buildConnector::ELSE');
return client.connectionFailed.bind(this);
}
}
Output:
INSIDE _buildConnector()
IF!!!
INSIDE _buildConnector()
IF!!!
_buildConnector::IF::TRY
INSIDE _buildConnector()
_buildConnector::ELSE
_buildConnector::IF::TRY
connectionFailed: function() {
console.log('INSIDE connectionFailed()');
if (this._errorCallback)
{
console.log('connectionFailed::IF');
console.log('this._errorCallback');
console.log(this._errorCallback);
console.log('arguments');
console.log(arguments);
this._errorCallback.apply(this._errorCallback, arguments);
}
else
{
console.log('connectionFailed::ELSE');
console.log('unable to connect :(');
Stomp.logger.log( "unable to connect" );
}
}
Output:
No output..

I discovered this node.js tool called wscat which can be used to send messages to a websocket server (http://einaros.github.io/ws/).
When entering the command using the external address:
wscat -c wss://my.ip.address:8676
I kept getting the following error: error: Error: Hostname/IP doesn't match certificate's altnames
But when I changed it to my domain name:
wscat -c wss://mydomain.com:8676
I was given this message: connected (press CTRL+C to quit)
Which led me to:
I am setting the host value in Javascript as the external ip address of the server where STOMP lives. But the SSL cert is configured for a domain name. This led me to think that I should use mydomain.com instead of my.ip.address for the host parameter in new Stomp.Client. But this is resulting in the following error in the Torquebox log file:
NoSuchHostException: No such host: mydomain.com
So then in the torquebox.yml file the host property needed to be configured to mydomain.com as well.
So the conclusion is: You need to use the domain name that was registered to the SSL cert when trying to connect to your websocket server, not the external ip address. huzzah!
Credit to: #bobmcwhirter who helped lead me to this solution here: https://github.com/projectodd/stilts/issues/20

Server authentication using Faye on Node.js

So I am extremely new to node.js, and faye - so I think this is pretty basic, but I'm not sure what I should be asking.
I have this setup for my faye server, running on node jitsu:
var http = require('http'),
faye = require('faye');
var bayeux = new faye.NodeAdapter({mount: '/faye', timeout: 45});
// Handle non-Bayeux requests
var server = http.createServer(function(request, response) {
response.writeHead(200, {'Content-Type': 'text/plain'});
response.write('Hello, non-Bayeux request');
response.end();
});
bayeux.attach(server);
server.listen(8000);
var fayeToken = "myToken";
var serverAuth = {
incoming: function(message, callback) {
// Add an error if the tokens don't match
if (fayeToken !== message.ext.auth_token)
message.error = 'Invalid auth token';
// Call the server back now we're done
callback(message);
}
};
bayeux.addExtension(serverAuth);
Then I have a rails app that connects to it.
Before I added the serverAuth stuff, it was working fine.
When I curl like so:
curl http://myapp.jit.su/faye -d 'message={"channel":"/alerts", "data":"hello", "ext":{"auth_token":"myToken"}}'
I get a success message.
In my JS file on the rails app I have:
$(function() {
//FAYE CLIENT
var faye = new Faye.Client('http://myapp.jit.su/faye');
faye.setHeader('ext.auth_token', 'myToken');
faye.subscribe("/alerts", function(data) {
alert(data);
});
});
Now I get a bad request 400 error when the rails app loads for the faye script.
The params being sent are specifically:
[{"channel":"/meta/handshake","version":"1.0","supportedConnectionTypes":["callback-polling"],"id":"1"}]
So it seems as though the header param that I'm setting isn't being sent in this "meta" handshake that faye does.
Before I had this server authentication in, it worked just fine, and I could curl messages into the app. No dice now.
Any ideas where I'm going wrong?
Thanks!

For future reference:
I had to make sure its not doing a meta call:
var serverAuth = {
incoming: function(message, callback) {
// Let non-subscribe messages through
if (message.channel.indexOf("/meta/") !== 0){
if (fayeToken !== message.ext.auth_token){
message.error = 'Invalid auth token';
}
}
callback(message);
}
};

There's two problems here that I can see here. First, your extension will fail if the message does not have an ext field. You should check for it before checking message.ext.auth_token:
var serverAuth = {
incoming: function(message, callback) {
// Let non-subscribe messages through
if (message.channel.indexOf("/meta/") !== 0){
if (!message.ext || fayeToken !== message.ext.auth_token){
message.error = 'Invalid auth token';
}
}
callback(message);
}
};
I'm also not sure what you're trying to authenticate. if (message.channel.indexOf("/meta/") !== 0) will match any non-meta message, i.e. all messages sent with publish() (meta messages are used for handshaking, polling, (un)subscribing, and disconnecting). This means anyone can publish messages, but only the server can subscribe to any channels.
Second, ext is part of the messages themselves, whereas setHeader() is used to set HTTP headers, i.e. it's part of the transport layer. You should use an extension to add ext to messages:
client.addExtension({
outgoing: function(message, callback) {
message.ext = message.ext || {};
message.ext.auth_token = THE_TOKEN;
callback(message);
}
});

Creating a map of ids to sockets and vice versa in Node.js

I'm trying to manage a bunch of socket connections. My app is basically an http server that receives posts and passes these along to a socket. When clients open a socket connection, they send a connect message with an id:
{"m":"connect","id":"1"}
The app then saves this id and socket in the id2socket and socket2id maps. On disconnect, the socket/id pair is deleted from the maps.
A post will also contain an id, which indicates the post data should be sent to the socket with that id.
That's great, and this works fine for a single open socket. However, when I have more than one socket open, and then I close a socket, that disconnect wipes everything from the map. I think my understanding of sockets in node is incomplete- is there only a single socket object that is used in the callback? Is there a better way to manage my open socket connections and ids?
start server:
>>node server.js
TCP server listening on 127.0.0.1:5280
HTTP server listening on 127.0.0.1:9002
telnet in:
>>telnet localhost 5280
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
{"m":"connect","id":"123"}
{"m":"connect","id":"123","success":"true"}
server after connection:
>>Connection from 127.0.0.1:57572
received data: {"m":"connect","id":"123"}
id: 1
m: connect
associating uid 1 with socket [object Object]
do a post:
python post.py {"foo":"bar"}
So this works fine for several open sockets (as long as 1 device is id 123, server has this hardwired for now). However, as soon as you close one connection all the socket connections are removed from the map.
Here's my code:
python script to do post:
import sys
import json
import httplib, urllib, urllib2
values = json.loads('{"foo":"bar"}')
headers = {"Content-type": "application/json"}
conn = httplib.HTTPConnection('127.0.0.1', 9002)
headers = {"Content-type": "application/json"}
conn.request("POST", "", json.dumps(values), headers)
response = conn.getresponse()
print "response.status: "+response.status
print "response.reason: "+response.reason
print "response.read: "+response.read()
conn.close()
node server (http and tcp), hardwired to send data to device '123' on post:
var net = require('net'); // tcp-server
var http = require("http"); // http-server
var qs = require('querystring'); // http-post
// Map of sockets to devices
var id2socket = new Object;
var socket2id = new Object;
// Setup a tcp server
var server_plug = net.createServer(function(socket) {
// Event handlers
socket.addListener("connect", function(conn) {
console.log("Connection from " + socket.remoteAddress + ":" + socket.remotePort );
});
socket.addListener("data", function(data) {
console.log("received data: " + data);
try {
request = JSON.parse(data);
response = request;
if(request.m !== undefined && request['id'] !== undefined){ // hack on 'id', id is js obj property
console.log("id: "+request['id']);
console.log("m: "+request.m);
if(request.m == 'connect'){
console.log("associating uid " + request['id'] + " with socket " + socket);
id2socket[request['id']] = socket;
socket2id[socket] = request['id'];
response.success = 'true';
} else {
response.success = 'true';
}
}
socket.write(JSON.stringify(response));
} catch (SyntaxError) {
console.log('Invalid JSON:' + data);
socket.write('{"success":"false","response":"invalid JSON"}');
}
});
socket.on('end', function() {
id = socket2id[socket]
console.log("socket disconnect by id " + id);
// wipe out the stored info
console.log("removing from map socket:"+socket+" id:"+id);
delete id2socket[id];
delete socket2id[socket];
});
socket.on('timeout', function() {
console.log('socket timeout');
});
});
// Setup http server
var server_http = http.createServer(
// Function to handle http:post requests, need two parts to it
// http://jnjnjn.com/113/node-js-for-noobs-grabbing-post-content/
function onRequest(request, response) {
request.setEncoding("utf8");
request.addListener("data", function(chunk) {
request.content += chunk;
});
request.addListener("end", function() {
console.log("post received!");
//console.log("Request received: "+request.content);
if (request.method == 'POST') {
//var json = qs.parse(request.content);
//console.log("Post: "+json);
// HACK TO TEST STUFF:
// send a message to one of the open sockets
try {
var socket = id2socket['123']; //hardwired
socket.write('{"m":"post"}');
} catch (Error) {
console.log("Cannot find socket with id "+'123');
}
}
});
}
);
// Fire up the servers
var HOST = '127.0.0.1';
var PORT = 5280;
var PORT2 = 9002;
server_plug.listen(PORT, HOST);
console.log("TCP server listening on "+HOST+":"+PORT);
server_http.listen(PORT2);
console.log("HTTP server listening on "+HOST+":"+PORT2);

Objects only take strings as keys for their properties. As your log shows, a socket object is converted into the string "[object Object]". As a result, socket #2 overwrites the id from socket #1 in the object, because all sockets are converted into the same string key. So, there is only one property in the object at all times, because all sockets come down to the same key. When you try to remove the id for socket #2, the single property is deleted and the object is empty.
You seem to want a custom property for each separate socket when used as a key. You can use WeakMaps for this. WeakMaps do allow objects as keys (as opposed to string-only keys), but as they're relatively new they may contain bugs at the moment.
(Note that the id2socket map can just be a plain object, because numbers are converted into strings just fine, and each number has its own, distinct string representation*.)
Using WeakMaps is as follows:
var socket2id = new WeakMap; // as if you were doing: var socket2id = {};
socket2id.set(socket, id); // as if you were doing: socket2id[socket] = id;
socket2id.get(socket); // as if you were doing: socket2id[socket];
socket2id.delete(socket); // as if you were doing: delete socket2id[socket];
Make sure to run with node --harmony (>= 0.7) or node --harmony_weakmaps (<= 0.6).
* 0 and -0 are exceptions, but you shouldn't be using -0 anyway because 0 === -0, so it's difficult to differ between them.