Develop Reference

Why does client.recv(1024) return an empty byte literal in this bare-bones WebSocket Server implementation?

I need a web socket client server exchange between Python and JavaScript on an air-gapped network, so I'm limited to what I can read and type up (believe me I'd love to be able to run pip install websockets). Here's a bare-bones RFC 6455 WebSocket client-server relationship between Python and JavaScript. Below the code, I'll pinpoint a specific issue with client.recv(1024) returning an empty byte literal, causing the WebSocket Server implementation to abort the connection.
Client:
<script>
const message = {
name: "ping",
data: 0
}
const socket = new WebSocket("ws://localhost:8000")
socket.addEventListener("open", (event) => {
console.log("socket connected to server")
socket.send(JSON.stringify(message))
})
socket.addEventListener("message", (event) => {
console.log("message from socket server:", JSON.parse(event))
})
</script>
Server, found here (minimal implementation of RFC 6455):
import array
import time
import socket
import hashlib
import sys
from select import select
import re
import logging
from threading import Thread
import signal
from base64 import b64encode
class WebSocket(object):
handshake = (
"HTTP/1.1 101 Web Socket Protocol Handshake\r\n"
"Upgrade: WebSocket\r\n"
"Connection: Upgrade\r\n"
"WebSocket-Origin: %(origin)s\r\n"
"WebSocket-Location: ws://%(bind)s:%(port)s/\r\n"
"Sec-Websocket-Accept: %(accept)s\r\n"
"Sec-Websocket-Origin: %(origin)s\r\n"
"Sec-Websocket-Location: ws://%(bind)s:%(port)s/\r\n"
"\r\n"
)
def __init__(self, client, server):
self.client = client
self.server = server
self.handshaken = False
self.header = ""
self.data = ""
def feed(self, data):
if not self.handshaken:
self.header += str(data)
if self.header.find('\\r\\n\\r\\n') != -1:
parts = self.header.split('\\r\\n\\r\\n', 1)
self.header = parts[0]
if self.dohandshake(self.header, parts[1]):
logging.info("Handshake successful")
self.handshaken = True
else:
self.data += data.decode("utf-8", "ignore")
playloadData = data[6:]
mask = data[2:6]
unmasked = array.array("B", playloadData)
for i in range(len(playloadData)):
unmasked[i] = unmasked[i] ^ mask[i % 4]
self.onmessage(bytes(unmasked).decode("utf-8", "ignore"))
def dohandshake(self, header, key=None):
logging.debug("Begin handshake: %s" % header)
digitRe = re.compile(r'[^0-9]')
spacesRe = re.compile(r'\s')
part = part_1 = part_2 = origin = None
for line in header.split('\\r\\n')[1:]:
name, value = line.split(': ', 1)
if name.lower() == "sec-websocket-key1":
key_number_1 = int(digitRe.sub('', value))
spaces_1 = len(spacesRe.findall(value))
if spaces_1 == 0:
return False
if key_number_1 % spaces_1 != 0:
return False
part_1 = key_number_1 / spaces_1
elif name.lower() == "sec-websocket-key2":
key_number_2 = int(digitRe.sub('', value))
spaces_2 = len(spacesRe.findall(value))
if spaces_2 == 0:
return False
if key_number_2 % spaces_2 != 0:
return False
part_2 = key_number_2 / spaces_2
elif name.lower() == "sec-websocket-key":
part = bytes(value, 'UTF-8')
elif name.lower() == "origin":
origin = value
if part:
sha1 = hashlib.sha1()
sha1.update(part)
sha1.update("258EAFA5-E914-47DA-95CA-C5AB0DC85B11".encode('utf-8'))
accept = (b64encode(sha1.digest())).decode("utf-8", "ignore")
handshake = WebSocket.handshake % {
'accept': accept,
'origin': origin,
'port': self.server.port,
'bind': self.server.bind
}
#handshake += response
else:
logging.warning("Not using challenge + response")
handshake = WebSocket.handshake % {
'origin': origin,
'port': self.server.port,
'bind': self.server.bind
}
logging.debug("Sending handshake %s" % handshake)
self.client.send(bytes(handshake, 'UTF-8'))
return True
def onmessage(self, data):
logging.info("Got message: %s" % data)
def send(self, data):
logging.info("Sent message: %s" % data)
self.client.send("\x00%s\xff" % data)
def close(self):
self.client.close()
class WebSocketServer(object):
def __init__(self, bind, port, cls):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind((bind, port))
self.bind = bind
self.port = port
self.cls = cls
self.connections = {}
self.listeners = [self.socket]
def listen(self, backlog=5):
self.socket.listen(backlog)
logging.info("Listening on %s" % self.port)
self.running = True
while self.running:
# upon first connection rList = [784] and the other two are empty
rList, wList, xList = select(self.listeners, [], self.listeners, 1)
for ready in rList:
if ready == self.socket:
logging.debug("New client connection")
client, address = self.socket.accept()
fileno = client.fileno()
self.listeners.append(fileno)
self.connections[fileno] = self.cls(client, self)
else:
logging.debug("Client ready for reading %s" % ready)
client = self.connections[ready].client
data = client.recv(1024) # currently, this results in: b''
fileno = client.fileno()
if data: # data = b''
self.connections[fileno].feed(data)
else:
logging.debug("Closing client %s" % ready)
self.connections[fileno].close()
del self.connections[fileno]
self.listeners.remove(ready)
for failed in xList:
if failed == self.socket:
logging.error("Socket broke")
for fileno, conn in self.connections:
conn.close()
self.running = False
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG,
format="%(asctime)s - %(levelname)s - %(message)s")
server = WebSocketServer("localhost", 8000, WebSocket)
server_thread = Thread(target=server.listen, args=[5])
server_thread.start()
# Add SIGINT handler for killing the threads
def signal_handler(signal, frame):
logging.info("Caught Ctrl+C, shutting down...")
server.running = False
sys.exit()
signal.signal(signal.SIGINT, signal_handler)
while True:
time.sleep(100)
server side logs:
INFO - Hanshake successful
DEBUG - Client ready for reading 664
DEBUG - Closing client 664
and on the client side I get
WebSocket connection to 'ws://localhost:8000' failed: Unknown Reason
The problem is traced here:
if data:
self.connections[fileno].feed(data)
else: # this is being triggered on the server side
logging.debug("Closing client %s" % ready)
So researching this I found a potential problem in the Python documentation for select used to retrieve rlist, wlist, xlist
select.select(rlist, wlist, xlist[, timeout]) This is a
straightforward interface to the Unix select() system call. The first
three arguments are iterables of ‘waitable objects’: either integers
representing file descriptors or objects with a parameterless method
named fileno() returning such an integer:
rlist: wait until ready for reading
wlist: wait until ready for writing
xlist: wait for an “exceptional condition” (see the manual page for
what your system considers such a condition)
Seeing that the feature is based on the Unix system call, I realized this code might not support Windows, which is my environment. I checked the values of rlist, wlist, xlist and found they're all empty lists on the first iteration rList = [784] (or another number, such as 664) and the other two are empty, after which the connection is closed.
The documentation goes on to note:
Note: File objects on Windows are not acceptable, but sockets are. On Windows, the underlying select() function is provided by the
WinSock library, and does not handle file descriptors that don’t
originate from WinSock.
But I'm not clear on the exact meaning of this.
So in the code logic, I did some logging and traced the issue here:
rList, wList, xList = select(self.listeners, [], self.listeners, 1)
for ready in rList: # rList = [836] or some other number
# and then we check if ready (so the 836 int) == self.socket
# but if we log self.socket we get this:
# <socket.socket fd=772, family=AddressFamily.AF_INET,
# type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 8000)>
# so of course an integer isn't going to be equivalent to that
if ready == self.socket:
logging.debug("New client connection")
#so lets skip this code and see what the other condition does
else:
logging.debug("Client ready for reading %s" % ready)
client = self.connections[ready].client
data = client.recv(1024) # currently, this results in: b''
fileno = client.fileno()
if data: # data = b'', so this is handled as falsy
self.connections[fileno].feed(data)
else:
logging.debug("Closing client %s" % ready)
And as to why client.recv(1024) returns an empty binary string, I have no idea. I don't know if rList was supposed to contain more than an integer, or if the protocol is working as intended up until recv
Can anyone explain what's causing the broken .recv call here? Is the client side JavaScript WebSocket protocol not sending whatever data should be expected? Or is the WebSocket Server at fault, and what's wrong with it?

I tried running your example and it seem to be working as expected. At least server logs end with the following line:
INFO - Got message: {"name":"ping","data":0}
My environment:
OS: Arch Linux;
WebSocket client: Chromium/85.0.4183.121 running the JS-code you provided;
WebSocket server: Python/3.8.5 running the Python code you provided;
select.select docstring indeed states that
On Windows, only sockets are supported
but most likely the OS is irrelevant since the server code uses only sockets as select.select arguments.
recv returns an empty byte string when the reading end of a socket is closed. From recv(3) man:
If no messages are available to be received and the peer has performed an orderly shutdown, recv() shall return 0.
An interesting thing is a message about a successful handshake in server logs you got:
INFO - Hanshake successful
It means that in your case the connection between the client and the server has been established and some data has flown in both directions. After that the socket got closed. Looking at the server code I see no reason for the server to stop the connection. So I assume that the client you are using is to blame.
To find out exactly what is going wrong, try intercepting the network traffic using tcpdump or wireshark and running the following Python WebSocket client script that reproduces the actions my browser did when I was testing:
import socket
SERVER = ("localhost", 8000)
HANDSHAKE = (
b"GET /chat HTTP/1.1\r\n"
b"Host: server.example.com\r\n"
b"Upgrade: websocket\r\n"
b"Connection: Upgrade\r\n"
b"Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==\r\n"
b"Sec-WebSocket-Protocol: chat, superchat\r\n"
b"Sec-WebSocket-Version: 13\r\n"
b"Origin: http://example.com\r\n"
b"\r\n\r\n"
)
# a frame with `{"name":"ping","data":0}` payload
MESSAGE = b"\x81\x983\x81\xde\x04H\xa3\xb0e^\xe4\xfc>\x11\xf1\xb7jT\xa3\xf2&W\xe0\xaae\x11\xbb\xeey"
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect(SERVER)
n = s.send(HANDSHAKE)
assert n != 0
data = s.recv(1024)
print(data.decode())
n = s.send(MESSAGE)
assert n != 0

Flask - Stream content keeping context

I am using the folllowing Flask code to stream the output of a command:
#app.route('/', methods=['GET', 'POST'])
def index():
if request.method == 'POST':
...
# some logic to get cmd from POST request
...
return redirect_to(url_to(stream, cmd=cmd))
return render_template('index.html')
#app.route('/stream/<cmd>')
def stream(cmd):
print("Executing %s" % cmd)
g = proc.Group()
p = g.run(cmd)
def stream_cmd():
while g.is_pending():
lines = g.readlines()
for proc, line in lines:
print(line)
yield line + '</br>'
return Response(stream_cmd(), mimetype='text/html') # text/html is required for most browsers to show th$
When I post my form, it redirects to a blank page where I see the output of my stream but I loose all my layout / css / html / etc ...
How can I keep the current layout in place while still seeing a streamed output ?
Ideally, I'd like to be able to update a <div> element in the current page (instead of redirect) with the stream output dynamically (Jquery), but I'm not sure that's even possible.

Following #reptilicus recommendation, I rewrote the code to use websockets.
Here is the working code:
Python
#socketio.on('message', namespace='/stream')
def stream(cmd):
# Streams output of a command
from shelljob import proc
g = proc.Group()
p = g.run(cmd)
while g.is_pending():
lines = g.readlines()
for proc, line in lines:
send(line, namespace='/stream')
eventlet.sleep(0) # THIS IS MANDATORY
The corresponding JavaScript that receives those messages sent by send calls is as follow:
JavaScript (JQuery)
var socket = io.connect('http://' + document.domain + ':' + location.port + '/stream')
socket.on('message', function(msg){
$('#streaming_text').append(msg);
})
...
jqxhr.done(function(cmd){ # This is executed after an AJAX post, but you can change this to whatever event you like
socket.send(cmd)
return false;
})

OSError: [Errno 98] Address already in use (Python program hosted by public server)

I am sorry. I have done python before, but hosting a python program in a webserver is a new thing I am learning. Your help will be much appreciated.
My aim is to host a pyhton program in a webpage such that user can give a set of inputs. Based on the inputs python program does some calculation and provides an output to the user. This whole thing is running fine on windows. In windows what I am doing is: I run cmd and
I type C:\Python34\python.exe -m http.server 8020
Then I run the python program (most important part is provided below) and it starts:
"serving at port
Server Starts - localhost:8020"
Then from chrome I type: http://127.0.0.1:8000, provide the input and get the output.
I have the same directory structure in the webserver (based on centos). However, I cannot run both http.server and python program as it gives the OSerror when I run the second one from the same port. How can I tackle this?
I am posting the javascript code first and then the python program.
When I run in the webpage, I get 404 error from inspect element of chrome.
<script>
function submitReq() {
// alert("calling post");
var singleValues = $( "#single" ).val();
var payload = {};
payload["Seq"] = $("#seq").val();
payload["Mut_Seq"] = $("#mut_seq").val();
payload["Ct"] = parseInt($("#ct").val());
payload["Sodium_con"] = parseInt($("#sodium_con").val());
// alert(JSON.stringify(payload));
$.post("/getDNAsequence",
JSON.stringify(payload),
function(data,status){
var res = data.replace(new RegExp("\n", 'g'), " <br/> ");
$("#output").html(res);
});
};
$("#submit").click(submitReq);
</script>
class MyServer(BaseHTTPRequestHandler):
def do_GET(self) :
if (self.path == "/"):
with open ("dna.html", "r") as myfile:
data=myfile.read().replace('\n', '\n')
self.wfile.write(bytes(data, "utf-8"))
def do_POST(self):
if(self.path=="/getDNAsequence"):
content_len = int(self.headers.get('content-length', 0))
post_body = self.rfile.read(content_len)
parsed_json = json.loads(post_body.decode("utf-8"))
Seq = parsed_json['Seq']
Mut_Seq = parsed_json['Mut_Seq']
Ct = parsed_json['Ct']
Sodium_con = parsed_json['Sodium_con']
baal = Forward_mamapcr_primer(Seq, Mut_Seq, Ct, Sodium_con)
self.wfile.write(bytes(baal, "utf-8"))
self.send_response(200)
self.send_header("Access-Control-Allow-Origin", "*")
def main():
hostName = "localhost"
hostPort = 8020
print ("serving at port")
myServer = HTTPServer((hostName, hostPort), MyServer)
print( "Server Starts - %s:%s" % (hostName, hostPort))
try:
myServer.serve_forever()
except KeyboardInterrupt:
pass
myServer.server_close()
if __name__=='__main__':
main()

Use cgi-bin of apache server to run python script with user provided parameters.

Using webpy with AJAX

I am relatively new to web development and am trying to get the client javascript to send GET requests to a python script running on the server and the server to return data based on that request. I have tried adapting the examples of the webpy library I found online to no avail. Whenever a GET request is sent, the responseText attribute of XMLHttpRequest() returns the text of the python file rather than the data. Any advise would be much appreciated!
The javascript function:
function sendSerialCommand(selection, command) {
var xmlhttp = new XMLHttpRequest();
xmlhttp.onreadystatechange = function () {
if (xmlhttp.readyState === 4 && xmlhttp.status === 200) {
if (command !== 5) {
document.getElementById("output2").innerHTML = xmlhttp.responseText;
document.getElementById("output2").style.color = "green";
} else {
document.getElementById("output1").innerHTML = xmlhttp.responseText;
console.log(xmlhttp.responseText);
document.getElementById("output1").style.color = "green";
}
}
};
xmlhttp.open("GET", pythonFileName + "?sel=" + selection + "?cmd=" + command, true);
xmlhttp.send();
}
...and the test python script:
import web
urls = (
'/', 'Index'
)
app = web.application(urls,globals())
#MAIN LOOP
class Index:
def GET(self):
webInput = web.input()
return 'message: GET OK!'
if __name__ == "__main__":
app.run()

The trick was to use the CGI library for python as such:
#!/usr/bin/python
# Import modules for CGI handling
import cgi, cgitb
# Create instance of FieldStorage
form = cgi.FieldStorage()
# Get data from fields
first_name = form.getvalue('cmd')
last_name = form.getvalue('sel')
print "Content-type:text/html\r\n\r\n"
print "Hello %s %s" % (first_name, last_name)
This captures the keys and data from the GET request and the print command returns data to the xmlhttp.responseText attribute on the client-side.
The script has to be placed into a file the websever is able to execute the script from. That is usually the default /cgi-bin folder located in either /var/www or /etc.

connecting python to javascript for two-direction communication

I would like to serve queries from a javascript code by python. But I am not experienced in this field at all. What I would like to build is something like this:
1. request.js:
open_connection('server.py');
for (var i=0; i<10; i++)
document.write(request_next_number());
close_connection('server.py')
2. server.py
x = 0
while connected:
if request:
send(x)
x = x + 1
I heard about JSON, but don't know if I should use it. (?)
Could you please give me some code examples or guides how to implement the two files above?

What you need is a socket server on the python end and a client/request server on the javascript end.
For the python server side, refer to SocketServer, (example taken from there as well), one thing you have to make sure is to have the socket go past NAT (possibly port forwarding). One other alternative is Twisted which is a very powerful framework, i believe it has functionality to send data through NAT.
import SocketServer
class MyTCPHandler(SocketServer.BaseRequestHandler):
"""
The RequestHandler class for our server.
It is instantiated once per connection to the server, and must
override the handle() method to implement communication to the
client.
"""
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print "{} wrote:".format(self.client_address[0])
print self.data
# just send back the same data, but upper-cased
self.request.sendall(self.data.upper())
if __name__ == "__main__":
HOST, PORT = "localhost", 9999
# Create the server, binding to localhost on port 9999
server = SocketServer.TCPServer((HOST, PORT), MyTCPHandler)
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
server.serve_forever()
On the JavaScript there are many frameworks that allow socket connections, here are a few
Socket IO
Example:
<script src="/socket.io/socket.io.js"></script>
<script>
var socket = io.connect('http://localhost');
socket.on('news', function (data) {
console.log(data);
socket.emit('my other event', { my: 'data' });
});
</script>
You can even use HTML5 Web Sockets
Example:
var connection = new WebSocket('ws://IPAddress:Port');
connection.onopen = function () {
connection.send('Ping'); // Send the message 'Ping' to the server
};
Also, take a look at a part of this book , Chapter 22 of Javascript: The Definitive Guide , https://www.inkling.com/read/javascript-definitive-guide-david-flanagan-6th/chapter-22/web-sockets
Finally, take a look at jssockets
Example:
_jssocket.setCallBack(event, callback);
_jssocket.connect(ip,port);
_jssocket.write(message);
_jssocket.disconnect();
Hope this help!

An example with Web Socket that i have used to transfer image to a web server and stream my screen.
stream.html
<!DOCTYPE HTML>
<meta charset = utf-8>
<html>
<header>
<title>Stream</title>
<script type="text/javascript" src="js/request.js"></script>
</header>
<body onload="doLoad()">
<div id="canvasWrapper">
<canvas id="display"></canvas>
</div>
</body>
</html>
request.js
var disp;
var dispCtx;
var im;
var ws;
function doLoad() {
disp = document.getElementById("display");
dispCtx = disp.getContext("2d");
im = new Image();
im.onload = function() {
disp.setAttribute("width", im.width);
disp.setAttribute("height", im.height);
dispCtx.drawImage(this, 0, 0);
};
im.src = "img/img_not_found.png";
ws = new WebSocket("ws://127.0.0.1:50007");
ws.onmessage = function (evt) {
im.src = "data:image/png;base64," + evt.data;
}
}
server.py
from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory
import base64
import sys
from twisted.python import log
from twisted.internet import reactor
class MyServerProtocol(WebSocketServerProtocol):
def onConnect(self, request):
print("Client connecting: {}".format(request.peer))
def onOpen(self):
print("WebSocket connection open.")
def hello():
with open("/var/www/html/img/image.png", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
self.sendMessage(encoded_string.encode('utf8'))
self.factory.reactor.callLater(0.2, hello)
# start sending messages every 20ms ..
hello()
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {} bytes".format(len(payload)))
else:
print("Text message received: {}".format(payload.decode('utf8')))
# echo back message verbatim
self.sendMessage(payload, isBinary)
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {}".format(reason))
if __name__ == '__main__':
log.startLogging(sys.stdout)
factory = WebSocketServerFactory(u"ws://127.0.0.1:50007")
factory.protocol = MyServerProtocol
# factory.setProtocolOptions(maxConnections=2)
# note to self: if using putChild, the child must be bytes...
reactor.listenTCP(50007, factory)
reactor.run()
You will need autobahn (you can install it with pip install autobahn)