Develop Reference

How can I emit Flask-SocketIO requests with callbacks that still work after a user rejoins and their sid changes?

Summarize the Problem
I am using Flask-SocketIO for a project and am basically trying to make it so that users can rejoin a room and "pick up where they left off." To be more specific:
The server emits a request to the client, with a callback to process the response and a timeout of 1 second. This is done in a loop so that the request is resent if a user rejoins the room.
A user "rejoining" a room is defined as a user joining a room with the same name as a user who has previously been disconnected from that room. The user is given their new SID in this case and the request to the client is sent to the new SID.
What I am seeing is this:
If the user joins the room and does everything normally, the callback is processed correctly on the server.
It a user rejoins the room while the server is sending requests and then submits a response, everything on the JavaScript side works fine, the server receives an ack but does not actually run the callback that it is supposed to:
uV7BTVtBXwQ6oopnAAAE: Received packet MESSAGE data 313["#000000"]
received ack from Ac8wmpy2lK-kTQL7AAAF [/]
This question is similar to mine but the solution for them was to update Flask-SocketIO and I am running a version newer than theirs: python flask-socketio server receives message but doesn't trigger event
Show Some Code
I have created a repository with a "minimal" example here: https://github.com/eshapiro42/socketio-example.
In case something happens to that link in the future, here are the relevant bits:
# app.py
from gevent import monkey
monkey.patch_all()
import flask_socketio
from collections import defaultdict
from flask import Flask, request, send_from_directory
from user import User
app = Flask(__name__)
socketio = flask_socketio.SocketIO(app, async_mode="gevent", logger=True, engineio_logger=True)
#app.route("/")
def base():
return send_from_directory("static", "index.html")
#app.route("/<path:path>")
def home(path):
return send_from_directory("static", path)
# Global dictionary of users, indexed by room
connected_users = defaultdict(list)
# Global dictionary of disconnected users, indexed by room
disconnected_users = defaultdict(list)
#socketio.on("join room")
def join_room(data):
sid = request.sid
username = data["username"]
room = data["room"]
flask_socketio.join_room(room)
# If the user is rejoining, change their sid
for room, users in disconnected_users.items():
for user in users:
if user.name == username:
socketio.send(f"{username} has rejoined the room.", room=room)
user.sid = sid
# Add the user back to the connected users list
connected_users[room].append(user)
# Remove the user from the disconnected list
disconnected_users[room].remove(user)
return True
# If the user is new, create a new user
socketio.send(f"{username} has joined the room.", room=room)
user = User(username, socketio, room, sid)
connected_users[room].append(user)
return True
#socketio.on("disconnect")
def disconnect():
sid = request.sid
# Find the room and user with this sid
user_found = False
for room, users in connected_users.items():
for user in users:
if user.sid == sid:
user_found = True
break
if user_found:
break
# If a matching user was not found, do nothing
if not user_found:
return
room = user.room
socketio.send(f"{user.name} has left the room.", room=room)
# Remove the user from the room
connected_users[room].remove(user)
# Add the user to the disconnected list
disconnected_users[room].append(user)
flask_socketio.leave_room(room)
#socketio.on("collect colors")
def collect_colors(data):
room = data["room"]
for user in connected_users[room]:
color = user.call("send color", data)
print(f"{user.name}'s color is {color}.")
if __name__ == "__main__":
socketio.run(app, debug=True)
# user.py
from threading import Event # Monkey patched
class User:
def __init__(self, name, socketio, room, sid):
self.name = name
self.socketio = socketio
self.room = room
self._sid = sid
#property
def sid(self):
return self._sid
#sid.setter
def sid(self, new_sid):
self._sid = new_sid
def call(self, event_name, data):
"""
Send a request to the player and wait for a response.
"""
event = Event()
response = None
# Create callback to run when a response is received
def ack(response_data):
print("WHY DOES THIS NOT RUN AFTER A REJOIN?")
nonlocal event
nonlocal response
response = response_data
event.set()
# Try in a loop with a one second timeout in case an event gets missed or a network error occurs
tries = 0
while True:
# Send request
self.socketio.emit(
event_name,
data,
to=self.sid,
callback=ack,
)
# Wait for response
if event.wait(1):
# Response was received
break
tries += 1
if tries % 10 == 0:
print(f"Still waiting for input after {tries} seconds")
return response
// static/client.js
var socket = io.connect();
var username = null;
var room = null;
var joined = false;
var colorCallback = null;
function joinedRoom(success) {
if (success) {
joined = true;
$("#joinForm").hide();
$("#collectColorsButton").show();
$("#gameRoom").text(`Room: ${room}`);
}
}
socket.on("connect", () => {
console.log("You are connected to the server.");
});
socket.on("connect_error", (data) => {
console.log(`Unable to connect to the server: ${data}.`);
});
socket.on("disconnect", () => {
console.log("You have been disconnected from the server.");
});
socket.on("message", (data) => {
console.log(data);
});
socket.on("send color", (data, callback) => {
$("#collectColorsButton").hide();
$("#colorForm").show();
console.log(`Callback set to ${callback}`);
colorCallback = callback;
});
$("#joinForm").on("submit", (event) => {
event.preventDefault();
username = $("#usernameInput").val();
room = $("#roomInput").val()
socket.emit("join room", {username: username, room: room}, joinedRoom);
});
$("#colorForm").on("submit", (event) => {
event.preventDefault();
var color = $("#colorInput").val();
$("#colorForm").hide();
colorCallback(color);
});
$("#collectColorsButton").on("click", () => {
socket.emit("collect colors", {username: username, room: room});
});
<!-- static/index.html -->
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<title>Socket.IO Example</title>
</head>
<body>
<p id="gameRoom"></p>
<form id="joinForm">
<input id="usernameInput" type="text" placeholder="Your Name" autocomplete="off" required>
<input id="roomInput" type="text" placeholder="Room ID" autocomplete="off" required>
<button id="joinGameSubmitButton" type="submit" btn btn-dark">Join Room</button>
</form>
<button id="collectColorsButton" style="display: none;">Collect Colors</button>
<form id="colorForm" style="display: none;">
<p>Please select a color.</p>
<input id="colorInput" type="color" required>
<button id="colorSubmitButton" type="submit">Send Color</button>
</form>
<script src="https://code.jquery.com/jquery-3.3.1.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
<script src="https://cdn.socket.io/4.4.1/socket.io.min.js" integrity="sha384-fKnu0iswBIqkjxrhQCTZ7qlLHOFEgNkRmK2vaO/LbTZSXdJfAu6ewRBdwHPhBo/H" crossorigin="anonymous"></script>
<script src="client.js"></script>
</body>
</html>
Edit
Steps to Reproduce
Start the server python app.py and visit localhost:5000 in your browser.
Enter any username and Room ID and click "Join Room."
Click "Collect Colors."
Select a color and click "Send." The selector should disappear and the server should print out a confirmation.
Reload everything.
Repeat steps 2 and 3 and copy the Room ID.
Exit the page and then navigate back to it.
Enter the same username and Room ID as you did in step 6 and click "Join Room."
Select a color and click "Send." The selector disappears briefly but then comes back, since the server did not correctly process the response and keeps sending requests instead.
Edit 2
I managed to work around (not solve) the problem by adding more state variables on the server side and implementing a few more events to avoid using callbacks entirely. I would still love to know what was going wrong with the callback-based approach though since using that seems cleaner to me.

The reason why those callbacks do not work is that you are making the emits from a context that is based on the old and disconnected socket.
The callback is associated with the socket identified by request.sid. Associating the callback with a socket allows Flask-SocketIO to install the correct app and request contexts when the callback is invoked.
The way that you coded your color prompt is not great, because you have a long running event handler that continues to run after the client goes aways and reconnects on a different socket. A better design would be for the client to send the selected color in its own event instead of as a callback response to the server.

JavaScript, JSP and JSON not working with POST

I'm trying to make a client-server application where from the client I send a request through a JSON object to the server to register. The thing is I should get another JSON with an "OK" field (which is actually being sent) but for some reason the client keeps going to the .fail function instead of the .done one (sorry if some of used terms are not very accurate, I'm new to this).
So I'll this is my code incase you can check if there's anything wrong causing this:
Client JS:
define(['ojs/ojcore', 'knockout', 'jquery', 'appController', 'jquery', 'ojs/ojknockout', 'ojs/ojinputtext'],
function(oj, ko, $, app) {
function RegistrarseViewModel() {
var self = this;
this.email = ko.observable();
this.pwd1 = ko.observable();
this.pwd2 = ko.observable();
this.registrar = function(){
alert("Se ha mandado el registro");
var p = {tipo:"Registrarse",email: this.email(), pwd1:this.pwd1(), pwd2:this.pwd2()};
$.ajax({
type:"POST",
url:"http://localhost:8080/ServidorWeb/Registrarse.jsp",
data: "p=" + JSON.stringify(p)
}).done(function(data, textStatus, jqXHR){
alert("Comprobando tipo");
if (data.tipo == "OK"){
//window.location="index.html?root=juegos"
sessionStorage.jugador=self.email();
app.router.go("login");
alert("Registro correcto");
}else
alert(respuesta.texto)
}).fail(function() {
alert("Sorry. Server unavailable. lol ");
});
}
this.cancelar = function(){
app.router.go("login");
}
}
return new RegistrarseViewModel();
}
);
Server JSP:
<%# page language="java" contentType="application/json ; charset=UTF-8"
pageEncoding="UTF-8"%>
<%# page import= "org.json.*,dominio.Manager"%>
<%
String p = request.getParameter("p");
JSONObject resultado=new JSONObject();
try{
JSONObject jso= new JSONObject(p);
if(!jso.getString("tipo").equals("Registrarse")){
resultado.put("tipo","NOK");
resultado.put("texto","Mensaje inesperado");
}else{
String email=jso.getString("email");
String pwd1=jso.getString("pwd1");
String pwd2=jso.getString("pwd2");
Manager.get().registrarse(email,pwd1,pwd2);
resultado.put("tipo","OK");
resultado.put("texto","Te has registrado con el email " + email);
}
}
catch(Exception e){
resultado.put("tipo","NOK");
resultado.put("texto","Mensaje Inesperadoo");
}
%>
<%=resultado.toString()%>
After executing Manager.get().registrarse(email,pwd1,pwd2); (which is the logic to register into a MongoDB) it just continues with the resultado.put("tipo","OK"); line which means the problem isn't in there.
Also if I send the request http://localhost:8080/ServidorWeb/Registrarse.jsp?p=%7Btipo:%22Registrarse%22,email:%2233%22,pwd1:%2220%22,pwd2:%2220%22%7D from a browser like Google Chrome it prints this: {"texto":"Te has registrado con el email 33","tipo":"OK"} but from the real client it just won't get into the .done function, idk why.
I really hope you can help me.
Thanks in advance.
EDIT 1: Added the server response from the browser console IMAGE

Okay I solved this finally.
I had to add this line at the beggining of the .jsp, this was an issu with TomCat which has something like 2 machines and without this line it doesn't allow communication among different machines because of security reasons it seems.
response.setHeader("Access-Control-Allow-Origin", "*");

if you use jquery the correct way is use serialize function from jquery
https://api.jquery.com/serialize/
first give a id for you form something like :
`
$("#myform form").submit(function(event){
event.preventDefault();
var sendData = $("#myform form").serialize();
$.post("your-PHP-handler.php", sendData);
});
<form id="myform" method="post" action="your-PHP-handler.php">
<input type="name" placeholder="name">
<input type="name" placeholder="age">
<input type="name" placeholder="address">
<button type="submit">send</button>
</form>
`
note when you submit your form via javascript the serialization jquery get all inputs in your post end send all together you cam handler the response php inside of $.post() you can make many things with this consulting jquery documentation.
anyway the basic is there , get everything inside my form and send to my php file

Retrieving data from database using js,php,ajax

I want to retrieve data from a database which is locally installed.
But the website is written and controlled with js, so I have to send an ajax request to a php-file, which then connects to the database, sends the query and later the result back to js.
I have a couple of files, because i followed this tutorial from 2012: https://www.youtube.com/watch?v=Yb3c-HljFro
The structure:
|-ajax (Folder)
| |-pkmn.php
|-db (Folder)
| |-connect.php
|-js (Folder)
| |-global.js
|-index.php
The files:
index.php:
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"></meta>
</head>
<body>
Type: <input type="text" id="type"></input>
Tier: <input type="text" id="tier"></input>
<input type="submit" id="submit" value="Suchen"></input>
<div id="pkmn-data"></div>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
<script src="js/global.js"></script>
</body>
</html>
global.js:
$("#submit").click(function () {
var type = $("type").val();
$.ajax("ajax/pkmn.php", {type: type}, function(data) {
alert(data);
});
});
connect.php
<?php
mysql_connect('localhost','root', '123456');
mysql_select_db('database');
pkmn.php:
<?php
echo 'Hello';
The user should enter something into the text-inputs, then click submit and the global.js-file sends that text to the file pkmn.php via an ajax request.
But when I click the submit-button, I receive an error:
XML-Verarbeitungsfehler: Ungeschlossenes Token
Adresse: file:///C:/Users/Jonathan%20Frakes/Documents/testDB/ajax/pkmn.php
Zeile Nr. 1, Spalte 1: pkmn.php:1:1
XML-Verarbeitungsfehler: Ungeschlossenes Token
Adresse: file:///C:/Users/Jonathan%20Frakes/Documents/testDB/index.php
Zeile Nr. 1, Spalte 1: index.php:1:1
Which says something like:
XML-Parseerror: unclosed token
at: file:///C:/Users/Jonathan%20Frakes/Documents/testDB/ajax/pkmn.php
Row 1, Column 1: pkmn.php
XML-Parseerror: unclosed token
at: file:///C:/Users/Jonathan%20Frakes/Documents/testDB/ajax/index.php
Row 1, Column 1: index.php
I have absolutely no idea, what could cause this error.
Furthermore: When I close the php-tag in pkmn.php the console says BOTH errors moved to Row 3 Column 3 and when I close the php-tag one line later (by adding another newline) the error again moves in both files one down. I can also add spaces in front of the closing tag, moving the errors horizontally.
I have never seen something like this before, so please help me to fix that.
Best regards,
PanCave
EDIT:
changing the global.js according to this tutorial (https://www.w3schools.com/php/php_ajax_database.asp) now produces an "no root element found" error :/
$("#submit").click(function () {
var type = $("#type").val();
var tier = $("#tier").val();
if(window.XMLHttpRequest) {
xmlhttp = new XMLHttpRequest();
} else {
xmlhttp = new ActiveXObject("Microsoft.XMLHTTP");
}
xmlhttp.onreadystatechange = function() {
if(this.readyState == 4 && this.status == 200) {
document.getElementById("pkmn-data").innerHTML = this.responseText;
}
};
xmlhttp.open("Get", "pkmn.php?type="+type+"&tier="+tier);
xmlhttp.send();
});

How to use AJAX with Google App Engine (Python)

I am completely novice at AJAX. I am familiar with HTML/CSS, jQuery and beginner at GAE and Python.
In an effort to understand how AJAX works, I would like to know how AJAX might be used (actual code) in this example below. Let's use a reddit-like example where vote ups/downs are ajaxified:
Here is the Story Kind:
class Story(ndb.Model):
title = ndb.StringProperty(required = True)
vote_count = ndb.IntegerProperty(default = 0)
The HTML would look like this:
<h2>{{story.title}}</h2>
<div>
{{story.vote_count}} | Vote Up Story
</div>
How does AJAX fit inside here?

Ok Sir here we go... A simple app with one story and infinite votes... ;-)
app.yaml
application: anotherappname
version: 1
runtime: python27
api_version: 1
threadsafe: true
default_expiration: "0d 0h 5m"
libraries:
- name: jinja2
version: latest
- name: webapp2
version: latest
handlers:
- url: .*
script: main.app
main.py
import logging
from controllers import server
from config import config
import webapp2
app = webapp2.WSGIApplication([
# Essential handlers
('/', server.RootPage),
('/vote/', server.VoteHandler)
],debug=True, config=config.config)
# Extra Hanlder like 404 500 etc
def handle_404(request, response, exception):
logging.exception(exception)
response.write('Oops! Naughty Mr. Jiggles (This is a 404)')
response.set_status(404)
app.error_handlers[404] = handle_404
models/story.py
from google.appengine.ext import ndb
class Story(ndb.Model):
title = ndb.StringProperty(required=True)
vote_count = ndb.IntegerProperty(default = 0)
controllers/server.py
import os
import re
import logging
import config
import json
import webapp2
import jinja2
from google.appengine.ext import ndb
from models.story import Story
class RootPage(webapp2.RequestHandler):
def get(self):
story = Story.get_or_insert('Some id or so', title='A voting story again...')
jinja_environment = self.jinja_environment
template = jinja_environment.get_template("/index.html")
self.response.out.write(template.render({'story': story}))
#property
def jinja_environment(self):
jinja_environment = jinja2.Environment(
loader=jinja2.FileSystemLoader(
os.path.join(os.path.dirname(__file__),
'../views'
))
)
return jinja_environment
class VoteHandler(webapp2.RequestHandler):
def post(self):
logging.info(self.request.body)
data = json.loads(self.request.body)
story = ndb.Key(Story, data['storyKey']).get()
story.vote_count += 1
story.put()
self.response.out.write(json.dumps(({'story': story.to_dict()})))
and finally
views/index.html
<!DOCTYPE html>
<html>
<head>
<base href="/">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
</head>
<body>
<h2>{{story.title}}</h2>
<div>
<span class="voteCount">{{story.vote_count}}</span> | <a href="javascript:VoteUp('{{story.key.id()}}');" >Vote Up Story</a>
</div>
<script>
function VoteUp(storyKey){
$.ajax({
type: "POST",
url: "/vote/",
dataType: 'json',
data: JSON.stringify({ "storyKey": storyKey})
})
.done(function( data ) { // check why I use done
alert( "Vote Cast!!! Count is : " + data['story']['vote_count'] );
$('.voteCount').text(data['story']['vote_count']);
});
};
</script>
</body>
</html>
Assemble, read it's simple enough and run. If you need a working git example just comment.
githublink (as from comments)

Here is a little prototype web app on GitHub to test how to handle error messages in HTML form submissions with AJAX, Python and Google App Engine. It will give a starting point to see how these three pieces of technology mesh together. You can test this "app" on https://ajax-prototype.appspot.com/
Here is how it works on the client-side:
This htlm form submission is used:
<form method="post" action="javascript:ajaxScript();">
<label>Please pick a name</label>
<input id="input" type="text">
<input type="submit">
<div id="error" style="color:red"></div>
It will trigger the JavaScript function ajaxScript:
function ajaxScript() {
var input = $("#input").val();
$.ajax({
type: "POST",
url: "/",
data: JSON.stringify({
"name": input
}),
dataType: "json"
})
.done(function(jsonResponse) {
$("#error").html(jsonResponse.message);
});
}
The jQuery .ajax() method handles the request while the .done() method will eventually handle the response that it gets from the server.
On the server-side:
The main.py file handles the server side of the business using our handler class AjaxHandler, which inherits from the GAE builtin class webapp2.RequestHandler
Within this class, the post method handles the AJAX request:
def post(self):
data = json.loads(self.request.body)
username = data["name"]
if not re.match(r"^[a-zA-Z0-9_-]{3,20}$", username):
if len(username) < 3:
message = "Your name must be at least 3 characters long."
else:
message = "Allowed characters are \
a-z, A-Z, 0-9, underscores \
and hyphens."
else:
message = "Congrats!"
self.response.write(json.dumps({"message": message}))
Basically, the handy json module provides the two key Python ingredients
json.loads handles the data that the browser sends to the server.
json.dumps handles the data sent by the server in response to the browser's request.
The self.request.body argument of json.loads is the only less common piece of GAE that is used in the process, as it is specific to the task. As its name suggests, it gets the body from the Ajax request sent by the client.

I am having difficulty reaching a web service. Any ideas what changes are needed on my code?

I have struggled with this task for so long that I am hoping for a reprieve.
We have a web service called Validate on a separate server/domain.
Due to same domain restrictions, I came up with proxy server (code) to work around it.
The code below called Validate.asp, points to the webservice:
Set http = Server.CreateObject("msxml2.ServerXMLHTTP")
http.Open "POST", "http://servername1/folder1/folder2/folder3/Validation/Validate", False
http.setRequestHeader "Content-type","application/x-www-form-urlencoded"
http.Send Request.Form
Response.Write http.ResponseText
Response.End
%>
Then the code markup below invokes Validate.asp.
<!DOCTYPE html>
<html>
<body>
<form>
Enter name: <input id="user" /><br/>
Enter Pass: <input id="pass" /><br/>
<input type="button" value="Get web response via AJAX" onclick="ajaxViaProxy()" />
</form>
<hr/>
Ressponse: <div id="result"></div>
<script type="text/javascript">
function ajaxViaProxy( )
{
var uname = document.getElementById("user").value;
var upass = document.getElementById("pass").value;
var http = new XMLHttpRequest();
http.open("POST", "Validate.asp", true );
http.setRequestHeader("Content-type","application/x-www-form-urlencoded");
http.onreadystatechange=function()
{
if (http.readyState==4 && http.status==200)
{
document.getElementById("result").innerHTML =
"Response from web server was '" + http.responseText + "'";
}
}
http.send("username=" + encodeURIComponent(uname)
+ "&password=" + encodeURIComponent(upass));
}
</script>
</body>
</html>
If everything goes well, we should get response similar to this one below:
{"Value":{"TokenId":35,"LoginName":"validUser","Created":"2013-12-03T09:53:35","Expires":"2013-12-24T09:53:35","LastUsed":"2013-12-03T09:53:35"},"Status":0,"Message":null}
This is the exact response we get when I paste the code below into an address bar and hit the enter key:
http://servername/folder1/folder2/folder3/Validation/Validate?data={"User":"validUserName","Password":"validPassword"}
However, when I use the markup to invoke proxyValidate.asp, rather than get the response with token, I get the following:
{"Value":null,"Status":2,"Message":"Invalid username or password"}
Does anyone know what I need to change in the markup or in proxyValidate.asp so that when I run the code, I get a properly formatted URL string like this below?
http://servername/folder1/folder2/folder3/Validation/Validate?data={"User":"validUserName","Password":"validPassword"}
Thanks alot in advance