NodeJS + Redis + WebSocket memory management? - javascript

I have a NodeJS that host a WebSocket Server. The WebSocket redistributes message from Redis.
The full line is, i have some python script that push some data in Redis and after that NodeJS is the WebSocket that reads the Redis newly input data to the connected clients. My problem is that the NodeJs is always taking up memory and after a while it just burst and stops.
I don't know what is my problem, since my code is pretty simple.
I don't need my WebSocket to receive message from the connected clients, since i only need to push them data, but alot of data.
var server = require('websocket').server,
http = require('http');
var redis = require("redis"),
client = redis.createClient();
var socket = new server({
httpServer: http.createServer().listen(443),
keepalive: false
});
client.subscribe("attack-map-production");
socket.on('request', function(request) {
var connection = request.accept(null, request.origin);
connection.on('message', function(message) {
console.log(message);
client.on("message", function(channel, message){
connection.send(message);
});
});
connection.on('close', function(connection) {
console.log('connection closed');
});
});
I'm looking to make this work without eating all the memory on my server and possibly make this much more fast, but i think it's fast enough.
Maybe NodeJS is not meant for this kind of work?
Any help is appreciated.
Thanks.
Update 2016-11-08
With the information provided below, i have "updated" my code. The problem is still there, i will continue to look around to find a answer... but i'm really not getting this.
var server = require('websocket').server,
http = require('http');
var redis = require("redis"),
client = redis.createClient();
var socket = new server({
httpServer: http.createServer().listen(443),
keepalive: false
});
client.subscribe("attack-map-production");
socket.on('request', function(request) {
var connection = request.accept(null, request.origin);
client.on("message", function(channel, message){
connection.send(message);
});
connection.on('close', function(connection) {
console.log('connection closed');
});
});
Update 2016-11-16
So here is my new code:
var io = require('socket.io').listen(443);
var sub = require('redis').createClient();
io.sockets.on('connection', function (sockets) {
sockets.emit('message',{Hello: 'World!'});
sub.subscribe('attack-map-production'); // Could be any patterni
sockets.on('disconnect', function() {
sub.unsubscribe('attack-map-production');
});
});
sub.on('message', function(channel, message) {
io.sockets.json.send(message);
});
Even this code, makes nodejs go at 100% CPU and even more, and it starts to go really slow, until everything just stops.
The complet flow of my data, is that a python script pushes data into Redis, and throught my subscribtion it pushes my data back to the browser by a webSocket and Socket.io.
That simple, how can this be slow? I just don't get it.

client = redis.createClient();
take a look at this line , everytime you invoke the variable client , you create an instance of redis client inside node , and you never close it. so if you recieve 10000 socket 'request' , you will also have 10000 redis instances.
You need to call the command client.quit() once the write or the read to redis is done
var server = require('websocket').server,
http = require('http');
var redis = require("redis"),
client = redis.createClient();
var socket = new server({
httpServer: http.createServer().listen(443),
keepalive: false
});
client.subscribe("attack-map-production");
socket.on('request', function(request) {
var connection = request.accept(null, request.origin);
client.on("message", function(channel, message){
connection.send(message);
});
client.quit(); // MISSING LINE
connection.on('close', function(connection) {
console.log('connection closed');
});
});
and i also noticed this piece of code
httpServer: http.createServer().listen(443)
the port 443 is for https ! so if you are using a secured connection you need to call the module https not http, like this
var socket = new server({
httpServer: https.createServer().listen(443),
keepalive: false
});
hope it helps !

Maybe NodeJS is not meant for this kind of work?
If node is meant for something it's this. I/O stream and reading/writing is the main advantage of node asynchronism.
On what kind of server are you running this ? In a too small EC2 instance you can hit some memory problem.
Else it's a leak. That's kind of hard to trace.
Code is small thought.
I would remove any console.log just in case.
connection.on('message', function(message) {
console.log(message);
client.on("message", function(channel, message){
connection.send(message);
});
});
This part feel suspicious, two variables with the same name, an unused variable, it calls for trouble, and I don't really get why you have to listen for connection message in order to wait for redis message.

Related

Socket.io unable to emit data to client's unique room

I am using Node.js to create a media upload microservice. This service works by taking in the binary data of the upload to a buffer, and then using the S3 npm package to upload to an S3 bucket. I am trying to use the eventEmitter present in that package which shows the amount of data uploaded to S3, and send that back to the client which is doing the uploading (so that they can see upload progress). I am using socket.io for this sending of progress data back to the client.
The problem I am having is that the .emit event in socket.io will send the upload progress data to all connected clients, not just the client which initiated the upload. As I understand it, a socket connects to a default room on 'connection', which is mirrored by the 'id' on the client side. According to the official docs, using socket.to(id).emit() should send the data scoped only to that client, but this is not working for me.
UPDATED Example code:
server.js:
var http = require('http'),
users = require('./data'),
app = require('./app')(users);
var server = http.createServer(app);
server.listen(app.get('port'), function(){
console.log('Express server listening on port ' + app.get('port'));
});
var io = require('./socket.js').listen(server);
socket.js:
var socketio = require('socket.io');
var socketConnection = exports = module.exports = {};
socketConnection.listen = function listen(app) {
io = socketio.listen(app);
exports.sockets = io.sockets;
io.sockets.on('connection', function (socket) {
socket.join(socket.id);
socket.on('disconnect', function(){
console.log("device "+socket.id+" disconnected");
});
socketConnection.upload = function upload (data) {
socket.to(socket.id).emit('progress', {progress:(data.progressAmount/data.progressTotal)*100});
};
});
return io;
};
s3upload.js:
var config = require('../config/aws.json');
var s3 = require('s3');
var path = require('path');
var fs = require('fs');
var Busboy = require('busboy');
var inspect = require('util').inspect;
var io = require('../socket.js');
...
var S3Upload = exports = module.exports = {};
....
S3Upload.upload = function upload(params) {
// start uploading to uploader
var uploader = client.uploadFile(params);
uploader.on('error', function(err) {
console.error("There was a problem uploading the file to bucket, either the params are incorrect or there is an issue with the connection: ", err.stack);
res.json({responseHTML: "<span>There was a problem uploading the file to bucket, either the params are incorrect or there is an issue with the connection. Please refresh and try again.</span>"});
throw new Error(err);
}),
uploader.on('progress', function() {
io.upload(uploader);
}),
uploader.on('end', function(){
S3Upload.deleteFile(params.localFile);
});
};
When using DEBUG=* node myapp.js, I see the socket.io-parser taking in this information, but it isn't emitting it to the client:
socket.io-parser encoding packet {"type":2,"data":["progress",{"progress":95.79422221709825}],"nsp":"/"} +0ms
socket.io-parser encoded {"type":2,"data":["progress",{"progress":95.79422221709825}],"nsp":"/"} as 2["progress",{"progress":95.79422221709825}] +0ms
However, if I remove the .to portion of this code, it sends the data to the client (albeit to all clients, which will not help at all):
io.sockets.on('connection', function(socket) {
socket.join(socket.id);
socket.emit('progress', {progress: (data.progressAmount/data.progressTotal)*100});
});
DEBUG=* node myapp.js:
socket.io:client writing packet {"type":2,"data":["progress",{"progress":99.93823786632886}],"nsp":"/"} +1ms
socket.io-parser encoding packet {"type":2,"data":["progress",{"progress":99.93823786632886}],"nsp":"/"} +1ms
socket.io-parser encoded {"type":2,"data":["progress",{"progress":99.93823786632886}],"nsp":"/"} as 2["progress",{"progress":99.93823786632886}] +0ms
engine:socket sending packet "message" (2["progress",{"progress":99.93823786632886}]) +0ms
engine:socket flushing buffer to transport +0ms
engine:ws writing "42["progress",{"progress":99.84186540937002}]" +0ms
engine:ws writing "42["progress",{"progress":99.93823786632886}]" +0ms
What am I doing wrong here? Is there a different way to emit events from the server to only specific clients that I am missing?
The second example of code you posted should work and if it does not, you should post more code.
As I understand it, a socket connects to a default room on
'connection', which is mirrored by the 'id' on the client side.
According to the official docs, using socket.to(id).emit() should send
the data scoped only to that client, but this is not working for me.
Socket.io is pretty much easier than that. The code below will send a 'hello' message to each client when they connect:
io.sockets.on('connection', function (socket) {
socket.emit('hello');
});
Everytime a new client connects to the socket.io server, it will run the specified callback using that particular socket as a parameter. socket.id is just an unique code to identify that socket but you don't really need that variable for anything, the code above shows you how to send a message through a particular socket.
Socket.io also provides you some functions to create namespaces/rooms so you can group connections under some identifier (room name) and be able to broadcast messages to all of them:
io.sockets.on('connection', function (socket) {
// This will be triggered after the client does socket.emit('join','myRoom')
socket.on('join', function (room) {
socket.join(room); // Now this socket will receive all the messages broadcast to 'myRoom'
});
...
Now you should understand socket.join(socket.id) just does not make sense because no socket will be sharing socket id.
Edit to answer the question with the new code:
You have two problems here, first:
socketConnection.upload = function upload (data) {
socket.to(socket.id).emit('progress', {progress:(data.progressAmount/data.progressTotal)*100});
};
Note in the code above that everything inside io.sockets.on('connection',function (socket) { will be run each time a clients connect to the server. You are overwriting the function to point it to the socket of the latest user.
The other problem is that you are not linking sockets and s3 operations. Here is a solution merging socket.js and s3upload.js in the same file. If you really need to keep them separated you will need to find a different way to link the socket connection to the s3 operation:
var config = require('../config/aws.json');
var s3 = require('s3');
var path = require('path');
var fs = require('fs');
var Busboy = require('busboy');
var inspect = require('util').inspect;
var io = require('socket.io');
var socketConnection = exports = module.exports = {};
var S3Upload = exports = module.exports = {};
io = socketio.listen(app);
exports.sockets = io.sockets;
io.sockets.on('connection', function (socket) {
socket.on('disconnect', function(){
console.log("device "+socket.id+" disconnected");
});
socket.on('upload', function (data) { //The client will trigger the upload sending the data
/*
some code creating the bucket params using data
*/
S3Upload.upload(params,this);
});
});
S3Upload.upload = function upload(params,socket) { // Here we pass the socket so we can answer him back
// start uploading to uploader
var uploader = client.uploadFile(params);
uploader.on('error', function(err) {
console.error("There was a problem uploading the file to bucket, either the params are incorrect or there is an issue with the connection: ", err.stack);
res.json({responseHTML: "<span>There was a problem uploading the file to bucket, either the params are incorrect or there is an issue with the connection. Please refresh and try again.</span>"});
throw new Error(err);
}),
uploader.on('progress', function() {
socket.emit('progress', {progress:(uploader.progressAmount/uploader.progressTotal)*100});
}),
uploader.on('end', function(){
S3Upload.deleteFile(params.localFile);
});
};
According to the documentation all the users join the default room identified by the socket id, so no need for you to join in on connection. Still according to that, if you want to emit to a room in a namespace from a specific socket you should use socket.broadcast.to(room).emit('my message', msg), given that you want to broadcast the message to all the clients connected to that specific room.
All new connections are automatically joined to room having the name that is equal to their socket.id. You can use it to send messages to specific user, but you have to know socket.id associated with connection initialized by this user. You have to decide how you will manage this association (via databases, or in memory by having an array for it), but once you have it, just send progress percentage via:
socket.broadcast.to( user_socket_id ).emit( "progress", number_or_percent );

Send data from websocket to socket.io

I used websocket interface to connect to websocket server . what if i want send data that i receive from the websocket server through my websocket interface to client connected to me through http server , should i use socket.io ?
so at the end i will have socket.io attached to to http server and websocket interface to get data and in case of message come will be send to client through socket.io . is that the best setup ?
Code Example :
// Require HTTP module (to start server) and Socket.IO
var http = require('http'),
io = require('socket.io');
var WebSocket = require('ws');
var ws = new WebSocket('ws://localhost:5000');
// Start the server at port 8080
var server = http.createServer(function (req, res) {
// Send HTML headers and message
res.writeHead(200, {
'Content-Type': 'text/html'
});
res.end('<h1>Hello Socket Lover!</h1>');
});
server.listen(8080);
// Create a Socket.IO instance, passing it our server
var socket = io.listen(server);
ws.on('open', function open() {
ws.send('something');
});
ws.on('message', function (data, flags) {
// here the data will be send to socket.io
});
// Add a connect listener
socket.on('connection', function (client) {
// Success! Now listen to messages to be received
client.on('message', function (event) {
console.log('Received message from client!', event);
});
client.on('disconnect', function () {
clearInterval(interval);
console.log('Server has disconnected');
});
});
Yes, your design is correct.
However, one thing that you should keep in mind is take care of sending the message to the correct client after authentication. In my opinion, it is very easy to make this mistake, partially because of the simplicity of messaging using websockets.

Laravel Broadcast with Redis not working/connecting?

So I'm trying to broadcast Laravel 5 Events with the help of Redis. No I don't wanna use a service like Pusher since it's not free (even if the free limit would be enough for me) and I wanna keep control of the broadcast server.
So what I've done so far is, I'Ve set up a redis server (listening on port 6379 -> default), I've set up the following event:
class MyEventNameHere extends Event implements ShouldBroadcast
{
use SerializesModels;
public $data;
/**
* Create a new event instance.
*
* #return \App\Events\MyEventNameHere
*/
public function __construct()
{
$this->data = [
'power' => 10
];
}
/**
* Get the channels the event should be broadcast on.
*
* #return array
*/
public function broadcastOn()
{
return ['pmessage'];
}
}
I registered a route to that event:
Route::get('test',function()
{
event(new App\Events\MyEventNameHere());
return "event fired";
});
I've created (more like copied :P) the node socket server:
var app = require('http').createServer(handler);
var io = require('socket.io')(app, {origins:'*:*'});
var Redis = require('ioredis');
var redis = new Redis();
app.listen(6379, function() {
console.log('Server is running!');
});
function handler(req, res) {
res.writeHead(200);
res.end('');
}
io.on('connection', function(socket) {
console.log(socket);
});
redis.psubscribe('*', function(err, count) {
});
redis.on('pmessage', function(subscribed, channel, message) {
console.log(message);
message = JSON.parse(message);
io.emit(channel + ':' + message.event, message.data);
});
And I created the view to actually receive the broadcast (testview.blade.php):
#extends('layout')
#section('content')
<p id="power">0</p>
<script>
var socket = io('http://localhost:6379');
socket.on("pmessage:App\\Events\\MyEventNameHere", function(message) {
console.log(message);
$('#power').text(message.data);
});
console.log(socket.connected);
</script>
#endsection
I can launch the redis server without any problems.
I can launch the node socket.js server and I'm getting the response "Server running"
When I hit the route to the event I get the return "event fired" in my browser.
When I hit the route to the actual view
Route::get('test/view',function()
{
return view('testview');
});
I can see the whole page (layout is rendered), and the webconsole does not show any errors.
However if I fire the event, the view won't change, which means, the broadcast is not received right?
Now I included an output for the console
console.log(socket.connected);
which should show me if the client is connected to the socket.io right?
Well, the output says false. What am I doing wrong here?
Further information on my setup: I'm running the whole project on the php built-in server, the whole thing is running on Windows (if ever that could matter), my firewall is not blocking any of the ports.
EDIT 1:
I forgot to say that my node server is not receiving the messages as well... It only says "Server running", nothing else.
EDIT 2:
I used another socket.js:
var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var Redis = require('ioredis');
var redis = new Redis();
app.get('/', function(req, res) {
res.sendFile(__dirname + '/index.html');
});
redis.subscribe('test-channel', function () {
console.log('Redis: test-channel subscribed');
});
redis.on('message', function(channel, message) {
console.log('Redis: Message on ' + channel + ' received!');
console.log(message);
message = JSON.parse(message);
io.emit(channel, message.payload)
});
io.on('connection', function(socket) {
console.log('a user connected');
socket.on('disconnect', function() {
console.log('user disconnected');
});
});
http.listen(3000, function() {
console.log('listening on *:3000');
});
And this time the console receives the messages.
So if the node socket.io receives the messages, then what's wrong with my client? Obviously the messages are being broadcasted correctly, the only thing is that they are not being received by the client...
I can't say what is exactly wrong and probably no one can't, because your problem is to broad and enviroment dependent. Using Wireshark Sniffer you can easily determinate part of solution that is not working correctly and then try find solution around actual problem.
If your question is about how to do that, I will suggest not involving node on server side and use .NET or Java language.
The problem with your code is you are connecting your client socket to the redis default port 6379 rather than the node port that is 3000.
So in your blade view change var socket = io('http://localhost:6379'); to var socket = io('http://localhost:3000');
have you tried to listen to the laravel queue, from command line, before to fire the event?
php artisan queue:listen

A severe memory leak with Socket.io 1.2.1 (and earlier)

I have a very simple socket.io server setup. My needs are to be able to communicate between server and client over sockets, but always on a one-to-one mode.
As such I created something simple as this on the client:
var socket = io();
socket.on('connect',function(){
socket.emit('connected', UserName);
});
And than on the server:
io.on('connection', function (socket) {
socket.on('connected', function (userName) {
if (userName !== null) {
socket.join("RoomFor:"+ userName);
}
});
});
So, each user has his own room I can now communicate with, and I can do this using my applicative userName (not socket id).
Now, on the same socket IO server, I also have express listening, with something like this:
app.get('/notify', function(req, res) {
var data = querystring.parse(req.url.split('?')[1]);
var userName = data.clientId;
io.sockets.to('RoomFor:'+ userName).emit('notice', {event: data.event });
res.writeHead(returnCode, {'Content-Type': 'text/plain'});
res.end();
});
So, a very simple server side signaling solution.
However, something in this setup is leaking badely, I have created heapdumps and tried to compare the results, but couldn't make out anything out of it.
Any thoughts?
(Btw, I went over all the threads related to node + socket.io memory leaks, none is my case... I don't use redis store, I am running with up to date node and socket.io versions.)

Update all clients using Socket.io?

Is it possible to force all clients to update using socket.io? I've tried the following, but it doesn't seem to update other clients when a new client connects:
Serverside JavaScript:
I'm attempting to send a message to all clients, which contains the current number of connected users, it correctly sends the amount of users.... however the client itself doesn't seem to update until the page has been refreshed. I want this to happen is realtime.
var clients = 0;
io.sockets.on('connection', function (socket) {
++clients;
socket.emit('users_count', clients);
socket.on('disconnect', function () {
--clients;
});
});
Clientside JavaScript:
var socket = io.connect('http://localhost');
socket.on('connect', function(){
socket.on('users_count', function(data){
$('#client_count').text(data);
console.log("Connection");
});
});
It's not actually sending an update to the other clients at all, instead it's just emitting to the client that just connected (which is why you see the update when you first load)
// socket is the *current* socket of the client that just connected
socket.emit('users_count', clients);
Instead, you want to emit to all sockets
io.sockets.emit('users_count', clients);
Alternatively, you can use the broadcast function, which sends a message to everyone except the socket that starts it:
socket.broadcast.emit('users_count', clients);
I found that using socket.broadcast.emit() will only broadcast to the current "connection", but io.sockets.emit will broadcast to all the clients.
here the server is listening to "two connections", which are exactlly 2 socket namespaces
io.of('/namespace').on('connection', function(){
socket.broadcast.emit("hello");
});
io.of('/other namespace').on('connection',function(){/*...*/});
i have try to use io.sockets.emit() in one namespace but it was received by the client in the other namespace. however socket.broadcast.emit() will just broadcast the current socket namespace.
As of socket.io version 0.9, "emit" no longer worked for me, and I've been using "send"
Here's what I'm doing:
Server Side:
var num_of_clients = io.sockets.clients().length;
io.sockets.send(num_of_clients);
Client Side:
ws = io.connect...
ws.on('message', function(data)
{
var sampleAttributes = fullData.split(',');
if (sampleAttributes[0]=="NumberOfClients")
{
console.log("number of connected clients = "+sampleAttributes[1]);
}
});
You can follow this example for implementing your scenario.
You can let all of clients to join a common room for sending some updates.
Every socket can join room like this:
currentSocket.join("client-presence") //can be any name for room
Then you can have clients key in you sockets which contains multiple client's data(id and status) and if one client's status changes you can receive change event on socket like this:
socket.on('STATUS_CHANGE',emitClientsPresence(io,namespace,currentSocket); //event name should be same on client & server side for catching and emiting
and now you want all other clients to get updated, so you can do something like this:
emitClientsPresence => (io,namespace,currentSocket) {
io.of(namespace)
.to(client-presence)
.emit('STATUS_CHANGE', { id: "client 1", status: "changed status" });
}
This will emit STATUS_CHANGE event to all sockets that have joined "client-presence" room and then you can catch same event on client side and update other client's status.
According to this Broadcasting.
With nodejs server, you can use this:
io.emit('event_id', {your_property: 'your_property_field'});
Be sure to initialise websocket, for example:
var express = require('express');
var http = require('http');
var app = express();
var server = http.Server(app);
var io = require('socket.io')(server);
app.get('/', function (req, res) {
res.send('Hello World!');
io.emit('event_hello', {message: 'Hello Socket'});
});
server.listen(3000, function () {
console.log('Example app listening on port 3000!');
});
In this case, when user reach your server, there will be "event_hello" broadcasted to all web-socket clients with a json object {message: 'Hello Socket'}.

Categories

Resources