im pretty new into javascript and node, currently working into a node.js app,
the app use express and mongoDB, the idea is listen to some third party services via webhook, websocket and mqtt and store all data into mongoDB.
but I have a litle problem, some of the third party apps send me data too often,
for example, the mqtt stream sends about 2 message every second, i need to store only one of those message every minute.
this is the way I instance mqtt into app.js
var mqttHandler = require('./mqtt/mqtt_handler'); //mqtt
var mqttClient = new mqttHandler(); //mqtt
mqttClient.connect(); //mqtt
this is my mqttHandler.js:
onst mqtt = require('mqtt');
class MqttHandler {
constructor() {
this.mqttClient = null;
this.host = 'mqtts://host';
this.username = 'foo'; // mqtt credentials if these are needed to connect
this.password = 'mypassqword';
this.port = 8083;
this.protocol = 'MQTTS';
this.client = 'bar'
}
connect() {
// Connect mqtt with credentials (in case of needed, otherwise we can omit 2nd param)
this.mqttClient = mqtt.connect(this.host, {password : this.password, username : this.username, port: this.port});
// Mqtt error calback
this.mqttClient.on('error', (err) => {
console.log(err);
this.mqttClient.end();
});
// Connection callback
this.mqttClient.on('connect', () => {
//console.log(`mqtt client connected`);
});
// mqtt subscriptions
this.mqttClient.subscribe('/the_subscription');
// When a message arrives, console.log it
this.mqttClient.on('message', function (topic, message) {
console.log(message.toString())
});
this.mqttClient.on('close', () => {
//console.log(`mqtt client disconnected`);
});
}
// Sends a mqtt message to topic: mytopic
sendMessage(message) {
this.mqttClient.publish('mytopic', message);
}
}
module.exports = MqttHandler;
i'veing reading about setInterval and setTimeout, but I can't figure out how to implement these to force a given function to only run once every X seconds (no mather how many times it is called)
could there be a similar / generic way to implement this feature for both, mqtt, webohooks and / or websocket?
I took this example about how to implement mqtt from a tutorial, its working perfect, as I said, im prettty new to javascript.
One naive approach using setInterval is to set a flag regularly and clear it once a message is posted. The ignore any other messages until the flag is set again by the interval function.
let readyToPost = false;
setInterval(function(){ readyToPost = true; }, 1000);
In your function:
function connect() {
if (!readyToPost) return; // do nothing
readyToPost = false;
// rest of your code
}
There is also a wrapper of the module mqtt:
const mqttNow = require('mqtt-now');
const options = {
host: 'localhost',
interval: 1000,
actions: [
{
topic: 'public',
message: 'my message'
},
{
topic: 'random',
message: () => ( 'random ' + Math.random() )
}
]
}
mqttNow.publish(options);
Related
First off let me start by saying I am a web developer who is very new to the Blockchain space so I apologize in advance if I am missing something obvious. With that being said I am having issues implementing a broadcast transaction method into a cryptocurrency project I am working on. Every time I am able to successfully make transaction requests to my API and I am able to see the correct transaction pool on the main dev network, however my peer network does not see any new transactions until I restart it. I know this means my broadcast transaction implementation is missing something but I am not sure what I need to fix it.
please refer to the following images
Here is the code snippets for my pubsub implementation using PUBNUB
const CHANNELS = {
TEST: "TEST",
BLOCKCHAIN: "BLOCKCHAIN",
TRANSACTION: "TRANSACTION"
};
class PubSub {
constructor({blockchain, transactionPool}) {
this.blockchain = blockchain;
this.transactionPool = transactionPool;
this.pubnub = new PubNub(credentials); // defined above but omitted for obvious reasons
this.pubnub.subscribe({channels: Object.values(CHANNELS)}); // defined in channels object
this.pubnub.addListener(this.listener());
};
listener() {
return {
message: messageObject => {
const { channel, message } = messageObject;
console.log(`Message received. Channel: ${channel}. Message: ${message}`);
const parsedMessage = JSON.parse(message);
switch(channel) {
case CHANNELS.BLOCKCHAIN:
this.blockchain.replaceChain(parsedMessage);
break;
case CHANNELS.TRANSACTION:
this.transactionPool.setTransaction(parsedMessage);
break;
default:
return;
}
}
};
}
publish({channel, message}) {
this.pubnub.unsubscribe(channel, () => {
this.pubnub.publish(channel, message, () => {
this.pubnub.subscribe(channel);
});
});
}
broadcastChain() {
this.publish({
channel: CHANNELS.BLOCKCHAIN,
message: JSON.stringify(this.blockchain.chain)
})
}
broadcastTransaction(transaction) {
this.publish({
channel: CHANNELS.TRANSACTION,
message: JSON.stringify(transaction)
});
}
};
And here is the snippets for where the broadcastTransaction method is called
const pubsub = new PubSub({blockchain, transactionPool});
let transaction = transactionPool.existingTransaction({inputAddress: wallet.publicKey}); // Creates global binding
// Sends transaction to the network
app.post("/api/transact", (req, res) => {
const {amount, recipient} = req.body;
try {
if(transaction) {
transaction.update({senderWallet: wallet, recipient, amount });
} else {
transaction = wallet.createTransaction({recipient, amount});
}
} catch(error) {
return res.status(400).json({type: "error", message: error.message});
};
transactionPool.setTransaction(transaction);
pubsub.broadcastTransaction(transaction); // Calls broadcastTransaction from pubsub class (does not work for peers)
res.json({type: "success", transaction});
});
I tried to be as specific as possible but If I missed anything please let me know. Thank you in advance!
Thank you for the input. logging helped me realize that my dev and peer ports were not subscribed when starting up the nodes. I added the following method to the PubSub class and was able to sync both the dev and peer ports
subscribeToChannels() {
this.pubnub.subscribe({
channels: [Object.values(CHANNELS)]
});
}
I have a small web application listening for incoming messages from a Websocket server. I receive them like so
const webSocket = new WebSocket("wss://echo.websocket.org");
webSocket.onopen = event => webSocket.send("test");
webSocket.onmessage = event => console.log(event.data);
but the sending server is more complex. There are multiple types of messages that could come e.g. "UserConnected", "TaskDeleted", "ChannelMoved"
How to detect which type of message was sent? For now I modified the code to
const webSocket = new WebSocket("wss://echo.websocket.org");
webSocket.onopen = event => {
const objectToSend = JSON.stringify({
message: "test-message",
data: "test"
});
webSocket.send(objectToSend);
};
webSocket.onmessage = event => {
const objectToRead = JSON.parse(event.data);
if (objectToRead.message === "test-message") {
console.log(objectToRead.data);
}
};
So do I have to send an object from the server containing the "method name" / "message type" e.g. "TaskDeleted" to identify the correct method to execute at the client? That would result in a big switch case statement, no?
Are there any better ways?
You can avoid the big switch-case statement by mapping the methods directly:
// List of white-listed methods to avoid any funny business
let allowedMethods = ["test", "taskDeleted"];
function methodHandlers(){
this.test = function(data)
{
console.log('test was called', data);
}
this.taskDeleted = function(data)
{
console.log('taskDeleted was called', data);
}
}
webSocket.onmessage = event => {
const objectToRead = JSON.parse(event.data);
let methodName = objectToRead.message;
if (allowerMethods.indexOf(methodName)>=0)
{
let handler = new methodHandlers();
handler[methodName](data);
}
else
{
console.error("Method not allowed: ", methodName)
}
};
As you have requested in one of your comments to have a fluent interface for the websockets like socket.io.
You can make it fluent by using a simple PubSub (Publish Subscribe) design pattern so you can subscribe to specific message types. Node offers the EventEmitter class so you can inherit the on and emit events, however, in this example is a quick mockup using a similar API.
In a production environment I would suggest using the native EventEmitter in a node.js environment, and a browser compatible npm package in the front end.
Check the comments for a description of each piece.
The subscribers are saved in a simple object with a Set of callbacks, you can add unsubscribe if you need it.
note: if you are using node.js you can just extend EventEmitter
// This uses a similar API to node's EventEmitter, you could get it from a node or a number of browser compatible npm packages.
class EventEmitter {
// { [event: string]: Set<(data: any) => void> }
__subscribers = {}
// subscribe to specific message types
on(type, cb) {
if (!this.__subscribers[type]) {
this.__subscribers[type] = new Set
}
this.__subscribers[type].add(cb)
}
// emit a subscribed callback
emit(type, data) {
if (typeof this.__subscribers[type] !== 'undefined') {
const callbacks = [...this.__subscribers[type]]
callbacks.forEach(cb => cb(data))
}
}
}
class SocketYO extends EventEmitter {
constructor({ host }) {
super()
// initialize the socket
this.webSocket = new WebSocket(host);
this.webSocket.onopen = () => {
this.connected = true
this.emit('connect', this)
}
this.webSocket.onerror = console.error.bind(console, 'SockyError')
this.webSocket.onmessage = this.__onmessage
}
// send a json message to the socket
send(type, data) {
this.webSocket.send(JSON.stringify({
type,
data
}))
}
on(type, cb) {
// if the socket is already connected immediately call the callback
if (type === 'connect' && this.connected) {
return cb(this)
}
// proxy EventEmitters `on` method
return super.on(type, cb)
}
// catch any message from the socket and call the appropriate callback
__onmessage = e => {
const { type, data } = JSON.parse(e.data)
this.emit(type, data)
}
}
// create your SocketYO instance
const socket = new SocketYO({
host: 'wss://echo.websocket.org'
})
socket.on('connect', (socket) => {
// you can only send messages once the socket has been connected
socket.send('myEvent', {
message: 'hello'
})
})
// you can subscribe without the socket being connected
socket.on('myEvent', (data) => {
console.log('myEvent', data)
})
Getting metric has already been registered when trying to publish metrics from service. To avoid that I used register.removeSingleMetric("newMetric"); but problem is that it clear register and past records every time a new call comes in.
Wondering what to address this problem:
export function workController(req: Request, res: Response) {
resolveFeaturePromises(featurePromises).then(featureAggregate => {
return rulesService.runWorkingRulesForHook(featureAggregate, hook.name, headers)
.then(([shouldLog, result]) => {
...
// publish metric
publishHookMetrics(result);
// publish metric
sendResponse(res, new CoreResponse(result.isWorking, result.responseData), req, featureAggregate, hook);
});
}).catch(err => {
sendResponse(res, new CoreResponse(Action.ALLOW, null), req, null, hook);
console.error(err);
});
}
function publishCustomMetrics(customInfoObject: CustomInfoObject) {
const counter = new promClient.Counter({
name: "newMetric",
help: "metric for custom detail",
labelNames: ["name", "isWorking"]
});
counter.inc({
name: customInfoObject.hook,
isWorking: customInfoObject.isWorking
});
}
stack trace
[Node] [2020-07-30T17:40:09+0500] [ERROR] Error: A metric with the name newMetric has already been registered.
application.ts
export async function startWebServer(): Promise<Server> {
if (!isGlobalsInitilized) {
throw new Error("Globals are noit initilized. Run initGlobals() first.");
}
// Setup prom express middleware
const metricsMiddleware = promBundle({
includeMethod: true,
includePath: true,
metricsPath: "/prometheus",
promClient: {
collectDefaultMetrics: {
}
}
});
// start http server
const app = require("express")();
app.use(bodyParser.json());
app.use(metricsMiddleware);
const routeConfig = require("./config/route-config");
routeConfig.configure(app);
const port = process.env.PORT || 3000;
return app.listen(port, function () {
console.log("service listening on port", port);
});
}
versions:
express-prom-bundle: 6.0.0
prom-client: 12.0.0
The counter should be initialize only once, then used for each call. Simplest modification from code you gave would be something like this.
const counter = new promClient.Counter({
name: "newMetric",
help: "metric for custom detail",
labelNames: ["name", "isWorking"]
});
export function publishCustomMetrics(customInfoObject: CustomInfoObject) {
counter.inc({
name: customInfoObject.hook,
isWorking: customInfoObject.isWorking
});
}
BTW, I found this thread while searching a way to avoid the "already been registered" error in unit tests. Many tests was instantiating the same class (from a library) that was initializing a counter in the constructor.
As it is in tests, and I didn't need consistency over the metrics, I found an easy solution is to clear metrics registers at the beginning of each test.
import { register } from "prom-client";
// ...
register.clear();
I'm trying to create game using socket io. I want to implement socket inside server.js but I would like to keep events inside Room.js.:
server.js
const io = require('socket.io').listen(server)
io.sockets.on('connection', socket => {
socket.on('join room', data => {
let room = new Room(data.id, socket)
socket.join(`room${data.id}`)
}
}
Room.js
class Room{
constuctor(id, socket){
this.id = id,
this.socket = socket,
this.handlerOfEvents()
}
handlerOfEvents() {
this.socket.on('new player connected', data => {
console.log('New player connected!')
}
}
}
I tried do as above but it doesn't work:
\node_modules\has-binary2\index.js:30
function hasBinary (obj) {
^
RangeError: Maximum call stack size exceeded
Is there any solution to do sth like this?
Or maybe there is another perfect way to implement events for particular room.
This problem occurs when I try assign socket to this.socket but when I put it directly as argument like below:
class Room{
constuctor(id, socket){
this.id = id,
//this.socket = socket,
this.handlerOfEvents(socket)
}
handlerOfEvents(socket) {
socket.on('new player connected', data => {
console.log('New player connected!')
}
}
}
then just invoke this method in server.js
socket.on('joined new player', data => {
...
room.handlerOfEvents(socket)
}
so, this solution is working for now. But if this is proper way?
Issue from link in comments didn't solve my problem unfortunately.
Thanks for helping!
I want to split my rabbitMQ connection code and call it across different components, so that it (the connection and channel) only initializes ONCE and I can use it whenever instead of having to open the connection again when I want to use it.
What happens right now is, I call the below's code function over and over again everytime I want to pass something to my exchange and queue. (so if I want to pass 20 individual data to rabbitMQ, I ended up opening and closing both the connection and channel 20 times)
Any solutions?
const exchange = "Exchange";
const queue = "Queue";
const passSomeData= async payload => {
amqp = require("amqplib").connect("amqp://localhost");
let ch;
let connection;
let publish = amqp
.then(function(conn) {
connection = conn;
return conn.createConfirmChannel();
})
.then(function(chn) {
ch = chn;
ch.assertQueue(queue, { durable: true });
return ch.assertExchange(exchange, "topic", { durable: true });
})
.then(function() {
const data = {
content: "x",
title: "y",
};
ch.bindQueue(queue, exchange, "routingKey");
return ch.publish(exchange, "routingKey", Buffer.from(JSON.stringify(data)), {
persistent: true
});
})
.then(() => {
setTimeout(function() {
connection.close();
}, 250);
});
};
module.exports = passSomeData;
Answer copied from here
This is a general Javascript question and not one specific to RabbitMQ or the amqplib library.
I believe you can open a connection at the module level and use that within your passSomeData method. Or, passSomeData can lazily open a connection if the module-level "connection" variable is null, and then re-use that connection.
At some point you may need to use a connection pool, but that depends on your use-case and workload.
NOTE: the RabbitMQ team monitors the rabbitmq-users mailing list and only sometimes answers questions on StackOverflow.