Develop Reference

How to keep multiple requests separate in Nodejs / Expressjs

I am developing a NodeJS / ExpressJS application on my computer. I have node running locally. I have a single page web app. When it needs information it makes ajax requests with jQuery.
The problem is when I have multiple requests for different sets of data.
Node/express will start processing the first request and then the second request comes in before the first request has been fulfilled, it sends the response to the second request from the first request instead of sending it to the first request like it is suppose to. If I put a pause ( using an alert ) in my app so that is slows it down so the next request doesn't get sent until the first request was fulfilled, everything works fine.
I don't understand why this is happening. I thought node / express was suppose to be able to handle this and keep the requests separate.
Also, I get a "Can't set headers after they are sent" error in node because it is apparently merging the requests....
Here is what happens
ajax request 1 -> server
ajax request 2 -> server
ajax request 3 -> server
server -> request1 ( no response )
server -> request2 ( request 1's data)
server -> request3 ( request 2's data)
server for request3 --> error: header's already sent
I am using Google Chrome 63 with jQuery 3.3.1 on a Windows 10 machine running Node 8.9.4 and Express 4.16.2
My work-around is to chain the ajax requests so that each request doesn't get called until the prior request has received a response from the server. But I shouldn't have to do that...
Here is the relevant server code:
var mysql = require("mysql");
var express = require('express');
var app = express();
var DEBUG = true;
var request = null;
var response = null;
var currentDataRowParser = null;
var con = mysql.createConnection(config);
function ParseMySqlRowData(rowData)
{
if (DEBUG) console.log("ParseMySqlRowData");
return JSON.stringify(rowData);
}
var ParseMySqlRowsDatatoResponse = function (err, rows)
{
if (DEBUG) console.log("ParseMySqlRowsDatatoResponse");
var MySQLRows;
try
{
if (!err)
{
MySQLRows = "[";
for (var i = 0; i < rows.length; i++)
{
if (i > 0)
MySQLRows += ", ";
MySQLRows += currentDataRowParser(rows[i]);
}
MySQLRows += "]";
if (DEBUG) console.log("write rows");
if (DEBUG) console.log(MySQLRows);
response.send(MySQLRows);
response.end();
}
}
catch (ex)
{
if (DEBUG) console.log("ParseMySQLRowsDatatoResponse: ERROR");
if (DEBUG) console.log(ex);
}
};
var GetQueryData = function (query, dataRowParser, parseDataCallbackFunction)
{
if (DEBUG) console.log("GetQueryData");
currentDataRowParser = dataRowParser;
if (parseDataCallbackFunction == null || parseDataCallbackFunction == undefined)
parseDataCallbackFunction = ParseDataCallback;
try
{
if (DEBUGQUERY)
{
console.log("before query");
console.log(con.query(query, parseDataCallbackFunction));
console.log("after query");
console.log(query.sql);
DEBUGQUERY = false;
}
else
{
con.query(query, parseDataCallbackFunction);
}
}
catch (ex)
{
console.log(ex);
}
};
app.post('/getdata', function(req, res)
{
request = req;
response = res;
var query;
switch (request.body.loaddata)
{
case "dataset1":
query = "SELECT * FROM table1 WHERE key='" + request.body.key + "'";
GetQueryData(query,ParseMySqlRowData,ParseMySqlRowsDatatoResponse);
break;
case "dataset2":
query = "SELECT * FROM table2 WHERE key='" + request.body.key + "'";
GetQueryData(query,ParseMySqlRowData,ParseMySqlRowsDatatoResponse);
break;
case "dataset3":
query = "SELECT * FROM table3 WHERE key='" + request.body.key + "'";
GetQueryData(query,ParseMySqlRowData,ParseMySqlRowsDatatoResponse);
break;
}
};

You cannot store req and res in global or module-level variables. When a second request comes in, it will immediately overwrite your globals and it will mix up the data for the various requests.
Does't node separate each request instance?
Yes, there is a separate request instance, but not a separate global or module-level namespace. So, when you assign the req into the global space, you are overwriting the previous one and your code will then use the wrong one.
It is very helpful to have the request and response as a global variable. Otherwise I would have to be passing them all over the place.
You HAVE to pass them to lower level functions that need them. That's how you keep each request separate from the others. ANY function that needs to operate on req or res should be passed those variables so it knows exactly which ones to operate on.
node.js has a shared global and module-level namespace. So, all requests in flight at the same time use that same namespace. The ONLY data that should ever be stored there is data that you specifically want to be shared between requests (such as session state, for example). Individual request or response objects should never be stored in a shared variable.
A more common way to code your type of code is to call a function like your GetQueryData() and have it return the data (likely via a promise) and then you send the response in the original request handler. Then, you don't have to pass req or res down multiple levels at all. Your helper functions just fetch data. The request handlers fetch data, then send the response. It's often a better encapsulation of functionality.
Here's one way to restructure your code along the lines described above.
GetQueryData() returns a promise that fulfills with the data
ParseMySqlRowsData() just returns a parsed result or null if a parsing error
app.post() just gets the data (via a promise) and then sends the appropriate response.
There's no global req or res and there's no need to pass them anywhere.
Here's the code:
var mysql = require("mysql");
var express = require('express');
var app = express();
var DEBUG = true;
var currentDataRowParser = null;
var con = mysql.createConnection(config);
function ParseMySqlRowData(rowData) {
if (DEBUG) console.log("ParseMySqlRowData");
return JSON.stringify(rowData);
}
var ParseMySqlRowsData = function(err, rows) {
if (DEBUG) console.log("ParseMySqlRowsDatatoResponse");
var MySQLRows;
try {
if (!err) {
MySQLRows = "[";
for (var i = 0; i < rows.length; i++) {
if (i > 0)
MySQLRows += ", ";
MySQLRows += currentDataRowParser(rows[i]);
}
MySQLRows += "]";
if (DEBUG) console.log("write rows");
if (DEBUG) console.log(MySQLRows);
return MySQLRows;
}
} catch (ex) {
if (DEBUG) console.log("ParseMySQLRowsDatatoResponse: ERROR");
if (DEBUG) console.log(ex);
return null;
}
};
var GetQueryData = function(query, dataRowParser, parseDataCallbackFunction) {
return new Promise((resolve, reject) =>{
if (DEBUG) console.log("GetQueryData");
let currentDataRowParser = dataRowParser;
if (parseDataCallbackFunction == null || parseDataCallbackFunction == undefined)
parseDataCallbackFunction = ParseDataCallback;
try {
if (DEBUGQUERY) {
console.log("before query");
console.log(con.query(query, parseDataCallbackFunction));
console.log("after query");
console.log(query.sql);
DEBUGQUERY = false;
} else {
con.query(query, function(err, rows) {
if (err) {
reject(err);
} else {
let result = parseDataCallbackFunction(rows);
if (result) {
resolve(result);
} else {
reject(new Error("ParseMySqlRowsData error"));
}
}
});
}
} catch (ex) {
console.log(ex);
reject(new Error("GetQueryData error"));
}
});
};
app.post('/getdata', function(req, res) {
var query;
let p;
switch (request.body.loaddata) {
case "dataset1":
query = "SELECT * FROM table1 WHERE key='" + request.body.key + "'";
p = GetQueryData(query, ParseMySqlRowData, ParseMySqlRowsData);
break;
case "dataset2":
query = "SELECT * FROM table2 WHERE key='" + request.body.key + "'";
p = GetQueryData(query, ParseMySqlRowData, ParseMySqlRowsData);
break;
case "dataset3":
query = "SELECT * FROM table3 WHERE key='" + request.body.key + "'";
p = GetQueryData(query, ParseMySqlRowData, ParseMySqlRowsData);
break;
default:
p = Promise.reject(new Error("Invalid request.body.loaddata"));
break;
}
p.then(data => {
res.send(data);
}).catch(err => {
console.log(err);
res.sendStatus(500);
});
};
P.S. I see you still have a module-level variable you should not have: currentDataRowParser. That needs to be replaced by passing the appropriate parser to ParseMySqlRowsData() as an argument, not using a module-level shared variable. I will leave that as an excercise for you to fix. Shared variables for operating on a specific request state are a bad idea in ANY server programming and specifically in node.js programming. Keep your request-specific state in your function arguments or on the req or res object itself. That's how you prevent overwritting the data from one request with the data from another while they are being processed.

How to stream tweets from specific accounts

I'm using the twitter npm package in an attempt to stream tweets from specified accounts.
I'm having trouble navigating the twitter api docs, so I'm a little confused.
I can hit the REST endpoint to get the specified user tweets info with:
var client = new Twitter({});
client.get('statuses/user_timeline', { screen_name: user }, function(error, tweets) {
if(error) throw error;
console.log(tweets);
});
How do I stream the tweets? Is it even possible? If not, how else could I accomplish this? I would like this to be as responsive and immediate as possible.

Figured it out...
var stream = client.stream('statuses/filter', { follow: userId });
stream.on('data', function(event) {
console.log(event && event.text);
});
This will client stream reader will display the tweets as they are made automatically.
Unfortunately, the screen_name of the user cannot be used, so you'll have to find that out beforehand.

I wrote the code below and was able to get last 20 tweets of a user and save them in a file log.txt.
var Twitter = require('twitter');
var fs = require("fs");
//Your keys go inside {}
var client = new Twitter({});
//These are the parameters for the API call
var params = {
screen_name: 'damiengold', //use damiengold instead of #damiengold
count: 20
};
//Perform the API call and return data
client.get('statuses/user_timeline.json', params, function(error, tweets, response) {
var dataToFile;
if (!error) {
for (var i = 0; i < tweets.length; i++) {
console.log(tweets[i].created_at + " " + tweets[i].text);
console.log("-------------------------------");
dataToFile = tweets[i].created_at + " " + tweets[i].text + "\n-----------\n";
//This block of code will append tweets to the file called "log.txt".
fs.appendFile("log.txt", dataToFile, function(err) {
// If the code experiences any errors it will log the error to the console.
if (err) {
return console.log(err);
}
})
}
}
console.log("Last 20 tweets are now displayed on the screen and saved in log.txt.");
});

Telegram API returning HTML instead of JSON

I'm writing a telegram bot to report fail2ban bans. It's very simple and dirty, written hastily, but it can be used to report any message to a single telegram user:
var TelegramBot = require('node-telegram-bot-api');
var fs = require('fs');
var store = {
get: function (key) {
return fs.readFileSync(__dirname + '/' + key, { encoding: 'utf-8' });
},
set: function (key, value) {
fs.writeFileSync(__dirname + '/' + key, value, { encoding: 'utf-8' });
}
};
var token = store.get('token');
var args = process.argv.slice(2);
if (args.length == 0) {
console.error('No mode specified');
process.exit(0);
}
TelegramBot.prototype.unregisterText = function (regexp) {
for (var i = 0; i < bot.textRegexpCallbacks.length; ++i) {
if (bot.textRegexpCallbacks[i].regexp.toString() == regexp) {
bot.textRegexpCallbacks.splice(i, 1);
return;
}
}
};
fs.appendFileSync(__dirname + '/logs',
'[' + (new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')) + '] '
+ args.join(' ') + '\n',
{ encoding: 'utf-8' });
switch (args[0]) {
case 'setup':
var bot = new TelegramBot(token, { polling: true });
var step = 'none';
bot.onText(/\/setup/, function (msg, match) {
var fromId = msg.from.id;
step = 'setup-started';
bot.sendMessage(fromId, 'Starting setup. Please enter the verification key.');
bot.onText(/(.+)/, function (msg, match) {
if (step == 'setup-started') {
var key = match[1];
var verification = store.get('key');
if (key == verification) {
store.set('owner', msg.from.id);
step = 'verified';
bot.sendMessage(msg.from.id, 'Correct. Setup complete.');
} else {
step = 'none';
bot.unregisterText(/(.+)/);
bot.sendMessage(msg.from.id, 'Wrong. Setup aborted.');
}
}
});
});
break;
case 'report':
var bot = new TelegramBot(token, { polling: false });
var owner = store.get('owner');
var subject = args[1];
if (subject == 'message') {
var message = args.slice(2).join(' ');
bot.sendMessage(owner, message);
} else if (subject == 'file') {
var content = fs.readFileSync(args[2], { encoding: 'utf-8' });
bot.sendMessage(owner, content);
}
break;
default:
console.error('Unrecognized mode', args[0]);
break;
}
On my developer machine it works fine. I invoke:
node bot.js report message whatever message i want
And I correctly received "whatever message i want" on telegram. However, once I gitted it on my digitalocean vps, it no longer worked. It turns out the problem is with the telegram library:
Unhandled rejection Error: Error parsing Telegram response: <!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Bots: An introduction for developers</title>
...
Which apparently returns an html page instead of json... I also tried to contact the same endpoint (api.telegram.org/bothash/sendMessage) with curl on my vps and it returned json (with an error message because i didnt send any parameters, but still json).
I cannot fathom why this happens. Any help?

It seems like either you don't have a file with token on your VPN or the token is incorrect.
You can check it by yourself:
When you make a request to api.telegram.org/{token}/sendMessage, and {token} is incorrect, it redirects you to this page, which responds with HTML you've mentioned in your question.
So you have to debug a behavior of your store.get and store.get functions along with files and tokens to make sure you are using a correct one.
Also, I'd recommend to run bot.getMe() before using any other Telegram API methods to ensure you specified a correct bot token.

Get subdomain and query database for results - Meteor

I am pretty new to Meteor now.
I want to:
get the sub-domain of the url
check if a client exists matching the sub-domain
if client exists query the database and get some results (say client settings) from the database.
I am sure that this would be a piece of cake if we use MongoDB, however, we have to move an existing application (built on PHP) that has MySQL backend.
I have found a package numtel:mysql for meteor and I have added it to the project.
Here is the source code written so far:
if(!Session.get('client_details')) {
var hostnameArray = document.location.hostname.split('.');
if(hostnameArray[1] === "localhost" && hostnameArray[2] === "local") {
var subdomain = hostnameArray[0];
}
if(subdomain) {
currentClientDetails = new MysqlSubscription('getClientDetailsFromUrl', subdomain).reactive();
Tracker.autorun(function() {
if(currentClientDetails.ready()) {
if(currentClientDetails.length > 0) {
var clientDetails = currentClientDetails[0];
Session.setPersistent('client_details', clientDetails);
var clientId = clientDetails.id;
if(!Session.get('client_settings')) {
clientSettings = new MysqlSubscription('clientSettings', clientId).reactive();
Tracker.autorun(function() {
if(clientSettings.ready()) {
if(clientSettings.length > 0)
Session.setPersistent('client_settings', clientSettings[0]);
else
Session.setPersistent('client_settings', {});
}
});
}
}
}
});
}
}
the session.setPersistent comes from u2622:persistent-session to store Sessions on client side
and here is the publish statement:
Meteor.publish("getClientDetailsFromUrl", function(url) {
if(typeof url === undefined) {
return;
}
var clientDetails = Meteor.readLiveDb.select(
'select * from clients where client_url = "'+ url +'"',
[{table: 'clients'}]
);
return clientDetails;
});
Meteor.publish("clientSettings", function(clientId) {
if(typeof clientId === undefined) {
throw new error('ClientId cannot be null');
return;
}
var clientSettings = Meteor.readLiveDb.select(
'select * from client_settings where client_id = ' + clientId, [{
table: 'client_settings'
}]);
return clientSettings;
});
and the database is initiated as
Meteor.readLiveDb = new LiveMysql(<MySQL Settings like host, user, passwords, etc>);
Problem
I get client_details into the session successfully, however, cannot get client_settings into the session. End up with an error:
Exception from Tracker recompute function:
Error: Subscription failed!
at Array.MysqlSubscription (MysqlSubscription.js:40)
at app.js?6d4a99f53112f9f7d8eb52934c5886a2b7693aae:28
at Tracker.Computation._compute (tracker.js:294)
at Tracker.Computation._recompute (tracker.js:313)
at Object.Tracker._runFlush (tracker.js:452)
at onGlobalMessage (setimmediate.js:102)
I know the code is messy and could get a lot better, please suggestions welcome

how does one combine kafka-node producer and node tail?

I want to combine kafka-node and node tail to watch some log files and send any new entry to a kafka cluster. I have the following code which works.
var info={
"zookeepers":"127.0.0.1:9500",
"topics":{
"server":[
"/tmp/server-1.log",
"/tmp/server-2.log"
],
"audit":[
"/tmp/audit-1.log",
"/tmp/audit-2.log"
]
}
}
function dldmProducer (logFiletoTail, fileTopic, zookeepers){
var kafka = require('kafka-node');
var Producer = kafka.Producer;
var client = new kafka.Client(zookeepers);
var producer = new Producer(client);
var partitionNo = 0
producer.on('ready', function () {
var Tail = require('tail').Tail;
var tail = new Tail(logFiletoTail);
tail.on("line", function(data) {
dataArray=data.split("\n")
var payloads = [{ topic: fileTopic, messages: dataArray, partition: partitionNo },];
partitionNo = (partitionNo +1)%3;
producer.send(payloads, function(err, data){
//console.log(payloads);
//console.log(data);
});
});
tail.on("error", function(error) {
//console.log('TAIL ERROR: ', error);
});
});
producer.on('error', function(error){
//console.log('PRODUCER ERROR: ', error)
});
}
for (var topicItem in info['topics']){
for (var i = 0; i<info['topics'][topicItem].length; i++){
dldmProducer(info['topics'][topicItem][i],topicItem,info['zookeepers'])
}
}
But then the problem is, error handling is non-existent, so i'm worried that if things go wrong during producer.send(payloads,function(err,data){}), the payloads would just be lost forever. I rewrote the function dldmProducer to the following trying to catch any unsuccessfully sent payloads and resend them. (if the payloads are successfully sent, the response would be in data. data is undefined otherwise, or at least that's my understanding)
function dldmProducer (logFiletoTail, fileTopic, zookeepers){
var kafka = require('kafka-node');
var Producer = kafka.Producer;
var client = new kafka.Client(zookeepers);
var producer = new Producer(client);
var partitionNo = 0
var buffer = []
var Tail = require('tail').Tail;
var tail = new Tail(logFiletoTail);
tail.on("line", function(data) {
var dataArray=data.split("\n")
console.log(dataArray)
buffer = buffer.concat(dataArray)
});
tail.on("error", function(error) {
console.log('TAIL ERROR: ', error);
});
producer.on('ready', function () {
while (true){
var logItem = buffer.shift()
if (logItem){
console.log(logItem)
var payloads = [{ topic: fileTopic, messages: [logItem], partition: partitionNo },];
partitionNo = (partitionNo +1)%3;
producer.send(payloads, function(error, data){
if (!data){
console.log(data)
buffer.push(logItem)
}
if (error){
console.log('KAFKA RESPONSE ERROR: ', error)
}
});
}
}
});
producer.on('error', function(error){
console.log('PRODUCER ERROR: ', error)
});
}
The function doesn't seem to work. It doesn't even tail on the files correctly and hardly any information from the file is logged on screen even though i'm constantly writing dummy contents into the files. I'm still not used to javascript kind of thinking so any pointer would be great.