I am developing a NodeJS / ExpressJS application on my computer. I have node running locally. I have a single page web app. When it needs information it makes ajax requests with jQuery.
The problem is when I have multiple requests for different sets of data.
Node/express will start processing the first request and then the second request comes in before the first request has been fulfilled, it sends the response to the second request from the first request instead of sending it to the first request like it is suppose to. If I put a pause ( using an alert ) in my app so that is slows it down so the next request doesn't get sent until the first request was fulfilled, everything works fine.
I don't understand why this is happening. I thought node / express was suppose to be able to handle this and keep the requests separate.
Also, I get a "Can't set headers after they are sent" error in node because it is apparently merging the requests....
Here is what happens
ajax request 1 -> server
ajax request 2 -> server
ajax request 3 -> server
server -> request1 ( no response )
server -> request2 ( request 1's data)
server -> request3 ( request 2's data)
server for request3 --> error: header's already sent
I am using Google Chrome 63 with jQuery 3.3.1 on a Windows 10 machine running Node 8.9.4 and Express 4.16.2
My work-around is to chain the ajax requests so that each request doesn't get called until the prior request has received a response from the server. But I shouldn't have to do that...
Here is the relevant server code:
var mysql = require("mysql");
var express = require('express');
var app = express();
var DEBUG = true;
var request = null;
var response = null;
var currentDataRowParser = null;
var con = mysql.createConnection(config);
function ParseMySqlRowData(rowData)
{
if (DEBUG) console.log("ParseMySqlRowData");
return JSON.stringify(rowData);
}
var ParseMySqlRowsDatatoResponse = function (err, rows)
{
if (DEBUG) console.log("ParseMySqlRowsDatatoResponse");
var MySQLRows;
try
{
if (!err)
{
MySQLRows = "[";
for (var i = 0; i < rows.length; i++)
{
if (i > 0)
MySQLRows += ", ";
MySQLRows += currentDataRowParser(rows[i]);
}
MySQLRows += "]";
if (DEBUG) console.log("write rows");
if (DEBUG) console.log(MySQLRows);
response.send(MySQLRows);
response.end();
}
}
catch (ex)
{
if (DEBUG) console.log("ParseMySQLRowsDatatoResponse: ERROR");
if (DEBUG) console.log(ex);
}
};
var GetQueryData = function (query, dataRowParser, parseDataCallbackFunction)
{
if (DEBUG) console.log("GetQueryData");
currentDataRowParser = dataRowParser;
if (parseDataCallbackFunction == null || parseDataCallbackFunction == undefined)
parseDataCallbackFunction = ParseDataCallback;
try
{
if (DEBUGQUERY)
{
console.log("before query");
console.log(con.query(query, parseDataCallbackFunction));
console.log("after query");
console.log(query.sql);
DEBUGQUERY = false;
}
else
{
con.query(query, parseDataCallbackFunction);
}
}
catch (ex)
{
console.log(ex);
}
};
app.post('/getdata', function(req, res)
{
request = req;
response = res;
var query;
switch (request.body.loaddata)
{
case "dataset1":
query = "SELECT * FROM table1 WHERE key='" + request.body.key + "'";
GetQueryData(query,ParseMySqlRowData,ParseMySqlRowsDatatoResponse);
break;
case "dataset2":
query = "SELECT * FROM table2 WHERE key='" + request.body.key + "'";
GetQueryData(query,ParseMySqlRowData,ParseMySqlRowsDatatoResponse);
break;
case "dataset3":
query = "SELECT * FROM table3 WHERE key='" + request.body.key + "'";
GetQueryData(query,ParseMySqlRowData,ParseMySqlRowsDatatoResponse);
break;
}
};
You cannot store req and res in global or module-level variables. When a second request comes in, it will immediately overwrite your globals and it will mix up the data for the various requests.
Does't node separate each request instance?
Yes, there is a separate request instance, but not a separate global or module-level namespace. So, when you assign the req into the global space, you are overwriting the previous one and your code will then use the wrong one.
It is very helpful to have the request and response as a global variable. Otherwise I would have to be passing them all over the place.
You HAVE to pass them to lower level functions that need them. That's how you keep each request separate from the others. ANY function that needs to operate on req or res should be passed those variables so it knows exactly which ones to operate on.
node.js has a shared global and module-level namespace. So, all requests in flight at the same time use that same namespace. The ONLY data that should ever be stored there is data that you specifically want to be shared between requests (such as session state, for example). Individual request or response objects should never be stored in a shared variable.
A more common way to code your type of code is to call a function like your GetQueryData() and have it return the data (likely via a promise) and then you send the response in the original request handler. Then, you don't have to pass req or res down multiple levels at all. Your helper functions just fetch data. The request handlers fetch data, then send the response. It's often a better encapsulation of functionality.
Here's one way to restructure your code along the lines described above.
GetQueryData() returns a promise that fulfills with the data
ParseMySqlRowsData() just returns a parsed result or null if a parsing error
app.post() just gets the data (via a promise) and then sends the appropriate response.
There's no global req or res and there's no need to pass them anywhere.
Here's the code:
var mysql = require("mysql");
var express = require('express');
var app = express();
var DEBUG = true;
var currentDataRowParser = null;
var con = mysql.createConnection(config);
function ParseMySqlRowData(rowData) {
if (DEBUG) console.log("ParseMySqlRowData");
return JSON.stringify(rowData);
}
var ParseMySqlRowsData = function(err, rows) {
if (DEBUG) console.log("ParseMySqlRowsDatatoResponse");
var MySQLRows;
try {
if (!err) {
MySQLRows = "[";
for (var i = 0; i < rows.length; i++) {
if (i > 0)
MySQLRows += ", ";
MySQLRows += currentDataRowParser(rows[i]);
}
MySQLRows += "]";
if (DEBUG) console.log("write rows");
if (DEBUG) console.log(MySQLRows);
return MySQLRows;
}
} catch (ex) {
if (DEBUG) console.log("ParseMySQLRowsDatatoResponse: ERROR");
if (DEBUG) console.log(ex);
return null;
}
};
var GetQueryData = function(query, dataRowParser, parseDataCallbackFunction) {
return new Promise((resolve, reject) =>{
if (DEBUG) console.log("GetQueryData");
let currentDataRowParser = dataRowParser;
if (parseDataCallbackFunction == null || parseDataCallbackFunction == undefined)
parseDataCallbackFunction = ParseDataCallback;
try {
if (DEBUGQUERY) {
console.log("before query");
console.log(con.query(query, parseDataCallbackFunction));
console.log("after query");
console.log(query.sql);
DEBUGQUERY = false;
} else {
con.query(query, function(err, rows) {
if (err) {
reject(err);
} else {
let result = parseDataCallbackFunction(rows);
if (result) {
resolve(result);
} else {
reject(new Error("ParseMySqlRowsData error"));
}
}
});
}
} catch (ex) {
console.log(ex);
reject(new Error("GetQueryData error"));
}
});
};
app.post('/getdata', function(req, res) {
var query;
let p;
switch (request.body.loaddata) {
case "dataset1":
query = "SELECT * FROM table1 WHERE key='" + request.body.key + "'";
p = GetQueryData(query, ParseMySqlRowData, ParseMySqlRowsData);
break;
case "dataset2":
query = "SELECT * FROM table2 WHERE key='" + request.body.key + "'";
p = GetQueryData(query, ParseMySqlRowData, ParseMySqlRowsData);
break;
case "dataset3":
query = "SELECT * FROM table3 WHERE key='" + request.body.key + "'";
p = GetQueryData(query, ParseMySqlRowData, ParseMySqlRowsData);
break;
default:
p = Promise.reject(new Error("Invalid request.body.loaddata"));
break;
}
p.then(data => {
res.send(data);
}).catch(err => {
console.log(err);
res.sendStatus(500);
});
};
P.S. I see you still have a module-level variable you should not have: currentDataRowParser. That needs to be replaced by passing the appropriate parser to ParseMySqlRowsData() as an argument, not using a module-level shared variable. I will leave that as an excercise for you to fix. Shared variables for operating on a specific request state are a bad idea in ANY server programming and specifically in node.js programming. Keep your request-specific state in your function arguments or on the req or res object itself. That's how you prevent overwritting the data from one request with the data from another while they are being processed.
Related
Employee webpage makes Ajax calls to the node.js web server in a loop. Code as given. All data values are correct. I expect the callback UpdateTeamArr to be called n times where n is equal to the loop max - document.getElementById("deptSelect").options.length. But its called only once. Thanks for your effort and support.
Client side code:
for (var i = 1; i < document.getElementById("deptSelect").options.length; i++) {
var objJSON = {
"deptid": document.getElementById("deptSelect").options[i].dataset.id,
"empid": selectedEmployeeId
}
var strJSON = JSON.stringify(objJSON);
var xmlhttp = new XMLHttpRequest();
xmlhttp.open("post", "../../GetEmployeesTeams", true);
xmlhttp.onreadystatechange = function() {
if(xmlhttp.readyState === XMLHttpRequest.DONE && xmlhttp.status === 200) {
UpdateTeamArr(xmlhttp);
}
}
xmlhttp.setRequestHeader("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8");
xmlhttp.send("strJSON=" + strJSON);
}
}
function UpdateTeamArr(xmlhttp) {
}
Server code:
app.post('/GetEmployeesTeams', function(req, res) {
var connection = mysql.createConnection({
host : "127.0.0.1",
port : 3306,
user : "root",
password : "root",
database : "lighting"
});
var strJSON = req.param('strJSON');
var objJSON = JSON.parse(strJSON);
connection.connect();
connection.query("SELECT db_teamemp.teamid, db_department.id AS deptid FROM lighting.db_teamemp, lighting.db_department, lighting.db_team WHERE db_teamemp.empid='" +
objJSON.empid + "' AND db_department.id='" + objJSON.deptid + "' AND db_teamemp.teamid=db_team.id AND db_team.dept=db_department.id;",
function(err, result, fields) {
if (err)
throw err;
else {
connection.end();
res.status(200);
return res.send(result);
}
});
});
Ah, you are using a var here for xmlhttp. A var is not block scoped, it's hoisted - that means this single var is used by all calls to UpdateTeamArr. I believe you are calling the function N times, but with the last response every time.
An easy test of this theory is simply changing var to let on that line.
Why don't you try to perform just a single request to the server by creating a uniq JSONArray with a list containing all Employees id's inside your 'deptSelect'?
This way you send a response with also a list containing the other attributes for the employees in another JSONArray format, that you can iterate in the UpdateTeamArr function
So I'm hosting a node.js file on my website, and I'm trying to get data through cookies.
I am using editmycookie, and I can see that the cookies ARE set.
I have a function that gets the cookies from a name
function parseCookies(request, finding){
rc = request.headers.cookie + ';';
rc && rc.split(';').forEach(function( cookie ) {
var parts = cookie.split('=');
if(parts.shift().trim() == finding){
return decodeURI(parts.join('=')).replace(/-/g, '=');
}
});
}
Then I run this, knowing the cookies are set
app.get('/', function(req, res){
user = parseCookies(req, 'user');
pass = parseCookies(req, 'pass');
console.log(user + pass);
and it logs as NaN
I'm still learning Node, sorry.
I'm on Ubuntu 16.04 if that helps!\
Weirdly enough, it works for some users on my site, and for some it doesn't. I haven't noticed a pattern of whose work and whose do not.
You do not return anything from the parseCookies function, the return decodeURI(parts.join('=')).replace(/-/g, '=') is called with the callback of the foreach, so this return is meaningless.
Because for that parseCookies returns undefined and undefined + undefined is NaN.
When using forEach you need to save the matched result in an temporary variable result, and return this one.
function parseCookies(request, finding) {
var result;
var rc = request.headers.cookie + ';';
rc && rc.split(';').forEach(function(cookie) {
var parts = cookie.split('=');
if (parts.shift().trim() == finding) {
result = decodeURI(parts.join('=')).replace(/-/g, '=');
}
});
return result;
}
Or use Array.prototype.find (I didn't have time to test that version so there might be a bug in it):
function parseCookies(request, finding) {
var rc = request.headers.cookie + ';';
var cookie = rc.split(';').find(cookie => cookie.split('=').shift().trim() == finding);
return decodeURI(cookie.split('=').join('=')).replace(/-/g, '=');
}
But why do you parse the cookies your self anyway. There are robust and well tested middlewares that will do this for you.
Set the cookies using this code and lets see how it works
var http = require('http');
function parseCookies (request) {
var list = {},
rc = request.headers.cookie;
rc && rc.split(';').forEach(function( cookie ) {
var parts = cookie.split('=');
list[parts.shift().trim()] = decodeURI(parts.join('='));
});
return list;
}
http.createServer(function (request, response) {
// To Read a Cookie
var cookies = parseCookies(request);
// To Write a Cookie
response.writeHead(200, {
'Set-Cookie': 'mycookie=test',
'Content-Type': 'text/plain'
});
response.end('Hello World\n');
}).listen(8124);
console.log('Server running at http://127.0.0.1:8124/');
And add your routes to which you want to call.Hope this hepls for you.
I'm using the twitter npm package in an attempt to stream tweets from specified accounts.
I'm having trouble navigating the twitter api docs, so I'm a little confused.
I can hit the REST endpoint to get the specified user tweets info with:
var client = new Twitter({});
client.get('statuses/user_timeline', { screen_name: user }, function(error, tweets) {
if(error) throw error;
console.log(tweets);
});
How do I stream the tweets? Is it even possible? If not, how else could I accomplish this? I would like this to be as responsive and immediate as possible.
Figured it out...
var stream = client.stream('statuses/filter', { follow: userId });
stream.on('data', function(event) {
console.log(event && event.text);
});
This will client stream reader will display the tweets as they are made automatically.
Unfortunately, the screen_name of the user cannot be used, so you'll have to find that out beforehand.
I wrote the code below and was able to get last 20 tweets of a user and save them in a file log.txt.
var Twitter = require('twitter');
var fs = require("fs");
//Your keys go inside {}
var client = new Twitter({});
//These are the parameters for the API call
var params = {
screen_name: 'damiengold', //use damiengold instead of #damiengold
count: 20
};
//Perform the API call and return data
client.get('statuses/user_timeline.json', params, function(error, tweets, response) {
var dataToFile;
if (!error) {
for (var i = 0; i < tweets.length; i++) {
console.log(tweets[i].created_at + " " + tweets[i].text);
console.log("-------------------------------");
dataToFile = tweets[i].created_at + " " + tweets[i].text + "\n-----------\n";
//This block of code will append tweets to the file called "log.txt".
fs.appendFile("log.txt", dataToFile, function(err) {
// If the code experiences any errors it will log the error to the console.
if (err) {
return console.log(err);
}
})
}
}
console.log("Last 20 tweets are now displayed on the screen and saved in log.txt.");
});
var client = require('http');
var endpoint = apiEndpoint;
var request = client.get(endpoint, function(responseFromApi) {
var responseString = '';
responseFromApi.setEncoding('utf-8');
responseFromApi.on('data', function(data) {
responseString += data;
});
// To reformat the string response into Json...
responseFromApi.on('end', function() {
var jsonResponse = JSON.parse(responseString);
callback(jsonResponse);
});
});
I am making API calls using the method above, however on random instances my call fails due to the Assertion fail like below. Anyone has any idea how to fix this?
Assertion failed: (handle->type == UV_TCP || handle->type == UV_TTY || handle->type == UV_NAMED_PIPE), function uv___stream_fd, file ../deps/uv/src/unix/stream.c, line 1568.
Environment: Mac, Nodejs
Note: I have tested the same code on an AWS lambda server and never faced this issue. I am guessing this is a Mac only instance. Lord Google informed me that it is a Mac desync issue.
Same is true if trying to get data from a dynamoDB sitting on Amazon server using the code below...
// To get userID.
var userId = getUserIdFromContext(this);
if (!userId) {
callback('userId is not set.');
}
// To get table name.
var table = constants.dynamoDBTableName;
if(!table) {
callback('DynamoDB Table name is not set.');
}
// To set the DocumentClient.
if(!doc) {
doc = new aws.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
}
// To set the params.
var params = {
Key: {
CustomerId: userId
},
TableName: table,
ConsistentRead: true
};
// To get data from table.
var skillContext = this;
doc.get(params, function(err, data){
if(err) {
console.log('get error: ' + JSON.stringify(err, null, 4));
callback(err);
} else {
if(isEmptyObject(data)) {
callback('The object is empty.');
} else {
var userData = JSON.parse(data.Item['Data']);
extractUserData(skillContext, userData, callback);
}
}
});
}
I'm writing a telegram bot to report fail2ban bans. It's very simple and dirty, written hastily, but it can be used to report any message to a single telegram user:
var TelegramBot = require('node-telegram-bot-api');
var fs = require('fs');
var store = {
get: function (key) {
return fs.readFileSync(__dirname + '/' + key, { encoding: 'utf-8' });
},
set: function (key, value) {
fs.writeFileSync(__dirname + '/' + key, value, { encoding: 'utf-8' });
}
};
var token = store.get('token');
var args = process.argv.slice(2);
if (args.length == 0) {
console.error('No mode specified');
process.exit(0);
}
TelegramBot.prototype.unregisterText = function (regexp) {
for (var i = 0; i < bot.textRegexpCallbacks.length; ++i) {
if (bot.textRegexpCallbacks[i].regexp.toString() == regexp) {
bot.textRegexpCallbacks.splice(i, 1);
return;
}
}
};
fs.appendFileSync(__dirname + '/logs',
'[' + (new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')) + '] '
+ args.join(' ') + '\n',
{ encoding: 'utf-8' });
switch (args[0]) {
case 'setup':
var bot = new TelegramBot(token, { polling: true });
var step = 'none';
bot.onText(/\/setup/, function (msg, match) {
var fromId = msg.from.id;
step = 'setup-started';
bot.sendMessage(fromId, 'Starting setup. Please enter the verification key.');
bot.onText(/(.+)/, function (msg, match) {
if (step == 'setup-started') {
var key = match[1];
var verification = store.get('key');
if (key == verification) {
store.set('owner', msg.from.id);
step = 'verified';
bot.sendMessage(msg.from.id, 'Correct. Setup complete.');
} else {
step = 'none';
bot.unregisterText(/(.+)/);
bot.sendMessage(msg.from.id, 'Wrong. Setup aborted.');
}
}
});
});
break;
case 'report':
var bot = new TelegramBot(token, { polling: false });
var owner = store.get('owner');
var subject = args[1];
if (subject == 'message') {
var message = args.slice(2).join(' ');
bot.sendMessage(owner, message);
} else if (subject == 'file') {
var content = fs.readFileSync(args[2], { encoding: 'utf-8' });
bot.sendMessage(owner, content);
}
break;
default:
console.error('Unrecognized mode', args[0]);
break;
}
On my developer machine it works fine. I invoke:
node bot.js report message whatever message i want
And I correctly received "whatever message i want" on telegram. However, once I gitted it on my digitalocean vps, it no longer worked. It turns out the problem is with the telegram library:
Unhandled rejection Error: Error parsing Telegram response: <!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Bots: An introduction for developers</title>
...
Which apparently returns an html page instead of json... I also tried to contact the same endpoint (api.telegram.org/bothash/sendMessage) with curl on my vps and it returned json (with an error message because i didnt send any parameters, but still json).
I cannot fathom why this happens. Any help?
It seems like either you don't have a file with token on your VPN or the token is incorrect.
You can check it by yourself:
When you make a request to api.telegram.org/{token}/sendMessage, and {token} is incorrect, it redirects you to this page, which responds with HTML you've mentioned in your question.
So you have to debug a behavior of your store.get and store.get functions along with files and tokens to make sure you are using a correct one.
Also, I'd recommend to run bot.getMe() before using any other Telegram API methods to ensure you specified a correct bot token.