Develop Reference

Formidable/NodeJS HTTP Server JPG save

Stackoverflow JS Genius's!
I have an issue with my current project, it's using node's HTTP createServer, using Formidable to parse the body data.
See code below. (http-listener.js)
var listenport = 7200;
const server = http.createServer((req, res) => {
// Set vars ready
var data = '';
var plateImg = '';
var overview1 = '';
var overview2 = '';
new formidable.IncomingForm().parse(req)
// I originally thought it was sent in files, but it isnt, it's fields.
.on('file', function(name, file) {
console.log('Got file:', name);
})
// This is the correct procedure for my issue.
.on('field', function(name, field) {
console.log('Got a field:', name);
if(name.toLowerCase() === "anpr.xml")
{
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(field, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Console log parsed json data.
console.log("Read: "+result.EventNotificationAlert.ANPR.licensePlate);
console.log(result);
data = result;
});
}
if(name.toLowerCase() === "licenseplatepicture.jpg")
{
plateImg = field
// This doesnt work?
// I need to store these fields as an image. ? Is this possible with it being sent as a field and not as a file upload.
// This is the only option I have as I can't control the client sending this data (It's a camera)
fs.writeFile(config.App.ImageDir+'/Plate.jpg', plateImg, function(err) {
if(err)console.log(err);
});
}
if(name.toLowerCase() === "detectionpicture.jpg")
{
if(overview1 == '')
{
overview1 = field;
}
else if(overview2 == '')
{
overview2 = field;
}
else
{
// do nothing else.
console.log("Couldn't send images to variable.");
}
}
})
.on('error', function(err) {
alert(err);
})
.on('end', function() {
// Once finished, send to ANPR data to function to handle data and insert to database. WORKS
// Call anpr function.
ANPR_ListenData(data, plateImg, overview1, overview2, function(result) {
if(result.Status > 0)
{
console.log("Accepted by: "+result.Example);
// reset var
data = '';
plateImg = '';
overview1 = '';
overview2 = '';
res.writeHead(200, {'content-type':'text/html'});
res.end();
}
});
});
});
server.listen(listenport, () => {
console.log('ANPR Server listening on port: ' + listenport);
});
Basically the images that are sent in the fields: licenseplatepicture.jpg etc I want to store them directly into my app image directory.
Unfortunately I have no control over how the chunks are sent to this server due to it being a network camera, I simply need to write a procedure.
The full request chunk is quite large so I will upload the file to OneDrive for you to glance at and understand the request.
Any help with this will be appreciated. I've tried everything I can possibly think of, but the file saves unreadable :(. I don't know where else to look or what else I can try, other than what I've already done & tried.
Request Txt File: https://1drv.ms/t/s!AqAIyFoqrBTO6hTwCimcHDHODqEi?e=pxJY00
Ryan.

I fixed this by using Busboy package instead of Formidable.
This is how my http listener looks like using Busboy.
var inspect = util.inspect;
var Busboy = require('busboy');
http.createServer(function(req, res) {
if (req.method === 'POST') {
//vars
var ref = Math.random().toString(36).substring(5) + Math.random().toString(36).substring(2, 15);;
var xml = '';
var parseXml = '';
var over1, over2 = '';
var i = 0;
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
if(filename.toLowerCase() === "licenseplatepicture.jpg")
{
var saveTo = config.App.ImageDir+"/"+ref+"_Plate.jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
if(filename.toLowerCase() === "detectionpicture.jpg")
{
i++;
var saveTo = config.App.ImageDir+"/"+ref+"_Front_"+i+".jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
file.on('data', function(data) {
if(filename.toLowerCase() === "anpr.xml")
{
xml += data;
}
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
});
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
// No fields according to busboy
});
busboy.on('finish', function() {
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(xml, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Set parsed var
parseXml = result;
});
var images = '';
if(i = 2)
{
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"${ref}_Front_2.jpg"}`;
} else {
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"null"}`;
}
// Once parsed, send on to ANPR listen function.
ANPR_ListenData(ref, parseXml, images, function(result) {
if(result.Status == 1)
{
console.log('Data transfered for: '+parseXml.EventNotificationAlert.ANPR.licensePlate);
console.log('Accepted Camera: '+result.Example);
res.writeHead(200, { Connection: 'close', Location: '/' });
res.end();
}
});
});
req.pipe(busboy);
}
}).listen(7200, function() {
console.log('Listening for requests');
});
Hope this helps someone else in the future. Certainly caused me a lot of a wasted time.
Busboy was the better package to use when I was reading into it more, it makes more sense for what I was attempting to achieve.
Ryan :).
All the best.

NodeJS finish writing the file with pipe before continuing with the next iteration

Similar to this question,
I have a script that downloads a file to a given url via http.get.
How can I make sure the pipe is finished before continuing to the next iteration with just the http/https module??
//nodejs default libs
var fs = require("fs");
var http = require('https');
function dlFile(fullFilePath, dlUrl, fsize, fname){
var file = fs.createWriteStream(fullFilePath); //fullFilePath will dictate where we will save the file + filename.
var rsult ='';
var downloadedFsize;
var stats; //stats of the file will be included here
var request = http.get( dlUrl, function(response) {
let rsult = response.statusCode;
//will respond with a 200 if the file is present
//404 if file is missing
response.pipe(file);
/*pipe writes the file...
how do we stop the iteration while it is not yet finished writing?
*/
console.log(" \n FILE : " + fname);
console.log("File analysis finished : statusCode: " + rsult + " || Saved on " + fullFilePath);
console.log(' \n Downloaded from :' + dlUrl);
console.log(' \n SQL File size is : ' + fsize);
//identify filesize
stats = fs.statSync(fullFilePath);
downloadedFsize = stats["size"]; //0 because the pipe isn't finished yet...
console.log(' actual file size is : ' + downloadedFsize);
}).on('error', function(e) {
console.error(e);
//log that an error happened to the file
}).on('end', function(e){
//tried putting the above script here but nothing happens
});
return rsult;
}
Is there a cleaner approach similar to what I have in mind above? or should I approach this differently? I tried putting the code on .on('end' but it does nothing

The end event is not triggered on the request, instead it is triggered on the response (docs):
response.on("end", function() {
console.log("done");
});

As #Jonas Wilms says, the trigger was indeed on response.
//nodejs default libs
var fs = require("fs");
var http = require('https');
function dlFile(fullFilePath, dlUrl, fsize, fname){
var file = fs.createWriteStream(fullFilePath); //fullFilePath will dictate where we will save the file + filename.
var rsult ='';
var downloadedFsize;
var stats; //stats of the file will be included here
var request = http.get( dlUrl, function(response) {
let rsult = response.statusCode;
//will respond with a 200 if the file is present
//404 if file is missing
response.pipe(file).on('finish', function(e){
console.log(" \n FILE : " + fname);
console.log("File analysis finished : statusCode: " + rsult + " || Saved on " + fullFilePath);
console.log(' \n Downloaded from :' + dlUrl);
console.log(' \n SQL File size is : ' + fsize);
//identify filesize
stats = fs.statSync(fullFilePath);
downloadedFsize = stats["size"];
console.log(' actual file size is : ' + downloadedFsize);
});
/*pipe writes the file above, and output the results once it's done */
}).on('error', function(e) {
console.error(e);
//log that an error happened to the file
}).on('end', function(e){
//tried putting the above script here but nothing happens
});
return rsult;
}

Node.js HTTP - TypeError: Header name must be a valid HTTP Token

when I'm trying to set the header in the following way ,it's working absolutely fine.
response.setHeader('Content-Type', 'application/json');
but when I'm trying to add variable instead of exact header name/value in the following way, it's showing error :-(
response.setHeader(result.headers);
if you console.log("headers -> " + result.header) the result would be the same.
headers -> 'Content-Type', 'application/json'
following are the exact error I'm getting , not able to figure out how to get around it.
_http_outgoing.js:487
throw new TypeError(`Header name must be a valid HTTP Token ["${name}"]`);
^
TypeError: Header name must be a valid HTTP Token ["'Content-Type', 'application/json'"]
at validateHeader (_http_outgoing.js:487:11)
at ServerResponse.setHeader (_http_outgoing.js:498:3)
at C:\service-mocker\src\main\app.js:54:22
at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:511:3)
Below is whole code I'm trying to implement :
var http = require('http');
var fs = require('fs');
var path = require('path');
var repl = require('repl');
var map={};
var key;
var value;
//create a server object:
var server = http.createServer(function(request, response) {
fs.readFile("./resources/FileData1.txt", function(err, data) {
if(err) throw err;
content = data.toString().split(/(?:\r\n|\r|\n)/g).map(function(line){
return line.trim();
}).filter(Boolean)
var result = processFile(content);
console.log("url -> " + result.url);
console.log("status -> " + result.status);
console.log("headers -> " + result.headers);
console.log("body -> " + result.body);
function objToString (obj) {
var str = '';
for (var p in obj) {
if (obj.hasOwnProperty(p)) {
str += obj[p] + '\n';
}
}
return str;
}
function processFile(nodes) {
nodes.forEach(function(node) {
if(node.startsWith("//")){
key = node.substring(2, node.length-2).toLowerCase().trim();
return;
}
else{
value = node;
}
// map[key] = value;
if(key in map){
map[key].push(value);
}else{
map[key]= [value];
}
});
return map;
// console.log(map);
}
if(request.url == result.url ){
response.setHeader(result.headers);
// response.setHeader('Content-Type', 'application/json');
response.write( objToString(result.body) );
response.statuscode = parseInt( result.status );
response.end();
}else {
// response.writeHead(404, {"Content-Type": "text/html"});
response.end("No Page Found");
}
});
});
var port = process.env.PORT || 8080;
server.listen(port, function() {
console.log('Listening on port ' + port);
});

It looks like result.headers is returning a single string, but response.setHeader needs two arguments: the header name and the header value.
Based on the error message, you're doing the equivalent of:
response.setHeader("'Content-Type', 'application/json'");
Instead, you need to split out the header into key and value:
//n.b. make sure that your result only has the one header!
var headerSplit = result.headers.split(',');
var headerKey = headerSplit[0];
var headerVal = headerSplit[1];
response.setHeader(headerKey, headerVal);

If you dont want to set headers one at a time with response.setHeader, you should use response.writeHead, which will write the status code and your headers into the response.
response.writeHead(200, headers)
Here headers must be a JavaScript object.
Replace
response.setHeader(result.headers);
response.statuscode = parseInt( result.status );
with
response.writeHead(parseInt(result.status), parseHeaders(result.headers));
where parseHeaders is a function that parses result.headers into an object.
function parseHeaders(headers) {
var o = {};
headers.forEach(header => {o[header.split(',')[0]] = header.split(',')[1]});
return o;
}
In this answer I've assumed result.headers is formatted like so
[
"header1, header1value",
"header2, header2value"
]

Template literals use a back-tick not quotes.

Telegram API returning HTML instead of JSON

I'm writing a telegram bot to report fail2ban bans. It's very simple and dirty, written hastily, but it can be used to report any message to a single telegram user:
var TelegramBot = require('node-telegram-bot-api');
var fs = require('fs');
var store = {
get: function (key) {
return fs.readFileSync(__dirname + '/' + key, { encoding: 'utf-8' });
},
set: function (key, value) {
fs.writeFileSync(__dirname + '/' + key, value, { encoding: 'utf-8' });
}
};
var token = store.get('token');
var args = process.argv.slice(2);
if (args.length == 0) {
console.error('No mode specified');
process.exit(0);
}
TelegramBot.prototype.unregisterText = function (regexp) {
for (var i = 0; i < bot.textRegexpCallbacks.length; ++i) {
if (bot.textRegexpCallbacks[i].regexp.toString() == regexp) {
bot.textRegexpCallbacks.splice(i, 1);
return;
}
}
};
fs.appendFileSync(__dirname + '/logs',
'[' + (new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')) + '] '
+ args.join(' ') + '\n',
{ encoding: 'utf-8' });
switch (args[0]) {
case 'setup':
var bot = new TelegramBot(token, { polling: true });
var step = 'none';
bot.onText(/\/setup/, function (msg, match) {
var fromId = msg.from.id;
step = 'setup-started';
bot.sendMessage(fromId, 'Starting setup. Please enter the verification key.');
bot.onText(/(.+)/, function (msg, match) {
if (step == 'setup-started') {
var key = match[1];
var verification = store.get('key');
if (key == verification) {
store.set('owner', msg.from.id);
step = 'verified';
bot.sendMessage(msg.from.id, 'Correct. Setup complete.');
} else {
step = 'none';
bot.unregisterText(/(.+)/);
bot.sendMessage(msg.from.id, 'Wrong. Setup aborted.');
}
}
});
});
break;
case 'report':
var bot = new TelegramBot(token, { polling: false });
var owner = store.get('owner');
var subject = args[1];
if (subject == 'message') {
var message = args.slice(2).join(' ');
bot.sendMessage(owner, message);
} else if (subject == 'file') {
var content = fs.readFileSync(args[2], { encoding: 'utf-8' });
bot.sendMessage(owner, content);
}
break;
default:
console.error('Unrecognized mode', args[0]);
break;
}
On my developer machine it works fine. I invoke:
node bot.js report message whatever message i want
And I correctly received "whatever message i want" on telegram. However, once I gitted it on my digitalocean vps, it no longer worked. It turns out the problem is with the telegram library:
Unhandled rejection Error: Error parsing Telegram response: <!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Bots: An introduction for developers</title>
...
Which apparently returns an html page instead of json... I also tried to contact the same endpoint (api.telegram.org/bothash/sendMessage) with curl on my vps and it returned json (with an error message because i didnt send any parameters, but still json).
I cannot fathom why this happens. Any help?

It seems like either you don't have a file with token on your VPN or the token is incorrect.
You can check it by yourself:
When you make a request to api.telegram.org/{token}/sendMessage, and {token} is incorrect, it redirects you to this page, which responds with HTML you've mentioned in your question.
So you have to debug a behavior of your store.get and store.get functions along with files and tokens to make sure you are using a correct one.
Also, I'd recommend to run bot.getMe() before using any other Telegram API methods to ensure you specified a correct bot token.

Parsing Unexpected End

Can someone please explain me why I am getting an issue with this one line because for some reason when I run it with node in the console I'm receiving the Unexpected end of input at Object.parse(native) response.
var profile = JSON.parse(body);
Full code:
//Problem: We need a simple way to look at a user's badge count and Javascript points
//Solution: Use Node.js to connect to Treehouse's API to get profile information to print out
var http = require("http");
var username = "testuser";
//Print out message
function printMessage(username, badgeCount, points) {
var message = username + " has " + badgeCount + " total badge(s) and " + points + " points in Javascript";
console.log(message);
}
//Print out error messages
function printError(error) {
console.error(error.message);
}
//Connect to API URL (http://teamtreehouse.com/username.json)
var request = http.get("http://teamtreehouse.com/" + username + ".json", function(response) {
var body = "";
//Read the data
response.on('data', function(chunk) {
body += chunk;
});
response.on('end', function() {
if(response.statusCode == 200){
try {
var profile = JSON.parse(body);
printMessage(username, profile.badges.length, profile.points.Javascript);
} catch(error) {
//Parse Error
printError(error);
}
} else {
//Status Code Error
printError({message: "There was an error getting the profile for " + username +". (" + http.SSTATUS_CODES[response.statusCode] + ")"});
}
});
//Parse the data
//Print the data
});
//Connection Error
request.on('error', printError);

When I try to browse to http://teamtreehouse.com/test.json, it redirects me to the corresponding HTTPS url. Use the nodejs https module and use the https version of the url : https://teamtreehouse.com/test.json.
Or use the popular request module that can handle redirects and https : https://github.com/request/request. It is easier to use as well.