Develop Reference

Formidable/NodeJS HTTP Server JPG save

Stackoverflow JS Genius's!
I have an issue with my current project, it's using node's HTTP createServer, using Formidable to parse the body data.
See code below. (http-listener.js)
var listenport = 7200;
const server = http.createServer((req, res) => {
// Set vars ready
var data = '';
var plateImg = '';
var overview1 = '';
var overview2 = '';
new formidable.IncomingForm().parse(req)
// I originally thought it was sent in files, but it isnt, it's fields.
.on('file', function(name, file) {
console.log('Got file:', name);
})
// This is the correct procedure for my issue.
.on('field', function(name, field) {
console.log('Got a field:', name);
if(name.toLowerCase() === "anpr.xml")
{
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(field, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Console log parsed json data.
console.log("Read: "+result.EventNotificationAlert.ANPR.licensePlate);
console.log(result);
data = result;
});
}
if(name.toLowerCase() === "licenseplatepicture.jpg")
{
plateImg = field
// This doesnt work?
// I need to store these fields as an image. ? Is this possible with it being sent as a field and not as a file upload.
// This is the only option I have as I can't control the client sending this data (It's a camera)
fs.writeFile(config.App.ImageDir+'/Plate.jpg', plateImg, function(err) {
if(err)console.log(err);
});
}
if(name.toLowerCase() === "detectionpicture.jpg")
{
if(overview1 == '')
{
overview1 = field;
}
else if(overview2 == '')
{
overview2 = field;
}
else
{
// do nothing else.
console.log("Couldn't send images to variable.");
}
}
})
.on('error', function(err) {
alert(err);
})
.on('end', function() {
// Once finished, send to ANPR data to function to handle data and insert to database. WORKS
// Call anpr function.
ANPR_ListenData(data, plateImg, overview1, overview2, function(result) {
if(result.Status > 0)
{
console.log("Accepted by: "+result.Example);
// reset var
data = '';
plateImg = '';
overview1 = '';
overview2 = '';
res.writeHead(200, {'content-type':'text/html'});
res.end();
}
});
});
});
server.listen(listenport, () => {
console.log('ANPR Server listening on port: ' + listenport);
});
Basically the images that are sent in the fields: licenseplatepicture.jpg etc I want to store them directly into my app image directory.
Unfortunately I have no control over how the chunks are sent to this server due to it being a network camera, I simply need to write a procedure.
The full request chunk is quite large so I will upload the file to OneDrive for you to glance at and understand the request.
Any help with this will be appreciated. I've tried everything I can possibly think of, but the file saves unreadable :(. I don't know where else to look or what else I can try, other than what I've already done & tried.
Request Txt File: https://1drv.ms/t/s!AqAIyFoqrBTO6hTwCimcHDHODqEi?e=pxJY00
Ryan.

I fixed this by using Busboy package instead of Formidable.
This is how my http listener looks like using Busboy.
var inspect = util.inspect;
var Busboy = require('busboy');
http.createServer(function(req, res) {
if (req.method === 'POST') {
//vars
var ref = Math.random().toString(36).substring(5) + Math.random().toString(36).substring(2, 15);;
var xml = '';
var parseXml = '';
var over1, over2 = '';
var i = 0;
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
if(filename.toLowerCase() === "licenseplatepicture.jpg")
{
var saveTo = config.App.ImageDir+"/"+ref+"_Plate.jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
if(filename.toLowerCase() === "detectionpicture.jpg")
{
i++;
var saveTo = config.App.ImageDir+"/"+ref+"_Front_"+i+".jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
file.on('data', function(data) {
if(filename.toLowerCase() === "anpr.xml")
{
xml += data;
}
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
});
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
// No fields according to busboy
});
busboy.on('finish', function() {
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(xml, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Set parsed var
parseXml = result;
});
var images = '';
if(i = 2)
{
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"${ref}_Front_2.jpg"}`;
} else {
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"null"}`;
}
// Once parsed, send on to ANPR listen function.
ANPR_ListenData(ref, parseXml, images, function(result) {
if(result.Status == 1)
{
console.log('Data transfered for: '+parseXml.EventNotificationAlert.ANPR.licensePlate);
console.log('Accepted Camera: '+result.Example);
res.writeHead(200, { Connection: 'close', Location: '/' });
res.end();
}
});
});
req.pipe(busboy);
}
}).listen(7200, function() {
console.log('Listening for requests');
});
Hope this helps someone else in the future. Certainly caused me a lot of a wasted time.
Busboy was the better package to use when I was reading into it more, it makes more sense for what I was attempting to achieve.
Ryan :).
All the best.

nodejs write after request end error

I am facing problem of write after request end in nodejs :
I have a server.js file , which sends request to other js file (say abc.js) which sends response back to server.js file and then server.js file writes the resoponse and then end response.
my problem is if I write response in abc.js and end it there only it works fine, but if it is in sererconf.js it doesn't.
Let me make it clear that I get this bug only when i send 20-30 requests at a time. I want to know the logic behind it, I searched a lot, but no nice answer found, any help will be appreciated.
server.js full code:
/* create HTTP server */
var httpd = http.createServer(function(req, res) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.writeHead(200, {"Content-Type" : "application/json"});
}).listen(3800);
/* send request to the file mentioned in url*/
httpd.on('request', function(req, res) {
urll = __dirname + '/..' + req.url;
fs.exists(urll, function (exists) {
if(exists){
var server = require(urll);
server.get(req,res);
}
});
module.exports = {
result : function(result){
if(Array.isArray(result)){
for(var key in result){
result[key] = JSON.parse(result[key]);
}
}
result = JSON.stringify(result);
res.write(result ,function(err) { if(!err) res.end(); });
},
};
});
**apps.js code**:
var constants = require('./lib/constant.js');
var APP_PATH = constants.APP_PATH;
module.exports = {
get : function(req) {
req.on('data', function(chunk) {
var hash = chunk;
hash = JSON.parse(hash);
var id = hash.id;
dirPath = APP_PATH + id;
fs.exists( dirPath, function (exists) {
if(exists)
read_app_dir(dirPath);
else
taskDone([]);
});
});
}
};
function read_app_dir(app_dir){
fs.readdir(app_dir,function(err, list){
if (err) {
httpd.log.info('cannot read apps dir at s_apps = '+err);
}else{
create_new_obj(list,app_dir);
}
});
}
function create_new_obj(list, app_dir){
appFilesObj = [];
var i = 0;
list.forEach(function(file) {
i=i+1;
file = app_dir +'/' +file;
appFilesObj.push(file);
if(i == Object.keys(list).length)
read_app_files(appFilesObj);
});
}
function read_app_files(appFilesObj,app_dir){
var apps = [];
var i = 0;
if(Object.keys(appFilesObj).length > 0){
appFilesObj.forEach(function(appfile) {
read_file(appfile,function(data){ i=i+1;
apps.push(data);
if(i == Object.keys(appFilesObj).length)
taskDone(apps);
});
});
}else{
taskDone([]);
}
}
function read_file(file,callback){
fs.readFile(file,'utf8', function (err, data) {
if (err)
httpd.log.info('cannot read file at s_apps = '+err);
else
callback(data);
});
}
function taskDone(apps){
var httpd = require(__dirname + '/server.js');
httpd.result(apps);
}
if I do res.write and res.end in this file in taskDone() then it works fine.
Thanks in advance :)

The problem with above code was, that I was sending back response by calling an exported function of server.js
like this:
var httpd = require(__dirname + '/server.js');
httpd.result(apps);
where result() is the function which I have exported in server.js to write response and end response
Instead of this, now I added a callback support while calling function of other files (ex-apps.js), so that I "res.write" and "res.end()" only when the actually called function gives back the response.
(I am not writing the whole code , please refer above code for difference in both)
httpd.on('request', function(req, res) {
urll = __dirname + '/..' + req.url;
fs.exists(urll, function (exists) {
if(exists){
var server = require(urll);
server.get(req,res,function(result){
res.write(result);
res.end();
});
}
});
**apps.js**
get : function(req, callback) {
req.on('data', function(chunk) {
//when task is done and taskDone() function is called I just callback() the result
function taskDone(result){
callback(result);
}
}
}
When I was sending result back by calling a function of server.js and then writing the response...I don't know how..but somehow server was getting confused in multiple requests and saying "write after end" error...while the end was called by some other user's request.
I may be wrong, but this is what I concluded from this :)
I hope this may help others.

Server proxy with Node JS

I want to build a proxy with node.js to use in my browser.
I searched at google and saw that example, but some Apis are deprecated.
So I changed the code following:
var http = require('http');
http.createServer(function(request, response) {
console.log("req for:"+ request.headers['host'] + "\nmethod: "+ request.method);
var options = {
host : request.headers['host'],
method : request.method,
headers : request.headers
}
var callback = function(res){
// get the proxyclient response and write to the original response
res.addListener('data', function(chunk){
console.log("writing response");
response.write(chunk, 'binary');
});
res.addListener('end', function(){
console.log("end response");
response.end();
});
};
// send the original request
var proxy = http.request(options, callback);
// intercept the request and write to proxyclient
request.addListener('data', function(chunk){
console.log("new request");
proxy.write(chunk, 'binary');
});
request.addListener('end', function(){
console.log("end request");
proxy.end();
});
}).listen(3333);
After that, I changed my browser proxy settings to use localhost:3333.
My problem is that event "data" isn't fired.
This should work as I thought?

Maybe if you write this code;
response.writeHead(res.statusCode, res.headers); after res.addListener('end', function(){
console.log("end response");
response.end();
});

With node.js how to I set a var to response from HTTP client?

I know this is probably Asynchronous Javascript 101 and I do have some books on the Kindle I could consult, but I am nowhere near my device.
I have a node app with a variable being assigned to a module that I'm loading. The module has one function that downloads a string of data from a URL.
The problem is, how do I not set the variable until the request has returned?
My code looks like this:
Downloader.js:
var http = require('http');
exports.downloadString = function(str) {
console.log("Downloading from " + str);
http.get(str, function(res) {
var data = [];
console.log("Got response: " + res.statusCode);
res.on('data', function (chunk) {
data.push(chunk);
});
res.on('end', function() {
return data.join('');
});
}).on('error', function(e) {
console.log("Got error: " + e.message);
});
}
app.js:
var downloader = require('./lib/Downloader')
, dateParser = require('./lib/DateParser')
, eventIdGenerator = require('./lib/EventIdGenerator');
var report = downloader.downloadString("http://exampleapi.com");
console.log(report);
I need to wait until the variable named "report" is populated.
Obviously this means my Mocha tests are also failing as I'm still unsure of how to tell the test to wait until the variable is filled.
I'm sure this is extremely simple, but I am drawing a blank!
Thanks!

Node.js is (mostly) asynchronous, so you'd need to pass a callback function to your module:
Downloader.js:
var http = require('http');
exports.downloadString = function(str, callback) {
console.log("Downloading from " + str);
http.get(str, function(res) {
var data = [];
console.log("Got response: " + res.statusCode);
res.on('data', function (chunk) {
data.push(chunk);
});
res.on('end', function() {
callback(data.join(''));
});
}).on('error', function(e) {
console.log("Got error: " + e.message);
});
};
app.js:
var downloader = require('./lib/Downloader')
, dateParser = require('./lib/DateParser')
, eventIdGenerator = require('./lib/EventIdGenerator');
downloader.downloadString("http://exampleapi.com", function(report) {
console.log(report);
});

Download large file with node.js avoiding high memory consumption

I`m trying to create a file downloader as a background service but when a large file is scheduled, it's first put in memory and then, at the end of the download the file is written to disk.
How can I make the file be wrote gradually to the disk preserving memory considering that I may have lots of files being downloaded at the same time?
Here's the code I`m using:
var sys = require("sys"),
http = require("http"),
url = require("url"),
path = require("path"),
fs = require("fs"),
events = require("events");
var downloadfile = "http://nodejs.org/dist/node-v0.2.6.tar.gz";
var host = url.parse(downloadfile).hostname
var filename = url.parse(downloadfile).pathname.split("/").pop()
var theurl = http.createClient(80, host);
var requestUrl = downloadfile;
sys.puts("Downloading file: " + filename);
sys.puts("Before download request");
var request = theurl.request('GET', requestUrl, {"host": host});
request.end();
var dlprogress = 0;
setInterval(function () {
sys.puts("Download progress: " + dlprogress + " bytes");
}, 1000);
request.addListener('response', function (response) {
response.setEncoding('binary')
sys.puts("File size: " + response.headers['content-length'] + " bytes.")
var body = '';
response.addListener('data', function (chunk) {
dlprogress += chunk.length;
body += chunk;
});
response.addListener("end", function() {
fs.writeFileSync(filename, body, 'binary');
sys.puts("After download finished");
});
});

I changed the callback to:
request.addListener('response', function (response) {
var downloadfile = fs.createWriteStream(filename, {'flags': 'a'});
sys.puts("File size " + filename + ": " + response.headers['content-length'] + " bytes.");
response.addListener('data', function (chunk) {
dlprogress += chunk.length;
downloadfile.write(chunk, encoding='binary');
});
response.addListener("end", function() {
downloadfile.end();
sys.puts("Finished downloading " + filename);
});
});
This worked perfectly.

does the request package work for your uses?
it lets you do things like this:
request(downloadurl).pipe(fs.createWriteStream(downloadtohere))

Take a look at http-request:
// shorthand syntax, buffered response
http.get('http://localhost/get', function (err, res) {
if (err) throw err;
console.log(res.code, res.headers, res.buffer.toString());
});
// save the response to 'myfile.bin' with a progress callback
http.get({
url: 'http://localhost/get',
progress: function (current, total) {
console.log('downloaded %d bytes from %d', current, total);
}
}, 'myfile.bin', function (err, res) {
if (err) throw err;
console.log(res.code, res.headers, res.file);
});

When downloading large file please use fs.write and not writeFile as it will override the previous content.
function downloadfile(res) {
var requestserver = http.request(options, function(r) {
console.log('STATUS: ' + r.statusCode);
console.log('HEADERS: ' + JSON.stringify(r.headers));
var fd = fs.openSync('sai.tar.gz', 'w');
r.on('data', function (chunk) {
size += chunk.length;
console.log(size+'bytes received');
sendstatus(res,size);
fs.write(fd, chunk, 0, chunk.length, null, function(er, written) {
});
});
r.on('end',function(){
console.log('\nended from server');
fs.closeSync(fd);
sendendstatus(res);
});
});
}

Instead of holding the content into memory in the "data" event listener you should write to the file in append mode.

Use streams like Carter Cole suggested. Here is a more complete example
var inspect = require('eyespect').inspector();
var request = require('request');
var filed = require('filed');
var temp = require('temp');
var downloadURL = 'http://upload.wikimedia.org/wikipedia/commons/e/ec/Hazard_Creek_Kayaker.JPG';
var downloadPath = temp.path({prefix: 'singlePageRaw', suffix: '.jpg'});
var downloadFile = filed(downloadPath);
var r = request(downloadURL).pipe(downloadFile);
r.on('data', function(data) {
inspect('binary data received');
});
downloadFile.on('end', function () {
inspect(downloadPath, 'file downloaded to path');
});
downloadFile.on('error', function (err) {
inspect(err, 'error downloading file');
});
You may need to install modules which you can do via
npm install filed request eyespect temp