Develop Reference

Formidable/NodeJS HTTP Server JPG save

Stackoverflow JS Genius's!
I have an issue with my current project, it's using node's HTTP createServer, using Formidable to parse the body data.
See code below. (http-listener.js)
var listenport = 7200;
const server = http.createServer((req, res) => {
// Set vars ready
var data = '';
var plateImg = '';
var overview1 = '';
var overview2 = '';
new formidable.IncomingForm().parse(req)
// I originally thought it was sent in files, but it isnt, it's fields.
.on('file', function(name, file) {
console.log('Got file:', name);
})
// This is the correct procedure for my issue.
.on('field', function(name, field) {
console.log('Got a field:', name);
if(name.toLowerCase() === "anpr.xml")
{
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(field, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Console log parsed json data.
console.log("Read: "+result.EventNotificationAlert.ANPR.licensePlate);
console.log(result);
data = result;
});
}
if(name.toLowerCase() === "licenseplatepicture.jpg")
{
plateImg = field
// This doesnt work?
// I need to store these fields as an image. ? Is this possible with it being sent as a field and not as a file upload.
// This is the only option I have as I can't control the client sending this data (It's a camera)
fs.writeFile(config.App.ImageDir+'/Plate.jpg', plateImg, function(err) {
if(err)console.log(err);
});
}
if(name.toLowerCase() === "detectionpicture.jpg")
{
if(overview1 == '')
{
overview1 = field;
}
else if(overview2 == '')
{
overview2 = field;
}
else
{
// do nothing else.
console.log("Couldn't send images to variable.");
}
}
})
.on('error', function(err) {
alert(err);
})
.on('end', function() {
// Once finished, send to ANPR data to function to handle data and insert to database. WORKS
// Call anpr function.
ANPR_ListenData(data, plateImg, overview1, overview2, function(result) {
if(result.Status > 0)
{
console.log("Accepted by: "+result.Example);
// reset var
data = '';
plateImg = '';
overview1 = '';
overview2 = '';
res.writeHead(200, {'content-type':'text/html'});
res.end();
}
});
});
});
server.listen(listenport, () => {
console.log('ANPR Server listening on port: ' + listenport);
});
Basically the images that are sent in the fields: licenseplatepicture.jpg etc I want to store them directly into my app image directory.
Unfortunately I have no control over how the chunks are sent to this server due to it being a network camera, I simply need to write a procedure.
The full request chunk is quite large so I will upload the file to OneDrive for you to glance at and understand the request.
Any help with this will be appreciated. I've tried everything I can possibly think of, but the file saves unreadable :(. I don't know where else to look or what else I can try, other than what I've already done & tried.
Request Txt File: https://1drv.ms/t/s!AqAIyFoqrBTO6hTwCimcHDHODqEi?e=pxJY00
Ryan.

I fixed this by using Busboy package instead of Formidable.
This is how my http listener looks like using Busboy.
var inspect = util.inspect;
var Busboy = require('busboy');
http.createServer(function(req, res) {
if (req.method === 'POST') {
//vars
var ref = Math.random().toString(36).substring(5) + Math.random().toString(36).substring(2, 15);;
var xml = '';
var parseXml = '';
var over1, over2 = '';
var i = 0;
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
if(filename.toLowerCase() === "licenseplatepicture.jpg")
{
var saveTo = config.App.ImageDir+"/"+ref+"_Plate.jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
if(filename.toLowerCase() === "detectionpicture.jpg")
{
i++;
var saveTo = config.App.ImageDir+"/"+ref+"_Front_"+i+".jpg";
if (!fs.existsSync(saveTo)) {
//file exists
file.pipe(fs.createWriteStream(saveTo));
}
}
file.on('data', function(data) {
if(filename.toLowerCase() === "anpr.xml")
{
xml += data;
}
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
});
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
// No fields according to busboy
});
busboy.on('finish', function() {
// DO PARSE INTO JSON! This works, all is well.
xml2js.parseString(xml, {explicitArray:false, ignoreAttrs:true}, function (err, result)
{
if(err)
{
alert('Parse: '+err);
}
// Set parsed var
parseXml = result;
});
var images = '';
if(i = 2)
{
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"${ref}_Front_2.jpg"}`;
} else {
images = `{"Plate":"${ref}_Plate.jpg", "Front":"${ref}_Front_1.jpg", "Overview":"null"}`;
}
// Once parsed, send on to ANPR listen function.
ANPR_ListenData(ref, parseXml, images, function(result) {
if(result.Status == 1)
{
console.log('Data transfered for: '+parseXml.EventNotificationAlert.ANPR.licensePlate);
console.log('Accepted Camera: '+result.Example);
res.writeHead(200, { Connection: 'close', Location: '/' });
res.end();
}
});
});
req.pipe(busboy);
}
}).listen(7200, function() {
console.log('Listening for requests');
});
Hope this helps someone else in the future. Certainly caused me a lot of a wasted time.
Busboy was the better package to use when I was reading into it more, it makes more sense for what I was attempting to achieve.
Ryan :).
All the best.

How to use node modules correctly

I am a CS student with a strong Java background, and javascript is resulting to be a challenging but fun experience so far, that is until I ran into the situation where I tried to use my own modules to return values that require the program to wait for the completion of some procedure before returning.So far no-one from any forum that I have posted this question on has been able to give an actual code fix to the problem, they have referred me to read further material that is not related to the problem at hand. Would anyone please read the code and provide a working correct standard solution to the problem I am facing?
Here is the code, a simple nodes server application, app.js and a weather module, weatherApp.js that uses an user provided zip code and returns a weather forecast in the area.
here is the code:
weatherApp.js
// The required modules.
var http = require("http");
var https = require("https");
//result object
var resultSet = {
googleRequestUrl:"",
forecastIOrequest:"",
latitude :"",
longitude:"",
localInfo:"",
weather:"",
humidity:"",
pressure:"",
time:""
};
//print out error messages
function printError(error){
console.error(error.message);
}
//Forecast API required information:
//key for the forecast IO app
var forecast_IO_Key = "bb9aac7c57877f8f5fab339e3b55669a";
var forecast_IO_Web_Adress = "https://api.forecast.io/forecast/";
//Create Forecast request string function
function createForecastRequest(latitude, longitude){
var request = forecast_IO_Web_Adress + forecast_IO_Key + "/"
+ latitude +"," + longitude;
return request;
}
//Google GEO API required information:
//Create Google Geo Request
var google_GEO_Web_Adress = "https://maps.googleapis.com/maps/api/geocode/json?address=";
function createGoogleGeoMapRequest(zipCode){
var request = google_GEO_Web_Adress+zipCode + "&sensor=false";
return request;
}
// 1- Need to request google for geo locations using a given zip
function connectToGoogleGEO(zipCode, afterCallback){
var googleRequest = https.get(createGoogleGeoMapRequest(zipCode), function(response){
//saving the Google request URL
resultSet.googleRequestUrl = createGoogleGeoMapRequest(zipCode);
var body = "";
var status = response.statusCode;
//a- Read the data.
response.on("data", function(chunk){
body+=chunk;
});
//b- Parse the data.
response.on("end", function(){
if(status === 200){
try{
var googleReport = JSON.parse(body);
resultSet.latitude = googleReport.results[0].geometry.location.lat;
resultSet.longitude = googleReport.results[0].geometry.location.lng;
resultSet.localInfo = googleReport.results[0].address_components[0].long_name + ", " +
googleReport.results[0].address_components[1].long_name + ", " +
googleReport.results[0].address_components[2].long_name + ", " +
googleReport.results[0].address_components[3].long_name + ". ";
// callback to forecast IO.
afterCallback(resultSet.latitude, resultSet.longitude);
}catch(error){
printError(error.message);
}finally{
// nothing here
}
}else{
printError({message: "Error with GEO API"+http.STATUS_CODES[response.statusCode]})
}
});
});
}
function connectToForecastIO(latitude,longitude){
var forecastRequest = https.get(createForecastRequest(latitude,longitude),function(response){
resultSet.forecastIOrequest = createForecastRequest(latitude,longitude);
var body = "";
var status = response.statusCode;
//read the data
response.on("data", function(chunk){
body+=chunk;
});
//parse the data
response.on("end", function(){
try{
var weatherReport = JSON.parse(body);
resultSet.weather = weatherReport.currently.summary;
resultSet.humidity = weatherReport.currently.humidity;
resultSet.temperature = weatherReport.currently.temperature;
resultSet.pressure = weatherReport.currently.pressure;
resultSet.time = weatherReport.currently.time;
}catch(error){
printError(error.message);
}finally{
console.log(resultSet);
}
});
});
}
function get(zipCode){
var results = connectToGoogleGEO(zipCode, connectToForecastIO);
return results;
}
//define the name of the outer module.
module.exports.get = get;
And here is the server code:
app.js
var express = require("express");
var weatherApp = require("./weatherApp.js");
var path = require("path");
var http = require("http");
var app = express();
//creating routes
//The home
app.get("/", function(req, res){
res.redirect("/weather");
});
app.get("/weather", function(req, res){
res.sendFile(path.join(__dirname + "/index.html"));
});
//------------------------------------------------------
//The resources, css, web js files, images etc.
app.get("/StyleSheets/style.css", function(req, res){
res.sendFile(path.join(__dirname + "/StyleSheets/style.css"));
});
app.get("/webScripts/app.js", function(req, res){
res.sendFile(path.join(__dirname + "/webScripts/app.js"));
});
app.get("/webImages/swirl_pattern.png", function(req, res){
res.sendFile(path.join(__dirname + "/webImages/swirl_pattern.png"));
});
//-------------------------------------------------------
//other requests
app.get("/zipcode.do", function(req, res){
var zipcode = req.query["zipcode"];
var response = "No report Available";
function getReport(zipCode, callback){
response = weatherApp.get(req.query["zipcode"]);
}
getReport(zipcode, ()=>{
res.send("<p>" + response+ "</p>");
});
});
//any other entry thats not listed as a valid to request
app.get("/:title", function(req,res){
var title = req.param.title;
if(title === undefined){
var status = res.status(503);
res.send("This page does not exists" + '"' + http.STATUS_CODES[503] + '"');
}else{
res.send(title);
}
});
app.listen(3000, function(){
console.log("Server running at port: 3000")
});
The main issue I am having right now is:
The program is not returning anything from the module even when final console.log in the weather module prints the right resultSet object.
The server is not waiting for the module to return, and continues to print no data.
Can someone provide a working fix to any of these problems I would be really grateful, This has really hindered my progress and broken down my morale a little :(

Your problem is that you are using asynchronous functions as if they were synchronous.
It might not be the only issue here, but this function is particularly problematic:
function get(zipCode){
var results = connectToGoogleGEO(zipCode, connectToForecastIO);
return results;
}
connectToGoogleGEO() calls the asynchronous https.get() function and does not return the data that is retrieved from Google. You need to rewrite your code so that it does not expect the data to be returned by the function. Instead, you need to pass a callback that will handle the data.
Take care to know when you are calling asynchronous functions and how their callbacks work. It is fundamental when working with Node.js

Have set up mongodb, mongoose etc but when I try and add something to the database it doesn't actually add it

I'm using localhost and have all the modules set up correctly and I've checked that the database exists. When I type in localhost:3000/pods/add?firstName='John' it's supposed to add it to the database, but for some reason it isn't working.
var express = require('express');
var _ = require('underscore');
var mongoose = require('mongoose');
var podStore = require('./lib/pod-handler.js');
var podsLibrary = require('./lib/pods-library.js');
var podList = [];
var mongoPort = 27017;
var app = express();
var port = 3000;
var router = express.Router();
var pods = podsLibrary.list();
mongoose.connect('mongodb://localhost:'+mongoPort+'/pods');
router.route('/').get(function(request, response) {
//console.log('You hit an empty URL .. :(');
response.status(503).send("Service Unavailable");
});
router.route('/lib').get(function(request, response) {
//console.log('You hit an empty URL .. :(');
response.status(200).send("Cool beans!");
});
router.route('/pods/list').get(function(request, response){
if(!pods){
return response.status(503).send("Service Unavailable");
}
return response.status(200).send(makeReadableList(pods));
function makeReadableList(pods){
var podsHtml = " ";
_.each(pods, function(value, key){podsHtml = podsHtml + key});
return podsHtml;
}
});
router.route('/pods/add').post(function(request, response){
if (!request.query){
return response.status(400).send("Please give first name");
}
var payload = request.query;
if (!payload.firstName){
return response.status(400).send("give name");
}
podStore.save({
firstName: payload.firstName,
lastName: payload.lastName
}, function(){
return response.status(200).send(payload.firstName + " has been added!");
var space = " ";
_.each(pods, function(value, key) {
key + space;
return space + payload.firstName + payload.secondName;
});
});
});
router.route('/pods').get(function(request, response) {
//console.log("We reached the POD page -- Yay! :D");
response.status(200).send("Server unavailable");
});
app.use('/', router);
app.listen(port, function () {
console.log('App listening on port %s', port);
});
I've checked over my code countless times and I can't seem to find the problem.
Here's the pod-handler file.
var PodDoc = require('../models/pods.js');
module.exports = {
save: save
}
function save(pod, callback){
var podToSave = new PodDoc();
podToSave.firstName = pod.firstName;
podToSave.lastName = pod.lastName;
/*podToSave.skills = pod.skills;
podToSave.avatarUrl = pod.avatarUrl;
podToSave.address = {
number: pod.address.number,
lineOne: pod.address.lineOne,
lineTwo: pod.address.lineTwo,
postcode: pod.address.postcode
}
podToSave.phoneNumbers = {
mobile: pod.phoneNumbers.mobile,
landline: pod.phoneNumbers.landline
}*/
podToSave.save(function(err){
if(err){
console.log(err);
} else {
console.log("Working");
callback();
}
})
}

When I type in localhost:3000/pods/add?firstName='John' it's supposed to add it to the database
If i understand correctly, you want to open this url in browser, and expect to have a record John in database.
Change router request type to GET, this
router.route('/pods/add').post(/*omitted*/);
to this
router.route('/pods/add').get(/*omitted*/);
Server is expecting POST request, but browser cannot handle it without FORM element or ajax request, browsers usually uses GET request, i mean when you open your url, it send GET request to server

There may be a typo error as in the url you are using port 300 where as the port configured for localhost is 3000.
When I type in localhost:300/pods/add?firstName='John'

nodejs write after request end error

I am facing problem of write after request end in nodejs :
I have a server.js file , which sends request to other js file (say abc.js) which sends response back to server.js file and then server.js file writes the resoponse and then end response.
my problem is if I write response in abc.js and end it there only it works fine, but if it is in sererconf.js it doesn't.
Let me make it clear that I get this bug only when i send 20-30 requests at a time. I want to know the logic behind it, I searched a lot, but no nice answer found, any help will be appreciated.
server.js full code:
/* create HTTP server */
var httpd = http.createServer(function(req, res) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.writeHead(200, {"Content-Type" : "application/json"});
}).listen(3800);
/* send request to the file mentioned in url*/
httpd.on('request', function(req, res) {
urll = __dirname + '/..' + req.url;
fs.exists(urll, function (exists) {
if(exists){
var server = require(urll);
server.get(req,res);
}
});
module.exports = {
result : function(result){
if(Array.isArray(result)){
for(var key in result){
result[key] = JSON.parse(result[key]);
}
}
result = JSON.stringify(result);
res.write(result ,function(err) { if(!err) res.end(); });
},
};
});
**apps.js code**:
var constants = require('./lib/constant.js');
var APP_PATH = constants.APP_PATH;
module.exports = {
get : function(req) {
req.on('data', function(chunk) {
var hash = chunk;
hash = JSON.parse(hash);
var id = hash.id;
dirPath = APP_PATH + id;
fs.exists( dirPath, function (exists) {
if(exists)
read_app_dir(dirPath);
else
taskDone([]);
});
});
}
};
function read_app_dir(app_dir){
fs.readdir(app_dir,function(err, list){
if (err) {
httpd.log.info('cannot read apps dir at s_apps = '+err);
}else{
create_new_obj(list,app_dir);
}
});
}
function create_new_obj(list, app_dir){
appFilesObj = [];
var i = 0;
list.forEach(function(file) {
i=i+1;
file = app_dir +'/' +file;
appFilesObj.push(file);
if(i == Object.keys(list).length)
read_app_files(appFilesObj);
});
}
function read_app_files(appFilesObj,app_dir){
var apps = [];
var i = 0;
if(Object.keys(appFilesObj).length > 0){
appFilesObj.forEach(function(appfile) {
read_file(appfile,function(data){ i=i+1;
apps.push(data);
if(i == Object.keys(appFilesObj).length)
taskDone(apps);
});
});
}else{
taskDone([]);
}
}
function read_file(file,callback){
fs.readFile(file,'utf8', function (err, data) {
if (err)
httpd.log.info('cannot read file at s_apps = '+err);
else
callback(data);
});
}
function taskDone(apps){
var httpd = require(__dirname + '/server.js');
httpd.result(apps);
}
if I do res.write and res.end in this file in taskDone() then it works fine.
Thanks in advance :)

The problem with above code was, that I was sending back response by calling an exported function of server.js
like this:
var httpd = require(__dirname + '/server.js');
httpd.result(apps);
where result() is the function which I have exported in server.js to write response and end response
Instead of this, now I added a callback support while calling function of other files (ex-apps.js), so that I "res.write" and "res.end()" only when the actually called function gives back the response.
(I am not writing the whole code , please refer above code for difference in both)
httpd.on('request', function(req, res) {
urll = __dirname + '/..' + req.url;
fs.exists(urll, function (exists) {
if(exists){
var server = require(urll);
server.get(req,res,function(result){
res.write(result);
res.end();
});
}
});
**apps.js**
get : function(req, callback) {
req.on('data', function(chunk) {
//when task is done and taskDone() function is called I just callback() the result
function taskDone(result){
callback(result);
}
}
}
When I was sending result back by calling a function of server.js and then writing the response...I don't know how..but somehow server was getting confused in multiple requests and saying "write after end" error...while the end was called by some other user's request.
I may be wrong, but this is what I concluded from this :)
I hope this may help others.

Reading file to disk error, "name and value are required for setHeader()"

Trying to allow users to upload image files to the Node.js server in a MEAN Stack application. I am using ng-file-upload for the client side angular directive. That seems to be working good enough. I run into an error when I pass the image to the server.
I use an API route to handle the work on the server side. The server will be responsible for saving the file to disk with node-multiparty module. It seems to hit route but when it tries to emit a close event I get the error. throw new Error('"name" and "value" are required for setHeader().'
The file I want is in my temp folder but it doesn't get saved to the target directory on my server plus I get the header error after the file should have been saved. So I need to stop the error and save the file with fs.rename() to the target image directory.
Here is the code that is breaking.
file api.js
// router to save images
router.route('/img/upload')
.post(function (req, res) {
console.log("image upload hits the router")
var options = {};
var count = 0;
var form = new multiparty.Form(options);
//save file to disk
form.on('file', function (name, file) {
var uploadDirectory = 'img/user/profile/';
var oldPath = file.path;
var newPath = uploadDirectory + file.originalFilename;
fs.rename(oldPath, newPath, function (err) {
if (err) throw err;
console.log('renamed complete');
});
});
// Close emitted after form parsed
form.on('close', function () {
console.log('Upload completed!');
res.setHeader('text/plain'); // Here is the line that gives an error.
res.end('Received ' + count + ' files');
});
// Parse req
form.parse(req);
});

So this is what I got to work for me
The actual line that gave me an error was setHeaders. It appears I needed to put the name and value as strings separated by a comma. This works perfectly for me now. I hope it saves everyone time coding.
// post
.post(function (req, res) {
var options = {};
var count = 0;
var form = new multiparty.Form(options);
form.on('error', function (err) {
console.log('Error parsing form: ' + err.stack);
});
//save file to disk
form.on('file', function (name, file) {
var uploadDirectory = '/img/user/profile/';
var oldPath = file.path;
var newPath = uploadDirectory + file.originalFilename;
fs.rename(oldPath, newPath, function (err) {
if (err) throw err;
console.log('renamed complete');
});
});
// Close emitted after form parsed
form.on('close', function () {
console.log('Upload completed!');
res.setHeader('Content-Type', 'text/plain');
res.end('Received ' + count + ' files');
});
// Parse req
form.parse(req);
});