Develop Reference

nodejs write after request end error

I am facing problem of write after request end in nodejs :
I have a server.js file , which sends request to other js file (say abc.js) which sends response back to server.js file and then server.js file writes the resoponse and then end response.
my problem is if I write response in abc.js and end it there only it works fine, but if it is in sererconf.js it doesn't.
Let me make it clear that I get this bug only when i send 20-30 requests at a time. I want to know the logic behind it, I searched a lot, but no nice answer found, any help will be appreciated.
server.js full code:
/* create HTTP server */
var httpd = http.createServer(function(req, res) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.writeHead(200, {"Content-Type" : "application/json"});
}).listen(3800);
/* send request to the file mentioned in url*/
httpd.on('request', function(req, res) {
urll = __dirname + '/..' + req.url;
fs.exists(urll, function (exists) {
if(exists){
var server = require(urll);
server.get(req,res);
}
});
module.exports = {
result : function(result){
if(Array.isArray(result)){
for(var key in result){
result[key] = JSON.parse(result[key]);
}
}
result = JSON.stringify(result);
res.write(result ,function(err) { if(!err) res.end(); });
},
};
});
**apps.js code**:
var constants = require('./lib/constant.js');
var APP_PATH = constants.APP_PATH;
module.exports = {
get : function(req) {
req.on('data', function(chunk) {
var hash = chunk;
hash = JSON.parse(hash);
var id = hash.id;
dirPath = APP_PATH + id;
fs.exists( dirPath, function (exists) {
if(exists)
read_app_dir(dirPath);
else
taskDone([]);
});
});
}
};
function read_app_dir(app_dir){
fs.readdir(app_dir,function(err, list){
if (err) {
httpd.log.info('cannot read apps dir at s_apps = '+err);
}else{
create_new_obj(list,app_dir);
}
});
}
function create_new_obj(list, app_dir){
appFilesObj = [];
var i = 0;
list.forEach(function(file) {
i=i+1;
file = app_dir +'/' +file;
appFilesObj.push(file);
if(i == Object.keys(list).length)
read_app_files(appFilesObj);
});
}
function read_app_files(appFilesObj,app_dir){
var apps = [];
var i = 0;
if(Object.keys(appFilesObj).length > 0){
appFilesObj.forEach(function(appfile) {
read_file(appfile,function(data){ i=i+1;
apps.push(data);
if(i == Object.keys(appFilesObj).length)
taskDone(apps);
});
});
}else{
taskDone([]);
}
}
function read_file(file,callback){
fs.readFile(file,'utf8', function (err, data) {
if (err)
httpd.log.info('cannot read file at s_apps = '+err);
else
callback(data);
});
}
function taskDone(apps){
var httpd = require(__dirname + '/server.js');
httpd.result(apps);
}
if I do res.write and res.end in this file in taskDone() then it works fine.
Thanks in advance :)

The problem with above code was, that I was sending back response by calling an exported function of server.js
like this:
var httpd = require(__dirname + '/server.js');
httpd.result(apps);
where result() is the function which I have exported in server.js to write response and end response
Instead of this, now I added a callback support while calling function of other files (ex-apps.js), so that I "res.write" and "res.end()" only when the actually called function gives back the response.
(I am not writing the whole code , please refer above code for difference in both)
httpd.on('request', function(req, res) {
urll = __dirname + '/..' + req.url;
fs.exists(urll, function (exists) {
if(exists){
var server = require(urll);
server.get(req,res,function(result){
res.write(result);
res.end();
});
}
});
**apps.js**
get : function(req, callback) {
req.on('data', function(chunk) {
//when task is done and taskDone() function is called I just callback() the result
function taskDone(result){
callback(result);
}
}
}
When I was sending result back by calling a function of server.js and then writing the response...I don't know how..but somehow server was getting confused in multiple requests and saying "write after end" error...while the end was called by some other user's request.
I may be wrong, but this is what I concluded from this :)
I hope this may help others.

send binary response in node from phantomjs child process

I have created a node endpoint to create rasterised version for my svg charts.
app.post('/dxexport', function(req, res){
node2Phantom.createPhantomProcess(req,res);
});
My node to phantom function uses spawn to run phantomjs
var spawn = require('child_process').spawn;
exports.createPhantomProcess = function(req,res){
var userRequest = JSON.stringify(req.body);
var bin = "node_modules/.bin/phantomjs"
var args = ['./dxexport/exporter-server.js', userRequest, res];
var cspr = spawn(bin, args);
cspr.stdout.on('data', function (data) {
var buff = new Buffer(data);
res.send(data);
});
cspr.stderr.on('data', function (data) {
data += '';
console.log(data.replace("\n", "\nstderr: "));
});
cspr.on('exit', function (code) {
console.log('child process exited with code ' + code);
process.exit(code);
});
};
when rendering is completed and file is successfully created I call the renderCompleted function inside phantomjs:
var renderCompleted = function (parameters) {
var exportFile = fileSystem.open(parameters.exportFileName, "rb"),
exportFileContent = exportFile.read();
parameters.response.statusCode = 200;
parameters.response.headers = {
"Access-Control-Allow-Origin": parameters.url,
"Content-Type": contentTypes[parameters.format],
"Content-Disposition": "attachment; fileName=" + parameters.fileName + "." + parameters.format,
"Content-Length": exportFileContent.length
};
parameters.response.setEncoding("binary");
parameters.response.write(exportFileContent);
/////// somehow send exportFileContent as node res object for download \\\\\\\\
exportFile.close();
parameters.format !== "svg" && fileSystem.remove(parameters.exportFileName);
for (var i = 0; i < parameters.filesPath.length; i++)
fileSystem.remove(parameters.filesPath[i]);
parameters.filesPath = [];
parameters.response.close()
};
this response is passed from nodejs however apparently this code is calling phantomjs methods so I get errors like
TypeError: 'undefined' is not a function (evaluating 'parameters.response.setEncoding("binary")')
How can I send the binary file response somehow to the node function so it can be sent with my node server to the user?
Any help is appreciated.

Ok after some struggle here is the working solution if someone stumbles on this post.
As Artjom B. mentioned I found that the easiest way was delegate the rendering and file creation of the visualisation to phantomjs. Then send all the parameters related to those operations once done through the console.
Also updated the answer based on #ArtjomB.'s advice to wrap the console message sent in a unique beginning and end string so the risk of the other possible future outputs being mistaken for the intended rendered file object is mitigated.
var renderCompleted = function (parameters) {
console.log("STRTORNDRD" + JSON.stringify(parameters) + "ENDORNDRD");
};
This then gets picked up by stdout and usable like this:
exports.exportVisual = function (req, res) {
var userRequest = JSON.stringify(req.body);
var bin = "node_modules/.bin/phantomjs"
var args = ['./dxexport/exporter-server.js', userRequest, res];
var cspr = spawn(bin, args);
var contentTypes = {
pdf: "application/pdf",
svg: "image/svg+xml",
png: "image/png"
};
cspr.stdout.on('data', function (data) {
var buff = new Buffer(data).toString('utf8');
var strData = buff.match(new RegExp("STRTORNDRD" + "(.*)" + "ENDORNDRD"));
if (strData) {
var parameters = JSON.parse(strData[1]);
var img = fs.readFileSync(parameters.exportFileName);
var headers = {
"Access-Control-Allow-Origin": parameters.url,
"Content-Type": contentTypes[parameters.format],
"Content-Disposition": "attachment; fileName=" + parameters.fileName + "." + parameters.format,
"Content-Length": img.length
};
res.writeHead(200, headers);
res.end(img, 'binary');
// delete files after done
if (parameters.format != "svg") {
fs.unlink(parameters.exportFileName);
}
for (var i = 0; i < parameters.filesPath.length; i++)
fs.unlink(parameters.filesPath[i]);
// done. kill it
cspr.kill('SIGINT');
}
});
cspr.stderr.on('data', function (data) {
data += '';
console.log(data.replace("\n", "\nstderr: "));
});
cspr.on('exit', function (code) {
console.log('child process exited with code ' + code);
process.exit(code);
});
};

Node.js Asynchronous read and write

I am having problem with asynchronous file read and write operations. only the last file is written to the server.
js:
function uploadassignment(req, res){
var path;
var multiparty = require("multiparty");
var form = new multiparty.Form();
console.log(req.query);
var filelength = req.query.filecount;
console.log(filelength);
form.parse(req, function(err, fields, files){
console.log(req.body);
for(i=0;i<filelength;i++){
var img = files.file[i];
console.log(img);
console.log('divide');
var fs = require('fs');
fs.readFile(img.path, function(err, data){
var originalfile = img.originalFilename.split('.');
console.log(originalfile);
var file_ext = originalfile[1];
path = "public/assignments/"+img.originalFilename;
console.log(path);
fs.writeFile(path, data, function(error){
if(error)console.log(error);
});
})
}
});
};

This is a common bug caused by using a loop variable without a closure. By the time the callback for the read operation is invoked, the loop has terminated and the index i points to the last element (and hence your img contains the last file). Create a function (a closure) that accepts the index as the parameter and call this function from the loop:
function blah(i) {
var img = files.file[i];
console.log(img);
console.log('divide');
var fs = require('fs');
fs.readFile(img.path, function(err, data){
var originalfile = img.originalFilename.split('.');
console.log(originalfile);
var file_ext = originalfile[1];
path = "public/assignments/"+img.originalFilename;
console.log(path);
fs.writeFile(path, data, function(error){
if(error)console.log(error);
});
})
}
for(i=0;i<filelength;i++) blah(i);

This isn't quite an answer, but it is too long for a comment.
What is not working? The file reading/writing bit of your code works fine:
var fs = require("fs")
img = {
path: "./test.txt",
originalFilename: "test.txt"
}
fs.readFile(img.path, function(err, data){
if(err)console.log(err);
var originalfile = img.originalFilename.split('.');
console.log(originalfile);
var file_ext = originalfile[1];
path = "public/assignments/"+img.originalFilename;
console.log(path);
fs.writeFile(path, data, function(error){
if(error)console.log(error);
});
})
With a directory structure like:
script.js
text.txt
public
assignments
I think your problem might be that you are assigning "fs" locally, then trying to call it from an async function. That might be why only the last one works (maybe.)
Try moving var fs = require('fs'); to the top of your code.

How to include javascript on client side of node.js?

I'm a beginner of node.js and javascript.
I want to include external javascript file in html code. Here is the html code, "index.html":
<script src="simple.js"></script>
And, here is the javascript code, "simple.js":
document.write('Hello');
When I open the "index.html" directly on a web browser(e.g. Google Chrome), It works.
("Hello" message should be displayed on the screen.)
However, when I tried to open the "index.html" via node.js http server, It doesn't work.
Here is the node.js file, "app.js":
var app = require('http').createServer(handler)
, fs = require('fs')
app.listen(8000);
function handler (req, res) {
fs.readFile(__dirname + '/index.html',
function (err, data) {
if (err) {
res.writeHead(500);
return res.end('Error loading index.html');
}
res.writeHead(200);
res.end(data);
});
}
("index.html", "simple.js" and "app.js" are on same directory.)
I started the http server. (by "bash$node app.js")
After then, I tried to connect "localhost:8000".
But, "Hello" message doesn't appear.
I think the "index.html" failed to include the "simple.js" on the http server.
How should I do?

Alxandr is right. I will try to clarify more his answer.
It happens that you have to write a "router" for your requests. Below it is a simple way to get it working. If you look forward www.nodebeginner.org you will find a way of build a proper router.
var fs = require("fs");
var http = require("http");
var url = require("url");
http.createServer(function (request, response) {
var pathname = url.parse(request.url).pathname;
console.log("Request for " + pathname + " received.");
response.writeHead(200);
if(pathname == "/") {
html = fs.readFileSync("index.html", "utf8");
response.write(html);
} else if (pathname == "/script.js") {
script = fs.readFileSync("script.js", "utf8");
response.write(script);
}
response.end();
}).listen(8888);
console.log("Listening to server on 8888...");

The problem is that nomatter what your browser requests, you return "index.html". So the browser loads your page and get's html. That html includes your script tag, and the browser goes asking node for the script-file. However, your handler is set up to ignore what the request is for, so it just returns the html once more.

Here is a working code.
There should be more cleaner simpler code, but this is very close to minimal.
This code suppose your index.html and other files are under /client dir.
Good luck.
var fs = require('fs');
var url = require("url");
var path = require('path');
var mime = require('mime');
var log = console.log;
var handler = function (req, res)
{
var dir = "/client";
var uri = url.parse(req.url).pathname;
if (uri == "/")
{
uri = "index.html";
}
var filename = path.join(dir, uri);
log(filename);
log(mime.lookup(filename));
fs.readFile(__dirname + filename,
function (err, data)
{
if (err)
{
res.writeHead(500);
return res.end('Error loading index.html');
}
log(data);
log(filename + " has read");
res.setHeader('content-type', mime.lookup(filename));
res.writeHead(200);
res.end(data);
});
}

Your handler is hardcoded to always return the content of /index.html. You need to pay attention to the resource that is being requested and return the right one. (i.e. if the browser asks for simple.js then you need to give it simple.js instead of index.html).

function contentType(ext) {
var ct;
switch (ext) {
case '.html':
ct = 'text/html';
break;
case '.css':
ct = 'text/css';
break;
case '.js':
ct = 'text/javascript';
break;
default:
ct = 'text/plain';
break;
}
return {'Content-Type': ct};
}
var PATH = 'C:/Users/DELL P26E/node_modules'
var http = require("http");
var fs = require('fs');
var url = require("url");
var path = require("path")
var fileName = "D:/index.html";
var server = http.createServer (function (request, response) {
var dir = "D:/";
var uri = url.parse(request.url).pathname;
if (uri == "/") {
uri = "index.html";
}
var filename = path.join(dir, uri);
fs.readFile(filename,
function (err, data) {
if (err) {
response.writeHead(500);
return response.end('Error loading index.html');
}
var ext = path.extname(filename)
response.setHeader('content-type',contentType(ext));
response.writeHead(200);
response.end(data);
});
}).listen(3000);
console.log('Server running on 8124') ;

How to execute the function return the value to called function across node js files

I have 3 node js files :
mysqlconnection.js to store the database connection properties:
var mysql = require('mysql');
var cjson = require('cjson');
var yaml_config = require('node-yaml-config');
// project files
var config = yaml_config.load(__dirname + '/billingv2.yaml');
exports.execute = function(callback){
var connection = mysql.createConnection(
{
host : config.host,
user : config.user,
password : config.password,
database : config.database,
}
);
connection.connect();
return callback(null,connection);
}
subscriptionRestService.js to handle the REST api calls:
var express = require('express');
var app = express();
app.use(express.bodyParser());
var fs = require('fs');
// Project files
var mysql = require('./mysqlRestService.js');
// Get Resource Subscription data by Resourceuri
app.post('/pricingdetails', function(req, res) {
var workload = req.body;
if(workload.elements && workload.elements.length > 0)
{
var arr = [];
for(var index in workload.elements)
{
arr[index] = workload.elements[index].uri;
}
var resourceIdentifiers = arr.join(',');
}
console.log(resourceIdentifiers);
mysql.getPricingDetail(function(resourceIdentifiers,callback){
});
});
mysqlRestService.js to handle mysql queries/stored procedures:
// packages
var mysql = require('mysql');
var cjson = require('cjson');
var fs = require('fs');
var yaml_config = require('node-yaml-config');
// project files
var dbconnection = require('./mysqlconnection');
exports.getPricingDetail = function (resourceIdentifiers,callback){
console.log('entered into mysql function');
console.log(resourceIdentifiers);
var pricingDetail = {};
dbconnection.execute(function(err,response){
if(err){
throw err;
}
else
{
var selectqueryString = "call SP_ExposePricingDetailforUI('" + resourceIdentifiers + "')";
response.query(selectqueryString, function(err,pricingDetail){
if(err) {
throw err;
}
else
{
console.log(pricingDetail);
pricingDetail = pricingDetail;
}
});
}
});
//console.log('printing pricing details');
//console.log(pricingDetail);
};
problems faced
Unable to send the variable resourceIdentifiers from subscriptionRestService to mysqlRestService.js
Unable to return the pricingdetail from mysqlRestService.js to calling function in subscriptionRestService.
Any guidance greatly appreciated.

Unable to send the variable resourceIdentifiers from subscriptionRestService to mysqlRestService.js
Well, you didn't send it. It currently is a parameter of your callback function in the invocation, not an argument for the parameter of getPricingDetails. Use
mysql.getPricingDetail(resourceIdentifiers, function callback(result){
// use result here
});
Unable to return the pricingdetail from mysqlRestService.js to calling function in subscriptionRestService.
I've got no idea what pricingDetail = pricingDetail; was supposed to do. You have to call (invoke) back the callback here! Use
callback(pricingDetail);