I have written a node.js server which creates a server and prints the output when done with an asynchronous function. While I am able to get the correct output always in the console.log. The same is not getting reflected in my response. Here is my code snippet :-
var request = require('request');
var http = require('http');
var cheerio = require('cheerio');
var url = require('url');
var Curl = require( 'node-libcurl' ).Curl;
var sleep = require('sleep');
isDone = 0;
globalImage = "";
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
var url_parts = url.parse(req.url, true);
var query = url_parts.query;
var main_url = query["link"];
if (req.url != '/favicon.ico') {
res.writeHead(200);
if(main_url != undefined ){
var position = parseInt(query["position"]);
// web_scrap()
web_scrap(main_url,position, function(image) {
console.log("Console log : " + image);
globalImage = image;
});
res.write(globalImage);
res.end("HEY");
}
}
else {//For favicon and other requests just write 404s
res.writeHead(404);
res.write('This URL does nothing interesting');
res.end();
}
}).listen(3000, '127.0.0.1');
console.log('Server running at http://127.0.0.1:3000/');
function web_scrap(url, position, callback){
// do something
callback(JSON.stringify(product));
}
Now on starting the server and accessing it in browser with parameters link and position as get, I am getting output on second or third refresh. I am getting perfect output in console.log though !
Can anyone help or guide me in this regard ?
Thanks !
From what I understand, you're loading an image from an external source, asynchronously.
Thus, your function continues to run, even though the load is not finished yet. And as your globalImage is a global variable, once it is loaded, it stays in memory, that's why you get the data after some tries.
Just move your res.write and res.end in the callback function, this way the content will be sent once the image is loaded.
web_scrap(main_url,position, function(image) {
console.log("Console log : " + image);
globalImage = image;
res.write(globalImage);
res.end("HEY");
});
Anyway, except if you want to cache your image, you should not have a globalImage variable, as it would stay in memory even though you would want it to be garbage collected. You can remove the variable and just make this:
web_scrap(main_url,position, function(image) {
console.log("Console log : " + image);
res.write(image);
res.end("HEY");
});
Related
Overarching goal is to save some JSON data I create on a webpage to my files locally. I am definitely sending something to the server, but not in format I seem to able to access.
JsonData looks like:
{MetaData: {Stock: "UTX", Analysis: "LinearTrend2"}
Projections: [2018-10-12: 127.62, 2018-10-11: 126.36000000000001, 2018-10-10: 132.17, 2018-10-09: 140.12, 2018-10-08: 137.73000000000002, …]}
XMLHttpRequest on my webpage:
function UpdateBackTestJSON(JsonUpdate){ //JsonUpdate being the JSON object from above
var request = new XMLHttpRequest();
request.open('POST', 'UpdateBackTestJSON');
request.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
// request.setRequestHeader("Content-Type", "text/plain;charset=UTF-8");
request.onload = function() {
console.log("Updated JSON File");
};
console.log("about to send request");
console.log(JsonUpdate);
request.send(JSON.stringify(JsonUpdate));
}
and I handle posts on my server (rather carelessly I realize, just going for functionality as a start here)
var http = require('http')
, fs = require('fs')
, url = require('url')
, port = 8008;
var server = http.createServer (function (req, res) {
var uri = url.parse(req.url)
var qs = require('querystring');
if (req.method == 'POST'){
var body = '';
req.on('data', function (data){
body += data;
// 1e6 === 1 * Math.pow(10, 6) === 1 * 1000000 ~~~ 1MB
if (body.length > 1e6){
// FLOOD ATTACK OR FAULTY CLIENT, NUKE REQUEST
req.connection.destroy();
}
});
req.on('end', function () {
var POST = qs.parse(body);
console.log(POST); // PARSED POST IS NOT THE RIGHT FORMAT... or something, idk whats going on
UpdateBackTestData(POST);
});
}
function UpdateBackTestData(TheJsonData){
console.log("UpdateBackTestData");
console.log(TheJsonData);
JsonUpdate = JSON.parse(TheJsonData);
console.log(JsonUpdate["MetaData"]);
//var Stock = JsonUpdate["MetaData"]["Stock"];
// var Analysis = JsonUpdate["MetaData"]["Analysis"];
fs.writeFile("/public/BackTestData/"+Analysis+"/"+Stock+".json", TheJsonData, function(err){
if(err){
console.log(err);
}
console.log("updated BackTest JSON!!!");
});
}
Most confusing to me is that when I run this, the Json object Im am trying to pass, does go through to the server, but the entirety of the data is a string used as a key for a blank value in an object. when I parse the body of the POST, I get: {'{MetaData:{'Stock':'UTX','Analysis:'LinearTrend2'},'Projections':[...]}': ''}. So my data is there... but not in a practical format.
I would prefer not to use express or other server tools, as I have a fair amount of other services set up in my server that I don't want to go back and change if I can avoid it.
Thanks for any help
I am a CS student with a strong Java background, and javascript is resulting to be a challenging but fun experience so far, that is until I ran into the situation where I tried to use my own modules to return values that require the program to wait for the completion of some procedure before returning.So far no-one from any forum that I have posted this question on has been able to give an actual code fix to the problem, they have referred me to read further material that is not related to the problem at hand. Would anyone please read the code and provide a working correct standard solution to the problem I am facing?
Here is the code, a simple nodes server application, app.js and a weather module, weatherApp.js that uses an user provided zip code and returns a weather forecast in the area.
here is the code:
weatherApp.js
// The required modules.
var http = require("http");
var https = require("https");
//result object
var resultSet = {
googleRequestUrl:"",
forecastIOrequest:"",
latitude :"",
longitude:"",
localInfo:"",
weather:"",
humidity:"",
pressure:"",
time:""
};
//print out error messages
function printError(error){
console.error(error.message);
}
//Forecast API required information:
//key for the forecast IO app
var forecast_IO_Key = "bb9aac7c57877f8f5fab339e3b55669a";
var forecast_IO_Web_Adress = "https://api.forecast.io/forecast/";
//Create Forecast request string function
function createForecastRequest(latitude, longitude){
var request = forecast_IO_Web_Adress + forecast_IO_Key + "/"
+ latitude +"," + longitude;
return request;
}
//Google GEO API required information:
//Create Google Geo Request
var google_GEO_Web_Adress = "https://maps.googleapis.com/maps/api/geocode/json?address=";
function createGoogleGeoMapRequest(zipCode){
var request = google_GEO_Web_Adress+zipCode + "&sensor=false";
return request;
}
// 1- Need to request google for geo locations using a given zip
function connectToGoogleGEO(zipCode, afterCallback){
var googleRequest = https.get(createGoogleGeoMapRequest(zipCode), function(response){
//saving the Google request URL
resultSet.googleRequestUrl = createGoogleGeoMapRequest(zipCode);
var body = "";
var status = response.statusCode;
//a- Read the data.
response.on("data", function(chunk){
body+=chunk;
});
//b- Parse the data.
response.on("end", function(){
if(status === 200){
try{
var googleReport = JSON.parse(body);
resultSet.latitude = googleReport.results[0].geometry.location.lat;
resultSet.longitude = googleReport.results[0].geometry.location.lng;
resultSet.localInfo = googleReport.results[0].address_components[0].long_name + ", " +
googleReport.results[0].address_components[1].long_name + ", " +
googleReport.results[0].address_components[2].long_name + ", " +
googleReport.results[0].address_components[3].long_name + ". ";
// callback to forecast IO.
afterCallback(resultSet.latitude, resultSet.longitude);
}catch(error){
printError(error.message);
}finally{
// nothing here
}
}else{
printError({message: "Error with GEO API"+http.STATUS_CODES[response.statusCode]})
}
});
});
}
function connectToForecastIO(latitude,longitude){
var forecastRequest = https.get(createForecastRequest(latitude,longitude),function(response){
resultSet.forecastIOrequest = createForecastRequest(latitude,longitude);
var body = "";
var status = response.statusCode;
//read the data
response.on("data", function(chunk){
body+=chunk;
});
//parse the data
response.on("end", function(){
try{
var weatherReport = JSON.parse(body);
resultSet.weather = weatherReport.currently.summary;
resultSet.humidity = weatherReport.currently.humidity;
resultSet.temperature = weatherReport.currently.temperature;
resultSet.pressure = weatherReport.currently.pressure;
resultSet.time = weatherReport.currently.time;
}catch(error){
printError(error.message);
}finally{
console.log(resultSet);
}
});
});
}
function get(zipCode){
var results = connectToGoogleGEO(zipCode, connectToForecastIO);
return results;
}
//define the name of the outer module.
module.exports.get = get;
And here is the server code:
app.js
var express = require("express");
var weatherApp = require("./weatherApp.js");
var path = require("path");
var http = require("http");
var app = express();
//creating routes
//The home
app.get("/", function(req, res){
res.redirect("/weather");
});
app.get("/weather", function(req, res){
res.sendFile(path.join(__dirname + "/index.html"));
});
//------------------------------------------------------
//The resources, css, web js files, images etc.
app.get("/StyleSheets/style.css", function(req, res){
res.sendFile(path.join(__dirname + "/StyleSheets/style.css"));
});
app.get("/webScripts/app.js", function(req, res){
res.sendFile(path.join(__dirname + "/webScripts/app.js"));
});
app.get("/webImages/swirl_pattern.png", function(req, res){
res.sendFile(path.join(__dirname + "/webImages/swirl_pattern.png"));
});
//-------------------------------------------------------
//other requests
app.get("/zipcode.do", function(req, res){
var zipcode = req.query["zipcode"];
var response = "No report Available";
function getReport(zipCode, callback){
response = weatherApp.get(req.query["zipcode"]);
}
getReport(zipcode, ()=>{
res.send("<p>" + response+ "</p>");
});
});
//any other entry thats not listed as a valid to request
app.get("/:title", function(req,res){
var title = req.param.title;
if(title === undefined){
var status = res.status(503);
res.send("This page does not exists" + '"' + http.STATUS_CODES[503] + '"');
}else{
res.send(title);
}
});
app.listen(3000, function(){
console.log("Server running at port: 3000")
});
The main issue I am having right now is:
The program is not returning anything from the module even when final console.log in the weather module prints the right resultSet object.
The server is not waiting for the module to return, and continues to print no data.
Can someone provide a working fix to any of these problems I would be really grateful, This has really hindered my progress and broken down my morale a little :(
Your problem is that you are using asynchronous functions as if they were synchronous.
It might not be the only issue here, but this function is particularly problematic:
function get(zipCode){
var results = connectToGoogleGEO(zipCode, connectToForecastIO);
return results;
}
connectToGoogleGEO() calls the asynchronous https.get() function and does not return the data that is retrieved from Google. You need to rewrite your code so that it does not expect the data to be returned by the function. Instead, you need to pass a callback that will handle the data.
Take care to know when you are calling asynchronous functions and how their callbacks work. It is fundamental when working with Node.js
I'm using nodejs with node-http-proxy along with harmon. I am using harmon to rewrite the proxied response to include a javascript file and a css file. When I set the target of the proxy to be http://nodejs.org or anything other than localhost, I receive a 301 or 302 redirect. The script is rewriting the 301 response instead of the fully proxied response. How can I use harmon to rewrite the end response instead of the 302 response?
Here is the example of the script I am running from the harmon example folder:
var http = require('http');
var connect = require('connect');
var httpProxy = require('http-proxy');
var selects = [];
var simpleselect = {};
//<img id="logo" src="/images/logo.svg" alt="node.js">
simpleselect.query = 'img';
simpleselect.func = function (node) {
//Create a read/write stream wit the outer option
//so we get the full tag and we can replace it
var stm = node.createStream({ "outer" : true });
//variable to hold all the info from the data events
var tag = '';
//collect all the data in the stream
stm.on('data', function(data) {
tag += data;
});
//When the read side of the stream has ended..
stm.on('end', function() {
//Print out the tag you can also parse it or regex if you want
process.stdout.write('tag: ' + tag + '\n');
process.stdout.write('end: ' + node.name + '\n');
//Now on the write side of the stream write some data using .end()
//N.B. if end isn't called it will just hang.
stm.end('<img id="logo" src="http://i.imgur.com/LKShxfc.gif" alt="node.js">');
});
}
selects.push(simpleselect);
//
// Basic Connect App
//
var app = connect();
var proxy = httpProxy.createProxyServer({
target: 'http://nodejs.org'
})
app.use(require('../')([], selects, true));
app.use(
function (req, res) {
proxy.web(req, res);
}
);
The problem is that a lot of sites are now redirecting HTTP to HTTPS.
nodejs.org is one of those.
I have updated the sample https://github.com/No9/harmon/blob/master/examples/doge.js to show how the http-proxy needs to be configured to deal with HTTPS.
If you still have problems with other arbitrary redirects please log an issue on harmon.
Thanks
My server.js is
// server.js - the outer server loop
var http = require('http')
, php = require("./phpServer");
function start() {
function onRequest(request, response) {
php.phpServer('D:/websites/coachmaster.co.uk/htdocs',request, response);
response.write('Ending');
response.end();
}
http.createServer(onRequest).listen(80);
console.log("Server started.");
}
exports.start = start;
That calls php.phpServer every request with response as the 3rd param.
phpServer contains.
//
// phpServer.js - a generic server to serve static files and
//
var fs = require('fs')
, pathfuncs = require('path')
, url = require('url')
, mimetypes = require('./mimetypes')
function phpServer(root, request, response) {
// serve static or pass to php.
var data = url.parse(request.url);
var ext = pathfuncs.extname(data.pathname);
fs.stat(root+request.url, function(err, stat) {
if (err || !stat.isFile()) { // error or not file.
console.log('404');
response.writeHead(404);
response.write('Not Found');
return;
}
// exists - serve.
console.log("serve("+root+request.url+", mimetypes.mimetype("+ext+"))");
response.writeHead(200, {'Content-Type': mimetypes.mimetype(ext)});
response.write('Somethign to serve');
// fs.createReadStream(root+request.url).pipe(response);
});
}
exports.phpServer = phpServer
As I see it, response is an object and is passed by reference, therefore the response.write() here should write to the response.
It doesn't. Response here is NOT the same as response in onRequest, so nothing in phpServer is sent to the browser - not code nor content.
The console.logs come out and show what I would expect.
How can I get the object response passed so I can call write on it?
------------- added later -------------------
I've tried to apply answers given and code for server.is now
// server.js - the outer server loop
var http = require('http')
, fs = require('fs')
, pathfuncs = require('path')
, url = require('url')
, mimetypes = require('./mimetypes')
function phpServer(root, request, res) {
// code adapted from page 118 of Smashing Node.js by Guillermo Rauch
// res is response provided to onRequest.
var data = url.parse(request.url);
var ext = pathfuncs.extname(data.pathname);
res.write('Start reply');
fs.stat(root+request.url, function(err,stat) {
// define delayed callback - reponse in scope
if (err || !stat.isFile()) { // error or not file.
console.log('404');
res.writeHead(404);
res.write('Not Found');
res.end
return;
};
// exists so serve.
console.log("serve("+root+request.url+", mimetypes.mimetype("+ext+"))");
res.writeHead(200, {'Content-Type': mimetypes.mimetype(ext)});
res.write('The file contents');
res.end;
} // end callback,
); // end fs.stat call.
} // end phpServer
function start() {
function onRequest(request, response) {
phpServer('D:/websites/coachmaster.co.uk/htdocs',request, response);
}
http.createServer(onRequest).listen(80);
console.log("Server started.");
}
exports.start = start;
This does not reply at all - it times out. However the call to res.writeHead will either
fail, if res is out of scope/does not exist/undefined, or succeed if re is the param passed in.
It succeeds, and is followed by write and end, so please - what have I got wrong.
If the file does not exist I get a start reply and then a timeout.
At the res.write('Start reply'); res is the response param, yet it isn't later in the fs.stat call-back.
Why not?
Damn - this is frustrating.
The call to response.end should be moved from the onRequest function to phpServer. As it stands phpServer cannot write anything else since the stream has been closed.
function onRequest(request, response) {
php.phpServer('D:/websites/coachmaster.co.uk/htdocs',request, response);
// response.end(); // move this to phpServer
}
As explained in the documentation for response.end
This method signals to the server that all of the response headers and body have been sent; that server should consider this message complete.
Your problem is not with parameter passing, it's with basic asynchronous control flow. The stat() function does not do its work immediately. Its callback parameter is called when it's done. You basically cannot structure the code the way you've done it. Instead, your "phpServer" code will need to take a callback parameter of its own, and call it after it does its work.
Technically this is my first try in nodejs and frankly I am not sure if I am doing it right. I am creating a local server that will stream the output from a distant server. However, when I run my code and I enter a URL in the browser, the program fails with the following message:
events.js:45
throw arguments[1]; // Unhandled 'error' event
^
Error: ENOTFOUND, Domain name not found
at IOWatcher.callback (dns.js:74:15)
The URL I used was: 127.0.0.1:9000/http://www.yahoo.fr. And in the browser I had the following message:
No data received
Unable to load the webpage because the server sent no data.
Here are some suggestions:
Reload this web page later.
Error 324 (net::ERR_EMPTY_RESPONSE): The server closed the connection without sending any data.
Any help is greatly appreciated.
Here is the code:
var base, dest, node_client,
count = 0,
url = require('url'),
util = require('util'),
http = require('http'),
http_client = require('http'),
request = require('request'),
events = require('events'),
httpProxy = require('./lib/node-http-proxy'),
data_emitter = new events.EventEmitter();
httpProxy.createServer(9000, 'localhost').listen(8000);
http.createServer(function (req, res) {
if(!count)
{
base = url.parse(req.url).pathname;
node_client = http_client.createClient(80, base);
count++;
} else {
dest = req.url.substr(1, req.url.length -1);
}
request = node_client.request("GET", dest, {"host": base});
request.addListener("response", function (response) {
var body = "";
response.addListener("data", function (data) {
body +=data;
});
response.addListener("end", function () {
var out = JSON.parse(body);
if(out.length > 0) {
data_emitter.emit("out", out);
}
});
});
// request.close();
var listener = data_emitter.addListener("data", function(out) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write(JSON.stringify(out));
res.close();
});
}).listen(9000);
Wild guess : your browser automatically requests 127.0.0.1:9000/favicon.ico and your program then tries to resolve favicon.ico which obviously fails and makes your program crash before it can send any data for the real request.
Why such tangled code?
This is a scenario where it makes sense to avoid nested callbacks, and use named functions. If you refactor the code, then people are more likely to be help you.
Can you do console.log(out) in your listener callback? Let us know if Node.js has any response data to return.
Well, for any newbie like me in this area, here is how I solved it. It's not clean and can be implemented in better way. Feel free to change, give suggestions.
Code:
var url = require('url'),
http = require('http'),
request = require('request'),
httpProxy = require('./lib/node-http-proxy'),
des = '',
util = require('util'),
colors = require('colors'),
is_host = true;
httpProxy.createServer(9000, 'localhost').listen(8000);
var server = http.createServer(function (req, res) {
var pathname = '';
if(is_host) {
dest = req.url.substr(0, req.url.length -1);
pathname = dest;
is_host = false;
} else {
pathname = req.url.substr(0, req.url.length);
if(pathname.charAt(0) == "/") {
console.log('new request');
console.log(pathname);
pathname = dest + pathname;
}
}
console.log(pathname);
request.get({uri: pathname}, function (err, response, html) {
res.end(html);
});
console.log('fetched from ' + pathname);
});
server.listen(9000);