Develop Reference

NodeJS trying to use a URL inside of a fs.createfilestream

I'm trying to do a post request onto my api, the api works perfectly ( I am able to post files, but not through a url), but now I'm trying to post through an url.
this is the code I have now, I removed some lines that aren't relevant to the question or were for testing.
request({
url: url + "gettoken"
, json: true
}, function (error, response, body) {
user = body;
var rs = fs.createReadStream(up.url);
var ws = request.post(url + "upload?token=" + `${user.token}&key=${user.key}&filename=${filename}`);
ws.on('drain', function () {
rs.resume();
});
rs.on('end', function () {
console.log(filename);
});
ws.on('error', function (err) {
console.error('cannot send file ' + err);
});
rs.pipe(ws);
})
Can anyone please help me.

So the idea is to upload a file that's located at up.url to another server at url + "upload?...".
Since fs.createReadStream is meant to read local files, and not URL's, you need something that can create a stream from a URL (or rather, retrieve that URL and stream the response).
You can also use request for that:
request({
url: url + "gettoken",
json: true
}, function (error, response, body) {
const user = body;
const rs = request.get(up.url);
const ws = request.post(url + "upload?token=" + `${user.token}&key=${user.key}&filename=${filename}`);
rs.on('end', function () {
console.log(filename);
});
ws.on('error', function (err) {
console.error('cannot send file ' + err);
});
rs.pipe(ws);
});
Typically, file uploads work through multipart/form-data, but your code doesn't suggest that being used here. If it is, the code would become something like this:
const ws = request.post(url + "upload?token=" + `${user.token}&key=${user.key}&filename=${filename}`, {
formData : {
the_file : rs
}
});
// no `rs.pipe(ws)`

How to use node modules correctly

I am a CS student with a strong Java background, and javascript is resulting to be a challenging but fun experience so far, that is until I ran into the situation where I tried to use my own modules to return values that require the program to wait for the completion of some procedure before returning.So far no-one from any forum that I have posted this question on has been able to give an actual code fix to the problem, they have referred me to read further material that is not related to the problem at hand. Would anyone please read the code and provide a working correct standard solution to the problem I am facing?
Here is the code, a simple nodes server application, app.js and a weather module, weatherApp.js that uses an user provided zip code and returns a weather forecast in the area.
here is the code:
weatherApp.js
// The required modules.
var http = require("http");
var https = require("https");
//result object
var resultSet = {
googleRequestUrl:"",
forecastIOrequest:"",
latitude :"",
longitude:"",
localInfo:"",
weather:"",
humidity:"",
pressure:"",
time:""
};
//print out error messages
function printError(error){
console.error(error.message);
}
//Forecast API required information:
//key for the forecast IO app
var forecast_IO_Key = "bb9aac7c57877f8f5fab339e3b55669a";
var forecast_IO_Web_Adress = "https://api.forecast.io/forecast/";
//Create Forecast request string function
function createForecastRequest(latitude, longitude){
var request = forecast_IO_Web_Adress + forecast_IO_Key + "/"
+ latitude +"," + longitude;
return request;
}
//Google GEO API required information:
//Create Google Geo Request
var google_GEO_Web_Adress = "https://maps.googleapis.com/maps/api/geocode/json?address=";
function createGoogleGeoMapRequest(zipCode){
var request = google_GEO_Web_Adress+zipCode + "&sensor=false";
return request;
}
// 1- Need to request google for geo locations using a given zip
function connectToGoogleGEO(zipCode, afterCallback){
var googleRequest = https.get(createGoogleGeoMapRequest(zipCode), function(response){
//saving the Google request URL
resultSet.googleRequestUrl = createGoogleGeoMapRequest(zipCode);
var body = "";
var status = response.statusCode;
//a- Read the data.
response.on("data", function(chunk){
body+=chunk;
});
//b- Parse the data.
response.on("end", function(){
if(status === 200){
try{
var googleReport = JSON.parse(body);
resultSet.latitude = googleReport.results[0].geometry.location.lat;
resultSet.longitude = googleReport.results[0].geometry.location.lng;
resultSet.localInfo = googleReport.results[0].address_components[0].long_name + ", " +
googleReport.results[0].address_components[1].long_name + ", " +
googleReport.results[0].address_components[2].long_name + ", " +
googleReport.results[0].address_components[3].long_name + ". ";
// callback to forecast IO.
afterCallback(resultSet.latitude, resultSet.longitude);
}catch(error){
printError(error.message);
}finally{
// nothing here
}
}else{
printError({message: "Error with GEO API"+http.STATUS_CODES[response.statusCode]})
}
});
});
}
function connectToForecastIO(latitude,longitude){
var forecastRequest = https.get(createForecastRequest(latitude,longitude),function(response){
resultSet.forecastIOrequest = createForecastRequest(latitude,longitude);
var body = "";
var status = response.statusCode;
//read the data
response.on("data", function(chunk){
body+=chunk;
});
//parse the data
response.on("end", function(){
try{
var weatherReport = JSON.parse(body);
resultSet.weather = weatherReport.currently.summary;
resultSet.humidity = weatherReport.currently.humidity;
resultSet.temperature = weatherReport.currently.temperature;
resultSet.pressure = weatherReport.currently.pressure;
resultSet.time = weatherReport.currently.time;
}catch(error){
printError(error.message);
}finally{
console.log(resultSet);
}
});
});
}
function get(zipCode){
var results = connectToGoogleGEO(zipCode, connectToForecastIO);
return results;
}
//define the name of the outer module.
module.exports.get = get;
And here is the server code:
app.js
var express = require("express");
var weatherApp = require("./weatherApp.js");
var path = require("path");
var http = require("http");
var app = express();
//creating routes
//The home
app.get("/", function(req, res){
res.redirect("/weather");
});
app.get("/weather", function(req, res){
res.sendFile(path.join(__dirname + "/index.html"));
});
//------------------------------------------------------
//The resources, css, web js files, images etc.
app.get("/StyleSheets/style.css", function(req, res){
res.sendFile(path.join(__dirname + "/StyleSheets/style.css"));
});
app.get("/webScripts/app.js", function(req, res){
res.sendFile(path.join(__dirname + "/webScripts/app.js"));
});
app.get("/webImages/swirl_pattern.png", function(req, res){
res.sendFile(path.join(__dirname + "/webImages/swirl_pattern.png"));
});
//-------------------------------------------------------
//other requests
app.get("/zipcode.do", function(req, res){
var zipcode = req.query["zipcode"];
var response = "No report Available";
function getReport(zipCode, callback){
response = weatherApp.get(req.query["zipcode"]);
}
getReport(zipcode, ()=>{
res.send("<p>" + response+ "</p>");
});
});
//any other entry thats not listed as a valid to request
app.get("/:title", function(req,res){
var title = req.param.title;
if(title === undefined){
var status = res.status(503);
res.send("This page does not exists" + '"' + http.STATUS_CODES[503] + '"');
}else{
res.send(title);
}
});
app.listen(3000, function(){
console.log("Server running at port: 3000")
});
The main issue I am having right now is:
The program is not returning anything from the module even when final console.log in the weather module prints the right resultSet object.
The server is not waiting for the module to return, and continues to print no data.
Can someone provide a working fix to any of these problems I would be really grateful, This has really hindered my progress and broken down my morale a little :(

Your problem is that you are using asynchronous functions as if they were synchronous.
It might not be the only issue here, but this function is particularly problematic:
function get(zipCode){
var results = connectToGoogleGEO(zipCode, connectToForecastIO);
return results;
}
connectToGoogleGEO() calls the asynchronous https.get() function and does not return the data that is retrieved from Google. You need to rewrite your code so that it does not expect the data to be returned by the function. Instead, you need to pass a callback that will handle the data.
Take care to know when you are calling asynchronous functions and how their callbacks work. It is fundamental when working with Node.js

Read raw http message in Nodejs

I'm sending an http request using the http.request function, and I would like to read the whole http response like text; that is, the raw http protocol text. Is it possible? I've written the below code but it's not working.
// Set up the request
console.log('Sending request');
var post_req = http.request(post_options, function(res) {
res.setEncoding('utf8');
console.log('Response statusCode: ' + res.statusCode);
// res.on('data', function (chunk) {
// console.log('Response: ' + chunk);
// });
// res.on('end', function() {});
});
post_req.on('socket', function (socket) {
var response = "";
socket.on('data', function(chunk){
console.log(chunk);
});
});
// post the data
post_req.write(post_data);
post_req.end();

If you want access to the raw http message, I'd suggest using the net module instead, and writing the request yourself. Something like this for a simple GET request:
var net = require('net');
var host = 'stackoverflow.com',
port = 80,
socket = net.connect(port, host, function() {
var request = "GET / HTTP/1.1\r\nHost: " + host + "\r\n\r\n",
rawResponse = "";
// send http request:
socket.end(request);
// assume utf-8 encoding:
socket.setEncoding('utf-8');
// collect raw http message:
socket.on('data', function(chunk) {
rawResponse += chunk;
});
socket.on('end', function(){
console.log(rawResponse);
});
});
For a POST request sending application/x-www-form-urlencoded data, you could write the request using something like:
function writePOSTRequest (data, host, path) {
return "POST " + path + " HTTP/1.1\r\n" +
"Host: " + host + "\r\n" +
"Content-Type: application/x-www-form-urlencoded\r\n" +
"Content-Length: " + Buffer.byteLength(data) + "\r\n\r\n" +
data + "\r\n\r\n";
}
var data = "name1=value1&name2=value2",
request = writePOSTRequest(data, host, "/path/to/resource");
where I'm using Buffer.byteLength because Content-Length requires the length in bytes, not in characters. Also, remember that data must be URL encoded.
If you don't know much about the format of HTTP messages, then this is a decent place to start:
http://jmarshall.com/easy/http/
Also, if you don't know what the encoding of the response will be then you'll have to parse the headers first to find out, but UTF-8 is by far the most common so it's a pretty safe bet.

Streams2 and Streams1 not always able to inter-operate well, see "problem: streams1 and streams2 duality" in this video.
I tried to listen data at a bit lower level than streams and this code prints raw http response with headers for me:
var http = require('http');
var raw = '';
console.log('Sending request');
var req = http.request({host: 'stackoverflow.com'}, function(res) {
watch(res, 'res');
res.on('end', function() {
console.log(raw);
});
res.on('data', function(data) {
// if we don't attach 'data' handler here 'end' is not called
});
});
req.on('socket', function (socket) {
socket.resume();
var oldOndata = socket.ondata;
socket.ondata = function(buf, start, end) {
raw += buf.slice(start, end).toString();
oldOndata.call(socket, buf, start, end);
};
});
req.end();

Assuming these kind of tools are allowed in your environment, you could run up an HTTP debug proxy such as Fiddler http://www.fiddler2.com/, which enables you to inspect the HTTP calls and responses.

NodeJS HTTP request POST ERROR socket hang up

Hi I'm having problems to perform HTTP request on NodeJS given a larger number array of json object. The request works fine given small array of json object. However, if I try to increase the size array of json, I received Error: socket hang up {"error":{"code":"ECONNRESET"}}. Is it required to perform multiple write? Or is it something wrong going on at the other end?
Thanks in advance for taking your time here!
// data is a json object
var post_data = JSON.stringify(data);
var buf = new Buffer(post_data);
var len = buf.length;
var options = {
hostname: address,
port: port,
path: pathName,
method: 'PUT',
headers: {
'Content-Type':'application/json',
'Content-Length': len,
'Transfer-Encoding':'chunked'
}
};
// http call to REST API server
var req = restHttp.request(options, function(res) {
console.log('server PUT response received.');
var resData = '';
res.on('data', function(replyData) {
// Check reply data for error.
console.log(replyData.toString('utf8'));
if(replyData !== 'undefined')
resData += replyData;
});
res.on('end', function() {
callback(JSON.parse(resData));
});
});
req.write(buf);
req.end();

You can stream the request body.
If the data in buf was in a readable stream then you can just do buf.pipe(req).
For example, if the current directory contains a file data.json with the JSON you can do
var buf = fs.createReadStream(__dirname + '/data.json');
to create a ReadStream object. Then you can pipe this to you req
buf.pipe(req);
The pipe command will call req.end once its done streaming.

Node.js http request

I am a beginner with node.js, and I'm not seeming to get this to work.
function sleep(milliSeconds){
var startTime = new Date().getTime();
while (new Date().getTime() < startTime + milliSeconds);
}
var isRequestComplete = false;
while(isRequestComplete == false){
console.log("in make request");
var querystring = require('querystring');
var data = querystring.stringify({
username: 'username',
password: 'password',
action: 'convert',
voice: 'engfemale1',
text: 'stuff and things, this should take longer than one request.'
});
var options = {
host: 'ws.ispeech.org',
port: 80,
path: '/api/rest/1.5',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': data.length
}
};
var http = require('http');
var req = http.request(options, function(res) {
console.log("got response");
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log("body: " + chunk);
if(chunk.indexOf("finished") != -1){
isRequestComplete = true;
}
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.write(data);
req.end();
console.log("completed");
sleep(5000);
}
For whatever reason the http request does not send a response back, ever. Unless the code is fully finished, so in the while loop I never get a response back. Thus the loop never ends. The username and password in my program are inputted, here they are not for confidentiality. Thanks for reading.

this is NOT the way to get your code to sleep! A while loop isn't "sleeping" it's processing as fast as it can. In you case it's grabbing date after date after date trying to get to your destination.
take a look here to see how this *should work.
http://nodejs.org/api/http.html

Have a look at the very first http-server example on http://nodejs.org/.
You have to create an http-server that listens for requests from browsers that arrive at the specified IP-Address:Port. Once a request arrives, the server send the specified response to the browser.