How to save the downloadable file link in zombie.js - javascript

I am scrapping a website using node.js and zombie.js. I am facing an issue where in a file I have an anchor which holds the link to download a pdf file.
If I click it using browser.clickLink() function, the result that I get in console is beyond my understanding. Is there a way to save this pdf file and have its link like in php? I want to save it for further processing. Here is my test js code
var http = require('http');
var browser = require('zombie');
var assert = require('assert');
const hostname = '127.0.0.1';
const port = 3000;
const server = http.createServer((req, res) => {
res.statusCode = 200;
//res.setHeader('Content-Type', 'text/plain');
//res.end('Hello World\n');
});
server.listen(port, hostname, () => {
console.log(`Server running at http://${hostname}:${port}/`);
});
var url = 'http://localhost/Node/HM_LandRegistry/downloadPdf.html'
browser.visit(url, function(error,browser) {
//browser.dump();
//console.log('browser.text (".link")', browser.text(".link"));
browser.clickLink("a.link");
browser.wait().then(function(){
console.log(browser.text());
browser.dump();
});
});

Here is something I found on google groups. It has solved my problem.
function getLinks(browser) {
var links = browser.querySelectorAll('.link');
return Array.prototype.map.call(links, function(e) {
return e.getAttribute('href'); // returns an array. Use .toString() to get string only
});
}
Save the link

Related

Javasctipt, Chunks and NodeJs

I need to upload a file on the Postgres database using the nodeJs server. On frontend (vueJs) I have <input ref="file_upload" type="file" multiple="true" #change="changeFile" > element where I pick files. After I select the wanted file I convert it to a base64 string with the following code:
var file_input = this.$refs.file_upload
var base64String
function changeFile() {
for(let i = 0; i < file_input.files.length; i++) {
var reader = new FileReader();
reader.onloadend = () => {
base64String = reader.result
.replace('data:', '')
.replace(/^.+,/, '');
console.log(base64String)
console.log("SIZE: " + base64String.length)
}
reader.readAsDataURL(file_input.files[i]);
}
}
file_input.addEventListener('change', changeFile);
After I convert it to a base64 string, on button click I create post request with this code:
btnSubmit.addEventListener("click", () => {
let dat_title = file_input.files[0].name;
let url_files = "http://localhost:3000/blobFile/" + dat_title + "/" + base64String
console.log("URL:\n" + url_files)
fetch(url_files, {
method: "POST"
})
.then(response => {
response.json().then(parsedJson => {
console.log(parsedJson);
})
})
})
And that's where problems start. If the size of the base64 string is less than 16kB, it will normally do a post request and will be inserted into the database table (column is bytea type, so before insert I decode base64 string). But, if the size of the base64 string is more than 16kB, it shows an error that says how it failed to fetch. So I figured out that the URL is too big to fetch and I need to split it into chunks. And my question is how can I do that. How can I split that base64 string into chunks and receive those chunks on the nodeJs server? I've tried millions of solutions but nothing worked. If you know how to tackle this problem please write it down. Under is nodeJs server configuration:
app.js
require('dotenv').config();
var express = require('express');
var cors = require('cors');
var app = express();
const pool = require('./dbConnect');
const port = 3000;
app.use(cors());
app.post("/blobFile/:title/:url(*)", pool.postBlobFile)
app.listen(port, () => {
var host = "localhost";
console.log(`Server listening on port http://%s:%s`, host, port);
})
dbConnect.js
const postBlobFile = (req, res) => {
const dat_title = req.params.title
var base64String = req.params.url
console.log("TITLE: " + dat_title)
console.log("STRING: " + base64String)
console.log("STRING_SIZE: " + base64String.length)
pool.query(`insert into test_blob (dat_naziv, dat_blob)
values ('${dat_title}', decode('${base64String}', 'base64'))`,
(err, results) => {
if (err) console.log(err);
else{
res.json(results.rows)
}
})
}
module.exports = {
pool,
postBlobFile,
}
THANK'S IN ADVANCE
POST is for a reason. you are using GET, POST is just sitting useless in your code
There are 2 Problems which I am seeing
I don't know what you are trying to do. but do note that there is a URL length limit. and you are trying to exploit it and that's why you are getting this error. I don't understand why you are using POST if you won't just want to use bas64 in the URL
It is a best practice that you don't use Postgres for blob or byte type of things. just a suggestion. use something like s3 or spaces.
btnSubmit.addEventListener("click", () => {
let dat_title = file_input.files[0].name;
let url_files = "http://localhost:3000/blobFile/"
console.log("URL:\n" + url_files)
fetch(url_files, {
method: "POST",
'data | body': {'**Your data over here**'}
})
.then(response => {
response.json().then(parsedJson => {
console.log(parsedJson);
})
})
})

Setting the correct content-type in HTML web browser [duplicate]

I have a logo that is residing at the public/images/logo.gif. Here is my nodejs code.
http.createServer(function(req, res){
res.writeHead(200, {'Content-Type': 'text/plain' });
res.end('Hello World \n');
}).listen(8080, '127.0.0.1');
It works but when I request for localhost:8080/logo.gif then I obviously don't get the logo.
What changes I need to do to serve an image.
2016 Update
Examples with Express and without Express that actually work
This question is over 5 years old but every answer has some problems.
TL;DR
Scroll down for examples to serve an image with:
express.static
express
connect
http
net
All of the examples are also on GitHub: https://github.com/rsp/node-static-http-servers
Test results are available on Travis: https://travis-ci.org/rsp/node-static-http-servers
Introduction
After over 5 years since this question was asked there is only one correct answer by generalhenry but even though that answer has no problems with the code, it seems to have some problems with reception. It was commented that it "doesn't explain much other than how to rely on someone else to get the job done" and the fact how many people have voted this comment up clearly shows that a lot of things need clarification.
First of all, a good answer to "How to serve images using Node.js" is not implementing a static file server from scratch and doing it badly. A good answer is using a module like Express that does the job correctly.
Answering comments that say that using Express "doesn't explain much other than how to rely on someone else to get the job done" it should be noted, that using the http module already relies on someone else to get the job done. If someone doesn't want to rely on anyone to get the job done then at least raw TCP sockets should be used instead - which I do in one of my examples below.
A more serious problem is that all of the answers here that use the http module are broken. They introduce race conditions, insecure path resolution that will lead to path traversal vulnerability, blocking I/O that will completely fail to serve any concurrent requests at all and other subtle problems - they are completely broken as examples of what the question asks about, and yet they already use the abstraction that is provided by the http module instead of using TCP sockets so they don't even do everything from scratch as they claim.
If the question was "How to implement static file server from scratch, as a learning exercise" then by all means answers how to do that should be posted - but even then we should expect them to at least be correct. Also, it is not unreasonable to assume that someone who wants to serve an image might want to serve more images in the future so one could argue that writing a specific custom static file server that can serve only one single file with hard-coded path is somewhat shortsighted. It seems hard to imagine that anyone who searches for an answer on how to serve an image would be content with a solution that serves just a single image instead of a general solution to serve any image.
In short, the question is how to serve an image and an answer to that is to use an appropriate module to do that in a secure, performant and reliable way that is readable, maintainable and future-proof while using the best practice of professional Node development. But I agree that a great addition to such an answer would be showing a way to implement the same functionality manually but sadly every attempt to do that has failed so far. And that is why I wrote some new examples.
After this short introduction, here are my five examples doing the job on 5 different levels of abstraction.
Minimum functionality
Every example serves files from the public directory and supports the minimum functionality of:
MIME types for most common files
serves HTML, JS, CSS, plain text and images
serves index.html as a default directory index
responds with error codes for missing files
no path traversal vulnerabilities
no race conditions while reading files
I tested every version on Node versions 4, 5, 6 and 7.
express.static
This version uses the express.static built-in middleware of the express module.
This example has the most functionality and the least amount of code.
var path = require('path');
var express = require('express');
var app = express();
var dir = path.join(__dirname, 'public');
app.use(express.static(dir));
app.listen(3000, function () {
console.log('Listening on http://localhost:3000/');
});
express
This version uses the express module but without the express.static middleware. Serving static files is implemented as a single route handler using streams.
This example has simple path traversal countermeasures and supports a limited set of most common MIME types.
var path = require('path');
var express = require('express');
var app = express();
var fs = require('fs');
var dir = path.join(__dirname, 'public');
var mime = {
html: 'text/html',
txt: 'text/plain',
css: 'text/css',
gif: 'image/gif',
jpg: 'image/jpeg',
png: 'image/png',
svg: 'image/svg+xml',
js: 'application/javascript'
};
app.get('*', function (req, res) {
var file = path.join(dir, req.path.replace(/\/$/, '/index.html'));
if (file.indexOf(dir + path.sep) !== 0) {
return res.status(403).end('Forbidden');
}
var type = mime[path.extname(file).slice(1)] || 'text/plain';
var s = fs.createReadStream(file);
s.on('open', function () {
res.set('Content-Type', type);
s.pipe(res);
});
s.on('error', function () {
res.set('Content-Type', 'text/plain');
res.status(404).end('Not found');
});
});
app.listen(3000, function () {
console.log('Listening on http://localhost:3000/');
});
connect
This version uses the connect module which is a one level of abstraction lower than express.
This example has similar functionality to the express version but using slightly lower-lever APIs.
var path = require('path');
var connect = require('connect');
var app = connect();
var fs = require('fs');
var dir = path.join(__dirname, 'public');
var mime = {
html: 'text/html',
txt: 'text/plain',
css: 'text/css',
gif: 'image/gif',
jpg: 'image/jpeg',
png: 'image/png',
svg: 'image/svg+xml',
js: 'application/javascript'
};
app.use(function (req, res) {
var reqpath = req.url.toString().split('?')[0];
if (req.method !== 'GET') {
res.statusCode = 501;
res.setHeader('Content-Type', 'text/plain');
return res.end('Method not implemented');
}
var file = path.join(dir, reqpath.replace(/\/$/, '/index.html'));
if (file.indexOf(dir + path.sep) !== 0) {
res.statusCode = 403;
res.setHeader('Content-Type', 'text/plain');
return res.end('Forbidden');
}
var type = mime[path.extname(file).slice(1)] || 'text/plain';
var s = fs.createReadStream(file);
s.on('open', function () {
res.setHeader('Content-Type', type);
s.pipe(res);
});
s.on('error', function () {
res.setHeader('Content-Type', 'text/plain');
res.statusCode = 404;
res.end('Not found');
});
});
app.listen(3000, function () {
console.log('Listening on http://localhost:3000/');
});
http
This version uses the http module which is the lowest-level API for HTTP in Node.
This example has similar functionality to the connect version but using even more lower-level APIs.
var path = require('path');
var http = require('http');
var fs = require('fs');
var dir = path.join(__dirname, 'public');
var mime = {
html: 'text/html',
txt: 'text/plain',
css: 'text/css',
gif: 'image/gif',
jpg: 'image/jpeg',
png: 'image/png',
svg: 'image/svg+xml',
js: 'application/javascript'
};
var server = http.createServer(function (req, res) {
var reqpath = req.url.toString().split('?')[0];
if (req.method !== 'GET') {
res.statusCode = 501;
res.setHeader('Content-Type', 'text/plain');
return res.end('Method not implemented');
}
var file = path.join(dir, reqpath.replace(/\/$/, '/index.html'));
if (file.indexOf(dir + path.sep) !== 0) {
res.statusCode = 403;
res.setHeader('Content-Type', 'text/plain');
return res.end('Forbidden');
}
var type = mime[path.extname(file).slice(1)] || 'text/plain';
var s = fs.createReadStream(file);
s.on('open', function () {
res.setHeader('Content-Type', type);
s.pipe(res);
});
s.on('error', function () {
res.setHeader('Content-Type', 'text/plain');
res.statusCode = 404;
res.end('Not found');
});
});
server.listen(3000, function () {
console.log('Listening on http://localhost:3000/');
});
net
This version uses the net module which is the lowest-level API for TCP sockets in Node.
This example has some of the functionality of the http version but the minimal and incomplete HTTP protocol has been implemented from scratch. Since it doesn't support chunked encoding it loads the files into memory before serving them to know the size before sending a response because statting the files and then loading would introduce a race condition.
var path = require('path');
var net = require('net');
var fs = require('fs');
var dir = path.join(__dirname, 'public');
var mime = {
html: 'text/html',
txt: 'text/plain',
css: 'text/css',
gif: 'image/gif',
jpg: 'image/jpeg',
png: 'image/png',
svg: 'image/svg+xml',
js: 'application/javascript'
};
var server = net.createServer(function (con) {
var input = '';
con.on('data', function (data) {
input += data;
if (input.match(/\n\r?\n\r?/)) {
var line = input.split(/\n/)[0].split(' ');
var method = line[0], url = line[1], pro = line[2];
var reqpath = url.toString().split('?')[0];
if (method !== 'GET') {
var body = 'Method not implemented';
con.write('HTTP/1.1 501 Not Implemented\n');
con.write('Content-Type: text/plain\n');
con.write('Content-Length: '+body.length+'\n\n');
con.write(body);
con.destroy();
return;
}
var file = path.join(dir, reqpath.replace(/\/$/, '/index.html'));
if (file.indexOf(dir + path.sep) !== 0) {
var body = 'Forbidden';
con.write('HTTP/1.1 403 Forbidden\n');
con.write('Content-Type: text/plain\n');
con.write('Content-Length: '+body.length+'\n\n');
con.write(body);
con.destroy();
return;
}
var type = mime[path.extname(file).slice(1)] || 'text/plain';
var s = fs.readFile(file, function (err, data) {
if (err) {
var body = 'Not Found';
con.write('HTTP/1.1 404 Not Found\n');
con.write('Content-Type: text/plain\n');
con.write('Content-Length: '+body.length+'\n\n');
con.write(body);
con.destroy();
} else {
con.write('HTTP/1.1 200 OK\n');
con.write('Content-Type: '+type+'\n');
con.write('Content-Length: '+data.byteLength+'\n\n');
con.write(data);
con.destroy();
}
});
}
});
});
server.listen(3000, function () {
console.log('Listening on http://localhost:3000/');
});
Download examples
I posted all of the examples on GitHub with more explanation.
Examples with express.static, express, connect, http and net:
https://github.com/rsp/node-static-http-servers
Other project using only express.static:
https://github.com/rsp/node-express-static-example
Tests
Test results are available on Travis:
https://travis-ci.org/rsp/node-static-http-servers
Everything is tested on Node versions 4, 5, 6, and 7.
See also
Other related answers:
Failed to load resource from same directory when redirecting Javascript
onload js call not working with node
Sending whole folder content to client with express
Loading partials fails on the server JS
Node JS not serving the static image
I agree with the other posters that eventually, you should use a framework, such as Express.. but first you should also understand how to do something fundamental like this without a library, to really understand what the library abstracts away for you.. The steps are
Parse the incoming HTTP request, to see which path the user is asking for
Add a pathway in conditional statement for the server to respond to
If the image is requested, read the image file from the disk.
Serve the image content-type in a header
Serve the image contents in the body
The code would look something like this (not tested)
fs = require('fs');
http = require('http');
url = require('url');
http.createServer(function(req, res){
var request = url.parse(req.url, true);
var action = request.pathname;
if (action == '/logo.gif') {
var img = fs.readFileSync('./logo.gif');
res.writeHead(200, {'Content-Type': 'image/gif' });
res.end(img, 'binary');
} else {
res.writeHead(200, {'Content-Type': 'text/plain' });
res.end('Hello World \n');
}
}).listen(8080, '127.0.0.1');
You should use the express framework.
npm install express
and then
var express = require('express');
var app = express();
app.use(express.static(__dirname + '/public'));
app.listen(8080);
and then the URL localhost:8080/images/logo.gif should work.
var http = require('http');
var fs = require('fs');
http.createServer(function(req, res) {
res.writeHead(200,{'content-type':'image/jpg'});
fs.createReadStream('./image/demo.jpg').pipe(res);
}).listen(3000);
console.log('server running at 3000');
It is too late but helps someone, I'm using node version v7.9.0 and express version 4.15.0
if your directory structure is something like this:
your-project
uploads
package.json
server.js
server.js code:
var express = require('express');
var app = express();
app.use(express.static(__dirname + '/uploads'));// you can access image
//using this url: http://localhost:7000/abc.jpg
//make sure `abc.jpg` is present in `uploads` dir.
//Or you can change the directory for hiding real directory name:
`app.use('/images', express.static(__dirname+'/uploads/'));// you can access image using this url: http://localhost:7000/images/abc.jpg
app.listen(7000);
Vanilla node version as requested:
var http = require('http');
var url = require('url');
var path = require('path');
var fs = require('fs');
http.createServer(function(req, res) {
// parse url
var request = url.parse(req.url, true);
var action = request.pathname;
// disallow non get requests
if (req.method !== 'GET') {
res.writeHead(405, {'Content-Type': 'text/plain' });
res.end('405 Method Not Allowed');
return;
}
// routes
if (action === '/') {
res.writeHead(200, {'Content-Type': 'text/plain' });
res.end('Hello World \n');
return;
}
// static (note not safe, use a module for anything serious)
var filePath = path.join(__dirname, action).split('%20').join(' ');
fs.exists(filePath, function (exists) {
if (!exists) {
// 404 missing files
res.writeHead(404, {'Content-Type': 'text/plain' });
res.end('404 Not Found');
return;
}
// set the content type
var ext = path.extname(action);
var contentType = 'text/plain';
if (ext === '.gif') {
contentType = 'image/gif'
}
res.writeHead(200, {'Content-Type': contentType });
// stream the file
fs.createReadStream(filePath, 'utf-8').pipe(res);
});
}).listen(8080, '127.0.0.1');
I like using Restify for REST services. In my case, I had created a REST service to serve up images and then if an image source returned 404/403, I wanted to return an alternative image. Here's what I came up with combining some of the stuff here:
function processRequest(req, res, next, url) {
var httpOptions = {
hostname: host,
path: url,
port: port,
method: 'GET'
};
var reqGet = http.request(httpOptions, function (response) {
var statusCode = response.statusCode;
// Many images come back as 404/403 so check explicitly
if (statusCode === 404 || statusCode === 403) {
// Send default image if error
var file = 'img/user.png';
fs.stat(file, function (err, stat) {
var img = fs.readFileSync(file);
res.contentType = 'image/png';
res.contentLength = stat.size;
res.end(img, 'binary');
});
} else {
var idx = 0;
var len = parseInt(response.header("Content-Length"));
var body = new Buffer(len);
response.setEncoding('binary');
response.on('data', function (chunk) {
body.write(chunk, idx, "binary");
idx += chunk.length;
});
response.on('end', function () {
res.contentType = 'image/jpg';
res.send(body);
});
}
});
reqGet.on('error', function (e) {
// Send default image if error
var file = 'img/user.png';
fs.stat(file, function (err, stat) {
var img = fs.readFileSync(file);
res.contentType = 'image/png';
res.contentLength = stat.size;
res.end(img, 'binary');
});
});
reqGet.end();
return next();
}
This may be a bit off-topic, since you are asking about static file serving via Node.js specifically (where fs.createReadStream('./image/demo.jpg').pipe(res) is actually a good idea), but in production you may want to have your Node app handle tasks, that cannot be tackled otherwise, and off-load static serving to e.g Nginx.
This means less coding inside your app, and better efficiency since reverse proxies are by design ideal for this.
This method works for me, it's not dynamic but straight to the point:
const fs = require('fs');
const express = require('express');
const app = express();
app.get( '/logo.gif', function( req, res ) {
fs.readFile( 'logo.gif', function( err, data ) {
if ( err ) {
console.log( err );
return;
}
res.write( data );
return res.end();
});
});
app.listen( 80 );
Let me just add to the answers above, that optimizing images, and serving responsive images helps page loading times dramatically since >90% of web traffic are images. You might want to pre-process images using JS / Node modules such as imagemin and related plug-ins, ideally during the build process with Grunt or Gulp.
Optimizing images means processing finding an ideal image type, and selecting optimal compression to achieve a balance between image quality and file size.
Serving responsive images translates into creating several sizes and formats of each image automatically and using srcset in your HTML allows you to serve optimal image set (that is, the ideal format and dimensions, thus optimal file size) for every single browser).
Image processing automation during the build process means incorporating it up once, and presenting optimized images further on, requiring minimum extra time.
Some great read on responsive images, minification in general, imagemin node module and using srcset.
//This method involves directly integrating HTML Code in the res.write
//first time posting to stack ...pls be kind
const express = require('express');
const app = express();
const https = require('https');
app.get("/",function(res,res){
res.write("<img src="+image url / src +">");
res.send();
});
app.listen(3000, function(req, res) {
console.log("the server is onnnn");
});
import http from "node:http";
import fs from "node:fs";
const app = http.createServer((req, res)=>{
if(req.url === "/" && req.method === "GET"){
res.writeHead(200, {
"Content-Type" : "image/jpg"
})
fs.readFile("sending.jpg", (err, data)=>{
if(err){
throw err;
}else{
res.write(data);
res.end();
}
})
}
}).listen(8080, ()=>{
console.log(8080)
})

How to send data between Node js and javascript in an html file

I am using an eye tracker and I want to create a website that displays this data in real time. I have the eye tracker notifying a Node js server and it provides data really consistently but when I used socket.io to send the data over it was buffering really slowly. I want a way to receive this data in a script in my index.html from the Node js server in real time or as close as possible. Any suggestions?
I have found my solution in socket.io-streams. Here's what I did:
in the app.js:
var ss = require('socket.io-stream');
var Readable = require('stream').Readable;
const io = require('socket.io')(http);
io.of('/data').on('connection', socket => {
var eyeTracker = ...
var listener = {
...
onGazeData:function(gazeData){ //trigger for recieving a gaze location
var s = new Readable()
s._read = function() {};
var stream = ss.createStream();
toSend = gazeData.x + "," + gazeData.y
s.push(toSend);
s.pipe(stream);
ss(socket).emit('gaze',stream);
}
}
eyeTracker.setListener(listener);
});
app.get('/', function (req, res) {
res.sendFile(path.join(__dirname, 'index.html'));
}
and in the index.js (linked to index.html) I put:
var ss = require('socket.io-stream');
$(function(){
var socket = io.connect('http://localhost:3000/data'); //or where ever you are running
socket.on('connect', function() {
ss(socket).on('gaze', function(stream) {
stream.on('data', function(data) {
//do what you want with data
})
})
});
//other parts of script outside of socket
});
This solution was able to keep up with the data being streamed.

How take the JSON in Node.JS

I have this in Node.JS file.
var express = require('express');
var app = express();
var http = require('http').Server(app);
var cfenv = require("cfenv");
var appEnv = cfenv.getAppEnv();
http.listen(appEnv.port, appEnv.bind);
var PersonalityInsightsV2 = require('watson-developer-cloud/personality-insights/v2');
var personality_insights = new PersonalityInsightsV2({
username: '<YOUR-USERNAME>',
password: '<YOUR-PASSWORD>'
});
personality_insights.profile({
text: "<YOUR-100-UNIQUE-WORDS>",
language: 'en' },
function (err, response) {
if (err)
console.log('error:', err);
else
console.log(JSON.stringify(response, null, 2));
});
I am sending an API call but as you can see, it shows me the result in JSON in the console.
How can I make this result in JSON that shows me in the console, show it to me in an HTML?
Thank you very much!
I supose that the problem is in console.log(JSON.stringify(res,null, 2));, but, I don't know how put this in HTML.
You can't just turn JSON into HTML. JSON is a data format. HTML is a markup language. You'll manually have to create some HTML with the way you want it, and then drop in values from the JSON.
For example, you could do something like this:
else {
const html =
`<!DOCTYPE html>
<body>
<p>${response.name}</p>
`;
console.log(html);
}
That would give you some HTML like:
<!DOCTYPE html>
<body>
<p>Bob</p>
assuming response has a value of name.
It sounds like you're wanting to view the JSON on an HTML page in a browser. Something like this should help. It will start your Express server listening on whatever port you specified using appEnv.port, and will serve up myJson (which will then be assigned in your code)
var express = require('express');
var app = express();
var http = require('http').Server(app);
var cfenv = require("cfenv");
var appEnv = cfenv.getAppEnv();
var myJson;
// respond with JSON when a GET request is made to the index
app.get('/', function (req, res) {
res.send(myJson)
})
app.listen(appEnv.port);
var PersonalityInsightsV2 = require('watson-developer-cloud/personality-insights/v2');
var personality_insights = new PersonalityInsightsV2({
username: '<YOUR-USERNAME>',
password: '<YOUR-PASSWORD>'
});
personality_insights.profile({
text: "<YOUR-100-UNIQUE-WORDS>",
language: 'en' },
function (err, response) {
if (err)
console.log('error:', err);
else
myJson = JSON.stringify(response, null, 2);
});
To try this, you would open your browser to "http://localhost:appEnv.port/" (where appEnv.port is the port you chose). You should see your JSON output

nodejs: node-http-proxy and harmon: rewriting the html response from the end point instead of the 302 redirected response.

I'm using nodejs with node-http-proxy along with harmon. I am using harmon to rewrite the proxied response to include a javascript file and a css file. When I set the target of the proxy to be http://nodejs.org or anything other than localhost, I receive a 301 or 302 redirect. The script is rewriting the 301 response instead of the fully proxied response. How can I use harmon to rewrite the end response instead of the 302 response?
Here is the example of the script I am running from the harmon example folder:
var http = require('http');
var connect = require('connect');
var httpProxy = require('http-proxy');
var selects = [];
var simpleselect = {};
//<img id="logo" src="/images/logo.svg" alt="node.js">
simpleselect.query = 'img';
simpleselect.func = function (node) {
//Create a read/write stream wit the outer option
//so we get the full tag and we can replace it
var stm = node.createStream({ "outer" : true });
//variable to hold all the info from the data events
var tag = '';
//collect all the data in the stream
stm.on('data', function(data) {
tag += data;
});
//When the read side of the stream has ended..
stm.on('end', function() {
//Print out the tag you can also parse it or regex if you want
process.stdout.write('tag: ' + tag + '\n');
process.stdout.write('end: ' + node.name + '\n');
//Now on the write side of the stream write some data using .end()
//N.B. if end isn't called it will just hang.
stm.end('<img id="logo" src="http://i.imgur.com/LKShxfc.gif" alt="node.js">');
});
}
selects.push(simpleselect);
//
// Basic Connect App
//
var app = connect();
var proxy = httpProxy.createProxyServer({
target: 'http://nodejs.org'
})
app.use(require('../')([], selects, true));
app.use(
function (req, res) {
proxy.web(req, res);
}
);
The problem is that a lot of sites are now redirecting HTTP to HTTPS.
nodejs.org is one of those.
I have updated the sample https://github.com/No9/harmon/blob/master/examples/doge.js to show how the http-proxy needs to be configured to deal with HTTPS.
If you still have problems with other arbitrary redirects please log an issue on harmon.
Thanks

Categories

Resources