I am troubling with nodejs proxy server modified(write) response.
I want to achieve auto login for one site via node proxy server and for that i have to query in database then i can modified response but it seems req ended before req.write and getting Error: write after end
Below is my implementation so far.
var express = require('express');
var proxy = require('http-proxy-middleware');
var options = {
target: 'http://example.com/', // target host
changeOrigin: true,
onProxyReq: function onProxyReq(proxyReq, req, res) {
var _write = res.write;
var body = "";
proxyReq.on('data', function(data) {
data = data.toString('utf-8');
body += data;
});
res.write = function (data) {
try{
//I have database query here instead of setTimeout
setTimeout(function(){
/* Modified response here and write */
_write.call(res, data); //can't write because req already end
},3000);
} catch (err) {
console.log('err',err);
}
}
}
}
// create the proxy (without context)
var exampleProxy = proxy(options);
// mount `exampleProxy` in web server
var app = express();
app.use('/', exampleProxy);
app.listen(8080);
Can anyone guide me how to achieve this ?
Related
I start learning Node.js and Express.js and I'm trying to create a simple API to list data from JSON file (using the GET method) and add a new user using the POST method.
the GET method works fine but the POST method does not work
when I request http://127.0.0.1:8080/listusers the API sends all users in a JSON file.
when I request http://127.0.0.1:8080/adduser the API has to add new User Info and send the new data back to the browser.
NOTE: I read all the questions on Stackoverflow about this problem but
non of them help me so I have to ask again.
the problem is when I request http://127.0.0.1:8080/adduser I get the following error
Cannot GET /adduser
here is the server.js:
var express = require('express');
var app = express();
var fs = require('fs');
var user = {
"user4" : {
"name" : "mounir",
"password" : "password4",
"profession" : "teacher",
"id": 4
}
};
app.post('/adduser', function (req, res) {
// First read existing users.
fs.readFile( __dirname + "/" + "users.json", 'utf8', function (err, data) {
data = JSON.parse( data );
data["user4"] = user["user4"];
console.log( data );
res.end(JSON.stringify(data) );
});
});
app.get('/listusers', function (req, res) {
fs.readFile( __dirname + "/" + "users.json", 'utf8', function (err, data) {
console.log(data);
res.end(data);
});
});
var server = app.listen(8080, function () {
var host = server.address().address;
var port = server.address().port;
console.log("listening at http://%s:%s", "0.0.0.0", port)
});
The answer is in the error. Cannot GET /adduser. Keyword GET! If you are making a post request, be sure you include the appropriate headers and that you are making a POST request, with a body, and not a GET request. For instance if you are using fetch:
const myInit = {
method: 'POST',
headers: myHeaders,
body: {
...
}
};
fetch("http://127.0.0.1:8080/adduser", myInit)
.then(res => {
...
});
I have an http-proxy to proxy any website and inject some custom JS file before to serve the HTML back to the client. Whenever I try to access the proxied website, it will hang up or the browser seems to load indeterminately. But when I check the HTML source, I successfully managed to inject my custom JavaScript file. Here is the code:
const cheerio = require('cheerio');
const http = require('http');
const httpProxy = require('http-proxy');
const { ungzip } = require('node-gzip');
_initProxy(host: string) {
let proxy = httpProxy.createProxyServer({});
let option = {
target: host,
selfHandleResponse: true
};
proxy.on('proxyRes', function (proxyRes, req, res) {
let body = [];
proxyRes.on('data', function (chunk) {
body.push(chunk);
});
proxyRes.on('end', async function () {
let buffer = Buffer.concat(body);
if (proxyRes.headers['content-encoding'] === 'gzip') {
try {
let $ = null;
const decompressed = await ungzip(buffer);
const scriptTag = '<script src="my-customjs.js"></script>';
$ = await cheerio.load(decompressed.toString());
await $('body').append(scriptTag);
res.end($.html());
} catch (e) {
console.log(e);
}
}
});
});
let server = http.createServer(function (req, res) {
proxy.web(req, res, option, function (e) {
console.log(e);
});
});
console.log("listening on port 5051");
server.listen(5051);
}
Can someone please tell me if I am doing anything wrong, it looks like node-http-proxy is dying a lot and can't rely much on it since the proxy can work sometimes and die at the next run, depending on how many times I ran the server.
Your code looked fine so I was curious and tried it.
Although you do log a few errors, you don't handle several cases:
The server returns a body with no response (cheerio will generate an empty HTML body when this happens)
The server returns a response that is not gzipped (your code will silently discard the response)
I made a few modifications to your code.
Change initial options
let proxy = httpProxy.createProxyServer({
secure: false,
changeOrigin: true
});
Don't verify TLS certificates secure: false
Send the correct Host header changeOrigin: true
Remove the if statement and replace it with a ternary
const isCompressed = proxyRes.headers['content-encoding'] === 'gzip';
const decompressed = isCompressed ? await ungzip(buffer) : buffer;
You can also remove the 2 await on cheerio, Cheerio is not async and doesn't return an awaitable.
Final code
Here's the final code, which works. You mentioned that "it looks like node-http-proxy is dying a lot [...] depending on how many times I ran the server." I experienced no such stability issues, so your problems may lie elsewhere if that is happening (bad ram?)
const cheerio = require('cheerio');
const http = require('http');
const httpProxy = require('http-proxy');
const { ungzip } = require('node-gzip');
const host = 'https://github.com';
let proxy = httpProxy.createProxyServer({
secure: false,
changeOrigin: true
});
let option = {
target: host,
selfHandleResponse: true
};
proxy.on('proxyRes', function (proxyRes, req, res) {
console.log(`Proxy response with status code: ${proxyRes.statusCode} to url ${req.url}`);
if (proxyRes.statusCode == 301) {
throw new Error('You should probably do something here, I think there may be an httpProxy option to handle redirects');
}
let body = [];
proxyRes.on('data', function (chunk) {
body.push(chunk);
});
proxyRes.on('end', async function () {
let buffer = Buffer.concat(body);
try {
let $ = null;
const isCompressed = proxyRes.headers['content-encoding'] === 'gzip';
const decompressed = isCompressed ? await ungzip(buffer) : buffer;
const scriptTag = '<script src="my-customjs.js"></script>';
$ = cheerio.load(decompressed.toString());
$('body').append(scriptTag);
res.end($.html());
} catch (e) {
console.log(e);
}
});
});
let server = http.createServer(function (req, res) {
proxy.web(req, res, option, function (e) {
console.log(e);
});
});
console.log("listening on port 5051");
server.listen(5051);
I ended up writing a small Python Server using CherryPy and proxied the web app with mitmproxy. Everything is now working smoothly. Maybe I was doing it wrong with node-http-proxy but I also became sceptic about using it in a production environment.
I have an app where I call an API to get headlines and digest that JSON and send the data back to the client via sockets. I have modularized all my code and used different files to stay organized. I have my index.js file, where I merely call the functions to grab to data from the API and newsDataFetcher.js is where I make the request to the API for the data using the request module. My issue is that since the request I am making is asynchronous, the data doesn't get passed to the main server file and therefore the socket doesn't send any data. After the request to the API finishes, I put the data into an array and that array is in the module.exports of my newsDataFetcher.js file, along with the function that makes the actual request. So since the array is in the module.exports, an empty array is passed when I require the newsDataFetcher.js file in the index.js file. So, that brings me to the question: is there anyway to call module.exports "on demand" so that it doesn't pass an empty array and it only passes the array when the data is finished downloading from the API?
index.js :
var newsDataFetcher = require('./newsDataFetcher');
var app = express();
var server = require('http').createServer(app);
var io = require('socket.io')(server);
var news = []; //array for news headlines and abstracts
app.set('view engine', 'ejs');
app.use(express.static(__dirname + '/views/routes/'));
server.listen(8888);
console.log("Running server on http://localhost:8888 ....");
newsDataFetcher.getNewsData();
news = newsDataFetcher.news;
setInterval(function () {
newsDataFetcher.getNewsData();
news = newsDataFetcher.news;
}, 6000000);
setInterval(function () {
sendAllData()
}, 1000);
function sendAllData() {
io.sockets.emit('news', news);
}
newsDataFetcher.js file:
var request = require('request');
var nyTimesApi = "http://api.nytimes.com/svc/topstories/v1/home.json?api-key=" + nyTimesApiKey_DEV;
var news = []; //array for news headlines and abstracts
function getNewsData() {
request({
url: nyTimesApi,
json: true
}, function (err, res, body) {
news = body.results;
});
}
module.exports = {
getNewsData: getNewsData,
news: news
}
You can do some like this:
// newsDataFetcher.js
module.exports = {
getNewsData: getNewsData,
news: function() {
return news;
}
}
// index.js
io.sockets.emit('news', newsDataFetcher.news());
You can use singleton pattern to achieve this.
class NewsDataFetcher {
constructor() {
this.news = [];
}
getNewsData() {
const self = this;
request({
url: nyTimesApi,
json: true
}, function (err, res, body) {
self.news = body.results;
});
}
}
module.exports = new NewsDataFetcher();
In the main page.
var newsDataFetcher = require('./newsDataFetcher');
newsDataFetcher.getNewsData(); // to fetch new data;
// To get news data
console.log(newsDataFetcher.news); // will always refer to one instance of news since you are using singleton.
I am using express.js and I need to make a call to HTTP GET request ,to fetch JSON data .Please suggest me some good node js/express js modules/lib to perform get/post request .
Node.js provides an extremely simple API for this functionality in the form of http.request.
var http = require('http');
//The url we want is: 'www.random.com/integers/?num=1&min=1&max=10&col=1&base=10&format=plain&rnd=new'
var options = {
host: 'www.random.com',
path: '/integers/?num=1&min=1&max=10&col=1&base=10&format=plain&rnd=new'
};
callback = function(response) {
var str = '';
//another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
//the whole response has been recieved, so we just print it out here
response.on('end', function () {
console.log(str);
});
}
http.request(options, callback).end();
Here I attach some more examples with POST and custom headers. If you don't need special things, I'd stick to the native code.
Besides, Request, Superagent or Requestify are pretty good libraries to use.
var express = require('express');
var app = express();
var fs = require('fs');
app.get('/', function (req, res) {
fs.readFile('./test.json', 'utf8', function (err, data) {
if (err) {
res.send({error: err});
}
res.send(data);
})
});
var server = app.listen(3001, function () {
console.log('Example app listening port 3001');
});
I am attempting to test drive an node.js application based on express. I want to return a simple 404.html, which I can successfully do, but afterward, calling close on the node http server gets this error:
Fatal error: Cannot call method 'call' of undefined
I am having a hard time tracking down what is undefined because the same method works beautifully when called elsewhere.
Here is my express code:
function Server() {
this.port = 9000;
this.staticDir = '/public';
}
function handleHomeRequest(req, res) {
var body = '<html><body>Home Page.</body></html>';
res.send(body);
}
Server.prototype.start = function () {
expServer = express();
expServer.get('/', function (req, res) { handleHomeRequest(req, res); });
expServer.use(function (req, res) {
res.status(404).sendfile('./src/public/404.html');
});
runningServer = expServer.listen(this.port);
};
Server.prototype.stop = function (cb) {
runningServer.close(cb);
};
Here is my nodeunit test code:
var ROOT_URL = 'http://localhost',
PORT = 9000,
URL = ROOT_URL + ':' + PORT + '/',
http = require('http'),
Server = require('./server.js'),
server;
exports.setUp = function(done) {
server = new Server();
done();
};
exports.tearDown = function (done) {
server = null;
done();
};
exports['Requesting a page that does not exist results in a 404.'] = function (test) {
server.start();
httpGet(URL + 'guaranteedNotToExistPage', function(res, data) {
test.equal(404, res.statusCode, 'Requesting a page that dne did not return with a status code of 404.');
test.ok(data.indexOf('404 Page Not Found') > -1, 'The 404 page was not returned.');
//test.done();
server.stop(test.done);
});
};
function httpGet(url, callback) {
var request = http.get(url),
receivedData = '';
request.on('response', function (response) {
response.setEncoding('utf8');
response.on('data', function (chunk) {
receivedData += chunk;
});
response.on('end', function () {
callback(response, receivedData);
});
});
}
The result of the http get request come back, the failure only occurs when I call server.stop(test.done); however, stopping the server is required to ensure my unit tests can be run in any order and independent.
First, where runningServer is defined? I can't see a
var runningServer;
anywhere in the first peace of code.
So, if you write a value in prototype.start I doubt you can access it on prototype.stop that is a different scope.
Second, {expressListener}.close() in node 0.6 was just synchronous, they added the callback on the 0.8. So, check the node.js version to be sure that the {cb} is correctly handled.