Develop Reference

How to get value of response headers in nodejs

const URL = 'https://www.imdb.com/title/tt0816222/?
ref_ = fn_al_tt_2 ';
(async() => {
const response = await request({
uri: URL,
headers: {
'Connection': 'keep-alive',
'User-Agent': 'Mozilla/0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
},
});
I need help from this code. How can I get the response header values in console of visual studio code for the following site.

Just handle the Promise from request library
request({
uri: 'https://www.imdb.com/title/tt0816222/?',
headers: /*your headers*/
})
.then(function(response){
console.log(response.headers)
})

You will get the response headers in response.headers

Print like this
console.log(response.headers)

This code prints headers:
const URL = 'https://www.imdb.com/title/tt0816222/?ref_ = fn_al_tt_2';
const request = require('request');
(async () => {
const response = await request({
uri: URL,
headers: {
'Connection': 'keep-alive',
'User-Agent': 'Mozilla/0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
},
});
console.log(response.headers);
})();

Because you are just fetching the body of response from request npm.
add resolveWithFullResponse: true in request options.
const URL = 'https://www.imdb.com/title/tt0816222/?
ref_ = fn_al_tt_2 ';
(async() => {
const response = await request({
uri: URL,
headers: {
'Connection': 'keep-alive',
'User-Agent': 'Mozilla/0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
},
resolveWithFullResponse: true
});
if you need to only headers
const URL = 'https://www.imdb.com/title/tt0816222/?
ref_ = fn_al_tt_2 ';
(async() => {
const {headers} = await request({
uri: URL,
headers: {
'Connection': 'keep-alive',
'User-Agent': 'Mozilla/0.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
},
resolveWithFullResponse: true
});
console.log(headers)

Automating Sign Up

I'm trying to login to Amazon via the node.js request module, and seem to be having difficulties.
My aim is to login to the site via their form, here is my code:
const request = require("request");
const rp = require("request-promise");
var querystring = require("querystring");
var cookieJar = request.jar();
var mainUrl = "https://www.amazon.com/";
var loginUrl = "https://www.amazon.co.uk/ap/signin";
let req = request.defaults({
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.61 Safari/537.36"
},
jar: cookieJar,
gzip: true,
followAllRedirects: true
});
var loginData =
"email=email#me.com&create=0&password=password123";
req.post(loginUrl, { data: loginData }, function(err, res, body) {
console.log(body);
});
I ran a debugger in the background, and found this seemed to be the URL called. I'm wondering if anyone knows what I may have done incorrectly.
Thank you.

Web scraping in Node.js

Lately I've been trying to scrape Information from a website using Nodejs, the request module and cheerio. The code it works well (statusCode = 200) on my localhost (127.0.0.1) but when I push the code to Heroku server, statusCode = 403.
Is it because of the cookie? If yes, why does it work on my localhost that doesn't add any cookie in the request?
request({
method: 'GET',
headers: {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
},
url: 'https://www.example.com/login',
json: true
}, (err, response, body) => {
if (err) {
return console.log('Failed to request: ', err);
}
console.log(response.statusCode);
});

Getting HPE_INVALID_TOKEN_ERROR when sending a get request in Node.js

This code was working flawlessly yesterday/earlier today, now it responds with empty response and body and a HPE_INVALID_TOKEN_ERROR in the error on the request callback. I really don't know what to do.
this.getVerifyAge = () => {
return new Promise((resolve, reject) => {
let url = 'https://example.org/api/';
//let url = 'https://www.google.ca'
let headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch, br',
'Accept-Language': 'fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4',
'Connection': 'keep-alive',
'Host': 'club.pokemon.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
};
var options = {
url: url,
//headers: headers,
jar: this.j
}
if (this.proxy) {
options.proxy = this.proxy;
}
request(options, (error, response, body) => {
if (error || response.statusCode === 502) {
reject(new Error('ERROR : getVerifyAge'));
}
else {
}
});
});
}

How to connect to a proxy server and make an http.request via the proxy using nodejs?

I'm not trying to make my server proxy anything i want to connect to a proxy and send http requests via that.
example:
proxy.connect(someip:someport,function(){
console.log('[PM]['+this._account+'] Logging in..');
var auth_re = /auth\.chatango\.com ?= ?([^;]*)/;
var data = querystring.stringify({user_id: this._account, password: this._password, storecookie: 'on', checkerrors: 'yes'});
var options = {hostname: 'chatango.com', port: 80, path: '/login', method: 'POST', headers: {'Connection': 'keep-alive', 'Content-Length': data.length, 'Cache-Control': 'max-age=0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'http://chatango.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded', 'Referer': 'http://chatango.com/login', 'Accept-Encoding': 'gzip,deflate', 'Accept-Language': 'en-US,en;q=0.8'}};
var self = this;
var req = http.request(options, function(res) {
res.setEncoding('utf8');
if (res.headers['set-cookie'][2]) {
var m = auth_re.exec(res.headers['set-cookie'][2]);
if (m) return callback(m[1]);
}
callback(false);
}).on('error', function(e) {
callback(false);
});
req.write(data);
req.end();
});
i dont know if it will look exactly like that but i'm tired of seeing answers for creating proxy
i just want to connect to one not create one
http proxy because the login page uses an http request

and i found my answer
console.log('[PM]['+this._account+'] Logging in..');
var auth_re = /auth\.chatango\.com ?= ?([^;]*)/;
var data = querystring.stringify({user_id: this._account, password: this._password, storecookie: 'on', checkerrors: 'yes'});
var options = {hostname: 'chatango.com', port: 80, path: '/login', method: 'POST', headers: {'Connection': 'keep-alive', 'Content-Length': data.length, 'Cache-Control': 'max-age=0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'http://chatango.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded', 'Referer': 'http://chatango.com/login', 'Accept-Encoding': 'gzip,deflate', 'Accept-Language': 'en-US,en;q=0.8'}};
var self = this;
if(self.proxy.url && self.proxy.port){
globaltunnel.initialize({
host: self.proxy.url,
port: self.proxy.port,
tunnel: 'neither',
protocol: 'http'
});
}
var req = http.request(options, function(res) {
res.setEncoding('utf8');
if (res.headers['set-cookie'][2]) {
var m = auth_re.exec(res.headers['set-cookie'][2]);
if (m) return callback(m[1]);
}
callback(false);
}).on('error', function(e) {
console.log(e);
callback(false);
});
req.write(data);
req.end();
if(self.proxy.url && self.proxy.port){
globaltunnel.end();
}