Puppeteer fails to post - javascript

I am trying to intercept a request, add some headers, change the method to post and pass some postData but seems to fail with the method that still being GET, also postData is undefined.
There is my code:
await page.setRequestInterception(true);
page.on('request', interceptedRequest => {
var overrides;
if (interceptedRequest.url() === 'https://www.example.com/') {
console.dir(querystring.stringify(query));
let headers = interceptedRequest.headers();
headers['host'] = 'example.com/';
headers['origin'] = 'https://www.example.com/';
headers['referer'] = 'https://www.example.com/test';
headers['x-requested-with'] = 'XMLHttpRequest';
headers['accept'] = 'application/json, text/javascript, */*; q=0.01';
headers['accept-encoding'] = 'gzip, deflate, br';
headers['content-type'] = 'application/json; charset=UTF-8';
overrides = {
'method': 'POST',
'postData': querystring.stringify(query),
'headers': headers
};
}
interceptedRequest.continue(overrides);
});
Then if I intercept the response:
...
_resourceType: 'document',
_method: 'GET',
_postData: undefined,
_headers:
{ 'upgrade-insecure-requests': '1',
'user-agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/72.0.3582.0 Safari/537.36',
host: 'example.com',
origin: 'https://www.example.com/',
referer: 'https://www.example.com/test',
'x-requested-with': 'XMLHttpRequest',
accept: 'application/json, text/javascript, */*; q=0.01',
'accept-encoding': 'gzip, deflate, br',
'content-type': 'application/json; charset=UTF-8' }
...
Thanks

Related

Unable to access the cookies from request headers

const headers = event.request.headers
console.log('headers: ', headers)
console.log('type is ', typeof headers)
console.log('keys are ', Object.keys(headers))
console.log('cookie are ', headers.cookie)
Output:
headers: {
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
'cache-control': 'max-age=0',
connection: 'keep-alive',
cookie: 'userid=7599271f-87de-4f44-8625-5d4dc4af4069; authToken=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6ImNsMWl4d2Y2aTAwMTdmNzNlcnVweWhqY3UiLCJpYXQiOjE2NDk1MTIzMjB9.ltrLZV7aFeNCmCtC5ZMVZvRhM5c285TAEWf_zFbVGs0',
host: 'localhost:3000',
'if-none-match': '"1l79fw0"',
referer: 'http://localhost:3000/profile',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'same-origin',
'sec-gpc': '1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.45 Safari/537.36'
}
type is object
keys are []
cookie are undefined
Click here to see my code screenshot
Can anyone say what's wrong with it?
It should be event.request.headers.get(“cookie”)

axios does not return the full data

axios.get(downloadUrl, Object.assign({}, this.tokens[token_nr].config, {responseType: stream}))
.then((response) => {
console.log("HEADERS", response.headers)
console.log(typeof response.data, "RESPONSE LENGTH", response.data.length)
const decryptedBuffer = encryptionService.decrypt(Buffer.from(response.data), infos);
resolve(decryptedBuffer);
})
This axios request should give the data of a mp3 file. I previously had it via the request package which gives a binary Buffer (using the encoding: null option) and I can use it in the encryptionService.decrypt() function.
In the response.headers I can see it gives the same content-length as it would with the request package. But when I print the length of response.data it's shorter. I tried both ArrayBuffer and stream as my ResponseType. Also leaving the ResponseType option out does not help. What should I do to get the full content.
Some logs: (not all headers)
HEADERS {
'accept-ranges': 'bytes',
'cache-control': 'public',
'content-type': 'audio/mpeg',
'content-length': '14175084',
connection: 'close'
}
string RESPONSE LENGTH 13495410
CONFIG HEADERS {
headers: {
Accept: 'application/json, text/plain, */*',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
'cache-control': 'max-age=0',
'accept-language': 'en-US,en;q=0.9,en-US;q=0.8,en;q=0.7',
'accept-charset': 'utf-8,ISO-8859-1;q=0.8,*;q=0.7',
cookie: 'arl=XXXX',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=1500, max=100',
'Content-Type': 'text/plain;charset=UTF-8'
},
}
When creating request try passing following headers Connection, Keep-Alive. Sometimes it close the connection before fully receiving the response
var axioRequest = await axios.create({
baseURL: url,
headers: {
Connection: 'keep-alive',
'Keep-Alive': 'timeout=1500, max=100'
}
});
It was resolved with this answer:
https://disjoint.ca/til/2017/09/20/how-to-download-a-binary-file-using-axios/
I missed the Content-Type header and the {ResponseType: 'arraybuffer'}

Getting HPE_INVALID_TOKEN_ERROR when sending a get request in Node.js

This code was working flawlessly yesterday/earlier today, now it responds with empty response and body and a HPE_INVALID_TOKEN_ERROR in the error on the request callback. I really don't know what to do.
this.getVerifyAge = () => {
return new Promise((resolve, reject) => {
let url = 'https://example.org/api/';
//let url = 'https://www.google.ca'
let headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch, br',
'Accept-Language': 'fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4',
'Connection': 'keep-alive',
'Host': 'club.pokemon.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
};
var options = {
url: url,
//headers: headers,
jar: this.j
}
if (this.proxy) {
options.proxy = this.proxy;
}
request(options, (error, response, body) => {
if (error || response.statusCode === 502) {
reject(new Error('ERROR : getVerifyAge'));
}
else {
}
});
});
}

Client side(Jquery&Angular) can't send request with x-access-token headers

I followed this article build a web api with token in Node.js:
var token = req.body.token || req.query.token || req.headers['x-access-token'];
if (token) {console.log("passed!");}
else
{console.log("No token provided.");}
I tested with Postman and tried with http://localhost:3000?token=eyJ0eXAiO
all work fine, but when call API in client side:
app.controller('myCtrl', function($scope, $http, $cookies) {
var apikey=$cookies.get('apikey');
$http({url: 'http://localhost:3000/', method: 'GET', headers: {'x-access-token': apikey}})
.success(function (data) {
console.log(data);
}).error(function(error){console.log(error);});
and jquery call:
jQuery.ajax( {
url: 'http://localhost:3000/',
type: 'GET',
beforeSend : function( xhr ) {
xhr.setRequestHeader( 'x-access-token', 'eyJ0eXAi');
},
success: function( response ) {
console.log(response);
},
error : function(error) {
console.log(error);
}
} );
What every Angular or jquery did not work and return "No token provided."
what did I miss? Please help me.
Here is headers from req.headers
Angular.JS
{ host: 'localhost:3000',
connection: 'keep-alive',
'cache-control': 'max-age=0',
'access-control-request-method': 'GET',
origin: 'localhost:3001',
'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/43.0.2357.132 Safari/537.36',
'access-control-request-headers': 'accept, max-age, x-access-token',
accept: '/',
referer: 'localhost:3001',
'accept-encoding': 'gzip, deflate, sdch',
'accept-language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2,zh-TW;q=0.
2' }
No token provided.
Postman
{ host: 'localhost:3000',
connection: 'keep-alive',
csp: 'active',
'cache-control': 'no-cache',
'x-access-token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJuYW1lIjoibGF3cmVuY2
UiLCJwYXNzd29yZCI6InNreTIwMDAiLCJhZG1pbiI6dHJ1ZSwiaWF0IjoxNDM2Mzc0NTYzLCJleHAiOj
E0MzY0NjA5NjN9.OycP6xdUlG3vLyZHcj4rLjyYKE1GnlWc3h-f0r1ZpZ0',
'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/43.0.2357.132 Safari/537.36',
'postman-token': 'ab2a26e3-f6a1-09e0-c21a-85e3cef0aff5',
accept: '/',
'accept-encoding': 'gzip, deflate, sdch',
'accept-language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2,zh-TW;q=0.
2' }
passed!
It turned out this is a Chrome issue, have to run Chrome with --disable-web-security then the headers request works.
CORS, Cordova, AngularJs $http and file:// confusion

How to connect to a proxy server and make an http.request via the proxy using nodejs?

I'm not trying to make my server proxy anything i want to connect to a proxy and send http requests via that.
example:
proxy.connect(someip:someport,function(){
console.log('[PM]['+this._account+'] Logging in..');
var auth_re = /auth\.chatango\.com ?= ?([^;]*)/;
var data = querystring.stringify({user_id: this._account, password: this._password, storecookie: 'on', checkerrors: 'yes'});
var options = {hostname: 'chatango.com', port: 80, path: '/login', method: 'POST', headers: {'Connection': 'keep-alive', 'Content-Length': data.length, 'Cache-Control': 'max-age=0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'http://chatango.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded', 'Referer': 'http://chatango.com/login', 'Accept-Encoding': 'gzip,deflate', 'Accept-Language': 'en-US,en;q=0.8'}};
var self = this;
var req = http.request(options, function(res) {
res.setEncoding('utf8');
if (res.headers['set-cookie'][2]) {
var m = auth_re.exec(res.headers['set-cookie'][2]);
if (m) return callback(m[1]);
}
callback(false);
}).on('error', function(e) {
callback(false);
});
req.write(data);
req.end();
});
i dont know if it will look exactly like that but i'm tired of seeing answers for creating proxy
i just want to connect to one not create one
http proxy because the login page uses an http request
and i found my answer
console.log('[PM]['+this._account+'] Logging in..');
var auth_re = /auth\.chatango\.com ?= ?([^;]*)/;
var data = querystring.stringify({user_id: this._account, password: this._password, storecookie: 'on', checkerrors: 'yes'});
var options = {hostname: 'chatango.com', port: 80, path: '/login', method: 'POST', headers: {'Connection': 'keep-alive', 'Content-Length': data.length, 'Cache-Control': 'max-age=0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'http://chatango.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36', 'Content-Type': 'application/x-www-form-urlencoded', 'Referer': 'http://chatango.com/login', 'Accept-Encoding': 'gzip,deflate', 'Accept-Language': 'en-US,en;q=0.8'}};
var self = this;
if(self.proxy.url && self.proxy.port){
globaltunnel.initialize({
host: self.proxy.url,
port: self.proxy.port,
tunnel: 'neither',
protocol: 'http'
});
}
var req = http.request(options, function(res) {
res.setEncoding('utf8');
if (res.headers['set-cookie'][2]) {
var m = auth_re.exec(res.headers['set-cookie'][2]);
if (m) return callback(m[1]);
}
callback(false);
}).on('error', function(e) {
console.log(e);
callback(false);
});
req.write(data);
req.end();
if(self.proxy.url && self.proxy.port){
globaltunnel.end();
}

Categories

Resources