Develop Reference

How to fetch and get response header with no cors?

I try to fetch file hosting with browser. i disable cors web security with extension https://chrome.google.com/webstore/detail/allow-cors-access-control/lhobafahddgcelffkeicbaginigeejlf?hl=en. i tested. fetch return the html. but the problem i dont get the response headers. like cookie. it just give me 4 key. i need that response header cookie to do next fetch.
const initHeader = new Headers()
initHeader.append('accept', '*/*')
initHeader.append('connection', 'keep-alive')
initHeader.append('user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36')
fetch('https://www56.zippyshare.com/v/d0M2gx7X/file.html', {
method: 'GET',
headers: initHeader
}).then(function(res) {
res.headers.forEach((val, key) => {
console.log(key + ': ' + val)
})
})

Request to mediamarkt.es always returns 403

So I'm trying to scrape mediamarkt.es with this code:
PORT = 8000;
const express = require("express");
const axios = require("axios").default;
const cors = require("cors")({ origin: true });
const app = express();
const ax = axios.create({
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "en-US,en;q=0.9",
},
});
const tc = () => {
ax.get("https://www.mediamarkt.es")
.then((response) => {
console.log(response.status);
})
.catch((err) => {
console.log(err.response.status);
});
};
app.listen(PORT, () => {
tc();
});
Every time I get 403 error. Tried on chrome with disabled javascript and cleaned cache, so it works and returns 200, but in nodejs code I always get 403. With 403 there is html which says captcha, but I believe I can make same request with nodejs as chrome do. Just can't imagine what I'm missing...
Any help would be greatly appreciated.

How can I convert a curl command into node.js

I have the following curl command. Curl in bash works ok but when I write equivalent in node.js (any http library), it doesn't seem to work
curl --location --request POST 'https://www.delta.com/shop/modals/flightspecific' \
--header 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36' \
--header 'Content-Type: application/json; charset=UTF-8' \
--header 'cachekey: d6469dbe-b47e-4f46-be02-12b900011c7c' \
--header 'Accept-Encoding: compressed' \
--data-raw '{"legList":[{"originAirportCode":"BOS","destinationAirportCode":"JFK","schedLocalDepartDate":"2020-03-10T08:30","marketingAirlineCode":"DL","operatingAirlineCode":"9E","classOfServiceList":["NE","NV","SN","OZ"],"flightNumber":"5419"}],"pageId":"dynamic-modal","appId":"sho","channelId":"ecomm"}'
Which works fine (returns JSON). But this code does not work
const axios = require("axios");
headers = {
"user-agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36",
"Content-Type": "application/json; charset=UTF-8",
cachekey: "d6469dbe-b47e-4f46-be02-12b900011c7c",
"Accept-Encoding": "compressed"
};
data =
'{"legList":[{"originAirportCode":"BOS","destinationAirportCode":"JFK","schedLocalDepartDate":"2020-03-10T08:30","marketingAirlineCode":"DL","operatingAirlineCode":"9E","classOfServiceList":["NE","NV","SN","OZ"],"flightNumber":"5419"}],"pageId":"dynamic-modal","appId":"sho","channelId":"ecomm"}';
url = "https://www.delta.com/shop/modals/flightspecific";
options = {
method: "POST",
headers: headers,
data: data,
url: url
};
axios(options).then(function(response) {
console.log(response);
});

Try something Like this :
var Request = require("request");
var headers = {
"user-agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36",
"Content-Type": "application/json; charset=UTF-8",
cachekey: "d6469dbe-b47e-4f46-be02-12b900011c7c",
"Accept-Encoding": "compressed"
};
data =
'{"legList":[{"originAirportCode":"BOS","destinationAirportCode":"JFK","schedLocalDepartDate":"2020-03-10T08:30","marketingAirlineCode":"DL","operatingAirlineCode":"9E","classOfServiceList":["NE","NV","SN","OZ"],"flightNumber":"5419"}],"pageId":"dynamic-modal","appId":"sho","channelId":"ecomm"}';
Request.post({
"headers": headers,
"url": "https://www.delta.com/shop/modals/flightspecific",
"body": JSON.stringify(data )
}, (error, response, body) => {
if (error) {
console.dir(error);
}
if (body) {
console.dir(JSON.parse(body));
}
});

Web scraping in Node.js

Lately I've been trying to scrape Information from a website using Nodejs, the request module and cheerio. The code it works well (statusCode = 200) on my localhost (127.0.0.1) but when I push the code to Heroku server, statusCode = 403.
Is it because of the cookie? If yes, why does it work on my localhost that doesn't add any cookie in the request?
request({
method: 'GET',
headers: {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
},
url: 'https://www.example.com/login',
json: true
}, (err, response, body) => {
if (err) {
return console.log('Failed to request: ', err);
}
console.log(response.statusCode);
});

Getting HPE_INVALID_TOKEN_ERROR when sending a get request in Node.js

This code was working flawlessly yesterday/earlier today, now it responds with empty response and body and a HPE_INVALID_TOKEN_ERROR in the error on the request callback. I really don't know what to do.
this.getVerifyAge = () => {
return new Promise((resolve, reject) => {
let url = 'https://example.org/api/';
//let url = 'https://www.google.ca'
let headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch, br',
'Accept-Language': 'fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4',
'Connection': 'keep-alive',
'Host': 'club.pokemon.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
};
var options = {
url: url,
//headers: headers,
jar: this.j
}
if (this.proxy) {
options.proxy = this.proxy;
}
request(options, (error, response, body) => {
if (error || response.statusCode === 502) {
reject(new Error('ERROR : getVerifyAge'));
}
else {
}
});
});
}