Develop Reference

How do I add parameters to https get request?

I am working on a little project to learn how to work with APIs by making GET requests to the Twitter API v2 via a node server.
For the get requests I am using Node's built in https package.
I made a basic GET request that returns a list of the last 10 tweets from a user.
I think in order to increase the amount of tweets I can get I have to make a separate parameter object, which I then implement in the get request.
Right now my function looks like this:
function getTweets() {
const options = {
host: "api.twitter.com",
path: `/2/users/${userId}/tweets`,
headers: {
authorization:
`Bearer ${bearerToken}`,
},
};
https
.get(options, (response) => {
let data = "";
response.on("data", (chunk) => {
data += chunk;
});
response.on("end", () => {
let jsonObject = JSON.parse(data);
tweetObjects = jsonObject.data;
tweetObjects.map((item) => {
let tweetWords = "";
tweetWords += item.text;
userTweets.push(tweetWords);
});
const result = userTweets.flatMap((str) => str.split(" "));
console.log(result);
});
})
.on("error", (error) => {
console.log(error);
});
}
Right now I only have the options object with host, path, and headers in the request.
This is what I am trying to do:
function getTweets() {
const options = {
host: "api.twitter.com",
path: `/2/users/${userId}/tweets`,
headers: {
authorization:
`Bearer ${bearerToken}`,
},
};
let params = {
max_results: 100,
};
https
.get(params, options, (response) => {
let data = "";
response.on("data", (chunk) => {
data += chunk;
});
response.on("end", () => {
let jsonObject = JSON.parse(data);
tweetObjects = jsonObject.data;
tweetObjects.map((item) => {
let tweetWords = "";
tweetWords += item.text;
userTweets.push(tweetWords);
});
const result = userTweets.flatMap((str) => str.split(" "));
console.log(result);
});
})
.on("error", (error) => {
console.log(error);
});
}
But I get
throw new ERR_INVALID_ARG_TYPE('listener', 'Function', listener);
^
TypeError [ERR_INVALID_ARG_TYPE]: The "listener" argument must be of type function. Received an instance of Object
at checkListener (events.js:131:11)
at ClientRequest.once (events.js:496:3)
at new ClientRequest (_http_client.js:215:10)
at request (https.js:326:10)
at Object.get (https.js:330:15)
at IncomingMessage.emit (events.js:388:22)
at endReadableNT (internal/streams/readable.js:1336:12)
at processTicksAndRejections (internal/process/task_queues.js:82:21) {
code: 'ERR_INVALID_ARG_TYPE'

You can either only pass the URL or an object containing options as stated in the docs:
https://nodejs.org/api/https.html#https_https_get_url_options_callback
So you might want to try something like this:
function getTweets() {
let params = {
max_results: 100,
};
const options = {
host: "api.twitter.com",
path: `/2/users/${userId}/tweets?max_results=${params.max_results}`,
headers: {
authorization:
`Bearer ${bearerToken}`,
},
};
https
.get(options, (response) => {
let data = "";
...
}

I am not sure to understand what you are calling parameters that you want to give to the get request.
Anyway a get request does not take some body or parameter. If you want to add some data to the core of your request you have to use the post method and not the get one.
Can you show us the import of https library just to check the functions that it propose?
Best Regards,
Hugo Delatte

Session cookie from node-fetch is invalid?

I am writing a javascript program (for a github action) right now but ran into a problem.
I was trying to log into www.overleaf.com and access the page https://www.overleaf.com/project after generating a session cookie by sending a POST request to https://www.overleaf.com/login with my credentials and the csrf token.
The response contained the requested token in the set-cookie header as expected, however, when I tried to access https://www.overleaf.com/project via GET, I get redirected back to https://www.overleaf.com/login
When copying a session cookie saved in my browser, the request works just fine as expected.
I tried doing the same thing in the command line with cURL and it worked there.
I am fairly certain my authentication request is accepted by Overleaf's server, because I have tried intentionally incorrectly sending the password or the csrf token and in both cases, the response does not give me a new session cookie but sends the old one.
If anyone has any clue what is going wrong, I'd be very thankful for your input.
This is what worked in the terminal, which I'm trying to replicate in javascript with node-fetch:
curl -v --header "Content-Type: application/json" --cookie "GCLB=someothercookie;overleaf_session2=firstsessioncookie" --data '{"_csrf":"the_csrf_token", "email": "MYEMAIL", "password":"MYPASSWORD"}' https://www.overleaf.com/login
to get the cookie and csrf token and
curl -v https://www.overleaf.com/project --cookie "overleaf_session2=returnedsessioncookie; GCLB=someothercookie" as the request that returns the html page of my projects.
This is my javascript code, I have double, triple, quadruple checked it but I think I'm missing something.
const fetch = require("node-fetch");
const parser = require("node-html-parser");
const scparser = require("set-cookie-parser");
async function run() {
const email = process.env.EMAIL;
const password = process.env.PASSWORD;
var cookies = await login(email, password);
console.log(await all_projects(cookies));
}
async function login(email, password) {
const login_get = await fetch("https://www.overleaf.com/login");
const get_cookies = login_get.headers.raw()["set-cookie"];
const parsed_get_cookies = scparser.parse(get_cookies, {
decodeValues: false
});
const overleaf_session2_get = parsed_get_cookies.find(
(element) => element.name == "overleaf_session2"
).value;
const gclb = parsed_get_cookies.find(
(element) => element.name == "GCLB"
).value;
console.log("overleaf_session2_get:", overleaf_session2_get, "gclb:", gclb);
const get_responsetext = await login_get.text();
const _csrf = parser
.parse(get_responsetext)
.querySelector("input[name=_csrf]")
.getAttribute("value");
login_json = { _csrf: _csrf, email: email, password: password };
console.log(login_json);
const login_post = await fetch("https://www.overleaf.com/login", {
method: "post",
body: JSON.stringify(login_json),
headers: {
"Content-Type": "application/json",
"Cookie": "GCLB=" + gclb + ";overleaf_session2=" + overleaf_session2_get
}
});
const post_cookies = login_post.headers.raw()["set-cookie"];
const parsed_post_cookies = scparser.parse(post_cookies, {
decodeValues: false
});
const overleaf_session2_post = parsed_post_cookies.find(
(element) => element.name == "overleaf_session2"
).value;
console.log(
"successful:",
overleaf_session2_get != overleaf_session2_post ? "true" : "false"
);
console.log(await fetch("https://www.overleaf.com/project", {
headers: {
"Cookie": "overleaf_session2=" + overleaf_session2_post
}
}))
return "overleaf_session2=" + overleaf_session2_post;
}
async function all_projects(cookies) {
const res = await fetch("https://www.overleaf.com/project", {
headers: {
Cookie: cookies
}
});
return res;
}
run();

Yes your authentication request is probably valid however this is likely to be a security issue which browsers do not allow you to do such thing and freely access another website's cookie.
Browsers do not allow you to access other domain's cookies, If they did then web would be an unsafe place because for example Stackoverflow could access my Facebook account cookie and extract my personal information.

I fixed my issue by not using node-fetch and switching to https.
Here is what worked:
async function login(email, password) {
//GET login page
const get = await get_login();
//get necessary info from response
const csrf = parser
.parse(get.html)
.querySelector(`meta[name="ol-csrfToken"]`)
.getAttribute("content");
const session1 = scparser
.parse(get.headers["set-cookie"], { decodeValues: false })
.find((cookie) => cookie.name == "overleaf_session2").value;
const gclb = scparser
.parse(get.headers["set-cookie"], { decodeValues: false })
.find((cookie) => cookie.name == "GCLB").value;
//POST login data
const post = await post_login(csrf, email, password, session1, gclb);
//get necessary data from response
const session2 = scparser
.parse(post["set-cookie"], { decodeValues: false })
.find((cookie) => cookie.name == "overleaf_session2").value;
//GET new csrf token from project page
const projects = await get_projects(session2, gclb);
const csrf2 = parser
.parse(projects.html)
.querySelector(`meta[name="ol-csrfToken"]`)
.getAttribute("content");
//return data
return {
session: session2,
gclb: gclb,
csrf: csrf2,
projects: projects.html
};
}
async function get_login() {
const url = "https://www.overleaf.com/login";
return new Promise((resolve) => {
https.get(url, (res) => {
var data;
res.on("data", (chunk) => {
data += chunk;
});
res.on("end", () => {
resolve({ html: data, headers: res.headers });
});
});
});
}
async function get_projects(session2, gclb) {
const url = "https://www.overleaf.com/project";
return new Promise((resolve) => {
https.get(
url,
{ headers: { Cookie: `GCLB=${gclb};overleaf_session2=${session2}` } },
(res) => {
var data;
res.on("data", (chunk) => {
data += chunk;
});
res.on("end", () => {
resolve({ html: data, headers: res.headers });
});
}
);
});
}
async function post_login(_csrf, email, password, session1, gclb) {
const url = "https://www.overleaf.com/login";
const options = {
method: "POST",
headers: {
"Content-Type": "application/json",
Cookie: `GCLB=${gclb};overleaf_session2=${session1}`
}
};
const postData = {
_csrf: _csrf,
email: email,
password: password
};
return new Promise((resolve) => {
var req = https.request(url, options, (res) => {
resolve(res.headers);
});
req.on("error", (e) => {
console.error(e);
});
req.write(JSON.stringify(postData));
req.end();
});
}

fetch has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource

i'm making this web app using api from https://www.football-data.org
when i make the boilerplate for the web i get this error in my console
const ENPOINT_JER = `${BASE_URL}competition/2002/standings`;
const ENPOINT_BEL = `${BASE_URL}competition/2003/standings`;
const ENPOINT_ING = `${BASE_URL}competition/2021/standings`;
const ENPOINT_SPA = `${BASE_URL}competition/2014/standings`;
const ENPOINT_PER = `${BASE_URL}competition/2015/standings`;
const fetchAPI = (url) => {
return fetch(url, {
headers: {
"X-Auth-Token": API_KEY,
},
})
.then((res) => {
if (res.status !== 200) {
console.log(`Error: ${res.status}`);
return Promise.reject(new Error(res.statusText));
} else {
return Promise.resolve(res);
}
})
.then((res) => res.json())
.catch((err) => {
console.log(err);
});
};
function getStandingJer() {
if ("caches" in window) {
caches.match(ENPOINT_JER).then(function (response) {
if (response) {
response.json().then(function (data) {
console.log("Competition Data: " + data);
showStanding(data);
});
}
});
}
fetchAPI(ENPOINT_JER)
.then((data) => {
showStanding(data);
})
.catch((error) => {
console.log(error);
});
}
i've already make the API_KEY variable

There are browser plugins for suppressing cors while your app is in development:
https://addons.mozilla.org/en-US/firefox/addon/cors-everywhere/
If you're using node and express there's a cors middleware package to help, but set up can still be tricky.
First import the module from the commandline with npm install cors;
then in your server.js file:
const express = require('express')
const app = express();
and then add cors() as an arg to the endpoint, before the callback.
More info in this guy's book

Create new product by calliing shopify api

I am trying to create a new product by calling the shopify product api (/admin/api/2020-01/products.json). I am trying to achieve this using the "https" module. Below is the sample code
const url1 = 'https://{api_token}#tuscstore.myshopify.com/admin/api/2020-01/products.json';
var obj = {
"product":[
{
"title": "Saturn",
"body_html": "<p>The epitome of elegance</p>",
"vendor": "Soltions inc",
"product_type": "Planets",
"handle": "saturn",
"tags": "",
"images": [
{
"src": "https://solarsystem.nasa.gov/system/stellar_items/image_files/38_saturn_1600x900.jpg"
}
]
}
]
};
const https = require('https');
var data = JSON.stringify(obj)
const options = new URL(url1);
var req = https.request(options, (res) => {
console.log('statusCode:', res.statusCode);
console.log('headers:', res.headers);
/* res.on('data', (d) => {
process.stdout.write(d);
}); */
});
req.on('error', (e) => {
console.error(e);
});
req.write(data);
req.end();
const Index = () => (
<div>
<p>Sample app using React and Next.js</p>
</div>
);
export default Index;
I am facing 2 problems,
when I do "process.stdout.write(d)", I receive cannot readproperty "write" undefined.
If I comment it out as I have done in
the code above, I don't get the error.
In either case I get the statuscode as 200, and not 201 which is what I shoudl receive according to shopify's docs.
Can someone please help me with what is going wrong?
Edit: Using Post,I get a type error
const https = require('https');
var data = JSON.stringify(obj)
var options = {
hostname: 'https://{apikey:password}#tuscstore.myshopify.com/admin/api/2020-01',
path: '/products.json',
method: 'POST',
headers: {
'Content-Type': 'application/json',
/*'Content-Length': data.length*/
'Authorization' : 'API_TOKEN'
}
};
var req = https.request(options, (res) => {
console.log('statusCode:', res.statusCode);
console.log('headers:', res.headers);
});
req.on('error', (e) => {
console.error(e);
});
req.write(data);
req.end();
TypeError: Failed to execute 'fetch' on 'Window': Failed to parse URL from https://[https://{APIKEY:PWD}#tuscstore.myshopify.com/admin/api/2020-01]/products.json

you creating a new product you have to make http POST request , and now your making http GET request you should update your options like so :
const options = {
hostname: 'https://apikey:password#<#store_url>/admin/api/2020-01', // your host name
path: '/shop.json', // your end point
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization' : 'YOUR_API_TOKEN'
}
}
OR you can use this package to solve all your problems https://www.npmjs.com/package/shopify-api-node

Adding Authorization header to GET request

I'm building an alexa skill using the new SDK 2.0 and I'm now having trouble implementing a simple http get request. How would I add an authorization header to the getRemoteData url request? The code below isn't working.
I'm trying to call Airtable API to get some remote data
const UserReplyIntent_Handler = {
canHandle(handlerInput) {
const request = handlerInput.requestEnvelope.request;
return request.type === 'IntentRequest' && request.intent.name === 'UserReplyIntent' ;
},
async handle(handlerInput) {
const response = await httpGet();
console.log(response);
return handlerInput.responseBuilder
.speak("Okay. Here is what I got back." + response.records.fields.Indication)
.reprompt("Would you like to learn more?")
.getResponse();
},
};
function httpGet() {
return new Promise(((resolve, reject) => {
const headers = {
Authorization: 'Bearer key12345678'
};
var options = {
host: 'api.airtable.com',
port: 443,
path: '/v0/appYqfJ3Rt2F0sRGn/Database?filterByFormula=(DrugName=%27azatadine%27)',
method: 'GET',
};
const request = https.request(options, {headers}, (response) => {
response.setEncoding('utf8');
let returnData = '';
response.on('data', (chunk) => {
returnData += chunk;
});
response.on('end', () => {
resolve(JSON.parse(returnData));
});
response.on('error', (error) => {
reject(error);
});
});
request.end();
}));
}

header go into the options object, not as a separate parameter:
var options = {
host: 'api.airtable.com',
port: 443,
path: '/v0/appYqfJ3Rt2F0sRGn/Database?filterByFormula=(DrugName=%27azatadine%27)',
method: 'GET',
headers: {
Authorization: 'Bearer key12345678'
}
};
https.request accepts the same options fields as http.reqest.
http.request options object allows headers to be defined.