Unable to parse info from webpage - javascript

I am trying to parse info from this link on my node.js project
https://stockx.com/api/products/nike-daybreak-undercover-black?includes=market
Im able to get info when I access the link through postman and going on the url on a web browser. However when I try accessing the request through my node.js project, it is saying access is denied. Any idea why?
Thanks.
Here is my code:
const express = require('express');
const request = require('request');
const cheerio = require('cheerio');
const axios = require('axios')
const app = express();
app.get('/', function(req, res){
let shoe =req.query.shoe;
let url = 'https://stockx.com/api/products/nike-daybreak-undercover-black?includes=market'
request(url, function(error, response, html) {
if (!error) {
var $ = cheerio.load(html);
console.log(html)
res.send();
}
});
});
app.listen('8080');
console.log('API is running on http://localhost:8080');
module.exports = app;

You just need to add "User-Agent" in the header. The website from which you are trying to get the data is denying all requests without User-Agent to avoid scrapers.
const options = {
url: 'https://stockx.com/api/products/nike-daybreak-undercover-black?includes=market',
headers: {
'User-Agent': 'request'
}
};
request(options, function(error, response, html) {
console.log('err: ', error);
if (!error) {
var $ = cheerio.load(html);
console.log(html)
res.send(html);
}
});

I have tried the following code and it works
// ...
app.get('/', function(req, res){
// let shoe =req.query.shoe;
let url = 'https://stockx.com/api/products/nike-daybreak-undercover-black?includes=market'
axios({
method : 'get',
url,
headers : { withCredentials: true, 'User-Agent' : 'Postman' }
})
.then(data => {
console.log('data', data.data);
})
.catch(err => {
console.log('err', err);
})
res.send().status(200);
});

Related

How to obtain Json data from different site

So, let's say i have /api/cat/fact.js directory.
I wanna to get JSON Data from catfact.ninja
The thing is, i can't use require() or request() package, because if i used require, it would saya Couldnt Found Module..., and if i used request one, instead of returning the JSON Data that you beable to sees in catfact.ninja, it return JSON about the api, like hostname, port, which is i don't need
/API/api/cat/fact.js:
const express = require('express');
const app = express.Router();
const request = require('request')
app.use('', (req, res) => {
const src = 'https://catfact.ninja/fact';
const facts = request({
uri: src,
hostname: 'catfact.ninja',
port: 443,
path: '/fact',
method: 'POST',
json: 'fact'
}, (error, response, body) => {
if (error) console.log(error)
console.log(body, '\n\n' + response.fact)
})
console.log(facts);
return res.jsonp(facts)
})
module.exports = app;
You are returning JSON in the wrong place. It should be returned inside of the callback function.
Here's the solution:
const express = require('express');
const request = require('request-promise')
const app = express();
app.use('', async (req, res) => {
const src = 'https://catfact.ninja/fact';
try {
const response = await request({
uri: src,
port: 443,
method: 'GET',
json: true
})
return res.jsonp(response)
} catch (err) {
return res.jsonp(err)
}
})
function startServer() {
const port = 3000
app.listen(port, () => {
console.info('Server is up on port ' + port)
})
app.on('error', (err) => {
console.error(err)
process.exit(1)
})
}
startServer()
TIP: I suggest using request-promise npm package instead of request package as it provides async-await approach, which is cleaner. Else, you can continue using callback function as second request() function parameter.

pass value from input to node js http request parameter

greeting kings
i have api request that have cors problem. I'm able to solve by using proxy setup using nodejs. Unfortunately im trying to pass certain query parameter from my main js to app.js(nodejs proxy) so my api can have certain value from main js. How to solve this or should point me where should i read more.below is my code
main js
const inputValue = document.querySelector('input').value
//this value is maybeline
app.js(node.js proxy)
const express = require('express')
const request = require('request');
const app = express()
app.use((req,res,next)=>{
res.header('Access-Control-Allow-Origin', '*');
next();
})
app.get('/api', (req, res) => {
request(
{ url: 'https://makeup-api.herokuapp.com/api/v1/products.json?brand=covergirl' },
(error, response, body) => {
if (error || response.statusCode !== 200) {
return res.status(500).json({ type: 'error', message: err.message });
}
res.json(JSON.parse(body));
}
)
});
const port = process.env.PORT || 5500
app.listen(5500,()=>console.log(`listening ${port}`))
I want to pass inputValue as query to api in app.js
as
https://makeup-api.herokuapp.com/api/v1/products.json?brand=${type}
How to do it or point me any direction?
note:this api work withour cors problem.This is an example api..tq
You can use stringify method from qs module.
const { stringify } = require("qs");
app.get("/api", (req, res) => {
request(
{
url: `https://makeup-api.herokuapp.com/api/v1/products.json?${stringify(
req.params
)}`,
}
/// Rest of the code
);
});

Unexpected file transfer behaviour when building API using Express

Here is how my API works:
You can find SeaweedFS here on GitHub.
And the code here:
// /drivers/seaweedfs.js Defines how API interacts with SeaweedFS
const { error } = require("console");
const http = require("http");
module.exports = class Weed {
constructor(mserver) {
this.mserver = new URL(mserver);
}
get(fileId) {
return new Promise((resolve, reject) => {
let options = {
hostname: this.mserver.hostname,
port: this.mserver.port,
method: "GET",
path: `/${fileId}`,
timeout: 6000,
};
let data;
const fileReq = http.request(options, (res) => {
console.log(`Statuscode ${res.statusCode}`);
res.on("data", (response) => {
data += response;
});
res.on("end", () => {
resolve(data);
});
});
fileReq.on("error", () => {
console.error(error);
reject();
});
fileReq.end();
});
}
};
// /routes/file.js An Express router
const express = require("express");
const router = express.Router();
const Weed = require("../drivers/seaweedfs");
let weedClient = new Weed("http://localhost:60002");
router.get("/:fileId", (req, res) => {
weedClient.get(req.params.fileId)
.then(data=>{
res.write(data)
res.end()
})
}
)
module.exports = router;
MongoDB driver not yet implemented.
When I try to GET a file(using Firefox, Hoppscotch says Could not send request: Unable to reach the API endpoint. Check your network connection and try again.), I get something whose MIME type is application/octet-stream for some reason. It's bigger than the original file. I know there must be some problems with my code, but I don't know where and how to fix it.

Undefined body of a fetch request (arraybuffer body) in node server

I've read a lot of similar questions, but any answer worked for me.
I'm trying to send an audio in the body of a fetch request (in vuejs) as an ArrayBuffer, but when I print it in the server side, the body is undefined.
Client code:
export async function makeQuery(data) {
let bufferAudio = await data.arrayBuffer();
console.log(bufferAudio);
const response = await fetch(`/api/query`, {
method: 'POST',
//headers: {'Content-Type': 'audio/mp3'},
body: bufferAudio,
})
.catch(function(e) {
console.log("error", e);
});
return response;
}
Node Server code:
const express = require('express');
const path = require('path');
const app = express(), port = 3080;
app.post('/api/query', async (req, res) => {
query = req.body;
console.log('query ', query); // Here the body is undefined
/*
Do some processing...
*/
});
When I send a simple json string in the request body (and app.use(express.json()) in the server)it works. But not with the arraybuffer audio. Thanks in advance

https request basic authentication node.js

I am really getting crazy looking for this over the web and stackoverflow.
Other posts about this topic talk of http request, not httpS.
I'm coding server side with node.js and I need to make an https request to another website to login
If I use postman tool in chrome trying with https://user:pass#webstudenti.unica.it/esse3/auth/Logon.do everything works fine and I log in.
If I use request library in node I can't login and I get a page with a custom error message about an error in my getting/sending data.
Maybe I am wrong setting the options to pass to request.
var request = require('request');
var cheerio = require('cheerio');
var user = 'xxx';
var pass = 'yyy';
var options = {
url : 'https://webstudenti.unica.it',
path : '/esse3/auth/Logon.do',
method : 'GET',
port: 443,
authorization : {
username: user,
password: pass
}
}
request( options, function(err, res, html){
if(err){
console.log(err)
return
}
console.log(html)
var $ = cheerio.load(html)
var c = $('head title').text();
console.log(c);
})
http://jsfiddle.net/985bs0sc/1/
You're not setting your http auth options correctly (namely authorization should instead be auth). It should look like:
var options = {
url: 'https://webstudenti.unica.it',
path: '/esse3/auth/Logon.do',
method: 'GET',
port: 443,
auth: {
user: user,
pass: pass
}
}
http/https should make no difference in the authentication. Most likely your user/pass needs to be base64 encoded. Try
var user = new Buffer('xxx').toString('base64');
var pass = new Buffer('yyy').toString('base64');
See: https://security.stackexchange.com/questions/29916/why-does-http-basic-authentication-encode-the-username-and-password-with-base64
Don't use npm package request because it is deprecated, use Node native https instead
const https = require('https')
var options = {
host: 'test.example.com',
port: 443,
path: '/api/service/'+servicename,
// authentication headers
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
//this is the call
request = https.get(options, function(res){
console.log(`statusCode: ${res.statusCode}`)
res.on('data', d => {
process.stdout.write(d)
})
})
req.on('error', error => {
console.error(error)
})
req.end()
With the updated version, I am able to make https call with basic auth.
var request = require('request');
request.get('https://localhost:15672/api/vhosts', {
'auth': {
'user': 'guest',
'pass': 'guest',
'sendImmediately': false
}
},function(error, response, body){
if(error){
console.log(error)
console.log("failed to get vhosts");
res.status(500).send('health check failed');
}
else{
res.status(200).send('rabbit mq is running');
}
})
Use Node.js' URL module to build your URL object. The httpsAgent module is required if you are calling servers w self-signed certificates.
const https = require('https');
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
});
// Allow SELF_SIGNED_CERT, aka set rejectUnauthorized: false
let options = {
agent: httpsAgent
}
let address = "10.10.10.1";
let path = "/api/v1/foo";
let url = new URL(`https://${address}${path}`);
url.username = "joe";
url.password = "password123";
url.agent = httpsAgent
let apiCall = new Promise(function (resolve, reject) {
var data = '';
https.get(url, options, res => {
res.on('data', function (chunk){ data += chunk })
res.on('end', function () {
resolve(data);
})
}).on('error', function (e) {
reject(e);
});
});
try {
let result = await apiCall;
} catch (e) {
console.error(e);
} finally {
console.log('We do cleanup here');
}

Categories

Resources