I would like to create a UI which is somehow similar to Telerik's Fiddler. For this purpose, I created a proxy with node-http-proxy which works perfectly :-)
But now I have the problem of how I match the response to my request and I have absolutely no idea what I could try.
Here is my current code:
let httpProxy = require('http-proxy');
let connect = require('connect');
let bodyParser = require('body-parser');
let http = require('http');
let proxy = httpProxy.createProxyServer();
let app = connect()
.use(bodyParser.json()) //json parser
.use((req, res) => {
let options = {
target: "myTargetHere"
};
proxy.web(req, res, options);
});
http.createServer(app).listen(8000);
proxy.on("proxyReq", (proxyReq, req, res) => {
if (req.body) {
let bodyData = JSON.stringify(req.body);
let bodyLength = Buffer.byteLength(bodyData)
proxyReq.setHeader("Content-Length", bodyLength);
proxyReq.write(bodyData);
}
});
proxy.on('proxyRes', (proxyRes, req, res) => {
let body = new Buffer.alloc(0);
let bodyLength = 0;
proxyRes.on('data' , data => {
process.stdout.write("c ");
body = Buffer.concat([body, data]);
});
proxyRes.on("end", () => {
bodyLength = Buffer.byteLength(body);
body = body.toString();
console.log("response body length: ", bodyLength);
});
});
Does anyone have an idea how I can match the response in proxy.on('proxyRes', ...) with my request in proxy.on('proxyReq', ...)? Is there something that is unique within every request/response?
Related
How to get data from my function Data() instead of JSON PLACE HOLDER mock API using HTTPS/HTTP node module and how to make an endpoint of this get data HTTP/HTTPS module to utilize response in front end just like Angular?
My mock backen.js file:
const https = require('https');
https.get(Data.data, res => {
let data = [];
const headerDate = res.headers && res.headers.date ? res.headers.date : 'no response date';
console.log('Status Code:', res.statusCode);
console.log('Date in Response header:', headerDate);
res.on('data', chunk => {
data.push(chunk);
});
res.on('end', () => {
console.log('Response ended: ');
const users = JSON.parse(Buffer.concat(data).toString());
for(user of users) {
console.log(`Got user with id: ${user.id}, name: ${user.name}`);
}
});
}).on('error', err => {
console.log('Error: ', err.message);
});
function Data() {
var data = {};
........
return data;
}
Your time and help will be really appreciated. Thanks :)
Hurray! I got it using the following code and Express in node js. I simply call my custom method that creates data into an express "get" endpoint. When I hit the end point the response will be my custom method result.
const express = require('express');
const cors = require('cors');
const app = express();
app.use(cors());
app.get('/data', (req, res) => {
res.send(
getDashboardData());
});
app.listen(3000, () => {
console.log('server is listening on port 3000');
});
function Data() {
var data = {};
..........
//console.log(JSON.stringify(data));
return JSON.stringify(data);
}
Here is how my API works:
You can find SeaweedFS here on GitHub.
And the code here:
// /drivers/seaweedfs.js Defines how API interacts with SeaweedFS
const { error } = require("console");
const http = require("http");
module.exports = class Weed {
constructor(mserver) {
this.mserver = new URL(mserver);
}
get(fileId) {
return new Promise((resolve, reject) => {
let options = {
hostname: this.mserver.hostname,
port: this.mserver.port,
method: "GET",
path: `/${fileId}`,
timeout: 6000,
};
let data;
const fileReq = http.request(options, (res) => {
console.log(`Statuscode ${res.statusCode}`);
res.on("data", (response) => {
data += response;
});
res.on("end", () => {
resolve(data);
});
});
fileReq.on("error", () => {
console.error(error);
reject();
});
fileReq.end();
});
}
};
// /routes/file.js An Express router
const express = require("express");
const router = express.Router();
const Weed = require("../drivers/seaweedfs");
let weedClient = new Weed("http://localhost:60002");
router.get("/:fileId", (req, res) => {
weedClient.get(req.params.fileId)
.then(data=>{
res.write(data)
res.end()
})
}
)
module.exports = router;
MongoDB driver not yet implemented.
When I try to GET a file(using Firefox, Hoppscotch says Could not send request: Unable to reach the API endpoint. Check your network connection and try again.), I get something whose MIME type is application/octet-stream for some reason. It's bigger than the original file. I know there must be some problems with my code, but I don't know where and how to fix it.
How to encode a request body using HMAC sha 256 and base64.
The request object that i receives from xero webhook.
HEADER:
"x-xero-signature" : HASH_VALUE
PAYLOAD:
{
"events": [],
"lastEventSequence": 0,
"firstEventSequence": 0,
"entropy": "S0m3r4Nd0mt3xt"
}
The note from xero documentation says "If the payload is hashed using HMACSHA256 with your webhook signing key and base64 encoded, it should match the signature in the header. This is a correctly signed payload. If the signature does not match the hashed payload it is an incorrectly signed payload."
And I followed this example : https://devblog.xero.com/using-xero-webhooks-with-node-express-hapi-examples-7c607b423379
const express = require("express");
const router = express.Router();
const base64 = require('base-64');
const crypto = require('crypto')
const bodyParser = require('body-parser')
const xero_webhook_key = '00fRRlJBYiYN4ZGjmTtG+g/pulyb1Eru68YYL3PFoLsa78dadfQtGrOMuISuVBHxpXeEYo0Yy1Gc+hHMhDkSI/EEcgtrA==';
let options = {
type: 'application/json'
};
let itrBodyParser = bodyParser.raw(options);
router.post("/", itrBodyParser, async (req, res, next) =>{
// console.log('::::WebhookPost:::');
const reSign = req.headers['x-xero-signature'];
console.log(req.headers['x-xero-signature']);
console.log('::::::::');
console.log(req.body);
console.log("Body: "+JSON.stringify(req.body))
console.log(req.body.toString());
console.log("Xero Signature: "+ reSign);
console.log('Server key::::',xero_webhook_key);
// Create our HMAC hash of the body, using our webhooks key
let hmac = crypto.createHmac("sha256", xero_webhook_key).update(req.body.toString()).digest('base64');
console.log("Resp Signature: ",hmac)
if (req.headers['x-xero-signature'] == hmac) {
res.statusCode = 200
} else {
res.statusCode = 401
}
console.log("Response Code: "+res.statusCode)
return res.send();
});
Hey I recently did a video on implementing webhooks with Xero, let me know if this gets you unstuck. I found that trying to pass itrBodyParser on the route the way you have wasn't working for me so I switched it with an app.use statement on my specific webhooks endpoint. If you prefer a written guide over video, here's the blog post
I solved it using this solution.! I was using express framework and the request were not getting as raw request also .toString of didn't worked as mentioned in xero documentation.
const server = http.createServer(async (req, resp) => {
try {
console.log(`::::Webhook::: ${webhookPort}`);
console.log("::::x-xero-signature:::");
console.log(req.headers["x-xero-signature"]);
console.log(`--------------------------------------`);
if (req.method === "POST") {
if(req.headers["x-xero-signature"]){
const rData = await new Promise((resolve, reject) => {
return collectRequestData(req, (result) => {
console.log(result);
let hmac = crypto
.createHmac("sha256", xero_webhook_key)
.update(result)
.digest("base64");
console.log("Resp Signature: ", hmac);
return resolve({
hmac,result
});
});
});
console.log(">>Resp Signature: ", rData);
console.log('>>x-xero-signature:::',req.headers["x-xero-signature"]);
if(rData.result){
const result = JSON.parse(rData.result);
console.log('result:::',result);
for(let { resourceId } of result.events) {
console.log('::INVOICE ID = ',resourceId);
getInvoiceData(resourceId);
}
}
if(rData.hmac == req.headers["x-xero-signature"] ){
console.log('::YES');
resp.statusCode = 200;
}else{
console.log('::NO');
resp.statusCode = 401;
}
}
resp.end();
}
console.log("::::Webhookgetsssss:::");
resp.message = 'Get API'
resp.end();
} catch (error) {
resp.statusCode = 200;
resp.end();
}
});
server.listen(webhookPort);
function collectRequestData(request, callback) {
let body = "";
request.on("data", (chunk) => {
body += chunk.toString();
});
request.on("end", () => {
callback(body);
});
}
I am creating a Recipe Search Engine/Culinary blog.
I want user to choose food category (e.g pork), send it to the server, pass that information to the API call link, then call the API and send the results back to front-end. I am up to sending category name to the server from client and got stuck on how i can handle that.
CLIENT-SIDE
function getRecipes(category){
const categorySearch = category.alt;
let data = {
categoryChoice: categorySearch
}
let options = {
method: 'POST',
headers: {
"Content-type": "application/json; charset=UTF-8"
},
body: JSON.stringify(data)
}
const promise = fetch('/data', options);
promise.then(response => {
if(!response.ok){
console.error(response)
} else {
return response.json();
}
}).then(result => {
console.log(result);
})
}
SERVER-SIDE
const express = require('express');
const app = express();
const fetch = require('node-fetch');
require('dotenv').config();
const API_KEY = process.env.API_KEY;
const port = process.env.PORT || 3100;
app.listen(port, () => console.log('listening at 3100'));
app.use(express.static('public'));
app.use(express.json({ limit: "1mb"}));
app.post('/data', async (request, response) => {
const data = await request.body;
const gotData = data.categoryChoice;
const category = gotData;
console.log(category);
response.json(category);
return category
})
app.get('/data', async (request, response, category) => {
const cat = await category;
const url = `https://edamam-recipe-search.p.rapidapi.com/search?q=${cat}&from=0&to=100`
const fetch_response = await fetch(url);
const json = await fetch_response.json();
response.json(json);
})
I am correctly getting the category depending on what i click in the front-end and that category is being send to server. How i can pass that variable to the API call ? And then how i can send it back to the front-end ?
Many Thanks
I am trying to get the username of the profile on the url of that users page. Just to figure out the basic of getting Instagram data
The code below is what I have tried to do and miserably failed(also I am a beginner sorry for bad code)
const express = require('express');
const bodyParser = require('body-parser');
const request = require('request-promise');
const path = require('path');
var cheerio = require('cheerio');
const app = express()
var followers = [];
app.use(express.static('public'));
app.use(bodyParser.urlencoded({ extended: true }));
app.set('view engine', 'ejs')
app.get('/', function (req, res) {
request({
method: 'GET',
url: 'https://www.instagram.com/unrivaledhype/'
}, function(err, response, body, callback) {
if(err) return console.error(err);
$ = cheerio.load(url);
var post = $("*");
var follwerCount = post.find('h1').text();
console.log(follwerCount);
followers.push({follwerCount: follwerCount})
});
res.send(JSON.stringify(followers, null, 4));
});
app.listen(3000, function () {
console.log('Example app listening on port 3000!')
})
It just displays follwerCount:"" but(even though I named it misspelled followerCount I went for the username) although if someone can show me how to get the follower count that would be greatly appreciated.
Unfortunately, Instagram pages are rendered on the client side, so it is not possible to get information like the number of followers through this way.
A solution to this problem would be to use puppeteer.
With puppeteer you can start a headless version of chrome, which also interprets javascript, so that Instagram pages get rendered completely.
Example code:
const puppeteer = require('puppeteer')
class InstagramClient {
async start() {
this.browser = await puppeteer.launch({
headless: true //When set to true, a new browser window will ge opened
})
}
async stop() {
await this.browser.close()
}
async getFollowers(username) {
if (!this.browser) throw new Error('Browser not started')
const page = await this.browser.newPage()
await page.goto(`https://instagram.com/${username}/`)
const pageExists = await page.evaluate(_ => {
return document.querySelector('.error-container') === null
})
if (!pageExists) {
throw new Error(`Page of ${username} doesn't exist`)
}
//Wait until the page got completly renderer
await page.waitForSelector('h1')
const followers = await page.evaluate(username => {
//This code will get executed on the instagram page
//Get the number of followers
const followers = document.querySelector(`a[href="/accounts/login/?next=%2F${username}%2Ffollowers%2F&source=followed_by_list"]`).querySelector('span').innerText
//Return the number of followers back to the node process
return followers
}, username)
page.close()
return followers
}
}
module.exports = InstagramClient
const InstagramClient = require('./utils/instagram-client')
async function start() {
const client = new InstagramClient()
await client.start()
console.log('#instagram:', await client.getFollowers('instagram'))
console.log('#unrivaledhype:', await client.getFollowers('unrivaledhype'))
console.log('#teslamotors:', await client.getFollowers('teslamotors'))
await client.stop()
}
start()
Output:
#instagram: 309m
#unrivaledhype: 3,230
#teslamotors: 6m
If you want a more detailed explanation, check out this video:
A Guide to Web Scraping with NodeJS
Express example:
const express = require('express')
const InstagramClient = require('./utils/instagram-client')
const app = express()
const client = new InstagramClient()
app.get('/:instagramName/followers', async (request, response) => {
const instagramName = request.params.instagramName
try {
const followers = await client.getFollowers(instagramName)
response.json({
success: true,
followers: followers
})
} catch (e) {
response.json({
success: false,
error: e.toString()
})
return
}
})
async function start() {
await client.start()
app.listen(3000, _ => console.log('Server is listening on port 3000'))
}
start()