Develop Reference

NodeJS API Proxy Server - Post Requests Error 404 - Routing

I am new at NodeJS and am trying to implement a proxy server for my GET requests. GET requests works fine can also update my UI as it should be by performAction and chained promises, however something is wrong with my POST request, I always get a 404 despite I defined the route, it pops up after UI update. Can anybody help me? Thanks!
SERVER
const express = require('express')
const cors = require('cors')
const rateLimit = require('express-rate-limit')
require('dotenv').config()
const errorHandler = require('./middleware/error')
const bodyParser = require('body-parser')
// support parsing of application/json type post data
//support parsing of application/x-www-form-urlencoded post data
const PORT = process.env.PORT || 5000
const app = express()
app.use(bodyParser.urlencoded({ extended: true }));
// Rate limiting
const limiter = rateLimit({
windowMs: 10 * 60 * 1000, // 10 Mins
max: 100,
})
app.use(limiter)
app.set('trust proxy', 1)
// Enable cors
app.use(cors())
// Set static folder
app.use(express.static('public'))
// Routes
app.use('/api', require('./routes'))
// Error handler middleware
app.use(errorHandler)
app.listen(PORT, () => console.log(`Server running on port ${PORT}`))
API PROXY SERVER
const url = require('url')
const express = require('express')
const router = express.Router()
const needle = require('needle')
const apicache = require('apicache')
// Env vars
const API_BASE_URL = process.env.API_BASE_URL
const API_KEY_NAME = process.env.API_KEY_NAME
const API_KEY_VALUE = process.env.API_KEY_VALUE
// Init cache
let cache = apicache.middleware
let projectData = {}
router.get('/', cache('2 minutes'), async (req, res, next) => {
try {
const params = new URLSearchParams({
[API_KEY_NAME]: API_KEY_VALUE,
...url.parse(req.url, true).query,
})
console.log('${API_BASE_URL}?${params}')
const apiRes = await needle('get', `${API_BASE_URL}?${params}`)
const data = apiRes.body
// Log the request to the public API
if (process.env.NODE_ENV !== 'production') {
console.log(`REQUEST: ${API_BASE_URL}?${params}`)
}
res.status(200).json(data)
} catch (error) {
next(error)
}
})
function sendForecastData(req, res) {
const { date, temp, content } = req.body;
let journal_entry_new = new Object();
journal_entry_new.date = date;
journal_entry_new.temp = temp + "°C";
journal_entry_new.content = content;
idx_entry = String("entry_" + idx)
idx = idx + 1
projectData[idx_entry] = JSON.stringify(journal_entry_new);
console.log(projectData)
res.send(projectData)
console.log("Post sucessful.")
}
router.post('/', cache('2 minutes'), async (req, res, next) => {
const postObject = needle.post('/addData', req.body, sendForecastData)
return postObject;
})
function readData(req, res) {
res.send(projectData)
console.log(projectData)
console.log("Read sucessful.")
}
router.get('/readData', readData)
module.exports = router
This is my app.js
//Event-Listener
document.getElementById('generate').addEventListener('click', performAction);
//Declare Fetch Function
//User Input
const the_date = document.getElementById('date');
const temp = document.getElementById('temp');
const content = document.getElementById('content');
// Create a new date instance dynamically with JS
let d = new Date();
let newDate = (d.getMonth() + 1) + "." + (d.getDate()) + '.' + (d.getFullYear());
//Proxy
const fetchWeather = async (zipcode) => {
const url = `/api?q=${zipcode}`
const res = await fetch(url)
const data = await res.json()
console.log(data)
return data;
}
const postData = async (url, data) => {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(data),
});
try {
const data = await response.json();
return data
}
catch (error) {
console.log("error", error);
}
}
const readData = async () => {
const request = await fetch('/readData');
try {
// Transform into JSON
const readData = await request.json()
return readData
} catch (error) {
console.log("error", error);
}
}
const UpdateUI = async (data) => {
console.log("Data received:")
console.log(data)
console.log("Date: " + data[Object.keys(data)[0]])
console.log("Temp: " + data[Object.keys(data)[1]])
console.log("Content: " + data[Object.keys(data)[2]])
document.getElementById('temp').innerText = "Temperature is: " + data[Object.keys(data)[1]]
document.getElementById('date').innerText = "Date is: " + data[Object.keys(data)[0]]
document.getElementById('content').innerText = "Feeling is: " + data[Object.keys(data)[2]]
console.log("Updated UI")
}
function performAction(e) {
//Check if user input is available
if (document.getElementById('zip').value == "") {
alert("Please type in a zipcode, then I will know where to look up the weather for
you!");
return
}
//Feeling now
let zipcode = (document.getElementById('zip').value).toString()
let feeling_now = (document.getElementById('feelings').value).toString()
fetchWeather(zipcode)
.then(data => {
/*let temp_k = parseFloat(data.list[0].main.temp)*/
/*let temp_c = String((temp_k - 273.15).toFixed(2)) + " °C"*/
let temp_c = parseFloat(data.main.temp) + " °C"
let feeling_now = (document.getElementById('feelings').value).toString()
console.log(temp_c)
console.log(newDate)
console.log(feeling_now)
console.log({ date: newDate, temp: temp_c, content: feelings.value })
return { date: newDate, temp: temp_c, content: feelings.value }
}).then(data => {
postData('/addData', data);
return data
})
.then(data => {
readData('/readData');
return data
})
.then(data => UpdateUI(data))
console.log(feeling_now)
return;
}
My UI is updated, however I get the following error and cannot access localhost:5000/addData - can you help why this is the case?
Defined the routes in the backend, however cannot call them from frontend

How to write file using fs.createWriteStream

am trying to build a web scraper that downloads all the pdfs in a website. i've written all the logic necessary to do this but for some reason it downloads an empty pdf file which is not suppose to be so, the problem seems to be coming from the downloadFile function when i try to pipe the data which for some reason seems not to be working because i get an empty pdf file after the function is ran. i'll would appreciate it if someone can help me out with this problem, thanks.
here's a sample of my code:
app.js
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");
const axiosInstance = require("./getAxios");
const axios = axiosInstance();
const Surl = "https://www.health.gov.ng/";
// linkList sample: "https://www.health.gov.ng/index.php?option=com_content&view=article&id=143&Itemid=512";
let = connectionFailCount = 0;
let linkList = [];
let dlinkList = [];
const getWebsiteLinks = async (Surl) => {
try {
console.log(`Crawling all links from: ${Surl}`);
const response = await axios.get(Surl);
const $ = cheerio.load(response.data);
const ranges = $("a").each(function (idx, el) {
if ($(el).attr("href")) {
return $(el).attr("href");
}
});
for (let index = 0; index < ranges.length; index++) {
let raw_links = $("a")[index].attribs.href;
if (raw_links.startsWith("/")) {
linkList.push(Surl + raw_links);
}
}
if (linkList.length > 0) {
console.log(`Finished crawling links: Found ${linkList.length} links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
return;
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
getWebsiteLinks(Surl);
console.log(`Connection error. \n
Reconnecting to server....`);
} else if (connectionFailCount === 5) {
console.error(`Can not connect to server. Try again later.`);
}
}
};
const downloadLinks = async (linkList) => {
try {
console.log("Crawling links to find pdf links. this may take a while...");
for (const link of linkList) {
const response = await axios.get(link);
// Skip where there's delayed server response
if (response.code === "ECONNRESET") continue;
const $ = cheerio.load(response.data);
$("a").each(function (idx, el) {
if ($(el)?.attr("href")?.endsWith(".pdf")) {
let addr = $(el).attr("href");
let dlink = Surl + addr;
dlinkList.push({
pathName: addr,
url: dlink,
});
}
});
}
console.log(dlinkList);
if (dlinkList.length > 0) {
console.log(`Crawling Finish: Found ${dlinkList.length} pdf links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
console.log(`Connection error. \n
Reconnecting to server: ${connectionFailCount} count`);
downloadLinks(linkList);
}
if (connectionFailCount === 3) {
console.error(`Can not connect to server. Try again later.`);
return;
}
// console.error("downloadLinksError: ", error);
}
};
const downloadFiles = async (dlinkList) => {
console.log("Creating directory to save PDF files");
const appRoot = path.dirname(path.resolve(__dirname));
// Had to change and restructure code due to error
const folderName = `PDF/${Surl.split("/").pop()}`;
const subFolderName = Surl.split("/").pop();
try {
if (!fs.existsSync(path.join(appRoot, folderName))) {
fs.mkdirSync(path.join(appRoot, "PDF"));
fs.mkdirSync(path.join(`${appRoot}/PDF`, subFolderName));
}
dlinkList.forEach(async (link) => {
let name = link.pathName;
let url = link.url;
let file = fs
.createWriteStream(
`${appRoot}/${folderName}/${name.split("/").pop()}`,
"utf-8"
)
.on("error", (err) => {
console.error("createWriteStreamError: ", err);
});
try {
console.log("Downloading PDF file...");
const { data } = await axios({
url,
method: "GET",
responseType: "stream",
});
if (data) {
console.log("PDF file Downloaded");
data.pipe(file);
}
} catch (error) {
console.error(error);
}
});
return;
} catch (error) {
console.error("downloadFilesError: ", error);
}
};
(async () => {
await getWebsiteLinks(Surl);
await downloadLinks(linkList);
await downloadFiles(dlinkList);
})();
getAxios.js
const axios = require("axios");
const https = require("https");
module.exports = function () {
const domain = "https://www.health.gov.ng/";
let instance;
if (!instance) {
//create axios instance
instance = axios.create({
baseURL: domain,
timeout: 60000, // Increase time out incase of network delay or delayed server response
maxContentLength: 500 * 1000 * 1000, // Increase maximum response ata length
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { "Content-Type": "application/xml" },
});
}
return instance;
};

NodeJs: `beginTransaction` is not a function error in mysql2

In my below code, I am getting the error is not a function. This error is thrown in the line await con.promise().beginTransaction(); please check my code below.
const mysql = require('mysql2');
const errorCodes = require('source/error-codes');
const PropertiesReader = require('properties-reader');
const prop = PropertiesReader('properties.properties');
const con = mysql.createPool({
connectionLimit : 10,
host: prop.get('server.host'),
user: prop.get("server.username"),
password: prop.get("server.password"),
port: prop.get("server.port"),
database: prop.get("server.dbname")
});
exports.checkInvestorBoost = async (event, context) => {
context.callbackWaitsForEmptyEventLoop = false;
con.config.namedPlaceholders = true;
const params = event.queryStringParameters;
if (!params || params.iduser == null) {
var response = errorCodes.not_null_parameters;
return response;
} else {
if(isNaN(params.iduser))
{
var response = errorCodes.missing_fields;
return response;
}
const iduser = Number(params.iduser);
const toDate = new Date();
console.log("iduser: " + iduser);
let sql = "SELECT * FROM golden_circle_member WHERE iduser= :iduser AND is_investor_boost = true AND to_date > :to_date";
try {
await con.promise().beginTransaction();
const [data, meta] = await con.promise().query(sql, {
iduser: iduser,
to_date: toDate
});
// commit and complete
await con.promise().commit();
let output= false;
if(data.length>0)
{
output = true;
}
var response = {
"statusCode": 200,
"headers": {
"Content-Type": "application/json"
},
"body": JSON.stringify({
"is_investor_boost": output
}),
"isBase64Encoded": false
};
return response;
} catch (error) {
console.log(error);
var response = errorCodes.internal_server_error;
return response;
}
}
};
Reading about the question, I figured out that when createPool is used, I need to get the connection first. Not sure how I can get that done. Plus, I prefer to do this with async/await
Appreciate your advice here

This should work:
const mysql = require('mysql2/promise');
…
const pool = mysql.createPool(…);
const connection = await pool.getConnection();
try {
await connection.beginTransaction();
…
const [data, meta] = await connection.query(…);
…
await connection.commit();
} catch(err) {
await connection.rollback();
}
This also uses the mysql2 promise wrapper so you don't have to use .promise() each time.

You've to get the connection from the promise pool before starting a transaction.
const conn = await pool.promise().getConnection();
await conn.beginTransaction();

NodeJS return value in axios get [duplicate]

This question already has answers here:
Return from a promise then()
(8 answers)
Closed 9 months ago.
I have this class written in javascript but I have difficulty in obtaining the result from the axios request, below my situation to better explain the problem:
i have a file called vtiger.js in the classes directory in the project root
vtiger.js
const axios = require('axios');
var md5 = require('md5');
var qs = require('qs');
const https = require('https');
class vTiger {
constructor() {
this.url = process.env.VTIGER_URL;
this.username = process.env.VTIGER_USERNAME;
this.password = process.env.VTIGER_PASSWORD;
}
async getchallengeTokenVtiger() {
var token;
var tokenmd5 = false;
var url = this.url + 'webservice.php?operation=getchallenge&username=' + this.username;
axios.get(url,
{
headers: {
"content-type": "application/x-www-form-urlencoded"
},
httpsAgent: new https.Agent(
{
rejectUnauthorized: false
})
}).then(response => {
if (response.data.success) {
token = response.data.result.token;
tokenmd5 = md5(token + this.password);
return tokenmd5;
}
});
}
}
module.exports = vTiger
then I have a file called api.js in the controllers folder with this content:
const http = require('http');
const axios = require('axios');
var qs = require('qs');
const vTiger = require('../classes/vtiger');
exports.welcome = (req, res, next) => {
const vtigerClass = new vTiger();
console.log(vtigerClass.getchallengeTokenVtiger())
res.status(200).json({
data: vtigerClass.getchallengeTokenVtiger()
});
}
from this file as an response I get:
{
"data": {}
}
while from the console.log(vtigerClass.getchallengeTokenVtiger()) line I get this:
Promise { undefined }
Where am I doing wrong?
Thanks

you probably don't want to use the .then in an async function. you should
const response = await axios.get(...)
if (response.data.success) {
token = response.data.result.token;
tokenmd5 = md5(token + this.password);
return tokenmd5;
}
else return null;
Or you can create a variable in your function called tokenmd5 i.e.
let tokenmd5 = ''
set its value in the .then, then return tokenmd5 at then end of your function NOT in the .then
THEN:
for your console.log you want:
exports.welcome = async (req, res, next) => {
const vtigerClass = new vTiger();
console.log(await vtigerClass.getchallengeTokenVtiger())
res.status(200).json({
data: vtigerClass.getchallengeTokenVtiger()
});
}

Unable to GET value from redis

I am trying to get a value from redis which I have set. When I call the checkCache function I get "CACHE: null" and afterwards it logs "NO ERROR + the data". I dont understand why since I used await.
const redis = require("redis");
const client = redis.createClient({
port : procces.env.PORT,
host : procces.env.HOST,
password : procces.env.PASSWORD,
});
const checkCache = async (key) => {
await client.get(key, (err, data) => {
if(err) throw err;
console.log("NO ERROR + " + data);
if(data !== null) { return JSON.parse(data); }
});
return null;
};
I call the methode like this:
const findAll = async(user) => {
const cache = await checkCache(user);
console.log('CACHE: ' + cache); // returns null
};

Module redis doesn't support Promise.
For this you have promisify it
const Promise = require('bluebird');
const redis = Promise.promisifyAll(require("redis"));
EDIT:
const Promise = require('bluebird');
const redis = Promise.promisifyAll(require("redis"));
const client = redis.createClient({
port : process.env.PORT,
host : process.env.HOST,
password : process.env.PASSWORD,
});
const checkCache = async (key) => {
const data = await client.getAsync(key);
console.log("NO ERROR + " + data);
if(data !== null) { return JSON.parse(data); }
return null;
};