I have a project in Node JS in which I have a form to add new users.
How can I view this information in JSON format?
These are the data that I see:
name age country city
------------------------------
user1 22 Spain Madrid button{View JSON}
When I press the 'View JSON' button, the following must be displayed below the table:
[
"id": 1,
"name": "user1",
"age": 22,
"country": "Spain" {
"city":"Madrid"
}
]
My problem: how can I create a function that performs this conversion? How do I call the function from index.ejs?
I cleared and merged the codes. And I created a new endpoint as /export to export the data as CSV file. I couldn't test it so let me know if it doesn't work.
const express = require('express');
const bodyParser = require('body-parser');
const app = express();
const MongoClient = require('mongodb').MongoClient;
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json());
app.use(express.static('public'));
app.set('views', './src/views');
app.get('/', async (req, res) => {
const db = await mongoDB();
const person = await db.collection('person').find().toArray();
res.render('index.ejs', { person: person })
})
app.get('/export', async (req, res) => {
await convertCSV();
res.status(200).send( { success: 1 });
})
app.post('/person', async (req, res) => {
res.redirect('/');
})
app.listen(process.env.PORT, function () {
console.log(`server: http://${process.env.HOST}:${process.env.PORT}`);
})
const mongoDB = () => {
return new Promise((resolve, reject) => {
const url = 'mongodb://127.0.0.1:27017';
MongoClient.connect(url, { useUnifiedTopology: true })
.then(client => {
const db = client.db('users')
resolve(db);
})
.catch(error => reject(error))
});
}
const convertCSV = () => {
return new Promise((resolve, reject) => {
const converter = require("json-2-csv");
const fetch = require("node-fetch");
const fs = require("fs");
const flatten = require('flat');
const maxRecords = 10;
const getJson = async () => {
const response = await fetch(`http://${process.env.HOST}:${process.env.PORT}/users.json`);
const responseJson = await response.json();
return responseJson;
};
const convertToCSV = async () => {
const json = await getJson();
let keys = Object.keys(flatten(json[0]));
let options = {
keys: keys
};
converter.json2csv(json, json2csvCallback, options);
};
let json2csvCallback = function (err, csv) {
if (err) throw err;
const headers = csv.split('\n').slice(0, 1);
const records = csv.split('\n').slice(0,);
for (let i = 1; i < records.length; i = i + maxRecords) {
let dataOut = headers.concat(records.slice(i, i + 3)).join('\n');
let id = Math.floor(i / maxRecords) + 1;
fs.writeFileSync('data' + id + '.csv', dataOut)
}
};
await convertToCSV();
resolve();
})
}
However, it is not a good practice at all to using controller, index and route in the same file. A better approach would be to create routes, controllers folders and put the codes in a more orderly form.
Something like this (You can find better ones of course mine is just advice):
- index.js
- router.js (A router to manage your endpoints)
- controllers (Controller when you call the endpoint)
-> export.controller.js
-> person.controller.js
- routes (Endpoints)
-> export.route.js
-> person.route.js
- helpers
-> databaseHandler.js (Database connection handler)
Related
I am new at NodeJS and am trying to implement a proxy server for my GET requests. GET requests works fine can also update my UI as it should be by performAction and chained promises, however something is wrong with my POST request, I always get a 404 despite I defined the route, it pops up after UI update. Can anybody help me? Thanks!
SERVER
const express = require('express')
const cors = require('cors')
const rateLimit = require('express-rate-limit')
require('dotenv').config()
const errorHandler = require('./middleware/error')
const bodyParser = require('body-parser')
// support parsing of application/json type post data
//support parsing of application/x-www-form-urlencoded post data
const PORT = process.env.PORT || 5000
const app = express()
app.use(bodyParser.urlencoded({ extended: true }));
// Rate limiting
const limiter = rateLimit({
windowMs: 10 * 60 * 1000, // 10 Mins
max: 100,
})
app.use(limiter)
app.set('trust proxy', 1)
// Enable cors
app.use(cors())
// Set static folder
app.use(express.static('public'))
// Routes
app.use('/api', require('./routes'))
// Error handler middleware
app.use(errorHandler)
app.listen(PORT, () => console.log(`Server running on port ${PORT}`))
API PROXY SERVER
const url = require('url')
const express = require('express')
const router = express.Router()
const needle = require('needle')
const apicache = require('apicache')
// Env vars
const API_BASE_URL = process.env.API_BASE_URL
const API_KEY_NAME = process.env.API_KEY_NAME
const API_KEY_VALUE = process.env.API_KEY_VALUE
// Init cache
let cache = apicache.middleware
let projectData = {}
router.get('/', cache('2 minutes'), async (req, res, next) => {
try {
const params = new URLSearchParams({
[API_KEY_NAME]: API_KEY_VALUE,
...url.parse(req.url, true).query,
})
console.log('${API_BASE_URL}?${params}')
const apiRes = await needle('get', `${API_BASE_URL}?${params}`)
const data = apiRes.body
// Log the request to the public API
if (process.env.NODE_ENV !== 'production') {
console.log(`REQUEST: ${API_BASE_URL}?${params}`)
}
res.status(200).json(data)
} catch (error) {
next(error)
}
})
function sendForecastData(req, res) {
const { date, temp, content } = req.body;
let journal_entry_new = new Object();
journal_entry_new.date = date;
journal_entry_new.temp = temp + "°C";
journal_entry_new.content = content;
idx_entry = String("entry_" + idx)
idx = idx + 1
projectData[idx_entry] = JSON.stringify(journal_entry_new);
console.log(projectData)
res.send(projectData)
console.log("Post sucessful.")
}
router.post('/', cache('2 minutes'), async (req, res, next) => {
const postObject = needle.post('/addData', req.body, sendForecastData)
return postObject;
})
function readData(req, res) {
res.send(projectData)
console.log(projectData)
console.log("Read sucessful.")
}
router.get('/readData', readData)
module.exports = router
This is my app.js
//Event-Listener
document.getElementById('generate').addEventListener('click', performAction);
//Declare Fetch Function
//User Input
const the_date = document.getElementById('date');
const temp = document.getElementById('temp');
const content = document.getElementById('content');
// Create a new date instance dynamically with JS
let d = new Date();
let newDate = (d.getMonth() + 1) + "." + (d.getDate()) + '.' + (d.getFullYear());
//Proxy
const fetchWeather = async (zipcode) => {
const url = `/api?q=${zipcode}`
const res = await fetch(url)
const data = await res.json()
console.log(data)
return data;
}
const postData = async (url, data) => {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(data),
});
try {
const data = await response.json();
return data
}
catch (error) {
console.log("error", error);
}
}
const readData = async () => {
const request = await fetch('/readData');
try {
// Transform into JSON
const readData = await request.json()
return readData
} catch (error) {
console.log("error", error);
}
}
const UpdateUI = async (data) => {
console.log("Data received:")
console.log(data)
console.log("Date: " + data[Object.keys(data)[0]])
console.log("Temp: " + data[Object.keys(data)[1]])
console.log("Content: " + data[Object.keys(data)[2]])
document.getElementById('temp').innerText = "Temperature is: " + data[Object.keys(data)[1]]
document.getElementById('date').innerText = "Date is: " + data[Object.keys(data)[0]]
document.getElementById('content').innerText = "Feeling is: " + data[Object.keys(data)[2]]
console.log("Updated UI")
}
function performAction(e) {
//Check if user input is available
if (document.getElementById('zip').value == "") {
alert("Please type in a zipcode, then I will know where to look up the weather for
you!");
return
}
//Feeling now
let zipcode = (document.getElementById('zip').value).toString()
let feeling_now = (document.getElementById('feelings').value).toString()
fetchWeather(zipcode)
.then(data => {
/*let temp_k = parseFloat(data.list[0].main.temp)*/
/*let temp_c = String((temp_k - 273.15).toFixed(2)) + " °C"*/
let temp_c = parseFloat(data.main.temp) + " °C"
let feeling_now = (document.getElementById('feelings').value).toString()
console.log(temp_c)
console.log(newDate)
console.log(feeling_now)
console.log({ date: newDate, temp: temp_c, content: feelings.value })
return { date: newDate, temp: temp_c, content: feelings.value }
}).then(data => {
postData('/addData', data);
return data
})
.then(data => {
readData('/readData');
return data
})
.then(data => UpdateUI(data))
console.log(feeling_now)
return;
}
My UI is updated, however I get the following error and cannot access localhost:5000/addData - can you help why this is the case?
Defined the routes in the backend, however cannot call them from frontend
it's nice to join the group of people brave enough to ask the questions on Stack, so that everyone cantake advantage :)
My problem is pretty strange. I'm writing an app in Express, I have two routes so far and everything is going pretty smoothly, yet I've encountered one problem, which I can not seem to solve. In one route, with the patch method, the incoming requests have emmpty body. The rest of app is running smoothly, everything is working fine and this one route seems to be broken, I can not figure out why. Strange enough, yet I found out that the requests DO have body in one case - when I'm sending requests with my tests (supertest) using the .send({ ... }) method. When I'm sending requests with .attach or .field - they come empty. Same with requests sent from Postman (empty). What is causing such strange behavior?
Here are my tests:
const request = require('supertest');
const Image = require('../models/image');
const app = require('../app');
const crypto = require('crypto');
const fs = require('fs')
const { setupImages } = require('./fixtures/db')
beforeEach(setupImages);
describe('[IMAGE] - ', () => {
test('Should get images', async () => {
const main_img = await Image.findOne({ main: true });
const image = await request(app)
.get(`/image/${main_img._id}`)
.expect(200);
expect(image.header['content-type']).toBe('image/png');
});
test('Should delete images', async () => {
const image = await Image.findOne({ description: 'Lorem ipsum' });
await request(app)
.delete(`/image/${image._id}`);
const imageFound = await Image.findById(image._id);
expect(imageFound).toBeNull();
});
//TEST THAT FAILS
test('Should edit images', async () => {
const image = await Image.findOne({ main: false });
await request(app)
.patch(`/image/${image._id}`)
.field('description', 'new desc')
.attach('image', './src/tests/fixtures/imgtest.png')
.expect(200);
const returnChecksum = file => {
return crypto
.createHash('md5')
.update(file, 'utf8')
.digest('hex')
}
const imageEdited = await Image.findById(image._id);
const newImageChecksum = returnChecksum(fs.readFileSync(__dirname + '/fixtures/imgtest.png'));
expect(returnChecksum(imageEdited.image)).toBe(newImageChecksum);
expect(imageEdited.description).toBe('new desc');
});
})
Here are image routes
const express = require('express');
const router = new express.Router();
const Image = require('../models/image');
const chalk = require('chalk');
router.get('/image/:id', async (req, res) => {
const { id } = req.params;
try {
const image = await Image.findById(id);
if (!image) {
return res.status(404).send()
}
res.set('Content-Type', 'image/png');
res.send(image.image);
} catch (e) {
console.log(chalk.red('Error serving image: ') + e);
res.send(500);
}
});
//THE ROUTE THAT FAILS
router.patch('/image/:id', async (req, res) => {
const { id } = req.params;
const updateFields = Object.entries(req.body);
console.log('image patch req body', req.body)
try {
const imageEdited = await Image.findById(id, function (err, doc) {
if (err) { return err; }
updateFields.forEach(field => doc[field[0]] = field[1])
doc.save(res.status(200).send(doc));
});
if (!imageEdited) {
res.status(400).send();
}
} catch (e) {
res.status(500).send();
console.log(chalk.red('Error editing image: ') + e);
}
});
router.delete('/image/:id', async (req, res) => {
const { id } = req.params;
try {
await Image.findByIdAndDelete(id);
res.status(200).send();
} catch (e) {
res.status(500).send();
console.log(chalk.red('Error deleting image: ') + e);
}
});
module.exports = router;
And my app.js file:
const express = require('express');
require('./db/mongoose');
const productRouter = require('./routers/product');
const imageRouter = require('./routers/image');
const app = express();
app.use(express.json());
app.use(productRouter);
app.use(imageRouter);
module.exports = app;
The result of console.log in image route:
console.log src/routers/image.js:30
image patch req body {}
And this is the behavior of the app with changed sending method in test:
test('Should edit images', async () => {
const image = await Image.findOne({ main: false });
await request(app)
.patch(`/image/${image._id}`)
// .field('description', 'new desc')
// .attach('image', './src/tests/fixtures/imgtest.png')
.send({description: 'new desc'})
.expect(200);
const returnChecksum = file => {
return crypto
.createHash('md5')
.update(file, 'utf8')
.digest('hex')
}
const imageEdited = await Image.findById(image._id);
const newImageChecksum = returnChecksum(fs.readFileSync(__dirname + '/fixtures/imgtest.png'));
expect(returnChecksum(imageEdited.image)).toBe(newImageChecksum);
expect(imageEdited.description).toBe('new desc');
});
console.log src/routers/image.js:30
image patch req body { description: 'new desc' }
Thanks in advance!
I've managed to fix it myself, the problem lied in fact that I've forgot about multer. So this strange behavior can be result of not using said library, even if you're not sending any files apparently.
I am trying to have a flexible Cloud Function that executes on different end points.
My original Cloud Function looks like this:
const functions = require('firebase-functions')
const admin = require('firebase-admin')
const _ = require('lodash')
const { getObjectValues } = require('./helper-functions.js')
admin.initializeApp()
const json2csv = require('json2csv').parse
exports.csvJsonReport = functions.https.onRequest((request, response) => {
const db = admin.firestore()
const userAnswers = db.collection('/surveys/CNA/submissions')
return (
userAnswers
.get()
// eslint-disable-next-line promise/always-return
.then(querySnapshot => {
let surveySubmissions = []
querySnapshot.forEach(doc => {
const userSubmission = doc.data()
surveySubmissions.push({
..._.mapValues(userSubmission.answers, getObjectValues), // format answers
...userSubmission.anonUser,
})
})
const csv = json2csv(surveySubmissions)
response.setHeader('Content-disposition', 'attachment; filename=cna.csv')
response.set('Content-Type', 'text/csv')
response.status(200).send(csv)
})
.catch(error => {
console.log(error)
})
)
})
I am trying to extend this function to work on multiple collections. In the above function I am targeting the CNA collection. so instead of db.collection('/surveys/CNA/submissions/') I would like it to be db.collection('/surveys/:surveyId/submissions/')
Below is my attempt at trying to extend my original Cloud Function:
const functions = require('firebase-functions')
const admin = require('firebase-admin')
const express = require('express')
const bodyParser = require('body-parser')
const _ = require('lodash')
const { getObjectValues } = require('./helper-functions.js')
admin.initializeApp(functions.config().firebase)
const db = admin.firestore()
const app = express()
const main = express()
main.use('/api/v1', app)
main.use(bodyParser.json())
exports.webApi = functions.https.onRequest(main)
app.get('surveys/:id', (request, response) => {
const surveyId = request.query
const userAnswers = db.collection(`/survey/${surveyId}/submissions`)
return (
userAnswers
.get()
// eslint-disable-next-line promise/always-return
.then(querySnapshot => {
let surveySubmissions = []
querySnapshot.forEach(doc => {
const userSubmission = doc.data()
surveySubmissions.push({
..._.mapValues(userSubmission.answers, getObjectValues), // format answers
...userSubmission.anonUser,
})
})
const csv = json2csv(surveySubmissions)
response.setHeader('Content-disposition', 'attachment; filename=cna.csv')
response.set('Content-Type', 'text/csv')
response.status(200).send(csv)
})
.catch(error => {
console.log(error)
})
)
})
When I request my endpoint: myapp.firebaseapp.com/api/v1/surveys/CNA
Cannot GET /api/v1/surveys/CNA is shown in my browser.
Could someone please point me in the right direction?
To crate a GET /survey/:id endpoint in order to fetch a submission by id, use the following code in your new Cloud Function:
app.get('surveys/:id', (request, response) => {
const surveyId = request.params.id
const userAnswers = db.collection(`/survey/${surveyId}/submissions`)
Let me know if it works for you.
I have written one POST endpoint in expressJS with node.when I a make call to API It runs a utility with setInterval() and I want to send the API response after utility executes clearInterval().
How I can I wait and send response after utility execution is finished?
Please see the code below
REST API code:
const router= express.Router();
const multer= require('multer');
const {readCSVFile}= require('../util/index');
var storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, 'uploads');
},
filename: (req, file, cb) => {
cb(null, file.fieldname + '-' + Date.now()+'.xlsx');
}
});
var upload = multer({storage: storage});
router.post('/fileUpload', upload.single('filename'), async (req, res) => {
readCSVFile();
res.status(201).json({id:1});
});
router.get('/',(req,res)=>{
res.sendFile(__dirname+'/index.html');
});
module.exports=router;
Utilty Code
const config = require('config')
const excelToJson = require('convert-excel-to-json')
const HttpsProxyAgent = require('https-proxy-agent')
const AWS = require('aws-sdk')
const json2xls = require('json2xls')
const fs = require('fs')
const awsConfig = {
httpOptions: {
agent: new HttpsProxyAgent(
config.get('aws.proxy')
),
}
}
AWS.config.credentials = new AWS.SharedIniFileCredentials({
profile: config.get('aws.profile'),
})
AWS.config.update(awsConfig)
let uuidv4 = require('uuid/v4')
let csv = [];
const lexRunTime = new AWS.LexRuntime({
region: config.get('aws.region'),
})
let refreshId
const readCSVFile = () => {
const csvSheet = excelToJson({
sourceFile: './Test.xlsx',
})
csvSheet.Sheet1.forEach(element => {
csv.push((element.A.slice(0, element.A.length)))
})
runTask()
refreshId = setInterval(runTask, 1000)
}
let botParams = {
botAlias: config.get('bot.alias'),
botName: config.get('bot.name'),
sessionAttributes: {},
}
const missedUtterancesArray = []
const matchedUtterancesArray = []
let start = 0
let end = 50
let count = 50
const runTask = () => {
let itemsProcessed = 0
console.log('executing...')
const arrayChunks = csv.slice(start, end)
arrayChunks.forEach((element) => {
botParams.inputText = element
botParams.userId = `${uuidv4()}`
lexRunTime.postText(botParams, function (err, data) {
itemsProcessed++
if (err) console.log(err, err.stack)
else {
if (data.intentName === null) {
missedUtterancesArray.push({
Utterance: element,
})
}
else{
matchedUtterancesArray.push({
Utterance: element,
})
}
}
if (itemsProcessed === arrayChunks.length) {
start = csv.indexOf(csv[end])
end = start + count
}
if (start === -1) {
let xls = json2xls(missedUtterancesArray)
fs.writeFileSync('./MissedUtterances.xlsx', xls, 'binary')
let matchedXls = json2xls(matchedUtterancesArray)
fs.writeFileSync('./MatchedUtterances.xlsx', matchedXls, 'binary')
console.log('File saved successfully!! ')
console.log('Total Matched utterances count: ',csv.length-missedUtterancesArray.length)
console.log('Total Missed utterances count: ',missedUtterancesArray.length)
console.log('Total Utterances count: ',csv.length)
clearInterval(refreshId)
}
})
})
}
I would have needed few more information to answer this but pardon my try if this does not work -
the setInterval method in the readCSVFile the reason. Being an asynchronous function, this will not stop the code progression.
lexRunTime.postText also looks like asynchronous. I think you'd be better off with using promises while responding to the client.
Currently I have axios and cheerio return data from a webpage. I then setup express to setup a few views. I double checked my index.hbs and it include {{data}} inside the body. This should allow the page to render the text from the index render data: dealss . Am I missing anything ? The dealss obj holds 4 different objects that I can access.
getdeals(result => console.log(result.totaldeals[0].date))
This returns [ 09/04/2019/ ] in the console.
const path = require('path')
const express = require('express')
const hbs = require('hbs')
const axios = require('axios');
const cheerio = require('cheerio');
const app = express()
// Define paths for express config
const publicDirPath = path.join(__dirname, '../public')
const viewsPath = path.join(__dirname, '../templates/views')
const partialsPath = path.join(__dirname, '../templates/partials')
// Setup handlebars engine and views location
app.set('view engine', 'hbs')
app.set('views', viewsPath)
hbs.registerPartials(partialsPath)
// Setup static directory to serve
app.use(express.static(publicDirPath))
// Views
app.get('', (req, res) => {
res.render('index', {
title: 'ClearVision',
data: dealss,
name: 'Chris'
})
})
app.get('/about', (req, res) => {
res.render('about', {
title: 'ClearVision - About Us',
header: 'About Us',
name: 'Chris'
})
})
app.get('/help', (req, res) => {
res.render('help', {
title: 'ClearVision - Help',
helptext: 'Please contact x for help.',
name: 'Chris'
})
})
app.get('/weather', (req, res) => {
res.send({
forecast: 'It is sunny.',
location: 'x, Ca'
})
})
app.listen(1337, () => {
console.log('Server is currently running on port 1337.')
})
const url = 'https://abcdef.com/';
axios.defaults.withCredentials = true
// Get the deals
const getdeals = (callback) => {
axios(url, {
headers: {
Cookie: "x=xx;"
}
})
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
// Deals Page
const statsTable = $('tbody > tr');
const totaldeals = [];
// Loop Table for data in each row
statsTable.each(function () {
const nwline = "\n"
let date = $(this).find('td:nth-child(1)').text()
let bodydeals = $(this).find('td:nth-child(2)').text()
let newdeal = $(this).find('td:nth-child(3)').text()
let revdeal = $(this).find('td:nth-child(4)').text()
let monthlydealrev = $(this).find('td:nth-child(5)').text()
// Clear /n
if (date.includes(nwline)) {
date = date.toString().replace("\n", ""),
date = date.toString().replace("\n", "")
}
// Clear /n
if (bodydeals.includes(nwline)) {
bodydeals = bodydeals.toString().replace("\n", ""),
bodydeals = bodydeals.toString().replace("\n", ""),
bodydeals = bodydeals.toString().replace("\n", "")
}
// Clear /n
if (newdeal.includes(nwline)) {
newdeal = newdeal.toString().replace("\n", ""),
newdeal = newdeal.toString().replace("\n", ""),
newdeal = newdeal.toString().replace("\n", "")
}
// Clear /n
if (revdeal.includes(nwline)) {
revdeal = revdeal.toString().replace("\n", ""),
revdeal = revdeal.toString().replace("\n", ""),
revdeal = revdeal.toString().replace("\n", "")
}
// Clear /n (lookup jquery table functions)
if (monthlydealrev.includes(nwline)) {
monthlydealrev = monthlydealrev.toString().replace("\n", ""),
monthlydealrev = monthlydealrev.toString().replace("\n", ""),
monthlydealrev = monthlydealrev.toString().replace("\n", "")
}
totaldeals.push({
date,
bodydeals,
newdeal,
revdeal,
monthlydealrev
})
})
callback({
totaldeals
})
//console.log(totaldeals[1].date)
})
.catch(console.error);
}
function newFunction() {[getdeals(result => console.log(result.totaldeals))]}
I added a data: dealss under the res.render for the index. I also checked the index.hbs which has {{data}} in there. Shouldn't this just add the text to the screen?
Any ideas on how to print it to the view?
You just need to pass it as a variable to your hbs file:
app.get('', (req, res) => {
getdeals(result => {
res.render('index', {
title: 'ClearVision',
data: result, // or result.totaldeals depending
name: 'Chris' // on what you really mean
})
});
})
Improvements
If you rewrite getdeals() to return a Promise instead of accepting a callback you can use async/await:
const getdeals = () => {
// NOTE THIS CHANGE, return axios promise:
return axios(url, {
/* ... */
})
.then(response => {
/* .. */
statsTable.each(function () {
/* .. */
})
return totaldeals; // NOTE we return the result instead
// of calling a callback. This will
// return a resolved Promise
})
// Don't catch here, your request will hang if an error occurs
}
Now with the change above (return axios and return the result) we can rewrite the route as:
app.get('', async (req, res, next) => { // must have async keyword!
try {
let result = await getdeals();
res.render('index', {
title: 'ClearVision',
data: result, // or result.totaldeals depending
name: 'Chris' // on what you really mean
})
}
catch (err) {
console.log(err);
next(err); // this will close the request socket
}
})