I created a web scraping function with cheerio.js that I want to get called when the user opens the home page. I got it to work with one function, but I actually have 3 scraping functions that all do slightly different things. How do I add those functions in my route call to the home page along with the first function call? (basically i want all the data to be scraped upon the user opening the home page)
I'm also not very familiar with async/await or promises and got this code to work by following examples online, so if something is wrong there please let me know!
express route call that works (index.js):
const getSUPResults = require("./routes/SUPstores");
app.use(getSUPResults);
app.get("/", async function(req, res, next){
const result = await getSUPResults();
res.render("index", result);
});
scraper function ("./routes/SUPstores")
const express = require("express");
const cheerio = require("cheerio");
const { default: Axios } = require("axios");
const fetchData = async () => {
const result = await Axios.get("http://example.com");
return cheerio.load(result.data);
};
const getSUPResults = async () => {
const $ = await fetchData();
const SUPStoreInfo = [];
$('.row > tbody > tr:contains("SUP")').each(function(i, element){
const SUPResult = {
"environment" : $(this).children().text(),
"store" : $(this).children().next().text(),
"version" : $(this).children().next().next().text()
};
SUPStoreInfo.push(SUPResult);
});
return SUPStoreInfo;
}
module.exports = getSUPResults;
I tried to do something like this, but it only seemed to call the first function (getSUPResults), and ignored the rest. Is there something similar to this I could try?
const getSUPResults = require("./routes/SUPstores");
const getQAResults = require("./routes/QAstores");
const getDEVResults = require("./routesDEVstores");
app.use(getSUPResults);
app.use(getQAResults);
app.use(getDEVResults);
app.get("/", async function(req, res, next){
const SUP = await getSUPResults();
const QA = await getQAResults();
const DEV = await getDEVResults();
const result = [SUP, QA, DEV];
res.render("index", result);
});
Related
My self and another developer are working on an API using node.js and we are not advanced coders yet. We ran into problem. I will try and explain the problem here with sample references.
We have an API route that would be called on the frontend, please this is just a sample and not the real code. client said the codes should be private. We want to stop the function when an error is detected in another function we called. I am sure that we are not doing it rightly. Here is the API route for this post request and we called another function that we exported and imported here.
We simply want the httpCreateHarsh function to end if there is an error in the anotherFunction. With what we have, the error is seen in console.log when the user is not an admin for example but the httpCreateHarshfunction keeps running until it reaches the last line. Is this possible? Or is there another way we can structure the code to achieve this?
Shared sample of my code
const callAnotherFunction = require("../anotherfunction")
const httpCreateHarsh = async(req, res) => {
await callAnotherFunction(req, res);
return res.status(200).json('created')
}
//This is the second function we called:
const obj = {
status: 'success',
code: '244'
}
const anotherFunction = async(req, res) => {
if (req.body.user !== 'admin') {
return res.status(401).json('Unauthorized')
}
return obj
}
export default = anotherFunction
//The route here:
const express = require('express');
const router = express.Router();
const httpCreateHarsh = require('../httpCreateHarsh');
router.post("/harsh", httpCreateHarsh)
You couldn't return the res object in your second function.
To solve this problem you could throw exceptions and catch them in your handler function.
// request handler function
const httpCreateHarsh = async (req, res) => {
try {
await callAnotherFunction(req, res);
} catch (e) {
return res.status(401).json('Unauthorized')
}
return res.status(200).json('created')
}
const anotherFunction = asyn(req, res) => {
if (req.body.user !== 'admin') {
throw new Error('Unauthorized')
}
return obj
}
What you can do is you can wrap the code in httpCreateHarsh in an try...catch so whenever there is error inside it it will trigger the catch block and u exit the api.
const httpCreateHarsh = async(req, res)=>{
try{
await callAnotherFunction(req, res);
return res.status(200).json('created')
} catch(err){
return res.status(401).json('Unauthorized')
}
}
As an addition to this code you can return a promise from anotherFunction so that the catch block will be triggered once the promise is rejected.
For Exmaple:
const anotherFunction = async(req, res) => {
return new Promise(function(myResolve, myReject) {
if (req.body.user !== 'admin') {
myReject();
}
myResolve(obj);
});
}
If the code runs as you want it, it will generate the "cannot set headers after they are sent to the client" error, because you will be returning 2 responses
the first will be "unauthorized" by "anotherFunction" function and then the other response which is "created" of the current function "httpCreateHarsh".
what you should do instead is to call the "anotherFunction" as a middleware before moving to the "httpCreateHarsh" function.
it can be done this way:
// anotherfunction.js file containing the function you want to import
module.exports = {
async anotherFunction(req, res) {
if (req.body.user !== 'admin') {
return res.status(401).json('Unauthorized')
}
// this way, you can access this object from the "httpCreateHarsh" function by using req.body.obj
req.body.obj = {
status: 'success',
code: '244'
}
// this next indicates that there were no errors, and the next function will be called
next();
}
}
const httpCreateHarsh = async(req, res) => {
// do wathever you want here
return res.status(200).json('created')
}
//The route here:
const express = require('express');
const router = express.Router();
const httpCreateHarsh = require('../httpCreateHarsh');
const callAnotherFunction = require("../anotherfunction")
router.post("/harsh", (req, res, next) => callAnotherFunction(req, res, next), httpCreateHarsh)
I have the following get request where I call a bunch of data and pass it through to my EJS view.
router.get('/currentrentals', ensureAuthenticated, async (req, res) => {
const companies = await Company.getCompanies();
const barges = await Barge.getBarges();
const parishes = await Parish.getParishes();
const states = await State.getStates();
const pickupdropoff = await PickupDropoff.getPickupDropoff();
var notifications = await Notification.getNotifications();
JSON.stringify(barges);
JSON.stringify(companies);
JSON.stringify(parishes);
JSON.stringify(states);
JSON.stringify(pickupdropoff);
JSON.stringify(notifications);
var notifications = await notifications.sort((a, b) => b.id - a.id).slice(0,3);
res.render('currentrentals', {
name: req.user.name, companies: companies, barges: barges, parishes: parishes, states: states, pickupdropoff : pickupdropoff, notifications : notifications
});
}
);
Two questions:
I have multiple get requests that requires the same information. Is there a way to make this data available across the entirety of my site, so I don't have to rewrite this for each get path?
Is there a more succinct way to write the existing code I have? Perhaps looping through them or something of the sort? Simply for learning purposes.
The code currently works as-is.
Thanks!
If the data is constant, you can try this:
let data = null;
async function getData() {
if (!data) {
data = {
companies: await Company.getCompanies(),
barges: await Barge.getBarges();
parishes: await Parish.getParishes(),
states: await State.getStates(),
pickupdropoff: await PickupDropoff.getPickupDropoff(),
notifications: (await Notification.getNotifications()).sort((a, b) => b.id - a.id).slice(0,3)
};
}
return data;
}
router.get('/currentrentals', ensureAuthenticated, async (req, res) => {
res.render('currentrentals', { name: req.user.name, ...(await getData()) });
}
);
// other routes
I have the following files:
My routes - where the orders_count route lives:
routes/index.js
const express = require('express');
const router = express.Router();
const transactionsController = require('../controllers/transactionsController');
const ordersController = require('../controllers/ordersController');
const ordersCountController = require('../controllers/ordersCountController');
router.get('/transactions', transactionsController);
router.get('/orders', ordersController);
router.get('/orders_count', ordersCountController);
module.exports = router;
I then have my orders count controller living in the controllers directory:
controllers/ordersCountController.js
const ordersCountService = require('../services/ordersCountService');
const ordersCountController = (req, res) => {
ordersCountService((error, data) => {
if (error) {
return res.send({ error });
}
res.send({ data })
});
};
module.exports = ordersCountController;
My controller then calls my order count service which fetches data from another API.
services/ordersService.js
const fetch = require('node-fetch');
// connect to api and make initial call
const ordersCountService = (req, res) => {
const url = ...;
const settings = { method: 'Get'};
fetch(url, settings)
.then(res => {
if (res.ok) {
res.json().then((data) => {
return data;
});
} else {
throw 'Unable to retrieve data';
}
}).catch(error => {
console.log(error);
});
}
module.exports = ordersCountService;
I'm trying to return the JSON response. I initially had it setup with requests but looking at the NPM site, it appears that it's depreciated so have been digging through how to use node-fetch.
I have tried both 'return data' and res.send({data}), but neither are solving the problem.
I am still new to this so I am likely missing something very obvious, but how come I am not sending the JSON back through so that it displays at the /api/orders_count endpoint?
I keep thinking I messed something up in my controller but have been looking at it for so long and can't seem to figure it out.
Any help would be greatly appreciated and if there is anything I can add for clarity, please don't hesitate to ask.
Best.
please learn promises and await syntax. life will be easier.
never throw a string. always prefer a real error object, like that : throw new Error('xxx'); that way you will always get a stack. its way easier to debug.
avoid the callback hell : http://callbackhell.com/
you need to decide if you want to catch the error in the controller or in the service. no need to do in both.
in the controller you call the service that way :
ordersCountService((error, data) => {
but you declare it like that :
const ordersCountService = (req, res) => {
which is not compatible. it should look like this if you work with callback style :
const ordersCountService = (callback) => {
...
if (error) return callback(error)
...
callback(null, gooddata);
here is an example to flatten your ordersCountService function to await syntax, which allows the "return data" you were trying to do :
const fetch = require('node-fetch');
// connect to api and make initial call
const ordersCountService = async (req, res) => {
const url = ...;
const settings = { method: 'Get'};
try {
const res = await fetch(url, settings);
if (!res.ok) throw new Error('Unable to retrieve data');
return await res.json();
} catch(error) {
console.log(error);
}
}
module.exports = ordersCountService;
in fact i would prefer to error handle in the controller. then this woud be sufficient as a service
const fetch = require('node-fetch');
// connect to api and make initial call
const ordersCountService = async () => {
const url = ...;
const settings = { method: 'Get'};
const res = await fetch(url, settings);
if (!res.ok) throw new Error('Unable to retrieve data');
return await res.json();
}
module.exports = ordersCountService;
then you can call this funtion like this :
try {
const data = await ordersCountService(req, res);
} catch(err) {
console.log(err);
}
//or
ordersCountService(req, res).then((data) => console.log(data)).catch((err) => console.error(err));
so I am building a website and using nodejs to do a bunch of api calls and populate all the info. I have a homepage which has a sidebar where people can sort by 'categories'. but when I run my code instead of the categories being displayed in the html I get [object Object]. I have tried a whole bunch of things but still it only returns [object Object]. here's the code:
index.js
const express = require('express')
const router = new express.Router();
const categoriesFunction = require('../controllers/categories')
// get index page
router.get('', async (req, res) => {
let requestString = '/apps'
allApps = await openChannelRequest(requestString, req, res)
await res.render('index', {
categories: categoriesFunction,
// this is where I try to add the categories to the homepage
})
})
here's the controller where I'm grabbing all categories data and storing it. I'm pretty sure I can do this in the index.js page but a few weeks ago when I started this I made controllers for some reason. if that is not the best way to do this please let me know.
categories.js
const express = require('express')
const app = express()
var request = require('request');
var categoricalNames = []
var options = {
'method': 'GET',
'url': 'a working url',
'headers': {
'Authorization': 'working authorization'
},
'contentType': 'application/json'
};
var categoriesFunction = async() => request(options, function (error, response) {
return new Promise(function(resolve, reject){
console.log('inside categories function')
var categoryName = ''
if(error || !response)
{
// report error
reject(new Error('Try Again'));
}
else
{
//process response
var body = JSON.parse(response.body);
var jsonResponse = body.values
jsonResponse.forEach(function(obj) {
// console.log(obj.label)
categoryName = obj.label
// JSON.stringify(categoryName)
categoricalNames.push(categoryName)
});
categoricalNames.push({categoryName});
// console.log(categoricalNames)
// report success
JSON.stringify(categoricalNames)
resolve(categoricalNames);
}
})
});
module.exports.getPlaceDetails = categoriesFunction;
for awhile I thought my code wasn't working but console.logs in my categoriesFunction function reveal that the array is populating correctly. it's just not being pushed to the index correctly. and trying that method on the index.js page does not do anything for me. still just get [object Object]. not really sure why. Can anyone explain this to me?
Thanks!
I figured out what I needed to do. basically needed to write a cleaner Promise and make sure not to return the array but to resolve the array. and also there needed to be a variable in index.js that awaits the results of categoriesFunction()
here's what index.js looks like now
const express = require('express')
const router = new express.Router();
const categoriesFunction = require('../controllers/categories')
// get index page
router.get('', async (req, res) => {
let requestString = '/apps'
allApps = await openChannelRequest(requestString, req, res)
const thing = await categoriesFunction()
// this is important cause the method needs to be called before its inserted
await res.render('index', {
categories: thing,
// this is where I try to add the categories to the homepage
})
})
and here's what the categories.js file looks like
let categoriesFunction = function() {
return new Promise(resolve =>
request(options,
(error, response) => {
var categoryName = ''
var body = JSON.parse(response.body);
var jsonResponse = body.values
jsonResponse.forEach(function(obj) {
categoryName = obj.label
JSON.stringify(categoryName)
categoricalNames.push(categoryName)
});
console.log(categoricalNames)
resolve(categoricalNames)
// now that this is resolved it can be returned
}
))
}
I am building a simple application for my portfolio using The Movie Database api. In my GET /movie route, I want to get and display the data about the movie, and the names and photos of the cast members, however the data for the movie and the data for the cast members belong two separate endpoints of the api, and I am at a complete loss as to how to access both response data sets under a single endpoint in my app.
I cannot call axios.get() on both endpoints under the /movie route because I will get a "Headers already sent" error, and I have tried to write a function that uses axios.get for 1 endpoint that returns the response and gets called in my GET /movie route, but that causes the entire GET route to return undefined.
Here is my current code for my /movie route that is incorrect, but closely conveys what I am trying to accomplish
const express = require('express');
const router = express.Router();
const axios = require('axios');
const api_key = require('../config/keys').api_key;
const imgURL = "http://image.tmdb.org/t/p/";
const dateFormat = require('../config/dateFormat');
getActors = movie_id => {
axios.get(`https://api.themoviedb.org/3/movie/${movie_id}/credits?api_key=${api_key}&language=en-US`)
.then(res => {
return res.data;
}).catch(err => console.log(err.message));
}
router.get('/:id', (req, res) => {
const id = req.params.id
axios.get(`https://api.themoviedb.org/3/movie/${id}?api_key=${api_key}&language=en-US`)
.then(res => {
const movieData = res.data;
const actors = getActors(id); //calling above function here, but returns undefined and hangs the application
res.render('movie', {movieInfo: movieData, imgURL: imgURL, releaseDate: dateFormat(movieData.release_date), actors: actors})
})
.catch(err => console.log(err.status_message))
});
module.exports = router;
any help is greatly appreciated.
You can use axios.all to concatenate several promises and executing them in parallel. Then, once all the added requests have been finished, you can handle with the then promise the result of all of them. For instance, in your code:
const express = require('express');
const router = express.Router();
const axios = require('axios');
const api_key = require('../config/keys').api_key;
const imgURL = "http://image.tmdb.org/t/p/";
const dateFormat = require('../config/dateFormat');
axios.all([
axios.get(`https://api.themoviedb.org/3/movie/${movie_id}/credits?api_key=${api_key}&language=en-US`),
axios.get(`https://api.themoviedb.org/3/movie/${id}?api_key=${api_key}&language=en-US`)
])
.then(axios.spread((actorsRes, moviesRes) => {
// Your logic with each response
});
module.exports = router;
I see that getActors is currently returning null. Change it to...
const getActors = movie_id => {
return axios.get(`https://api.themoviedb.org/3/movie/${movie_id}/credits?api_key=${api_key}&language=en-US`)
.then(res => {
return res.data;
}).catch(err => console.log(err.message));
}
Another problem is calling of getActors function. It's a function that contains asynchronous function axios.get(). Change that to ..
router.get('/:id', (req, res) => {
let movieData;
const id = req.params.id
axios.get(`https://api.themoviedb.org/3/movie/${id}?api_key=${api_key}&language=en-US`)
.then(res => {
movieData = res.data;
return getActors(id);
})
.then(actors => {
res.render('movie', {movieInfo: movieData, imgURL: imgURL, releaseDate: dateFormat(movieData.release_date), actors: actors})
})
.catch(err => console.log(err.status_message))
});