Develop Reference

Puppeteer scraper taking too long

I'm trying to scrape little pieces of data from a webpage, but it is taking soo long to scrape... any reason why this is happening ? Initially, it stops scraping, then i had to set the default timeout to 0. This same code was working perfectly fine earlier..
Now it's taking forever to scrape the data i need
code below
const puppeteer = require("puppeteer");
const express = require("express");
//const ejs = require("ejs");
const port = 5000;
const app = express();
app.set('view engine', 'ejs');
app.use(express.static(__dirname + '/public', {
types: {
'js': "text/javascript"
}
}));
var usdBuy;
var usdSell;
var gbpBuy;
var gbpSell;
var eurBuy;
var eurSell;
app.get('/', function(req, res) {
async function start() {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0);
await page.goto("URL");
const prices = await page.evaluate(() => {
return Array.from(document.querySelectorAll(".overlay-text")).map(x => x.textContent)
})
usdBuy = prices[0]
usdSell = prices[1]
gbpBuy = prices[2]
gbpSell = prices[3]
eurBuy = prices[4]
eurSell = prices[5]
console.log(usdBuy);
console.log(usdSell);
console.log(gbpBuy);
console.log(gbpSell);
console.log(eurBuy);
console.log(eurSell);
await page.close();
await browser.close();
}
start();
res.render("home", {
usdBuy: usdBuy,
usdSell: usdSell,
gbpBuy: gbpBuy,
gbpSell: gbpSell,
eurBuy: eurBuy,
eurSell: eurSell
});
})
app.listen(process.env.PORT || port, function () {
console.log(`Listening on port ${port}`)
})

MongoDb query return empty object

I'm trying to perform a simple .find() query on my mongodbAtlas, but the result of this query is an empty object.
this is my server file:
require("dotenv").config({ path: "./config.env" });
const { MongoClient, ServerApiVersion } = require("mongodb");
const express = require("express");
const { ServiceBroker } = require("moleculer");
const AUTH_SERVICE = require("./controller/services/auth/auth.service");
global.broker = new ServiceBroker({
nodeID: "auth",
});
const app = express();
app.use(express.json());
app.use("/auth", require("./routes/auth"));
const { PORT, URI } = process.env || 5000;
global.broker.createService(AUTH_SERVICE);
const start = async () => {
const dba = await MongoClient.connect(URI, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
global.db = dba.db("Auth");
try {
await dba.connect();
console.log("DATABASE CONNESSO CON SUCCESSO📡");
} catch (e) {
console.error(e);
}
await global.broker.start();
app.listen(PORT, () => console.log(`PORT IT'S UP AND RUNNING 🚀 ON ${PORT}`));
};
start();
this is my routes file:
const express = require("express");
const router = express.Router();
router.get("/register", async (req, res) => {
const data = global.db.collection("Users").find({}).toArray();
res.send(data);
});
module.exports = router;
this is how my document is populated:
{"_id":{"$oid":"6297bbc83a95b81d74882f65"},"username":"Test","email":"test#gmail.com","password":"1234"}

I think you are missing the "await" keyword after const data..... as API data fetching calls are asynchronous and required promise/ async-await to handle. Being async in nature, it moves forward to the next instruction and returns an empty array.
const express = require("express");
const router = express.Router();
router.get("/register", async (req, res) => {
const data = await global.db.collection("Users").find({}).toArray();
res.send(data);
});
module.exports = router;

Node.js Express show a Json file with Class Method

Just started with node.js and express and i was trying to show a json file containing an array of 3 objects with a class method
Here is the Class structure
const fs = require('fs')
class GrupoArchivo {
constructor(ruta) {
this.ruta = ruta
this.productos = []
}
_leer() {
return fs.promises.readFile(this.ruta, 'utf-8')
.then(texto => {
const productosComoArray = JSON.parse(texto)
this.productos = productosComoArray
})
}
async obtenerTodas() {
await this._leer()
return [...this.productos]
}
}
module.exports = GrupoArchivo
And here is the Js file with the express server
With the second "app.get" i was trying to get the method showing the json file with no luck
const express = require("express");
const GrupoArchivo = require('./GrupoArchivo.js')
const path = require("path");
const app = express();
app.get("/", (req, res)=>{
res.sendFile(path.join(__dirname + "/index.html"))
})
const grupo = new GrupoArchivo('./productos.json')
app.get("/productos", (req, res)=>
res.json(grupo.obtenerTodas()) //this returned me an {}
)
app.listen(8080, ()=>{
console.log("server listening on port", 8080)
})

All async functions return a promise, so when you call it, you have to use await or .then() to get the resolved value from that promise. You can add async and await like this:
app.get("/productos", async (req, res) => {
try {
res.json(await grupo.obtenerTodas());
} catch(e) {
console.log(e);
res.sendStatus(500);
}
});
Note, this also adds error handling so you can catch any errors from grupo.obtenerTodas().

Error sending json in puppeteer and express

When I try to pass a parameter through the URL in express to send a json with puppeteer, another browser opens after a while and does not send the json, returns something empty.
const express = require('express');
const path = require('path'); // NEW
const puppeteer = require('puppeteer');
const fs = require('fs');
const config = require('../src/config.js');
const app = express();
const port = process.env.PORT || 3006;
const DIST_DIR = path.join(__dirname, '../dist'); // NEW
const HTML_FILE = path.join(DIST_DIR, 'index.html'); // NEW
const mockResponse = {
foo: 'bar',
bar: 'foo'
};
app.use(express.static(DIST_DIR)); // NEW
app.get('/api', (req, res) => {
res.send(mockResponse);
});
app.get('/getapi', async (req, res) => {
function randomIntFromInterval(min, max) { // min and max included
return Math.floor(Math.random() * (max - min + 1) + min);
}
const next_class = 'snByac';
const browser = await puppeteer.launch({
headless: false,
dumpio: true
});
const page = await browser.newPage();
await page.goto('https://adwords.google.com/ko/KeywordPlanner/Home?', {waitUntil: 'networkidle2'});
await page.waitFor(randomIntFromInterval(20000,25000));
await page.type('input[name=identifier]', config.mail);
await page.click('span.' + next_class);
await page.waitFor(randomIntFromInterval(20000,23000));
await page.type('input[name=password]', config.password)
await page.click('span.' + next_class)
await page.waitFor(randomIntFromInterval(39000,49000));
await page.click('[icon="arrow_forward"]')
await page.waitFor(randomIntFromInterval(3800,6000));
await page.type('[aria-autocomplete="list"]', req.query.palabra)
await page.waitFor(randomIntFromInterval(1400,2400));
await page.keyboard.press('Enter');
await page.waitFor(randomIntFromInterval(10000,14000));
await page.keyboard.press('Enter');
await page.waitFor(randomIntFromInterval(10000,14000));
res.send(mockResponse);
await browser.close();
});
app.get('/', (req, res) => {
res.sendFile(HTML_FILE); // EDIT
});
app.listen(port, function () {
console.log('App listening on port: ' + port);
});
I enclose the complete code, I don't get any console errors.
localhost has not sent any data.
ERR_EMPTY_RESPONSE
This, i get
Update: I solved it by placing within the route, the following: req.setTimeout (500000);

not able to hit the router in the browser when I hit the url

I have a working codebase where they have already setup node proxy and jwt.
my server is running at 1010 port
but when I hit this url http://localhost:1010/sampletest
I am getting an error in the browser screen Cannot GET /sampletest
even I hit this url http://localhost:1010/jump/api/v1 but still I am getting same error
can you tell me how to fix it or am I missing anthing in the configurations
providing my index.js and server.js code below
sports.js
const express = require('express');
const axios = require('axios');
const mime = require('mime-types');
const router = express.Router();
const ResponseUtil = require('../../utils/ResponseUtil');
const AppConstants = require('../../../constants/AppConstants');
const credentials = require('../../../internals/credentials.json');
const memberGroupingHelper = require('../../helpers/rank/memberGrouping');
const exportHelper = require('../../helpers/rank/rankExportHelper');
const formatExportData = require('../../helpers/rank/formatExportData');
const rankCommonHelper = require('../../helpers/rank/rankCommonHelper');
const rankProvDataHelper = require('../../helpers/group/getProvData');
//const aggregateHelper = require('../../helpers/group/aggregateFilter');
const { rankAggregatelastrsApi } = require('jump-svc-utils');
//router.get('/:searchMode/:lastrSearch', (req, res, next) => {
router.get('/sampletest', (req, res, next) => {
const { originalUrl } = req;
//console.log(" originalUrl ", originalUrl);
const mode = req.params.searchMode;
const value = encodeURIComponent(req.params.lastrSearch);
console.log("document 40--->", mode);
console.log("for document Testing0--->", mode);
const url = `/jkjkjk/sdjksdjkjksdjksd/sdklsdlksdklsdkl`;
axios.get(AppConstants.GET_JWT_TOKEN_URL, {
auth: {
username: credentials.auth.racfId, password: credentials.auth.password
}
})
.then((jwtResponse) => {
// var jwtToken = `Bearer ${jwtResponse.data.jwt}`;
var jwtToken = `Bearer 787878bjhbnmnmmwqdqwqwqwqwqwqwqwqwqwqwqwqwqwqwqwqwqwqw`;
axios.get(url, { headers: { "Authorization": jwtToken } })
.then((response) => {
console.log("document then0--->", response);
const file = Buffer.from(response.data.content, 'base64');
const fileType = mime.contentType(response.data.contentInfo.fileType);
const fileExtension = response.data.contentInfo.fileType.toLowerCase();
const fileName = `filename=${response.data.contentInfo.id}.${fileExtension}`;
res.set('Content-Type', fileType);
res.set('Content-disposition', `attachment; ${fileName}`);
res.send(file);
})
.catch((e) => {
console.log("e catch document0--->", e);
console.log("e.message catch document0--->", e.message);
console.log("catch document--->", e.response);
if (e.response) {
return res.status(e.response.status).send(e.response.data);
}
res.status(500).send(e.message || 'Something wrong');
});
});
ResponseUtil.callService(res, url);
});
module.exports = router;
index.js
const express = require('express')
const app = express()
const port = 1010
const jumpServices = require('./services/jump');
const compression = require('compression');
var BodyParser = require('body-parser');
const glob = require('glob');
const path = require('path');
app.use('/jump/api/v1', jumpServices);
app.use(BodyParser.json());
app.use(BodyParser.urlencoded({
extended: true
}));
//app.use(compress())
// app.use(compression());
// include all the controllers
const controllers = glob.sync(path.join(__dirname, '/controllers/**/*.js'));
console.log("controllers--->", controllers);
controllers.forEach((controllerFileName) => {
require(controllerFileName)(app); //eslint-disable-line
});
app.listen(port, (err) => {
if (err) {
return console.log('something bad happened', err)
}
console.log(`server is listening on ${port}`)
})

I don't see any calls registering your sub Routers with the top level express application. You should need to call app.use for each of your sub routers.