Develop Reference

Puppeteer scraper taking too long

I'm trying to scrape little pieces of data from a webpage, but it is taking soo long to scrape... any reason why this is happening ? Initially, it stops scraping, then i had to set the default timeout to 0. This same code was working perfectly fine earlier..
Now it's taking forever to scrape the data i need
code below
const puppeteer = require("puppeteer");
const express = require("express");
//const ejs = require("ejs");
const port = 5000;
const app = express();
app.set('view engine', 'ejs');
app.use(express.static(__dirname + '/public', {
types: {
'js': "text/javascript"
}
}));
var usdBuy;
var usdSell;
var gbpBuy;
var gbpSell;
var eurBuy;
var eurSell;
app.get('/', function(req, res) {
async function start() {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0);
await page.goto("URL");
const prices = await page.evaluate(() => {
return Array.from(document.querySelectorAll(".overlay-text")).map(x => x.textContent)
})
usdBuy = prices[0]
usdSell = prices[1]
gbpBuy = prices[2]
gbpSell = prices[3]
eurBuy = prices[4]
eurSell = prices[5]
console.log(usdBuy);
console.log(usdSell);
console.log(gbpBuy);
console.log(gbpSell);
console.log(eurBuy);
console.log(eurSell);
await page.close();
await browser.close();
}
start();
res.render("home", {
usdBuy: usdBuy,
usdSell: usdSell,
gbpBuy: gbpBuy,
gbpSell: gbpSell,
eurBuy: eurBuy,
eurSell: eurSell
});
})
app.listen(process.env.PORT || port, function () {
console.log(`Listening on port ${port}`)
})

how to send multiple script at once

I am using puppeteer and express js to make an api. I have done my code stuff and now i am testing the api on postman but i have a problem which is i could not send multiple scripts at once i mean i can send but just returning the last one.So what is the problem.
const express = require("express");
const app = express();
app.use(express.json())
const puppeteer = require("puppeteer");
const port = 3000;
let browser = null;
var target_url = ""
app.post('/get/script_result', async (req, res) => {
if (browser == null) browser = await puppeteer.launch({
headless: false, defaultViewport: null, args: ['--disable-infobars', '--disable-notifications', '--disable-default-apps',
'--no-sandbox', '--mute-audio', '--ignore-certificate-errors', '--disable-features=LookalikeUrlNavigationSuggestionsUI'], ignoreHTTPSErrors: true
});
try {
var page = await browser.newPage()
await page.goto(req.body.target_url);
var script = ""
var script_delay = ""
await page.waitForTimeout(`${req.body.script_delay}`)
const handle = await page.evaluate(`${req.body.script}`)
res.send(handle)
res.end()
//await page.close();
return res.status(200)
}
catch (e) {
console.log(e);
res.send(e)
res.status(500)
}
})
app.listen(port, () => {
console.log(`app is running on port: ${port}`);
});

MongoDb query return empty object

I'm trying to perform a simple .find() query on my mongodbAtlas, but the result of this query is an empty object.
this is my server file:
require("dotenv").config({ path: "./config.env" });
const { MongoClient, ServerApiVersion } = require("mongodb");
const express = require("express");
const { ServiceBroker } = require("moleculer");
const AUTH_SERVICE = require("./controller/services/auth/auth.service");
global.broker = new ServiceBroker({
nodeID: "auth",
});
const app = express();
app.use(express.json());
app.use("/auth", require("./routes/auth"));
const { PORT, URI } = process.env || 5000;
global.broker.createService(AUTH_SERVICE);
const start = async () => {
const dba = await MongoClient.connect(URI, {
useNewUrlParser: true,
useUnifiedTopology: true,
});
global.db = dba.db("Auth");
try {
await dba.connect();
console.log("DATABASE CONNESSO CON SUCCESSO📡");
} catch (e) {
console.error(e);
}
await global.broker.start();
app.listen(PORT, () => console.log(`PORT IT'S UP AND RUNNING 🚀 ON ${PORT}`));
};
start();
this is my routes file:
const express = require("express");
const router = express.Router();
router.get("/register", async (req, res) => {
const data = global.db.collection("Users").find({}).toArray();
res.send(data);
});
module.exports = router;
this is how my document is populated:
{"_id":{"$oid":"6297bbc83a95b81d74882f65"},"username":"Test","email":"test#gmail.com","password":"1234"}

I think you are missing the "await" keyword after const data..... as API data fetching calls are asynchronous and required promise/ async-await to handle. Being async in nature, it moves forward to the next instruction and returns an empty array.
const express = require("express");
const router = express.Router();
router.get("/register", async (req, res) => {
const data = await global.db.collection("Users").find({}).toArray();
res.send(data);
});
module.exports = router;

cannot run async function inside express, how to fix?

Async function not run inside express view, console log, before and after function, separately code working, but with express it doesn't.
const path = require('path')
const fs = require('fs')
const puppeteer = require('puppeteer');
const express = require('express');
const bodyParser = require('body-parser');
const app = express()
const port = 9596;
app.use(bodyParser.json());
app.post('/', async function(req, res){
siteName = req.body.siteName;
links = req.body.links;
screenshotPageSource = req.body.screenshotPageSource;
await createScreenshots(siteName, links, screenshotPageSource);
res.end('Hello World!');
});
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
})
async function createScreenshots(siteName, links, screenshotPageSource){
const browser = await puppeteer.launch({defaultViewport: { width: 1366, height: 768 }});
const page = await browser.newPage();
linksLength = links.Length;
for (var i = 0; i < linksLength; i++) {
link = links[i]
if(screenshotPageSource){
currentLink = `view-source:${link}`;
} else {
currentLink = link;
}
pageNumber = i + 1;
console.log(pageNumber);
await page.goto(currentLink, {waitUntil: 'load', timeout: 60000}).then(() => {
console.log(`Страница ${pageNumber}/${linksLength}, удалось сделать скриншот.`)
}).catch((res) => {
console.log(`Страница ${pageNumber}/${linksLength}, не удалось сделать скриншот.`)
});
await page.screenshot({ path: `${siteName}__${pageNumber}.png` });
}
await browser.close();
return links;
};
It looks like your post is mostly code; please add some more details.It looks like your post is mostly code; please add some more details.It looks like your post is mostly code; please add some more details.

The problem I guess is in app.post.
Below is have added a code for your reference which might help you to edit your code.
app.post('/', async(req, res, next) => {
const { error } = validateBody(req.body);
if (error) {
return res.status(400).send(error.details[0].message);
}
const newData= new createScreenshots({
siteName = req.body.siteName;
links = req.body.links;
screenshotPageSource = req.body.screenshotPageSource; });
await newData.save();
console.log('saving the document');
res.send(newData);
})
The above code is just for your reference as to how you can write app.post()

not able to hit the router in the browser when I hit the url

I have a working codebase where they have already setup node proxy and jwt.
my server is running at 1010 port
but when I hit this url http://localhost:1010/sampletest
I am getting an error in the browser screen Cannot GET /sampletest
even I hit this url http://localhost:1010/jump/api/v1 but still I am getting same error
can you tell me how to fix it or am I missing anthing in the configurations
providing my index.js and server.js code below
sports.js
const express = require('express');
const axios = require('axios');
const mime = require('mime-types');
const router = express.Router();
const ResponseUtil = require('../../utils/ResponseUtil');
const AppConstants = require('../../../constants/AppConstants');
const credentials = require('../../../internals/credentials.json');
const memberGroupingHelper = require('../../helpers/rank/memberGrouping');
const exportHelper = require('../../helpers/rank/rankExportHelper');
const formatExportData = require('../../helpers/rank/formatExportData');
const rankCommonHelper = require('../../helpers/rank/rankCommonHelper');
const rankProvDataHelper = require('../../helpers/group/getProvData');
//const aggregateHelper = require('../../helpers/group/aggregateFilter');
const { rankAggregatelastrsApi } = require('jump-svc-utils');
//router.get('/:searchMode/:lastrSearch', (req, res, next) => {
router.get('/sampletest', (req, res, next) => {
const { originalUrl } = req;
//console.log(" originalUrl ", originalUrl);
const mode = req.params.searchMode;
const value = encodeURIComponent(req.params.lastrSearch);
console.log("document 40--->", mode);
console.log("for document Testing0--->", mode);
const url = `/jkjkjk/sdjksdjkjksdjksd/sdklsdlksdklsdkl`;
axios.get(AppConstants.GET_JWT_TOKEN_URL, {
auth: {
username: credentials.auth.racfId, password: credentials.auth.password
}
})
.then((jwtResponse) => {
// var jwtToken = `Bearer ${jwtResponse.data.jwt}`;
var jwtToken = `Bearer 787878bjhbnmnmmwqdqwqwqwqwqwqwqwqwqwqwqwqwqwqwqwqwqwqw`;
axios.get(url, { headers: { "Authorization": jwtToken } })
.then((response) => {
console.log("document then0--->", response);
const file = Buffer.from(response.data.content, 'base64');
const fileType = mime.contentType(response.data.contentInfo.fileType);
const fileExtension = response.data.contentInfo.fileType.toLowerCase();
const fileName = `filename=${response.data.contentInfo.id}.${fileExtension}`;
res.set('Content-Type', fileType);
res.set('Content-disposition', `attachment; ${fileName}`);
res.send(file);
})
.catch((e) => {
console.log("e catch document0--->", e);
console.log("e.message catch document0--->", e.message);
console.log("catch document--->", e.response);
if (e.response) {
return res.status(e.response.status).send(e.response.data);
}
res.status(500).send(e.message || 'Something wrong');
});
});
ResponseUtil.callService(res, url);
});
module.exports = router;
index.js
const express = require('express')
const app = express()
const port = 1010
const jumpServices = require('./services/jump');
const compression = require('compression');
var BodyParser = require('body-parser');
const glob = require('glob');
const path = require('path');
app.use('/jump/api/v1', jumpServices);
app.use(BodyParser.json());
app.use(BodyParser.urlencoded({
extended: true
}));
//app.use(compress())
// app.use(compression());
// include all the controllers
const controllers = glob.sync(path.join(__dirname, '/controllers/**/*.js'));
console.log("controllers--->", controllers);
controllers.forEach((controllerFileName) => {
require(controllerFileName)(app); //eslint-disable-line
});
app.listen(port, (err) => {
if (err) {
return console.log('something bad happened', err)
}
console.log(`server is listening on ${port}`)
})

I don't see any calls registering your sub Routers with the top level express application. You should need to call app.use for each of your sub routers.