Develop Reference

Puppeteer map push data and then scraping

How can I put the crawling result value in the cafecrawling array?
await page.waitForTimeout('5000')
console.log('crawlling start')
const cafecrawlling = await page.evaluate(() => {
const cafecrawlling = []
document.querySelector('#cafe_main')
const Post_name = Array.from(document.querySelector('#cafe_main').contentDocument.querySelectorAll('tr > td:first-of-type .board-list .inner_list .article')).map((v) => v.textContent.replace(/\s+/g, '')) // title
const Post_link = Array.from(document.querySelector('#cafe_main').contentDocument.querySelectorAll('tr > td:first-of-type .board-list .inner_list .article')).map((v)=> v.href) // link
const Post_nickname = Array.from(document.querySelector('#cafe_main').contentDocument.querySelectorAll('tr > td:nth-of-type(2) .p-nick .m-tcol-c')).map((v) => v.textContent) //nick name
const Post_date = Array.from(document.querySelector('#cafe_main').contentDocument.querySelectorAll('tr > td:nth-of-type(3)')).map((v)=> v.textContent) // date
return Post_name.map((v, i) => {
return {
postname: v,
postlink: Post_link[i],
nickname: Post_nickname[i],
postdate: Post_date[i],
}
})
});
after contents scraping code is
const articlePage = await browser.newPage()
for (let article of cafecrawlling) {
await articlePage.goto(article.href)
const content = await articlePage.$eval('#postContent', element => element.innerText)
article.content = content;
}

JS API client throws 'Uncaught (in promise) TypeError: e.join is not a function' after 40 requests

Here is my code (it takes a csv file as input, does some API calls, and outputs the results as another csv file):
let allRows = [];
async function fileToLines(file) {
return new Promise((resolve, reject) => {
reader = new FileReader();
reader.onload = function (e) {
parsedLines = e.target.result.split(/\r|\n|\r\n/);
resolve(parsedLines);
};
reader.readAsText(file);
});
}
document
.getElementById('fileInput')
.addEventListener('change', async function (e) {
var file = e.target.files[0];
if (file != undefined) {
fileToLines(file).then(async id => {
console.log(id)
console.log(parsedLines)
console.log(typeof id);
var idInt = id.map(Number);
var idFiltered = id.filter(function (v) { return v !== '' });
console.log(idFiltered)
if (file != undefined) {
allRows = [];
}
for (let id of idFiltered) {
const row = await getRelease(id);
allRows.push(row);
}
download();
});
}
});
function debounce(inner, ms = 0) {
let timer = null;
let resolves = [];
return function (...args) {
// Run the function after a certain amount of time
clearTimeout(timer);
timer = setTimeout(() => {
/* Get the result of the inner function, then apply it to the resolve function of
each promise that has been created since the last time the inner function was run */
let result = inner(...args);
resolves.forEach(r => r(result));
resolves = [];
}, ms);
return new Promise(r => resolves.push(r));
};
}
const throttleFetch = debounce(fetch, 2500);
function getRelease(idFiltered) {
return throttleFetch(`https://api.discogs.com/releases/${idFiltered}`, {
headers: {
'User-Agent': '***/0.1',
'Authorization': 'key=***, secret=***',
},
}).then(response => response.json())
.then(data => {
if (data.message === 'Release not found.') {
return { error: `Release with ID ${idFiltered} does not exist` };
} else {
const id = data.id;
const delimiter = document.getElementById("delimiter").value || "|";
const artists = data.artists ? data.artists.map(artist => artist.name) : [];
const barcode = data.identifiers.filter(id => id.type === 'Barcode')
.map(barcode => barcode.value);
var formattedBarcode = barcode.join(delimiter);
const country = data.country || 'Unknown';
const genres = data.genres || [];
const formattedGenres = genres.join(delimiter);
const labels = data.labels ? data.labels.map(label => label.name) : [];
const formattedLabels = labels.join(delimiter);
const catno = data.labels ? data.labels.map(catno => catno.catno) : [];
const formattedCatNo = catno.join(delimiter);
const styles = data.styles || [];
const formattedStyles = styles.join(delimiter);
const tracklist = data.tracklist ? data.tracklist
.map(track => track.title) : [];
const formattedTracklist = tracklist.join(delimiter);
const year = data.year || 'Unknown';
const format = data.formats ? data.formats.map(format => format.name) : [];
const qty = data.formats ? data.formats.map(format => format.qty) : [];
const descriptions = data.formats ? data.formats
.map(descriptions => descriptions.descriptions) : [];
const preformattedDescriptions = descriptions.toString()
.replace('"', '""').replace(/,/g, ', ');
const formattedDescriptions = '"' + preformattedDescriptions + '"';
console.log(idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
)
return [idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
];
}
});
}
function download() {
const ROW_NAMES = [
"release_id",
"artist",
"format",
"qty",
"format descriptions",
"label",
"catno",
"country",
"year",
"genres",
"styles",
"barcode",
"tracklist"
];
var csvContent = "data:text/csv;charset=utf-8,"
+ ROW_NAMES + "\n" + allRows.map(e => e.join(",")).join("\n");
var encodedUri = encodeURI(csvContent);
var link = document.createElement("a");
link.setAttribute("href", encodedUri);
link.setAttribute("download", "my_data.csv");
document.body.appendChild(link); // Required for Firefox
link.click();
}
The app passes tests for 5, 10, 20, or 30 lines of input, but when I try 40, I get this error:
Uncaught (in promise) TypeError: e.join is not a function
csvContent file:///D:/DOWNLOADS/CODE/***/csv.js:151
download file:///D:/DOWNLOADS/CODE/***/csv.js:151
<anonymous> file:///D:/DOWNLOADS/CODE///***/csv.js:38
async* file:///D:/DOWNLOADS/CODE/***/csv.js:20
I really don't know why that would be. Any ideas please? TIA.
Apparently I need to add more details but IDK what?

Result won't update VAR

I am trying to run a query, inside AXIOS which gets data from a 3rd party URL. Then uses some of that data to search our mongoDB database.
However it seems it won't update var total = 0
While the query below does function correctly, the return result won't allow me to set that it to the query.
Promise.all(arr.forEach( async (id,index) => {
//(CODE REMOVED JUST TO GET THIS FUNCTION TO WORK)
const search = await geoLocation.find({
'location': {
'$geoWithin': {
'$box': [
[-35.2418503, -13.5076852], [112.8656697, 129.0020486]
]
}
}}).toArray();
total = search.length;
}));
See the full code below
var array = [];
var pointarray = []
var total = 0;
areas.forEach((id,index) => {
if(id.type == "Point"){
pointarray[index] = "N"+id.id;
}else{
array[index] = "R"+id.id;
}
});
var arraySearch = "https://nominatim.openstreetmap.org/lookup?osm_ids="+array.toString()+"&polygon_geojson=1&bbox=1&format=json";
var pointSearch = "https://nominatim.openstreetmap.org/lookup?osm_ids="+pointarray.toString()+"&polygon_geojson=1&bbox=0&format=json"
const requestOne = axios.get(arraySearch);
const requestTwo = axios.get(pointSearch);
axios.all([requestOne, requestTwo])
.then(axios.spread((...responses) => {
const responseOne = responses[0]
const responseTwo = responses[1]
/*
process the responses and return in an array accordingly.
*/
return [
responseOne.data,
responseTwo.data,
];
}))
.then(arr => {
Promise.all(arr.forEach( async (id,index) => {
//const middleIndex = id[index].boundingbox.length / 2;
//const firstHalf = id[index].boundingbox.splice(0, middleIndex);
//const secondHalf = id[index].boundingbox.splice(-middleIndex);
//res.send(secondHalf[0]);
const query = [{
$match: {
location: {
$geoWithin: {$box:[[Number(firstHalf[0]),Number(firstHalf[1])],[Number(secondHalf[0]),Number(secondHalf[1])]]
}
}
}
},{
$count: 'id'
}]
const search = await geoLocation.find({
'location': {
'$geoWithin': {
'$box': [
[-35.2418503, -13.5076852], [112.8656697, 129.0020486]
]
}
}}).toArray();
total = search.length;
// total = search.length;
// const search = geoLocation.aggregate(query).toArray.length;
}));
})
.catch(errors => {
console.log("ERRORS", errors);
})
.then(function () {
res.send(total);
});

Console.log(appointment[0].apps.date) gives a date, but I get 'Cannot read property 'date' of undefined' when I use it in my function

I keep getting Cannot read property 'date' of undefined when I call my getDataPrevDay function, but console.log(appointment[0].apps.date) logs the correct date
const admin = ({data}) => {
const [appointment, setAppointment] = useState(data);
console.log(appointment[0].apps.date)
const getDataPrevDay = async () => {
let currentDate = dayjs(appointment[0].apps.date).format('YYYY-MM-DD')
let newDate = dayjs(currentDate).subtract(1, 'day').format('YYYY-MM-DD')
const res = await fetch(`http://localhost:3000/api/appointmentsdb?
date=${newDate}`)
let json = await res.json();
setAppointment(json);
}
....
return (
<button onClick={getDataPrevDay}> Prev Day </button>
)
}
export async function getServerSideProps(context){
const currentDate = dayjs().format('YYYY-MM-DD')
const res = await fetch(`http://localhost:3000/api/appointmentsdb?date=${currentDate}`)
let json = await res.json();
json = json.map(apps => ({apps}))
return {
props: {
data: json
},
};}
this is the handler
handler.get(async (req, res) => {
const {date, first_name, last_name, number, email, time} =
req.query;
const dataModel =
{'_id': new ObjectID(),
'first_name': first_name,
'last_name': last_name,
'number': number,
'email': email,
'date': date,
'time': time
};
let doc = {}
if(date) {
doc = await req.db.collection('Info').find({date: date},
{projection : {_id: 0}}).toArray()
} else {
doc = await req.db.collection('Info').findOne()
} if(doc == null) {
doc = dataModel;
}
res.json(doc);
});

The prop is rendered and updated dynamically. You can add a fail safe to avoid uniterable values by skipping if they are null.
const admin = ({data}) => {
const [appointment, setAppointment] = useState(data);
console.log(appointment[0].apps.date)
const getDataPrevDay = async () => {
if(appointment && appointment.length && appointment[0].apps && appointment[0].apps.date) {
let currentDate = dayjs(appointment[0].apps.date).format('YYYY-MM-DD')
let newDate = dayjs(currentDate).subtract(1, 'day').format('YYYY-MM-DD')
const res = await fetch(`http://localhost:3000/api/appointmentsdb?
date=${newDate}`)
let json = await res.json();
setAppointment(json);
}
}
....
return (
<button onClick={getDataPrevDay}> Prev Day </button>
)
}
export async function getServerSideProps(context){
const currentDate = dayjs().format('YYYY-MM-DD')
const res = await fetch(`http://localhost:3000/api/appointmentsdb?date=${currentDate}`)
let json = await res.json();
json = json.map(apps => ({apps}))
return {
props: {
data: json
},
};}

How to bypass jest setTimeout error of 5000ms by managing promises (Async and Await)

I wrote an Async/Await function to return promises for drivers report and analysis.
I have three different promise API files I extracted details from to do my analysis. However running test ith jest I get the error
Timeout - Async callback was not invoked within the 5000ms timeout specified by jest.setTimeout.Error:
I have refactored my code more than three times in two days but the error returns.
I will like to know how to manage my promises, perhaps there is something am not doing well and I am keen on this for optimization.
Is there a way to manage the promises in the code below to bypass the jest error?
any other suggestion will be highly appreciated.
NB: sorry I have post all the code for better insight.
code
const { getTrips } = require('api');
const { getDriver } = require('api')
const { getVehicle } = require('api')
/**
* This function should return the data for drivers in the specified format
*
* Question 4
*
* #returns {any} Driver report data
*/
async function driverReport() {
// Your code goes here
let trip = await getTrips()
trip = trip.map(item => {
item.billedAmount = parseFloat(item.billedAmount.toString().replace(',', '')).toFixed(2);
return item;
})
let getId = trip.reduce((user, cur) => {
user[cur.driverID] ? user[cur.driverID] = user[cur.driverID] + 1 : user[cur.driverID] = 1
return user
}, {})
// console.log(getId)
let mapId = Object.keys(getId)
// console.log(mapId)
let eachTripSummary = mapId.reduce((acc, cur) => {
let singleTrip = trip.filter(item => item.driverID == cur)
acc.push(singleTrip)
return acc
}, [])
// eachTripSummary = eachTripSummary[0]
// console.log(eachTripSummary)
// console.log(trip)
let reducedReport = eachTripSummary.reduce(async(acc, cur) =>{
acc = await acc
// console.log(acc)
let user = {}
let cash = cur.filter(item => item.isCash == true)
// console.log(cash.length)
let nonCash = cur.filter(item => item.isCash == false)
let driverSummary = await getDriverSummary(cur[0]['driverID'])
let trips = []
let customer = {}
cur[0].user ? (customer['user'] = cur[0]['user']['name'], customer['created'] = cur[0]['created'], customer['pickup'] = cur[0]['pickup']['address'],
customer['destination'] = cur[0]['destination']['address'], customer['billed'] = cur[0]['billedAmount'], customer['isCash'] = cur[0]['isCash']) : false
trips.push(customer)
let vehicles = []
if(driverSummary == undefined){
// console.log(cur)
user = {
id: cur[0]['driverID'],
vehicles: vehicles,
noOfCashTrips: cash.length,
noOfNonCashTrips: nonCash.length,
noOfTrips: cur.length,
trips: trips
}
acc.push(user)
// console.log(user)
return acc
}
let driverInfo = driverSummary[0]
let vehicleInfo = driverSummary[1]
let { name, phone } = driverInfo
let { plate, manufacturer } = vehicleInfo[0]
// console.log(plate)
let vpm = {
plate,
manufacturer
}
vehicles.push(vpm)
// console.log(cash.length)
user ={
fulName: name,
phone,
id: cur[0]['driverID'],
vehicles: vehicles,
noOfCashTrips: cash.length,
noOfNonCashTrips: nonCash.length,
noOfTrips: cur.length,
trips: trips
}
acc.push(user)
// console.log(acc)
return acc
}, [])
// reducedReport.then(data =>{console.log(data)})
return reducedReport
}
async function getDriverSummary(param) {
let driverDetails = await getDriver(param)
.then(data => {return data}).catch(err => {return err})
// console.log(driverDetails)
let vehicleDetails;
let { vehicleID } = driverDetails
if(driverDetails != "Error" & vehicleID != undefined){
// console.log(vehicleID)
vehicleDetails = vehicleID.map(async item => {
let vehicleSummary = getVehicle(item)
return vehicleSummary
})
// console.log(await vehicleDetails)
return await Promise.all([driverDetails, vehicleDetails])
}
}
driverReport().then(data => {
console.log(data)
})
module.exports = driverReport;

Use jest.setTimeout(30000); to increase the timeout. It will increase the timeout globally.
// jest.config.js
module.exports = {
setupTestFrameworkScriptFile: './jest.setup.js'
}
// jest.setup.js
jest.setTimeout(30000)
Or you can use user test example like this
describe("...", () => {
test(`...`, async () => {
...
}, 30000);
});