Develop Reference

How to select specific button in puppeteer

So I'm building a program that scrapes Poshmark webpages and extracts the usernames of each seller on the page!
I want it to go through every page using the 'next' button, but theres 6 buttons all with the same class name...
Heres the link: https://poshmark.com/category/Men-Jackets_&_Coats?sort_by=like_count&all_size=true&my_size=false
(In my google chrome this page has an infinite scroll (hence the scrollToBottom async function i started writing) but i realized inside puppeteer's chrome it has 'next page' buttons.)
The window displays page 1-5 and then the 'next page' button.
The problem is that all of the buttons share the same html class name, so I'm confused on how to differentiate.
const e = require('express');
const puppeteer = require('puppeteer');
const url = "https://poshmark.com/category/Men-Jackets_&_Coats?sort_by=like_count&all_size=true&my_size=false";
let usernames = [];
const initItemArea = async (page) => {
const itemArea = await page.evaluate(() => {
return Array.from(document.querySelectorAll('.tc--g.m--l--1.ellipses')).map(x => x.textContent);
});
}
const pushToArray = async (itemArea, page) => {
itemArea.forEach(function (element) {
//console.log('username: ', $(element).text());
usernames.push(element);
});
};
const scrollToBottom = async (itemArea, page) => {
while (true) {
previousHeight = await page.evaluate('document.body.scrollHeight');
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
await page.waitForFunction(`document.body.scrollHeight > ${previousHeight}`);
await new Promise((resolve) => setTimeout(resolve, 1000));
await page.screenshot({path : "ss.png"})
}
};
const gotoNextPage = async (page) => {
await page.waitForSelector(".button.btn.btn--pagination");
const nextButton = await page.evaluate((page) => {
document.querySelector(".button.btn.btn--pagination")
});
await page.click(nextButton);
console.log('Next Page Loading')
};
async function main() {
const client = await puppeteer.launch({
headless: false,
executablePath: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
});
const page = await client.newPage();
await page.goto(url);
await page.waitForSelector(".tc--g.m--l--1.ellipses");
const itemArea = await page.evaluate(() => {
return Array.from(document.querySelectorAll('.tc--g.m--l--1.ellipses')).map(x => x.textContent);
});
gotoNextPage(page)
};
main();
Currently, my gotoNextPage function doesnt even find the button, so i thought i'd entered the selector wrong...
Then when I went to find the selector, I realized all buttons have the same one anyway...
My html knowledge is basically nonexistent, but I want to finish this project out. All help is very appreciated.
Bonus: my initPageArea function doesn't work when I call as a function like that, so I hardcoded it into main()...
I'll be diving deep into this problem later on, as I've seen it before, but any quick answers / direction would be awesome.
Thanks a lot.

you can try selecting the buttons using their position in the page.
For example, you can select the first button using the following CSS selector:
.button.btn.btn--pagination:nth-child(1)
to select the second button:
.button.btn.btn--pagination:nth-child(2)
Got the idea? :)
you can refactor your gotoNextPage function to use this approach, consider this example:
const gotoNextPage = async (page, buttonIndex) => {
await page.waitForSelector(".button.btn.btn--pagination");
// Select the button using its position in the page
const nextButton = await page.evaluate((buttonIndex) => {
return document.querySelector(`.button.btn.btn--pagination:nth-child(${buttonIndex})`);
}, buttonIndex);
// Click on the button
await page.click(nextButton);
console.log("Next Page Loading");
};

Whenever you're messing with buttons and scroll, it's a good idea to think about where the data is coming from. It's usually being delivered to the front-end via a JSON API, so you might as well try to hit that API directly rather than mess with the DOM.
const url = maxId => `https://poshmark.com/vm-rest/channel_groups/category/channels/category/collections/post?request={%22filters%22:{%22department%22:%22Men%22,%22category_v2%22:%22Jackets_%26_Coats%22,%22inventory_status%22:[%22available%22]},%22sort_by%22:%22like_count%22,%22facets%22:[%22color%22,%22brand%22,%22size%22],%22experience%22:%22all%22,%22sizeSystem%22:%22us%22,%22max_id%22:%22${maxId}%22,%22count%22:%2248%22}&summarize=true&pm_version=226.1.0`;
(async () => {
const usernames = [];
for (let maxId = 1; maxId < 5 /* for testing */; maxId++) {
const response = await fetch(url(maxId)); // Node 18 or install node-fetch
if (!response.ok) {
throw Error(response.statusText);
}
const payload = await response.json();
if (payload.error) {
break;
}
usernames.push(...payload.data.map(e => e.creator_username));
}
console.log(usernames.slice(0, 10));
console.log("usernames.length", usernames.length);
})()
.catch(err => console.error(err));
The response blob has a ton of additional data.
I would add a significant delay between requests if I were to use code like this to avoid rate limiting/blocking.
If you're set on Puppeteer, something like this should work as well, although it's slower and I didn't have time to run to the end of the 5k (or more?) users:
const puppeteer = require("puppeteer"); // ^19.1.0
const url = "Your URL";
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto(url, {waitUntil: "domcontentloaded"});
const usernames = [];
const sel = ".tc--g.m--l--1.ellipses";
for (;;) {
try {
await page.waitForSelector(sel);
const users = await page.$$eval(sel, els => {
const text = els.map(e => e.textContent);
els.forEach(el => el.remove());
return text;
});
console.log(users); // optional for debugging
usernames.push(...users);
await page.$$eval(
".btn--pagination",
els => els.find(el => el.textContent.includes("Next")).click()
);
}
catch (err) {
break;
}
}
console.log(usernames);
console.log(usernames.length);
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
I don't think navigations are triggered by the "Next" button, so my strategy for detecting when a page transition has occurred involves destroying the current set of elements after scraping the usernames, then waiting until the next batch shows up. This may seem inelegant, but it's easy to implement and seems reliable, not making assumptions about the usernames themselves.
It's also possible to use Puppeteer and make or intercept API requests, armed with a fresh cookie. This is sort of halfway between the two extremes shown above. For example:
const puppeteer = require("puppeteer");
const url = "Your URL";
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto(url, {waitUntil: "domcontentloaded"});
const usernames = await page.evaluate(async () => {
const url = maxId => `https://poshmark.com/vm-rest/channel_groups/category/channels/category/collections/post?request={%22filters%22:{%22department%22:%22Men%22,%22category_v2%22:%22Jackets_%26_Coats%22,%22inventory_status%22:[%22available%22]},%22sort_by%22:%22like_count%22,%22facets%22:[%22color%22,%22brand%22,%22size%22],%22experience%22:%22all%22,%22sizeSystem%22:%22us%22,%22max_id%22:%22${maxId}%22,%22count%22:%2248%22}&summarize=true&pm_version=226.1.0`;
const usernames = [];
try {
for (let maxId = 1; maxId < 5 /* for testing */; maxId++) {
const response = await fetch(url(maxId)); // node 18 or install node-fetch
if (!response.ok) {
throw Error(response.statusText);
break;
}
const json = await response.json();
if (json.error) {
break;
}
usernames.push(...json.data.map(e => e.creator_username));
}
}
catch (err) {
console.error(err);
}
return usernames;
});
console.log(usernames);
console.log("usernames.length", usernames.length);
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
The above code limits to 4 requests to keep it simple and easy to validate.
Blocking images and other unnecessary resources can help speed the Puppeteer versions up, left as an exercise (or just use the direct fetch version shown at top).

How to parse images posted by user in google reviews?

I am working on a scraping project to scrape google maps reviews, but I got struck when I have to parse images posted by users. I tried this method which only gives me the first image posted by user :
$('.gws-localreviews__google-review').each((i,el) => {
images[i] = $(el)
.find(".EDblX .JrO5Xe").attr("style")
})
I am scraping google reviews by this URL: https://www.google.com/async/reviewDialog?hl=en&async=feature_id:0x47e66e2964e34e2d:0x8ddca9ee380ef7e0,next_page_token:,sort_by:,start_index:,associated_topic:,_fmt:pc
Here is my response:
[
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipPgClEw3JwTLJOuf-DqC2xtZRodoavkpYVFBYqu=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipOs9TSNoyYmW1GL4SH9PlkAihvWsUbMTn-8O2Sj=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipMBRGdJb3zL1rME20osajG-bosdIV8U82VTYS1n=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipOBuGDXFDhJP69LNo6yI9cZWcjSVHpVfPBNoKyL=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipP7wBZt8Kilm8VF75T8amjMrZ7ZkOpmtb0nHChF=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipOixJabuSd4mSHnveU5JSQ1ZszHJ6Hn-pkeosiY=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipPyOXO1vnyTXVnlkPJNLlnoHYHEna36vYnrqwE=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipPReBboes7S7lNklRT21pwn096JUQVJbTX3VRRA=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipNPLvARJu1vDk03r_y4fp8f7aDDvzRX-7yJklW8=w100-h100-p-n-k-no)',
'background-image:url(https://lh5.googleusercontent.com/p/AF1QipMW3jp20hjKwuvhogH9ZC8IeH8QhQTUESH_ycNX=w100-h100-p-n-k-no)'
]
But what I want an object containing one user images, then a second a object containing a second user images and so on.
I want the results like this:
[
{
"All image's links posted by user 1"
},
{
"All images links posted by user 2"
}
{
"All images links posted by user 3"
},
{
"All images links posted by user 4"
},
{
........
}
]

So I found the answer, it was a bit tricky but yeah it will be solved like this :
$(".gws-localreviews__google-review").each((i, el) => {
images[i] = $(el)
.find(".EDblX .JrO5Xe")
.toArray()
.map($)
.map(d => d.attr("style").substring(21 , d.attr("style").lastIndexOf(")")))
});
})
It will return the response like this:
[
[
All images posted by user 1,
],
[
All images posted by user 2,
],
[.........
]

I don't see your full code, but as an alternative, to scrape all Google Maps Users Reviews images, you can use some browser automation, e.g. Puppeteer. In the code below I show you how you can do this (also check it on the online IDE):
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
puppeteer.use(StealthPlugin());
const placeUrl =
"https://www.google.com/maps/place/Starbucks/data=!4m7!3m6!1s0x549069a98254bd17:0xb2f64f75b3edf4c3!8m2!3d47.5319688!4d-122.1942498!16s%2Fg%2F1tdfmzpb!19sChIJF71UgqlpkFQRw_Tts3VP9rI?authuser=0&hl=en&rclk=1";
async function scrollPage(page, scrollContainer) {
let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
while (true) {
await page.evaluate(`document.querySelector("${scrollContainer}").scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
await page.waitForTimeout(2000);
let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
if (newHeight === lastHeight) {
break;
}
lastHeight = newHeight;
}
}
async function getReviewsFromPage(page) {
const reviews = await page.evaluate(() => {
return Array.from(document.querySelectorAll(".jftiEf")).map((el, i) => {
return {
[`All image's links posted by user ${i + 1}`]: Array.from(el.querySelectorAll(".KtCyie button")).length
? Array.from(el.querySelectorAll(".KtCyie button")).map((el) => getComputedStyle(el).backgroundImage.slice(5, -2))
: undefined,
};
});
});
return reviews;
}
async function getLocalPlaceReviews() {
const browser = await puppeteer.launch({
headless: false,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
});
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(60000);
await page.goto(placeUrl);
await page.click(".HHrUdb");
await page.waitForTimeout(2000);
await page.waitForSelector(".jftiEf");
await scrollPage(page, ".DxyBCb");
const rawReviews = await getReviewsFromPage(page);
const reviews = rawReviews.filter((el) => Object.keys(el).length);
await browser.close();
return reviews;
}
getLocalPlaceReviews().then(console.log);
Output
[
{
"All image's links posted by user 3":[
"https://lh5.googleusercontent.com/p/AF1QipNw_1HUpO-tbdPg0wS_hlMGnzhRFrDt8-CGE7yf=w600-h450-p-k-no"
]
},
{
"All image's links posted by user 5":[
"https://lh5.googleusercontent.com/p/AF1QipNIUP-aOWRElmfVOjnf5lJJYFiLKBaSx7MSkhg8=w300-h450-p-k-no",
"https://lh5.googleusercontent.com/p/AF1QipPcTFJIW9JAZxZ0PU0WC2U5rPnESv7OnrnSANwV=w300-h225-p-k-no",
"https://lh5.googleusercontent.com/p/AF1QipN_LkT7MCwx-oaf1yXkMnc_D-gm6HrWa7Kqoep8=w300-h225-p-k-no"
]
},
{
"All image's links posted by user 6":[
"https://lh5.googleusercontent.com/p/AF1QipPrI2xvgjFNh2vxFmBxRJBYvw553mORZdRZYwdZ=w300-h450-p-k-no",
"https://lh5.googleusercontent.com/p/AF1QipPVZ4YJqXjLvL-XTFBpB0oo4lVaBdrAGv2Ohyux=w300-h450-p-k-no"
]
},
{
"All image's links posted by user 9":[
"https://lh5.googleusercontent.com/p/AF1QipNTPaghKZbvv5aouQjzCq7no46UiiCa8IbsNmCZ=w600-h450-p-k-no"
]
}
...and other reviews
]
You can read more about scraping Google Maps Users Reviews from my blog post Web Scraping Google Maps Reviews with Nodejs.

When I try to web scrape a dynamic website, i get back an empty array

I am trying to web scrape a dynamic website with puppeteer, using this code:
const puppeteer = require('puppeteer');
async function getTokoPedia(){
const browser = await puppeteer.launch({ headless: false }); // for test disable the headlels mode,
const page = await browser.newPage();
await page.setViewport({ width: 1000, height: 926 });
await page.goto("https://store.401games.ca/collections/pokemon-singles",{waitUntil: 'networkidle2'});
console.log("start evaluate javascript")
var productNames = await page.evaluate(()=>{
var div = document.querySelectorAll('.info-container');
console.log(div) // console.log inside evaluate, will show on browser console not on node console
var productnames = []
div.forEach(element => {
var price = element.querySelector(' .fs-result-page-3sdl0h')
if(price != null){
productnames.push(price.innerText);
}
});
return productnames
})
console.log(productNames)
browser.close()
}
getTokoPedia();
However, upon running it, I get back an empty array. How can I fix this?

Two problems:
The elements you want are in a shadow root, so you have to pierce the root as described in Puppeteer not giving accurate HTML code for page with shadow roots.
The cards lazy-load, so you'd have to scroll down to be able to populate their data into the DOM.
But there's an easier way to get the initial set of data, which is in the static HTML as a JSON blob in var meta = {"products":...};. You can scrape it with a regex, as described in this tutorial.
Here's an example showing both approaches:
const puppeteer = require("puppeteer"); // ^14.1.1
let browser;
(async () => {
browser = await puppeteer.launch({headless: true});
const [page] = await browser.pages();
const url = "https://store.401games.ca/collections/pokemon-singles";
await page.goto(url, {waitUntil: "domcontentloaded"});
// here's the hard way for illustration:
const el = await page.waitForSelector("#fast-simon-serp-app");
await page.waitForFunction(({shadowRoot}) =>
shadowRoot.querySelector(".product-card .title")
, {}, el);
const items = await el.evaluate(({shadowRoot}) =>
[...shadowRoot.querySelectorAll(".product-card")]
.map(e => ({
title: e.querySelector(".title")?.textContent,
price: e.querySelector(".price")?.textContent,
}))
);
console.log(items); // just the first 6 or so
// TODO scroll the page to get the rest;
// I didn't bother implementing that...
// ...or do it the easy way:
const html = await page.content();
const pat = /^[\t ]*var meta = ({"products":[^\n]+);$/m;
const data = JSON.parse(html.match(pat)[1]);
console.log(JSON.stringify(data, null, 2));
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
At this point, since we're not dealing with anything but the static HTML, you can dump Puppeteer and use axios or fetch to get the data more efficiently:
const axios = require("axios");
axios.get("https://store.401games.ca/collections/pokemon-singles")
.then(({data: body}) => {
const pat = /^[\t ]*var meta = ({"products":[^\n]+);$/m;
const data = JSON.parse(body.match(pat)[1]);
console.log(JSON.stringify(data, null, 2));
})
.catch(err => console.error(err))
;
Now, the data.products array contains 50 but the UI shows 26466 results. If you want more than those initial items from the static HTML's var meta, which appears to be the same on all 1000+ pages, I suggest using the API. A URL looks like https://ultimate-dot-acp-magento.appspot.com/categories_navigation?request_source=v-next&src=v-next&UUID=d3cae9c0-9d9b-4fe3-ad81-873270df14b5&uuid=d3cae9c0-9d9b-4fe3-ad81-873270df14b5&store_id=17041809&cdn_cache_key=1654217982&api_type=json&category_id=269055623355&facets_required=1&products_per_page=5000&page_num=1&with_product_attributes=true. You can see there are ids and keys that probably protect against usage by parties other than the site, but I didn't see any change other than cdn_cache_key after a few tries. I'm not sure how long a URL is valid, but while it is, you can set products_per_page=1000 for example, then move page_num=1 forward 27 times or so. This gets you all of the data while avoiding all of the difficulties of scraping from the page itself.
Here's a pessimistic approach that uses Puppeteer to get an up-to-date URL, in case a URL goes stale:
const axios = require("axios");
const puppeteer = require("puppeteer"); // ^14.1.1
let browser;
(async () => {
browser = await puppeteer.launch({headless: true});
const [page] = await browser.pages();
const url = "https://store.401games.ca/collections/pokemon-singles";
const reqP = page.waitForRequest(res =>
res.url()
.startsWith("https://ultimate-dot-acp-magento.appspot.com/categories_navigation")
);
await page.goto(url, {waitUntil: "domcontentloaded"});
const req = await reqP;
const apiUrl = req
.url()
.replace(/(?<=products_per_page=)(\d+)/, 1000);
const {data} = await axios.get(apiUrl);
console.log(JSON.stringify(data, null, 2));
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
And tossing in the loop:
const axios = require("axios");
const fs = require("fs").promises;
const puppeteer = require("puppeteer"); // ^14.1.1
let browser;
(async () => {
browser = await puppeteer.launch({headless: true});
const [page] = await browser.pages();
const url = "https://store.401games.ca/collections/pokemon-singles";
const reqP = page.waitForRequest(res =>
res.url()
.startsWith("https://ultimate-dot-acp-magento.appspot.com/categories_navigation")
);
await page.goto(url, {waitUntil: "domcontentloaded"});
const req = await reqP;
const apiUrl = req
.url()
.replace(/(?<=products_per_page=)(\d+)/, 1000);
const items = [];
for (let i = 1;; i++) {
const pageUrl = apiUrl.replace(/(?<=page_num=)(\d+)/, i);
const response = await axios.get(pageUrl);
if (response.status !== 200 ||
items.length >= response.data.total_results) {
break;
}
items.push(...response.data.items);
}
await fs.writeFile("data.json", JSON.stringify(items));
console.log(items.slice(0, 10));
console.log(items.length);
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
This hammers the site, pulling a ton of data in a short amount of time, so consider this script for educational purposes, or modify it to throttle your requests way back.

How to get links from multiple pages in a single array

I have a working code that successfully obtains all product links from multiple pages that have at least a 20% discount. The only problem is that it returns links in the arrays for each page separately. However, I would like it to return links for all pages in a single array and then transfer them to another function. I tried to create a string var all_links = [] and push all the links from each page into it and then return them like return all_links, as I know from a simpler example. However, I have not been successful in this case because I have no experience with coding. I started learning the basics three weeks ago. I would be very grateful if you could help me with the whole code as I don't have the necessary prior knowledge.
const puppeteer = require('puppeteer')
const minDiscount = 20;
async function getLinks() {
const browser = await puppeteer.launch({
headless: false,
defaultViewport: null,
});
const page = await browser.newPage();
const url = 'https://www.mytoys.de/spielzeug-spiele/holz/';
await page.goto(url);
// getting all the products, this will return an array of ElementHandle
while(await page.$(".pager__link--next")){
await page.waitForSelector(".pager__link--next")
await page.waitForTimeout(1000);
await page.click('.pager__link--next')
await page.waitForTimeout(1500);
const products = await page.$$('.prod-grid.js-prod-grid .prod-grid__item.js-prod-grid_item');
const proms = await Promise.allSettled(
products.map(async (prod) => {
// searching for a discount on each product
const disc = await prod.$$eval(
'.prod-grid.js-prod-grid .prod-flag.prod-flag-sale',
(discount) =>
discount.map((discItem) =>
discItem.innerText.replace(/[^0-9.]/g, '').replace(/\D+/g,'0')
)
);
// if it has a discount
if (disc.length > 0) {
// we parse the discount to Integer type to compare it to minDiscount
const discountInt = parseInt(disc[0], 10);
if (discountInt >= minDiscount) {
// we get the link of the product
const link = await prod.$$eval('.prod-grid.js-prod-grid .prod-tile__link.js-prodlink', (allAs) => allAs.map((a) => a.href));
if (link.length > 0) {
// push an object containing the discount and the link of the product
return link[0];
}
}
}
return null;
})
);
const bulkArray = proms.map((item) => {
if (item.status === 'fulfilled') return item.value;
});
const endArray = bulkArray.filter(item => item !== null);
console.log(endArray);
}
}
getLinks();
An example of the result I am currently obtaining
[
'https://www.mytoys.de/erzi-kinderwurst-sortiment-spiellebensmittel-6749036.html',
'https://www.mytoys.de/chr-tanner-spiellebensmittel-wurststaender-1031946.html',
'https://www.mytoys.de/hape-xylophon-und-hammerspiel-2503719.html',
'https://www.mytoys.de/erzi-kinderparty-spiellebensmittel-6749035.html',
]
[
'https://www.mytoys.de/brio-holzeisenbahnset-landleben-5501952.html',
'https://www.mytoys.de/brio-brio-33277-bahn-ir-reisezug-set-4592516.html',
'https://www.mytoys.de/brio-parkhaus-strassen-schienen-3175226.html',
'https://www.mytoys.de/mytoys-steckwuerfel-12-tlg-11389814.html',
'https://www.mytoys.de/brio-schienen-und-weichensortiment-1758325.html',
]
[
'https://www.mytoys.de/hape-grosser-baukran-4141517.html',
'https://www.mytoys.de/noris-mein-buntes-tuermchenspiel-3421170.html',
'https://www.mytoys.de/goki-ziehtier-schaf-suse-2488933.html',
'https://www.mytoys.de/eichhorn-colorsoundzug-mit-licht-1521635.html',
]
An example of the result you would like to obtain
[
'https://www.mytoys.de/erzi-kinderwurst-sortiment-spiellebensmittel-6749036.html',
'https://www.mytoys.de/chr-tanner-spiellebensmittel-wurststaender-1031946.html',
'https://www.mytoys.de/hape-xylophon-und-hammerspiel-2503719.html',
'https://www.mytoys.de/erzi-kinderparty-spiellebensmittel-6749035.html',
'https://www.mytoys.de/brio-holzeisenbahnset-landleben-5501952.html',
'https://www.mytoys.de/brio-brio-33277-bahn-ir-reisezug-set-4592516.html',
'https://www.mytoys.de/brio-parkhaus-strassen-schienen-3175226.html',
'https://www.mytoys.de/mytoys-steckwuerfel-12-tlg-11389814.html',
'https://www.mytoys.de/brio-schienen-und-weichensortiment-1758325.html',
'https://www.mytoys.de/hape-grosser-baukran-4141517.html',
'https://www.mytoys.de/noris-mein-buntes-tuermchenspiel-3421170.html',
'https://www.mytoys.de/goki-ziehtier-schaf-suse-2488933.html',
'https://www.mytoys.de/eichhorn-colorsoundzug-mit-licht-1521635.html',
]

Declare new variable for links collecting before your loop:
const allLinks = []; // <--
while(await page.$(".pager__link--next")){ ... }
Push all links into it:
...
const endArray = bulkArray.filter(item => item !== null);
console.log(endArray);
allLinks.push(endArray); // <--
Return / log result after loop execution:
async function getLinks() {
...
return allLinks.flat(); // <--
}
console.log(await getLinks()) // result array
Refs: Array.prototype.flat()

Can't get url from list of images using pupetter

I'm trying to make a scraper using puppeteer using node and everything seems to work fine. I want to get an array of objects looking like this:
[{
title,
price,
link,
image,
}]
and the following code accomplish it, I got lucky and there was a data attribute with the image src on the page and was able to get it like this:
img: item.querySelector('.imagebox').dataset.imgsrc,.
Nevertheless I would like to know why this code fails when I want to get the src like this
image: item.querySelector('img').src,
here is the code I use and the url for the website I'm trying to scrape.
import puppeteer from 'puppeteer'
async function getHTML(url) {
const browser = await puppeteer.launch()
const page = await browser.newPage()
await page.goto(url)
const listItem = await page.evaluate(() =>
[...document.querySelectorAll('.aditem')].map(item => ({
title: item.querySelector('.text-module-begin').textContent.trim(),
price: item.querySelector('.aditem-details strong').textContent.trim(),
link: item.querySelector('.ellipsis').href,
img: item.querySelector('.imagebox').dataset.imgsrc,
image: item.querySelector('img').src,
}))
)
console.log(listItem)
await browser.close()
}
const searchArea = `s-kreuzberg`
const searchParam = `bike`
const url = `https://www.ebay-kleinanzeigen.de/${searchArea}/seite:1/${searchParam}/k0l3375r5`
async function go() {
await getHTML(url)
}
go()
thanks in advance for any help 👍🏽

Images of the page are lazy-loaded as soon as they are scrolled into view. So we need to scroll to them and to wait a bit.
Even then some images are not added to the DOM due to some reason, so we need to add a check for these cases.
You can try something like this:
import puppeteer from 'puppeteer'
async function getHTML(url) {
const browser = await puppeteer.launch()
const page = await browser.newPage()
await page.goto(url)
const listItem = await page.evaluate(async () => {
function delay(ms) {
return new Promise((resolve) => { setTimeout(resolve, ms) })
}
const items = [...document.querySelectorAll('.aditem')]
for (const item of items) {
item.scrollIntoView()
await delay(100)
}
return items.map(item => ({
title: item.querySelector('.text-module-begin').textContent.trim(),
price: item.querySelector('.aditem-details strong').textContent.trim(),
link: item.querySelector('.ellipsis').href,
img: item.querySelector('.imagebox').dataset.imgsrc,
image: item.querySelector('img')? item.querySelector('img').src : null,
}));
}
)
console.log(listItem)
await browser.close()
}
const searchArea = `s-kreuzberg`
const searchParam = `bike`
const url = `https://www.ebay-kleinanzeigen.de/${searchArea}/seite:1/${searchParam}/k0l3375r5`
async function go() {
await getHTML(url)
}
go()