Develop Reference

What is wrong is the puppeteer script that stops it saving the scraped data to a file?

What is wrong here? I need to be able to scrape the data CSS selectors from the URLs in the const URLs array, and then save that information back into a file in an array. I've been playing with getting this working for ages, but I think I don't understand something correctly and thus am missing something, any pointers please?
const puppeteer = require('puppeteer');
const colors = require('colors/safe');
const fs = require('fs');
const bluebird = require("bluebird");
const https = require('https');
const querystring = require('querystring');
(async() => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
// Now you can go wherever you want
const urls = ['/news/england/', '/news/wales/']
const results = await bluebird.map(urls, async(url) => {
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0);
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0')
await page.goto('https://www.bbc.com' + url);
await page.setViewport({
width: 1920,
height: 1080,
deviceScaleFactor: 1,
});
await sleep(60000);
await page.waitForSelector('#topos-component');
const [a, b] = await page.evaluate(() => {
const a = document.querySelector("a.gel-paragon-bold");
const aText = a.innerText;
const b = document.querySelectorAll(".gs-o-responsive-image--lead > img:nth-child(1)");
const bText = b.href;
return [aText, bText];
}, {
concurrency: 3
})
});
fs.writeFileSync('test.html', JSON.stringify(results));
// Then when you're done, just close
await browser.close();
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
})();

I have solved it and got it working how I need it
const puppeteer = require('puppeteer');
const colors = require('colors/safe');
const fs = require('fs');
const bluebird = require("bluebird");
const https = require('https');
const querystring = require('querystring');
(async () => {
const args = ['--proxy-server=socks5://127.0.0.1:9050'];
const browser = await puppeteer.launch({
args
});
const page = await browser.newPage();
await page.goto('https://check.torproject.org/');
const isUsingTor = await page.$eval('body', el =>
el.innerHTML.includes('Congratulations. This browser is configured to use Tor')
);
if (!isUsingTor) {
console.log(colors.red.bold('Not using Tor. Closing...'));
return await browser.close();
}
console.log(colors.green.bold('Using Tor. Continuing... '));
// Now you can go wherever you want
const urls = ['/xxxx123','/xxxxxx345','/xxxxxx567']
const results = await bluebird.map(urls, async (url) => {
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0);
await page.setUserAgent('xxxxxxxxxxxxxxxxxx')
await page.goto('https://aaaaaaaa.com'+url);
await page.setViewport({
width: 1920,
height: 1080,
deviceScaleFactor: 1,
});
await sleep(60000);
await page.waitForSelector('.xxxx', { timeout: 30000 });
const xx1 = await page.evaluate(() => document.querySelector('.xx1').innerText);
const xx2= await page.evaluate(() => document.querySelector('.xx2').innerText);
const xx3 = await page.evaluate(() => document.querySelector('xx3').innerText);
const publish_date = await page.evaluate(() => document.querySelector('.post-banner-p').innerText);
const xx4= await page.evaluate(() => document.querySelector('.xx4').innerText);
const xx5= await page.evaluate(() => document.querySelector('.xx5').innerText);
const xx6 = await page.evaluate(() => document.querySelector('xx6)').innerText);
const xx7 = await page.evaluate(() => document.querySelector('xx7').innerText);
const xx8 = await page.evaluate(() => document.querySelector('.xx8').innerText);
const xx9 = await page.evaluate(() => document.querySelector('.xx9').innerText);
return [xx1,xx2,xx3,xx4,xx5,xx6,xx7,xx8,xx9];
}, {
concurrency: 3
});
fs.writeFileSync('xxxxxxx.html', JSON.stringify(results));
// Then when you're done, just close
await browser.close();
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
})();

Loop through pages to return all the product links

The goal is to obtain all product links from all pages in the pagination. So far I have managed to print information to the console with console.log (links). However, since I am completely new to this field and completely inexperienced, I have a problem how to pass some value back with the return command. return links.
With console.log(links) I get a warning: getLinks is not iterable
const puppeteer = require('puppeteer')
async function getLinks(){
const browser = await puppeteer.launch({headless: false, defaultViewport: null});
const page = await browser.newPage();
const url = "https://example.com/product-category?p=1&nidx"
await page.goto(url)
while(await page.$('.change-country-buttons > button:nth-child(1)')){
await page.waitForTimeout(2000);
await page.keyboard.press('ArrowDown');
await page.waitForSelector('.change-country-buttons');
await page.waitForTimeout(2000);
await page.click('.change-country-buttons > button:nth-child(1)');
await page.waitForTimeout(2000);
}
while(await page.$(".pagination .pagination--next")){
await page.waitForTimeout(2000);
await page.evaluate(() => {
document.querySelector(".pagination .pagination--next").scrollIntoView();
});
await page.waitForTimeout(1000);
await page.waitForSelector(".pagination .pagination--next")
await page.waitForTimeout(500);
await page.click('.pagination .pagination--next')
const links = await page.$$eval('.item__info > .mtc-link:nth-child(2)', (allAs) => { return allAs.map((a) => a.href) });
await page.waitForTimeout(1500);
console.log(links)
}
}
return links // Is returning links only form the first page and then the loop stops
I tried something with Promise.all () but it wasn't entirely clear to me how to do it.
Please help and be gentle as I am just starting to learn the basics

You need to create an array and push all the helmet links from each page onto it.
This tested successfully for me.
const puppeteer = require('puppeteer')
async function getLinks(){
const browser = await puppeteer.launch({headless: false, defaultViewport: null});
const page = await browser.newPage();
const url = "https://www.motocard.com/en/motorcycle-road-gear/helmets/precio_150-3200/full-face?p=1&nidx"
var all_links = [];
await page.goto(url);
while(await page.$('.change-country-buttons > button:nth-child(1)')){
await page.waitForTimeout(2000);
await page.keyboard.press('ArrowDown');
await page.waitForSelector('.change-country-buttons');
await page.waitForTimeout(2000);
await page.click('.change-country-buttons > button:nth-child(1)');
await page.waitForTimeout(2000);
}
while(await page.$(".pagination .pagination--next")){
await page.waitForTimeout(2000);
await page.evaluate(() => {
document.querySelector(".pagination .pagination--next").scrollIntoView();
});
await page.waitForTimeout(1000);
await page.waitForSelector(".pagination .pagination--next")
await page.waitForTimeout(500);
await page.click('.pagination .pagination--next')
const links = await page.$$eval('.item__info > .mtc-link:nth-child(2)', (allAs) => { return allAs.map((a) => a.href) });
await page.waitForTimeout(1500);
//console.log(links)
all_links.push(...links);
}
return all_links;
}
(async ()=>{
var links = await getLinks();
console.log('done');
console.log(links);
})();

store puppeteer page into session

i want to store puppeteer page into session to use in another middleware in express
here is my code in first middleware:
(async () => {
let solve;
const browser = await puppeteer.launch({
headless: false
});
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0)
await page.goto(URL, { waitUntil: 'domcontentloaded' });
await page.click('button.imeiInquiry');
const images = await page.$eval(('.captcha-image[src]'),node => node.src);
res.send({
type: 'img',
url: images,
});
req.session.page = page;
})();
and here is the code in 2nd middleware:
(async () => {
const page = await req.session.page;
res.send(await req.session.page.title);
await page.type('#CaptchaInputText', req.body.capchaNumber);
await page.type('#ImeiNumber', req.body.imei);
await page.click('.swal2-confirm')
})
but 2nd middleware doesnt work

No node found for selector in (Puppeteer Error)

This is the code I have given:
const puppeteer = require('puppeteer');
(async () => {
const baseUrl = 'example.com';
const browser = await puppeteer.launch({ headless: true});
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage();
function delay(time) {
return new Promise(function (resolve) {
setTimeout(resolve, time)
});
}
await page.goto(baseUrl);
await page.waitForSelector('[name="loginfmt"]');
await page.type('[name="loginfmt"]', 'abc#amazon.com');
await page.click('[type="submit"]');
delay(1000);
await page.authenticate({ username: `abc#amazon.com`, password: `abc#123` });
await page.click((`[data-row-id="111-222-333-444"]`));
await delay(1000);
await page.pdf(
{
path: "Z1.pdf",
printBackground: true,
width: '1300px',
height: '14200px'
}
)
browser.close();
})()
When inspected the page with document.querySelectorAll([data-row-id="111-222-333-444"]); it returned a NodeList array which consisted the details which I have been looking for.
However when tried the same via Puppeteer it terminates with an error saying "Error: No node found for selector: [data-row-id="111-222-333-444"]"
Tried using:
"const button= await page.$(([data-row-id="111-222-333-444"]);
button.click();"
I have been trying to fix this since a long time. It would be really helpful if I get a fix for this.

Try using this code:
const puppeteer = require('puppeteer');
(async () => {
const baseUrl = 'example.com';
const browser = await puppeteer.launch({ headless: true});
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage();
await page.goto(baseUrl);
await page.waitForSelector('[name="loginfmt"]');
await page.type('[name="loginfmt"]', 'abc#amazon.com');
await page.click('[type="submit"]');
await page.waitForTimeout(1000)
await page.authenticate({ username: `abc#amazon.com`, password: `abc#123` });
await page.waitForTimeout(5000)
await page.$(`[data-row-id="111-222-333-444"]`);
await page.click((`[data-row-id="111-222-333-444"]`));
await page.waitForTimeout(1000)
await page.pdf(
{
path: "Z1.pdf",
printBackground: true,
width: '1300px',
height: '14200px'
}
)
browser.close();
})()

How to Click a button in an Iframe using puppeteer?

I'm new to puppeteer so dont know much about it. This is my code so far and everything works.
But I want it to click the login button on the page after it has put the text in the fields, but I cannot figure out how to do it for the life of me. I've tried many different things and none work. Any help with this would be awesome.
just incase you need it
https://server.nitrado.net/usa/rent-gameserver
(async () => {
console.log('launch browser');
const browser = await pup.launch({headless: false});
console.log('new page');
const page = await browser.newPage();
console.log('goto');
await page.setViewport({ width: 1920, height: 1080 });
await page.goto('https://server.nitrado.net/usa/rent-gameserver', { waitUntil: "networkidle2", timeout: 60000 });
await page.waitFor(5000);
console.log('extract login iframe');
var iframes = await page.frames();
var loginFrame = iframes.find(f => f.url().indexOf("oauth.nitrado.net") > -1);
await page.waitFor(5000);
console.log('evaluate iframe');
await loginFrame.evaluate(() => {
document.getElementById('username').value = 'test';
document.getElementById('password').value = '12345';
});
await page.waitFor(300000);
console.log('done');
await browser.close();
})()```

I tried a workaround entering the frame url, I'm not sure if this will help, but here goes the code (main.js):
const pup = require('puppeteer');
mainFunc = async function () {
return new Promise(async (resolve, reject) => { //Wrap de promise
var browser;
try {
//Wrap de tratamento de erros
const browser = await pup.launch({ headless: false });
const page = await browser.newPage();
//SELECTORS:
var userInputSel = '#username';
var passInputSel = '#password';
var loginBtnSel = '#auth_login_ws_header > form > button';
var myUser = "myusername"; //PUT YOUR USERNAME HERE!!!
var myPass = "MyPaSsWoRd123"; //PUT YOUR PASSWORD HERE!!!
await page.goto('https://server.nitrado.net/usa/rent-gameserver',
{ waitUntil: "networkidle2", timeout: 60000 });
await page.waitFor(5000);
console.log('extract login iframe');
var iframes = await page.frames();
var loginFrame = iframes.find(f => f.url().indexOf("oauth.nitrado.net") > -1);
console.log(loginFrame.url())
await page.goto(loginFrame.url(),
{ waitUntil: "networkidle2", timeout: 60000 });
await page.waitFor(5000);
console.log('evaluate iframe');
/*await loginFrame.evaluate(() => {
document.getElementById('username').value = 'test';
document.getElementById('password').value = '12345';
});*/
await page.waitForSelector(userInputSel);
await page.type(userInputSel, myUser);
await page.waitForSelector(passInputSel);
await page.type(passInputSel, myPass);
await page.waitForSelector(loginBtnSel);
await page.click(loginBtnSel);
await page.waitFor(300000);
console.log('done');
await browser.close();
} catch (e) {
if(browser!=undefined){
browser.close();//Close browser if error
}
return reject(e);
}
});//Wrap de promise
}
mainFunc();
This is a running version for you to test. Just type "node main". (of course you need puppeteer installed (npm i puppeteer))...