I'm trying to create a webscraper in Node, using Puppeteer.
My first challange (which I thought would be easy), it's pass by a pagination "Load more" button.
But, when I run the following code, Puppeteer click on all "Load more" and after click on a content, when I need to stop clicking.
Why this happens?
let loadMore = true;
while (loadMore) {
selector = 'ul.pager > li > a.button';
await page.waitForSelector(selector, { timeout: 600 }).then(() => {
page.click(selector);
}).catch(() => {
loadMore = false;
});
}
Thx all!
I would do it like this.
I think timeout doesn't make a difference here, if the await inside the if's condition is not enough then there could be other problem with the script.
let loadMore = true;
while (loadMore) {
const selector = 'ul.pager > li > a.button';
if ((await page.$(selector)) !== null) {
await page.click(selector);
} else {
loadMore = false;
}
}
Related
I have an index page and a dashboard, on the index I'm using typewriter and particlesjs on the index only, and on the dashboard I have a sidebar.
If I have all the code as-is, I get errors as the page is still looking for typewriter and particlesjs on all pages.
So I have attempted to wrap each section around an if so the plan is if that class or id exists on that page it will only render that JS. So I've created the following code.
edited code below based on groovy_guy's answer
document.querySelector('nav .toggle').addEventListener('click', e => {
document.querySelector('nav .hidden').classList.toggle('show')
});
let checkTypewriter = document.getElementById('typewriter');
if (checkTypewriter.length > 0) {
new Typewriter('#typewriter', {
strings: ['Website Developer', 'Freelancer' , 'System Admin'],
autoStart: true,
loop: true
});
}
let checkParticlesjs = document.getElementsByClassName('particlesjs');
if (checkParticlesjs.length > 0) {
let particles = Particles.init({
selector: '.particlesjs',
color: ['#48F2E3', '#48F2E3', '#48F2E3'],
connectParticles: true,
maxParticles: 200
});
}
let checkSidebar = document.getElementsByClassName('sidebar');
if (checkSidebar.length > 0) {
user_wants_collapse = false;
// Fetch all the details element.
const details = document.querySelectorAll("details");
// Add the onclick listeners.
details.forEach((targetDetail) => {
targetDetail.addEventListener("click", () => {
// Close all the details that are not targetDetail.
details.forEach((detail) => {
if (detail !== targetDetail) {
detail.removeAttribute("open");
};
});
});
});
document.querySelector('section').addEventListener('click', (ev) => {
// close any open details elements that this click is outside of
let target = ev.target;
let detailsClickedWithin = null;
while (target && target.tagName != 'DETAILS') {
target = target.parentNode;
};
if (target && target.tagName == 'DETAILS') {
detailsClickedWithin = target;
};
Array.from(document.getElementsByTagName('details')).filter(
(details) => details.open && details != detailsClickedWithin
).forEach(details => details.open = false);
// if the sidebar collapse is true and is re-expanded by clicking a menu item then clicking on the body should re-close it
if (user_wants_collapse == true && (document.querySelectorAll('.sidebar details'))) {
document.querySelector('body').classList.add('is--expand');
};
});
// when the sidebar menu is clicked this sets the user_wants_collapse var to true or false and toggles is--expand class on body
document.querySelector('.sidebar .menu-link').addEventListener('click', () => {
document.querySelector('body').classList.toggle('is--expand');
user_wants_collapse = !user_wants_collapse
document.querySelector('.sidebar').classList.toggle('is--expand');
// show all text
document.querySelectorAll('.sidebar .title').forEach((el) => {
el.classList.toggle('hidden');
});
// changing sidebar menu items and menu collapse icons
const icon = document.querySelector('.menu-link-arrows span');
if (icon.classList.contains('fa-angle-double-left')) {
icon.classList.remove('fa-angle-double-left');
icon.classList.add('fa-angle-double-right');
} else {
icon.classList.remove('fa-angle-double-right');
icon.classList.add('fa-angle-double-left');
}
});
// making sure the sidebar menu items re-expands the sidebar on click
let x = document.querySelectorAll('.sidebar details');
let i;
for (i = 1; i < x.length; i++) {
x[i].addEventListener('click', () => {
// changing sidebar menu items and menu collapse icons
// change menu items and menu collapse icons
const icon = document.querySelector('.sidebar-drop-parent-arrow span');
if (icon.classList.contains('fa-chevron-down')) {
icon.classList.remove('fa-chevron-down');
icon.classList.add('fa-chevron-up');
} else {
icon.classList.remove('fa-chevron-up');
icon.classList.add('fa-chevron-down');
}
if (document.querySelector('body').classList.contains('is--expand')) {
document.querySelector('body').classList.remove('is--expand');
};
});
};
};
when loading the JS I'm not getting any console errors but I'm not seeing any result of the JS.
Why don't you use querySelector()? I think that's more uniform across your codebase. Besides, I see that you only care about one element and not a list of elements, so this method is ideal since it gets the first element that encounters.
const checkTypewriter = document.querySelector('#typewriter')
if (checkTypewriter) {
// Element with an ID 'typewriter' exist in the DOM.
}
const checkParticlesjs = document.querySelector('.particlesjs')
if (checkParticlesjs) {
// Element with a class named "particlesjs" exist in the DOM.
}
Also, make sure to check if an element exist before attaching an event listener:
const toggleNav = document.querySelector('nav .toggle')
if (toggleNav) {
toggleNav.addEventListener('click', (e) => {
document.querySelector('nav .hidden').classList.toggle('show')
});
}
For Javascript:
var checkTypewriter = document.getElementsByClassName('typewriter');
if (checkTypewriter.length > 0) {
// Here write your code
}
var checkParticlesjs = document.getElementsByClassName('particlesjs');
if (checkParticlesjs.length > 0) {
// Here write your specific code
}
For JQuery:
if ($("#typewriter")[0]){
// Here write your code
}
if ($(".particlesjs")[0]){
// Here write your specific code
}
This is how you can check if your classes exist,
I'm using cypress since a week, and I succesfully did an integration with the stripe iframe: I've used this code:
in cypress/support/command.js
Cypress.Commands.add('iframeLoaded', { prevSubject: 'element' }, $iframe => {
const contentWindow = $iframe.prop('contentWindow')
return new Promise(resolve => {
if (contentWindow && contentWindow.document.readyState === 'complete') {
resolve(contentWindow)
} else {
$iframe.on('load', () => {
resolve(contentWindow)
})
}
})
})
Cypress.Commands.add('getInDocument', { prevSubject: 'document' }, (document, selector) =>
Cypress.$(selector, document),
)
In cypress/integration/staging-web/web.test.js
cy.get('iframe')
.eq(1)
.iframeLoaded()
.its('document')
.getInDocument('[name="cardnumber"]')
.then($iframe => {
if ($iframe.is(':visible')) {
$iframe.type('4242424242424242')
}
})
cy.get('iframe')
.eq(1)
.iframeLoaded()
.its('document')
.getInDocument('[name="exp-date"]')
.then($iframe => {
if ($iframe.is(':visible')) {
$iframe.type('1225')
}
})
cy.get('iframe')
.eq(2)
.iframeLoaded()
.its('document')
.getInDocument('[name="cvc"]')
.then($iframe => {
if ($iframe.is(':visible')) {
$iframe.type('123')
}
})
cy.get('.mt1 > .relative > .input').type('utente')
My problem is that during the page loading, cypress does not wait until stripe fields are fully loaded, and I get an error because happens this (sorry for not-english language, but it's a screenshot):
Those lines are:
cardnumber
expiration date , pin number
4th line is card owner
I've tried with .should('be.visibile') but it does nothing; plus I've tried with
cy.get('iframe')
.eq(1)
.iframeLoaded()
.its('document')
.getInDocument('[name="cardnumber"]')
.then($iframe => {
if ($iframe.is(':visible')) {
$iframe.type('4242424242424242')
}
})
but no way, it always gives me an error; in this latter case, it doesn't even give an error, it just goes on without filling the fields and after this it stops because it cant go on in the test.
If I add cy.wait(800) before the code in web.test.js it works fine, but I don't want to use wait, because it's basically wrong (what happens if it loads after 5 seconds?).
is there a way to check that those elements must have an height?
Remember that they are in an iframe (sadly).
If I add cy.wait(800) ... it works fine.
This is because you are not using Cypress commands with auto-retry inside getInDocument().
Cypress.$(selector) is jQuery, it just attempts to grab the element, not retry for async loading, and no test failure when not found.
Should use a proper command with retry, like
Cypress.Commands.add('getInDocument', { prevSubject: 'document' }, (document, selector) =>
cy.wrap(document).find(selector)
)
or you might need to work from body
Cypress.Commands.add('getInDocument', { prevSubject: 'document' }, (document, selector) =>
cy.wrap(document).its('body')
.find(selector)
.should('be.visible')
)
Without a test system I'm not sure exactly which one is correct syntax, but you get the idea.
Also, too many custom commands. You always follow .iframeLoaded() with .its('document'), so just wrap it all up in iframeLoaded custom command.
In fact, resolve(contentWindow.document.body) because it's a better point to chain .find(selector).
This is my test against the Stripe demo page,
Cypress.Commands.add('iframeLoaded', { prevSubject: 'element' }, $iframe => {
const contentWindow = $iframe.prop('contentWindow')
return new Promise(resolve => {
if (contentWindow && contentWindow.document.readyState === 'complete') {
resolve(contentWindow.document.body)
} else {
$iframe.on('load', () => {
resolve(contentWindow.document.body)
})
}
})
})
it('finds card number', () => {
cy.viewport(1000, 1000)
cy.visit('https://stripe-payments-demo.appspot.com/')
cy.get('iframe')
.should('have.length.gt', 1) // sometimes 2nd iframe is slow to load
.eq(1)
.iframeLoaded()
.find('input[name="cardnumber"]')
.should('be.visible')
})
#Nanker Phelge by the way I updated my function in this way, I posted it here because there was no space in comments:
UPDATE: I put waitForMillisecs inside the function -_- it HAVE TO stay outside! Correction made.
const ADD_MS = 200
const STACK_LIMIT = 15
let stackNumber = 0
let waitForMillisecs = 500
function checkAgainStripe() {
stackNumber++
if (stackNumber >= STACK_LIMIT) {
throw new Error('Error: too many tries on Stripe iframe. Limit reached is', STACK_LIMIT)
}
cy.get('.btn').scrollIntoView().should('be.visible').click()
cy.get('iframe')
.eq(1)
.then($elem => {
//console.log($elem[0].offsetHeight)
//console.log($elem[0])
cy.log('now wait for ' + waitForMillisecs + '; stack number is ' + stackNumber)
cy.wait(waitForMillisecs)
waitForMillisecs = waitForMillisecs + ADD_MS
if (!$elem[0] || !$elem[0].offsetHeight || $elem[0].offsetHeight < 19) {
console.log('entered in if')
cy.reload()
return checkAgainStripe()
}
})
}
this.router.routeReuseStrategy.shouldReuseRoute = () => false;
I have applied this sort of line in order to make the component UI updated everytime. But in some other cases it start to refreshing the page event if it should reuse the route.
How can we overcome this issue?
Actually in my application there are three tabs in left panel. In each tab there are some listings clicking on list items opens the content on right panel. But in one of the listing there is a common UI that is getting open on some list item, but the problem is that when we don't apply above sort of code then the UI is not getting updated. But if we apply the code then the UI is updated everytime we click on other list item. But the problem is that when we apply this code it start to refresh the page everytime we click on other list in different tabs also, that should not be the case.
If we apply this code this.router.routeReuseStrategy.shouldReuseRoute = () => false; then how can we revert this functionality under this.router?
To take less risks I'm just reverting it back to what it was once the reload is done:
refresh() {
const prev = this.router.routeReuseStrategy.shouldReuseRoute;
const prevOSN = this.router.onSameUrlNavigation;
this.router.routeReuseStrategy.shouldReuseRoute = () => false;
this.router.onSameUrlNavigation = 'reload';
this.router.navigate([this.router.url]);
setTimeout(() => {
this.router.routeReuseStrategy.shouldReuseRoute = prev;
this.router.onSameUrlNavigation = prevOSN;
}, 0);
}
I have the same issue, I changed that line for this:
// override the route reuse strategy
this.router.routeReuseStrategy.shouldReuseRoute = function () {
return false;
};
this.router.events.subscribe((evt) => {
if (evt instanceof NavigationEnd) {
// trick the Router into believing it's last link wasn't previously loaded
this.router.navigated = false;
// if you need to scroll back to top, here is the right place
window.scrollTo(0, 0);
}
});
I don't even know if this works well or do the same thing.
private saveRouterStrategyReuseLogic: any;
ngOnInit() {
// Save logic
this.saveRouterStrategyReuseLogic = this.router.routeReuseStrategy.shouldReuseRoute;
this.router.routeReuseStrategy.shouldReuseRoute = (future, curr) => { return false; };
}
ngOnDestroy() {
this.router.routeReuseStrategy.shouldReuseRoute =
this.saveRouterStrategyReuseLogic;
}
I need to verify that after a button click an alert is displayed.
I tried the following:
it('should display an alert when the user tries to add empty value', () => {
jest.setTimeout(100000);
page.alert = jest.fn(text => text);
const addButtonSelector = '#root > div > div > div.ToDoInput > button';
expect(page).toClick(addButtonSelector);
expect(page.alert).toHaveBeenCalledWith('Please enter a todo!');
})
But the test Fails with zero calls to the page.alert (in reality the alert is shown after the click on this button in this conditions).
Also I tried window.alert - it is not recognized.
Please help.
Update: I tried like that, and the alert function is not replaced with the mock function (???)
it('should display an alert when the user tries to add empty value', async() => {
jest.setTimeout(50000);
const dialogHandler = jest.fn();
page.on('dialog', dialogHandler);
const addButtonSelector = '#root > div > div > div.ToDoInput > button';
await expect(page).toClick(addButtonSelector);
await expect(dialogHandler).toHaveBeenCalled();
const [firstCall] = dialogHandler.mock.calls;
const [dialog] = firstCall;
expect(dialog.message()).toEqual('Please enter a todo!');
})
Take a look at Pupeteer's Dialog. You can set event handler once a Dialog is displayed and if you pass jest.fn() you can have reference to the dialog that it has been called with
e.g.:
describe('page with dialog', () => {
const dialogHandler = jest.fn(dialog => dialog.dismiss());
beforeAll(() => {
page.on('dialog', dialogHandler);
});
describe('when the ToDoInput button is clicked', () => {
beforeAll(async () => {
await page.click('#root > div > div > div.ToDoInput > button');
});
it('should display a dialog', () => {
expect(dialogHandler).toHaveBeenCalled();
});
it('should have message "Please enter a todo!"', () => {
const [firstCall] = dialogHandler.mock.calls;
const [dialog] = firstCall;
expect(dialog.message()).toEqual('Please enter a todo!');
});
});
});
Edit: after testing it the only thing that was not working is that the test script was becoming idle until the dialog is either accepted or dismissed and after changing the dialogHandler to dismiss the dialog made it work flawlessly:
const dialogHandler = jest.fn(dialog => dialog.dismiss());
here is a working example
My idea is to have a simple search function, where people can type text in a search box and see if there is anything matched with the text in the html. If it does, the matched part(let's say it might be in later half of the whole HTML page that's not seenable on your current screen/viewport) will jump/scroll into view. If it doesn't, you get "no match found" message in the #feedback tag.
HTML looks like this:
<input id="search-text">
<p id="feedback"></p>
<p>title1</p>
<p>title2</p>
<p>title3</p>
JS is like this:
let searchText = ''
const titlesNodes = document.querySelectorAll('p')
const titles = Array.from(titlesNodes)
document.querySelector('#search-text').addEventListener('input', (e) => {
searchText = e.target.value
searchFunction()
})
const searchFunction = () => {
const filteredTitles = titles.filter((title) => title.innerText.toLowerCase().includes(searchText.toLowerCase()) )
const msg = document.querySelector('#feedback')
msg.innerHTML = ''
if (filteredTitles.length > 0){
filteredTitles[0].scrollIntoView()
} else {
msg.textContent = 'No match found'
}
}
So far I'm able to get the "no matched found" but when I type something that's only in the later half of the page, it doesn't jump to the view. I tested on both Chromium and Firefox and I use Linux.
What is wrong with my code? Thanks!
This is because by default, browser will set your <input> element into screen when you type in. So your own call is overridden by browser's default one.
Not sure what's the best way to avoid that though...
You may try to call scrollIntoView after a small timeout, so it occurs after the one of the input. If you use requestAnimationFrame timing function, it should occur just before the next paint, so you shouldn't notice the one of the input occurred.
let searchText = '';
const titlesNodes = document.querySelectorAll('p')
const titles = Array.from(titlesNodes)
document.querySelector('#search-text').addEventListener('input', (e) => {
searchText = e.target.value
searchFunction()
})
const searchFunction = () => {
const filteredTitles = titles.filter((title) => title.innerText.toLowerCase().includes(searchText.toLowerCase()))
const msg = document.querySelector('#feedback')
msg.innerHTML = ''
if (filteredTitles.length > 0) {
requestAnimationFrame(() => { // wait just a bit
filteredTitles[0].scrollIntoView()
});
} else {
msg.textContent = 'No match found'
}
}
#feedback { margin-bottom: 125vh }
p { margin-bottom: 50vh }
<input id="search-text">
<p id="feedback"></p>
<p>title1</p>
<p>title2</p>
<p>title3</p>
Try changing:
- filteredTitles[0].scrollIntoView()
+ filteredTitles[0].scrollIntoView({alignToTop: true})
document.querySelector('#search-text').addEventListener('input', (e) => {
searchText = e.target.value
- searchFunction()
+ setTimeout(searchFunction, 500)})
})