I want to get the web vital performance of Cumulative Layout Shift using layout instability., I have written a playwright test case for it but when executed its throws the timeout exceeded and page.evaluate error.
test.only('CLS', async ({ page }) => {
await page.goto(HOME_PAGE_URL, { waitUntil: 'networkidle' });
const cummulativeLayoutShift = await page.evaluate(() => {
return new Promise((resolve) => {
const observer = new PerformanceObserver((list) => {
const result = list.getEntries();
resolve(result);
});
observer.observe({ type: 'layout-shift', buffered: true });
});
});
console.log(`SHIFT - ${cummulativeLayoutShift}`);
});
Related
I'm fetching data from a website but the process sometimes fails so I make the function to retry it. But I'm using a 15 seconds timeout, so when the timeout is triggered it will stop retrying the function and it will return some error message from the website.
This works fine in my local machine but when I deploy my code to Vercel (running on AWS Lambda as far as I know), the setTimeout is being completely ignored, so the fetchData function keeps running until it gets the correct response or until the server default 60 seconds timeout triggers.
Here's the code:
router.get('/test', async (req, res) => {
try {
const browser = await playwright.launchChromium({
headless: false });
const context = await browser.newContext();
const page = await context.newPage();
let timeout = false;
let fetchDataTimeout = setTimeout(() => {
timeout = true;
}, 15000);
const fetchData = async () => {
await page.type('#code', '6824498040');
let data = await (
await Promise.all([
page.waitForResponse(
(response) =>
response.url() === `${env.API_URL2}` && response.status() === 200,
{ timeout: 10000 },
),
page.click('#btn-check'),
])
)[0].json();
if (data.errors && !timeout) {
await page.reload();
return await fetchData();
} else {
clearTimeout(fetchDataTimeout);
return data;
}
};
let data = await fetchData();
await browser.close();
res.json({
status: 200,
message: data,
});
} catch (error) {
console.error(error);
return res.status(500).send({ 'Server Error': `${error}` });
}
});
I read that you have to wrap the setTimeout into a Promise and return it as an async function. Like this:
const timeOut = async (t) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(`Completed in ${t}`)
}, t)
})
}
await timeOut(15000).then((result) => console.log(result))
But this will trigger the 15 seconds wait always. I need to discard the waiting if I get the correct response from fetchData and trigger the timeout if I don't 15 seconds after I start trying.
Any ideas???
The solution was quite simple. Since AWS Lambda requires an async function to properly wait the timeouts, I just had to wrap the timeout in a Promise. But then what did the trick was wrapping both functions, the timeout and the fetchData, in a Promise.race() method. By doing this, the promise that resolves first will stop the other one from running.
The code is now like this:
router.get('/test', async (req, res) => {
try {
const browser = await playwright.launchChromium({headless: false });
const context = await browser.newContext();
const page = await context.newPage();
const timeout = () =>
new Promise((resolve, reject) => {
setTimeout(() => {
reject('FUNCTION TIMEOUT');
}, 12000);
});
const fetchData = async () => {
await page.type('#code', '6824498040');
let response = await (
await Promise.all([
page.waitForResponse(
(res) =>
res.url() === `${env.API_URL2}` && res.status() === 200,
{ timeout: 10000 },
),
page.click('#btn-check'),
])
)[0].json();
if (data.error) {
await page.reload();
return await fetchData();
} else return data;
}
};
let data = await Promise.race([fetchData(), timeout()]);
await browser.close();
res.json({
status: 200,
message: data,
});
} catch (error) {
console.error(error);
return res.status(500).send({ 'Server Error': `${error}` });
}
});
I implemented a 12 seconds timeout instand of 15. I tested this on Vercel (AWS Lambda) and it works fine.
Thanks everybody, especially dandavis, this implementation was his idea.
Anyway, I hope it helps.
I have a list of proxies which I want to test if they are still active or not, after creating a promise list I run through the results, but after the last promise result the process doesnt stop and the programm freezes even though it should finish because there isnt any code after that.
The only fix I found was to put a process.exit() after running through all results, but this seems really odd to me. I want to automate the process but because the result function never ends its really difficult to workaround it.
const axios = require('axios')
const HttpsProxyAgent = require('https-proxy-agent');
const fs = require('fs');
const data = fs.readFileSync("./proxies.txt", "utf-8");
const proxyList = data.split("\n")
const axiosGet = (url, proxy) => {
const abort = axios.CancelToken.source()
const id = setTimeout(() => abort.cancel(`Timeout of 60000ms.`), 60000)
return axios.get(url, { cancelToken: abort.token, httpsAgent: new HttpsProxyAgent(`http://${proxy.trim()}`), proxy: false })
.then(response => { clearTimeout(id); return true })
.catch(error => { clearTimeout(id); return false })
}
const promises = proxyList.map((proxy, index) => {
return axiosGet(`https://api.ipify.org`, proxy, index)
}).map(p => p.catch(e => { e; return false }))
Promise.all(promises)
.then((results) => {
results.forEach((result, index) => console.log(index + 1, result))
//not exiting after running through all promises, only process.exit() works
})
.catch(err => console.log(err));
I have a part of my code that makes several API calls to different endpoints and I want to know if any of those calls fail so I can display an appropriate error message. Right now, if an error happens in one() it will stop all other calls from happening, but that's not what I want; If an error occurs, I want it to populate the errors object and have the program continue on.
async function gatherData() {
let errors = { one: null, two: null, three: null };
const responseOne = await one(errors);
const responseTwo = await two(errors);
const responseThree = await three(errors);
if (!_.isNil(errors.one) || !_.isNil(errors.two) || !_.isNil(errors.three)) {
// an error exists, do something with it
} else {
// data is good, do things with responses
}
}
gatherData();
async function one(errors) {
await axios
.get("https://jsonplaceholder.typicode.com/comment")
.then(res => {
return res;
})
.catch(err => {
errors.one = err;
return err;
});
}
async function two(errors) {
await axios
.get("https://jsonplaceholder.typicode.com/comments")
.then(res => {
return res;
})
.catch(err => {
errors.two = err;
return err;
});
}
async function three(errors) {
await axios
.get("https://jsonplaceholder.typicode.com/comments")
.then(res => {
return res;
})
.catch(err => {
errors.three = err;
return err;
});
}
If you pass the errors to the async functions, so pass the errors object as parameter
const responseOne = await one(errors);
const responseTwo = await two(errors);
const responseThree = await three(errors);
I try to run puppeteer in a firebase function. As I understand this
https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-on-google-cloud-functions
it should work adding
"engines": {
"node": "8"
},
in package.json. I get positive feedback it use firebase deploy
functions: updating Node.js 8 function helloWorld(us-central1)...
Sadly it cashes with
Error: Error: Failed to launch chrome!
[0408/080847.149912:ERROR:zygote_host_impl_linux.cc(89)] Running as root without --no-sandbox is not supported. See https://crbug.com/638180.
TROUBLESHOOTING: https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md
at startPuppeteer.then.then.then.catch.error (/srv/index.js:42:15)
at <anonymous>
at process._tickDomainCallback (internal/process/next_tick.js:229:7)
Tested the same code on local which worked.
That' the cloud functions code which fails.
const functions = require('firebase-functions');
const puppeteer = require('puppeteer');
let browser = ""
let page = ""
const startPuppeteer = async () => {
browser = await puppeteer.launch();
page = await browser.newPage()
}
const usePageInPuppeteer = async (url) => {
await page.goto(url);
return await page.title()
}
const closePuppeteer = async () => {
return await browser.close();
}
const runtimeOpts = {
timeoutSeconds: 500,
memory: '2GB'
}
exports.helloWorld = functions
.runWith(runtimeOpts)
.https.onRequest((request, response) => {
//response.send()
startPuppeteer()
.then(() => {
return usePageInPuppeteer('https://www.google.com')
})
.then(returnUse => {
console.log(returnUse)
return response.send(returnUse)
})
.then(() => {
return closePuppeteer()
})
.catch(error => {
throw new Error(error)
});
});
That the local test, which works
const puppeteer = require('puppeteer');
let browser = ""
let page = ""
const startPuppeteer = async () => {
browser = await puppeteer.launch();
page = await browser.newPage()
}
const usePageInPuppeteer = async (url) => {
await page.goto(url);
return await page.title()
}
const closePuppeteer = async () => {
return await browser.close();
}
startPuppeteer()
.then(() => {
return usePageInPuppeteer('https://www.google.com')
})
.then(returnUse => {
console.log(returnUse)
return closePuppeteer()
})
.catch(error => {
throw new Error(error)
});
const startPuppeteer = async () => {
browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox']});
page = await browser.newPage()
}
works great. Thanks to https://stackoverflow.com/users/2911633/igor-ilic for the hint
you need to launch the chrome in sandbox mode. Running chrome as root is not supported directly, you can pass the argument --no-sandbox to launch it as a root user.
The final code will look like this
browser = await puppeteer.launch({args: ['--no-sandbox']});
so im new to async functions and promises , imagine a promiselike this (pleas ignore syntax errors)
await new Promise(async (resolve, reject ) => {
const page = await browser.newPage();
await page.goto('https://example.com').catch( ()=>reject('ERROR -> LINK 1 TIMEOUT '));
// INSERT USERNAME AND PASSWORD
await page.$eval('form', form => form.submit()).catch( ()=>reject('ERROR -> FORM SUBMIT ERROR '));
if( await page.$("#username"))
{
reject(" ERROR -> LOGIN FAILED !!!!!!!!!!!!!! ");
}
await page.waitForSelector('#xxx').catch( ()=>reject('ERROR -> WAITING FOR ELEMENT TIMEOUT '));
var scraped_data = // dop some page evaluate and scrap data;
resolve(scraped_data);
}).then(function(scraped_data){
await page.close();
console.log('all done');
insert_data_in_databas(scraped_data);
})
.catch(function(error){
console.log(' tab failed : ');
console.log(error);
});
i want to convert this to a async function ... what is proper way to do this ? should i just put all of them in a try/catch block like
async function do_stuff(){
try {
const page = await browser.newPage();
await page.setViewport({ width: 1000, height: 1100});
await page.goto( 'https://example.com' );
// INSERT USERNAME AND PASSWORD
await page.$eval('form', form => form.submit());
await page.waitForSelector('#xxx');
var scraped_data = // dop some page evaluate and scrap data;
await page.close();
console.log('all done');
insert_data_in_databas(scraped_data);
}
catch (e) {
await page.close();
console.log('error');
console.log(e);
}
}
how can i reject when there is a error so the rest of the code wouldnt execute ? can i have custom error text in catche block like
ERROR -> FORM SUBMIT ERROR
how should i to this
if( await page.$("#username"))
{
reject(" ERROR -> LOGIN FAILED !!!!!!!!!!!!!! ");
}
which is not an actual error (i mean its not code error) in try/catche ?
------------------------------------------ edit --------------------
i tried
async function open_tab(){
try {
const page = await browser.newPage();
await page.setViewport({ width: 1000, height: 1100});
await page.goto( 'https://google.com' );
await page.waitForSelector('#xxx').catch(()=> { throw new Error('ERROR -> LOGIN FAILED')});
await page.close();
console.log('all done');
}
catch (e) {
console.log('error');
console.log(e);
await page.close();
}
}
its almost working but i cant close the tab in the catch block i get
UnhandledPromiseRejectionWarning: ReferenceError: page is not defined
and the tab remains open which is not ideal
There should be no new Promise because a promise already exists and can be chained.
If resulting promise should be rejected in async function, it is:
if (await page.$("#username")) {
throw new Error('ERROR -> LOGIN FAILED');
}
It should be
let page;
try {
page = await browser.newPage();
instead of
try {
const page = await browser.newPage();
// Best solution for error handling in puppeteer
const puppeteer = require("puppeteer");
const param_puppeteer = {
args: [
"--incognito",
"--ignore-certificate-errors",
"--no-sandbox",
"--disable-setuid-sandbox",
"--window-size=1920,1080",
"--disable-accelerated-2d-canvas",
"--disable-gpu",
// '--unlimited-storage',
// '--no-startup-window',
// '--disable-dev-shm-usage',
// '--disable-crash-reporter',
// '--disable-breakpad'
],
headless: false,
};
async function start() {
return puppeteer
.launch(param_puppeteer)
.then(async (browser) => {
const page = await browser.newPage();
return await task(page)
.catch((err) => console.log(err))
.finally(() => browser.close());
})
.catch((err) => console.log(err));
}
async function task(page) {
await page.setViewport({ width: 1000, height: 1100 });
await page.goto("https://google.com");
await page.waitForSelector("#hplogo");
let exist = await page.$("#hplogo").then((res) => !!res);
if (exist) {
return new Promise((resolve, reject) => resolve("success"));
} else {
return new Promise((resolve, reject) => reject("failed"));
}
}
start();