Develop Reference

Login > Go to Specific Page > Search > Grab Specific Content From Page Using Node JS and Puppeteer

I want to scrape some data from login protected page using Node Js and Puppeteer. When I try to scrape data, the shows page is not reachable. I don't have any idea about why I am getting output like this. Here is my code.
I am new to Puppeteer and Node Js, its been 8 hours I am trying to modify that code, but not working.
async function EmailLookup(MlsNumber)
{
resp = new { Success = false };
(page = await singletonBrowser.Instance.newPage())
{
await page.setRequestInterception(true);
page.Request += (sender, e) =>
{
if (e.Request.resourceType === resourceType.document || e.Request.resourceType === resourceType.Script || e.Request.resourceType === resourceType.Xhr)
e.Request.Continue();
else
e.Request.Abort();
};
await page.async("https://example.com/idp/login");
if (!page.Url.Contains("layouts"))
{
await page.waitForSelector("#username");
await page.waitForSelector("#password");
await page.waitForSelector("#loginbtn");
await page.async("#username", "xxxxx");
await page.async("#password", "xxxxx");
await page.async("#password", "\r");
await page.waitForNavigation(new navigationOptions { Timeout = 0 });
}
await page.goto("https://example.com/launch?layoutid=61&appid=433");
await page.waitForSelector("#ctl02_m_ucSpeedBar_m_tbSpeedBar");
await page.async("#ctl02_m_ucSpeedBar_m_tbSpeedBar", MlsNumber);
await page.async("#ctl02_m_ucSpeedBar_m_tbSpeedBar", "\r");
await page.waitForNavigation(new navigationOptions { Timeout = 0 });
var MLSLink = await page.waitForXPath("//a[text()='" + MlsNumber + "']");
if (MLSLink != null)
{
await MLSLink.click();
await page.waitForNavigation(new navigationOptions{ Timeout = 0 });
var Content = await page.get();
htmldoc = new htmldoc();
htmldoc.load(Content);
var parcelNode = htmldoc.document.selectSingleNode("//a[contains(#href,'AssessorParcelDetail')]");
var emailNode = htmldoc.document.selectSingleNode("//a[contains(#href,'mailto:')]");
if (emailNode != null && parcelNode != null)
{
resp.Success = true;
resp.Email = emailNode.innerText;
resp.Parcel = parcelNode.innerText;
}
}
return json(resp);
}
}

Getting SyntaxError with async/await

I have the following the following code I am trying to run with node.js:
ethereum-block-by-date.js
module.exports = class {
constructor(web3) {
this.web3 = web3.constructor.name === 'Web3' ? web3 : { eth: web3 };
this.checkedBlocks = {};
this.savedBlocks = {};
this.requests = 0;
}
.
.
.
async getEvery(duration, start, end, every = 1, after = true) {
start = moment(start), end = moment(end);
let current = start, dates = [];
while (current.isSameOrBefore(end)) {
dates.push(current.format());
current.add(every, duration);
}
if (typeof this.firstBlock == 'undefined' || typeof this.latestBlock == 'undefined' || typeof this.blockTime == 'undefined') await this.getBoundaries();
return await Promise.all(dates.map((date) => this.getDate(date, after)));
}
.
.
.
And
getBlockNumber.js
const EthDater = require('ethereum-block-by-date');
const { ethers } = require('ethers');
const url = "node url";
const provider = new ethers.providers.WebSocketProvider(url);
const dater = new EthDater(
provider
);
let blocks = await dater.getEvery('hours', '2018-11-15T00:00:00Z', '2018-11-22T00:00:00Z');
console.log(blocks);
When I run node getBlockNumber.js, I get the SyntaxError: await is only valid in async function referring to the let blocks = await dater.getEvery() function call.
If getEvery() is defined as aync in ethereum-block-by-date.js, then why is it returning this SyntaxError? When I remove await from dater.getEvery() it returns a Promise object.

The issue is resolved when I put the await dater.getEvery() inside an async function and then call the function:
async function getBlocks() {
let blocks = await dater.getEvery('hours', '2018-11-15T00:00:00Z', '2018-11-22T00:00:00Z');
console.log(blocks);
}
getBlocks();

Read files from directory and save to array of object

I have a directory of the tax file of employees. Each file has a filename as employee code. I am reading each file and extract some components and save to an array of employee objects.
const readline = require('readline');
let empArr = [];
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
for await (file of filenames) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone =true;
empArr.push(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
})
}
readFiles(directoryPath);
I am not able to get empArr. After getting the array, I need to save to excel. That part I will try after getting the array of employee objects.

I got it working after reading several articles on closure and promises. The below code works for me and sends me array of employees that are processed.
const directoryPath = './tax/';
function readFiles(dirname) {
fs.readdir(dirname, async function (err,filenames) {
if(err) {
return err;
}
let promiseArr = filenames.map( file=> {
return new Promise((resolve)=>{
processFile(file, resolve)
})
});
Promise.all(promiseArr).then((ret)=>console.log(ret));
})
}
function processFile(file, callback) {
const filePath = path.join(__dirname,directoryPath,file);
const readStream = fs.createReadStream(filePath);
const fileContent = readline.createInterface({
input: readStream
});
let employeeObj = {
empId : '',
TotalEarning:'',
ProfessionalTax:0,
GrossIncome:0,
isDone:false
};
fileContent.on('line', function(line) {
if(!employeeObj.empId && line.includes("Employee:")) {
const empId = line.replace('Employee: ','').split(" ")[0];
employeeObj.empId = empId;
}
else if(line.includes('Total Earnings')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.TotalEarning = amount;
}
else if(line.includes('Profession Tax')) {
const amount = line.split(" ").pop() || 0;
employeeObj.ProfessionalTax = amount;
}
else if(line.includes('Gross Income')) {
const amount = line.replace(/[^0-9.]/g,'');
employeeObj.GrossIncome = amount ||0;
}
else if(line.includes('finance department immediately')) {
employeeObj.isDone = true;
return callback(employeeObj);
}
});
fileContent.on('close', function() {
fileContent.close();
});
}
readFiles(directoryPath);
Surely, the code can be improved further.

async/await, not waiting for the variable to populate before printing

async function filterusers(users){
let usersfiltered=[]
for(var i = 0;i < users.length; i++){
let userref = db.collection('usernames').doc(users[i]);
let getDoc = userref.get()
.then(doc => {
if (doc.exists) {
usersfiltered.push(users[i])
}
})
}
return await usersfiltered;
}
filterusers(users).then(console.log);
i am looking to wait for the filtered result but it always prints blank array ie before the result is being returned.

async function filterusers(users){
let usersfiltered=[]
// You should use let or const instead of var.
for(let i = 0;i < users.length; i++){
// I believe getting doc needs await.
let userref = await db.collection('usernames').doc(users[i]);
await userref.get()
.then(doc => {
if (doc.exists) {
usersfiltered.push(users[i])
}
})
}
return usersfiltered;
}
filterusers(users).then(console.log);

First, you have to turn the array of users into an array of Promises ( asynchronous operations) by using Array.map:
const checkUserPromises = users.map((user) => {
const userref = db.collection("usernames").doc(user);
return userref.get().then((doc) => {
if (doc.exists) {
return user;
} else {
return null;
}
});
});
Then, you need to await these promises with Promises.all:
const checkedUsers = await Promise.all(checkUserPromises);
Lastly, you may want to filter out the users that are not existing:
const existingUsers = checkedUsers.filter((user) => user !== null);

await should be with Promise
async function filterusers(users) {
let usersfiltered = [];
for (let i = 0; i < users.length; i++) {
let userref = db.collection("usernames").doc(users[i]);
let getDoc = await userref.get();
if (getDoc.exists) {
usersfiltered.push(users[i]);
}
}
return usersfiltered;
}
let filterdUsers = filterusers(users);
console.log(filterdUsers);

Firstly, do not mix async/await with .then.
Secondly, use new ES6 for loops, to make the code work properly with async/await.
async function filterusers(users) {
let usersfiltered = [];
filterusers(users).then(console.log);
for (const user of users) {
let userref = db.collection('usernames').doc(user);
const doc = await userref.get();
if (doc.exists) {
usersfiltered.push(user);
}
}
return await usersfiltered;
}

Solved it myself by moving the await to before userref.get()
async function filterusers(users){
let usersfiltered=[]
for(var i = 0;i < users.length; i++){
let userref = db.collection('usernames').doc(users[i]);
let getDoc = await userref.get()
.then(doc => {
if (doc.exists) {
usersfiltered.push(users[i])
}
})
}
return usersfiltered;
}
filterusers(users).then(console.log);

JS Pagination Using Promises

I'm attempting to make an API call using promises. The API is paginated and as such, depending on the headers in that first API call make more to get the rest of the results if need be.
Here's what I have so far:
const get = (url, pageNo) => {
var options = {
url: url,
headers: {
'Authorization': `Token token=${apiToken}`
},
json: true,
page: pageNo
};
return new Promise((resolve, reject) => {
request.get(options, (err, resp) => {
err ? reject(err) : resolve(resp);
})
});
};
Using get() to loop and get all responses:
const getAll = (plannerId, timestamp, range) => {
const plannerBookingsUrl = new URL(
`/api/planners/${plannerId}/bookings?since=${timestamp}&range=${range}`,
baseUrl
);
let response = get(plannerBookingsUrl, 1);
let bookings = [];
bookings.push(response);
response.then(results => {
let moreRequests = true;
let currentPage = 1;
const totalPages = parseInt(results.headers['x-total-pages']);
while (moreRequests) {
if (currentPage < totalPages) {
nextBatch = get(plannerBookingsUrl, currentPage + 1);
bookings.push(nextBatch);
currentPage++;
} else {
moreRequests = false;
}
}
});
return Promise.all(bookings);
};
Main() where I'm using getAll(...):
const main = () => {
const response = getAll(
'11716',
'2020-02-27',
'7'
);
response.then(results => {
console.log(results);
.catch(error => console.log(error))
};
main();
This returns the initial promise but not the remaining promises.
What I'm really have a problem with is reading the first API, making the remainder and returning them all together to be using in my main function.
Any help would be much appreciated!
Thanks.

You could put all your fetching logic inside the while loop. The way you get your bookings is the same, except for the first time where you need to get a little more information on the amount of pages.
Accomplish this by making your function async and check the first time of the loop if the totalPages value is already known. If it's not, await the response and get the info from the headers, and otherwise just push the response to the bookings array.
const getAll = async (plannerId, timestamp, range) => {
const plannerBookingsUrl = new URL(
`/api/planners/${plannerId}/bookings?since=${timestamp}&range=${range}`,
baseUrl
);
let bookings = [];
let currentPage = 1;
let totalPages = null;
while (totalPages === null || currentPage < totalPages) {
let response = get(plannerBookingsUrl, currentPage);
if (totalPages === null) {
let results = await response;
totalPages = parseInt(results.headers['x-total-pages']);
}
bookings.push(response);
currentPage++;
}
return Promise.all(bookings);
};

The problem is that you are returning Promise.all(bookings) outside response.then callback, so at this point bookings contains only the first call get(plannerBookingsUrl, 1).
Here is a possible solution using async:
const getAll = async (plannerId, timestamp, range) => {
const plannerBookingsUrl = new URL(
`/api/planners/${plannerId}/bookings?since=${timestamp}&range=${range}`,
baseUrl
);
let response = get(plannerBookingsUrl, 1);
let bookings = [];
bookings.push(response);
const results = await response; // wait for results here
let moreRequests = true;
let currentPage = 1;
const totalPages = parseInt(results.headers['x-total-pages']);
while (moreRequests) {
if (currentPage < totalPages) {
nextBatch = get(plannerBookingsUrl, currentPage + 1);
bookings.push(nextBatch);
currentPage++;
} else {
moreRequests = false;
}
}
return Promise.all(bookings); // bookings now contains every next batch
};
adapt on main() function:
const main = async () => {
const results = await getAll(
'11716',
'2020-02-27',
'7'
);
...
};
main();