async traverse(url) {
const ts = new TournamentScraper()
const ms = new MatchScraper()
const results = []
const tournaments = await ts.run(url)
for(let href of tournaments.map(t => t.href)){
let matches = await ms.run(href)
let pages = ms.getPages()
let seasons = ms.getSeasons()
//console.log(pages)
//console.log(seasons)
results.push(matches)
for(let href of pages) {
//console.log(href)
matches = await ms.run(href)
//console.log(matches)
results.push(matches)
}
}
return results
}
TournamentScraper returns an array of objects, which typically looks like this:
{name: 'Foo', href: 'www.example.org/tournaments/foo/'}
The link points to the tournament's last season's first page. This page contains the links to the other seasons and a paginator (if any).
MatchScraper's run returns some data, and sets the instance's dom property. getPages() and getSeasons() consumes this property and each returns an array of links.
The problem that results contains only the first batch of matches. I can see the 2nd page's matches in the console log, but they are not in the results array when traverse returns.
I found this rule which is against await in for loop. The problem, that I have to wait for ms.run(href), because it sets dom, and getPages() and getSeasons() needs it to be set, to extract the needed links.
I think this should work. It utilizes Promise all rather than for loops
const run = href => ms.run(href);
async function getMatches(href) {
const out = [];
const matches = await run(href);
const pages = ms.getPages();
out.push(matches);
if(pages.length) {
const pageResults = await Promise.all(pages.map(href => run(href)));
out.push(...pageResults);
}
return out;
}
async function traverse(url) {
const ts = new TournamentScraper();
const ms = new MatchScraper();
const tournaments = await ts.run(url)
const matches = await Promise.all(tournaments.map(t => getMatches(t.href)));
return matches.reduce((a, b) => {
a.push(...b);
return a;
}, []);
}
Related
Trying to clear my search result after I submit a new API call. Tried implementing gallery.remove(galleryItems); at different points but to no avail.
A bit disappointed I couldn't figure it out but happy I was able to get a few async functions going. Anyway, here's the code:
'use strict';
const form = document.querySelector('#searchForm');
const gallery = document.querySelector('.flexbox-container');
const galleryItems = document.getElementsByClassName('flexbox-item');
form.addEventListener('submit', async (e) => {
e.preventDefault();
const userSearch = form.elements.query.value; // grab user input
const res = await getRequest(userSearch); // async func that returns a fully parsed Promise
tvShowMatches(res.data); // looks for matches, creates and appends name + image;
form.elements.query.value = '';
});
const getRequest = async (search) => {
const config = { params: { q: search } };
const res = await axios.get('http://api.tvmaze.com/search/shows', config);
return res;
};
const tvShowMatches = async (shows) => {
for (let result of shows) {
if (result.show.image) {
// new div w/ flexbox-item class + append to gallery
const tvShowMatch = document.createElement('DIV')
tvShowMatch.classList.add('flexbox-item');
gallery.append(tvShowMatch);
// create, fill & append tvShowName to tvShowMatch
const tvShowName = document.createElement('P');
tvShowName.textContent = result.show.name;
tvShowMatch.append(tvShowName);
// create, fill & append tvShowImg to tvShowMatch
const tvShowImg = document.createElement('IMG');
tvShowImg.src = result.show.image.medium;
tvShowMatch.append(tvShowImg);
}
}
};
Thanks
Instead of gallery.remove(galleryItems); consider resetting gallery.innerHTML to an empty string whenever a submit event occurs
Like this:
form.addEventListener('submit', async (e) => {
e.preventDefault();
gallery.innerHTML = ''; // Reset here
const userSearch = form.elements.query.value; // grab user input
const res = await getRequest(userSearch); // async func that returns a fully parsed Promise
tvShowMatches(res.data); // looks for matches, creates and appends name + image;
form.elements.query.value = '';
});
I believe this will do it.. you were close.
const galleryItems = document.getElementsByClassName('flexbox-item');
// to remove
galleryItems.forEach(elem => elem.remove() );
I´m trying to loop through the content of a DataSnapshot and then depending on a condition do some work FOR EACH one of the elements but currently, the ForEach is only doing the work in the first item. The "serverStatus" sometimes is waiting and sometimes in "onCall". When the first item is "onCall" does not go through the rest of the items as I think is supposed to do. Below a snapchot of where I get the information from:
And here is my function:
exports.manageCallRequests = functions.database.ref('/resquests/{userId}').onCreate((snap, context) => {
const event = snap.val();
console.log("function manageCallRequests is being called")
var rootPath = admin.database().ref();
var userOnCall = context.params.userId;
var serversRef = rootPath.child('servers');
var callRequest = event;
var userTime = callRequest["time"];
var waiting= "waiting";
//We first get all the servers in ascending order depending on the last time they were used
var serversSorted = serversRef.orderByChild('lastTimeUsed')
//Gets the children on the "serversSorted" Query
return serversSorted.once("value").then(allServers =>{
//Checks if there is any child
if(allServers.hasChildren()){
allServers.forEach(async function(server) {
//we extract the value from the server variable, this contains all the information
//about each one of the servers we have
var serverInfo = server.val();
var serverKey = server.key;
var serverNumber = serverInfo["serverNumber"];
var serverStatus = serverInfo["serverStatus"];
console.log("server status "+serverStatus)
if(serverStatus === waiting){
const setCallRequest = await serversRef.child(serverKey).child("current").child("callRequest").set(callRequest);
const removeUserOnCall = await rootPath.child("resquests").child(userOnCall).remove();
const setServerStatus = await serversRef.child(serverKey).child("serverStatus").set("onCall");
}
});
}else{
console.log("No servers available")
}
});
});
I had the same behavior because my cloud function was exited before that all iterations were executed in the forEach loop.I get rid of it using this snippet of code:
for (const doc of querySnapshot.docs) {
// Do wathever you want
// for instance:
await doc.ref.update(newData);
}
I found 2 ways of getting this done. The first one is useful if we have a DataSnapshot without any OrderBy* call, in this case, would be:
var allServers = await serversRef.once("value");
for (let serverKey of Object.keys(allServers.val())){
var server = allServers[serverKey];
//Do some work
}
We need to first get the keys of the object to then be able to extract it from within the for loop, as explained here otherwise we´ll get a "TypeError: 'x' is not iterable"
Now the problem with this particular case is that a have a DataSnapshot that was previously sorted at var serversSorted = serversRef.orderByChild('lastTimeUsed') so when we call Object.keys(allServers.val()) the value returned is no longer sorted and that´s where forEach() comes in handy. It guarantees the children of a DataSnapshot will be iterated in their query order as explained here however for some reasons when doing some async work within the forEach loop this seems not to work, that´s why I had to do this:
var serversSorted = serversRef.orderByChild('lastTimeUsed')
var allServers = await serversSorted.once("value");
//Checks if there is any children
if (allServers.hasChildren()) {
//if there is iterate through the event that was passed in containing all
// the servers
var alreadyOnCall = false;
var arrayOfServers = []
var arrayOfKeys = []
allServers.forEach(function(individualServer){
arrayOfKeys.push(individualServer.key)
arrayOfServers.push(individualServer)
})
for (var serveIndex = 0; serveIndex < arrayOfServers.length;serveIndex++){
var serverObj = arrayOfServers[serveIndex]
var serverObject = serverObj.val()
var serverKey = arrayOfKeys[serveIndex]
var serverStatus = serverObject["serverStatus"];
var serverNumber = serverObject["serverNumber"];
console.log("server info "+serverStatus+" "+serverKey);
if (serverStatus === waiting && alreadyOnCall === false) {
const setCallRequest = await serversRef.child(serverKey).child("current").child("callRequest").set(callRequest);
const removeUserOnCall = await rootPath.child("resquests").child(userOnCall).remove();
const setServerStatus = await serversRef.child(serverKey).child("serverStatus").set("onCall");
alreadyOnCall= true
console.log("Call properly set");
}
}
}
I have different document id for every loop and when I query inside the forEach loop query is working but not pushing the obj into the array
function getAllDonations() {
donations = [];
const user_session_data = sessionStorage.getItem('LoginInfo');
const parse_user_login_data = JSON.parse(user_session_data);
let TABLE_NAME = "donation_favourites";
let get_requests_qry = App.db.collection(TABLE_NAME);
get_requests_qry.where('user_id', '==', parse_user_login_data.user_id).get().then(snapshot => {
let changes = snapshot.docChanges();
changes.forEach(change => {
var one_item = change.doc.data();
let TABLE_NAME1 = "donation_requests";
let get_requests_qry1 = App.db.collection(TABLE_NAME1);
get_requests_qry1.doc(one_item.donationId).get().then(snapshot => {
donations.push(snapshot.data())
});
});
console.log("checking the data",donations.length) //this length is not coming
});
}
If you want to read the files in use forloop but it is not recommended for large loop for small loop it is ok
if you want to read files parallel use forEach
You can also do it with async and await instead forLoop
await Promise.all(changes.map(async (change) => {
var one_item = change.doc.data()
let TABLE_NAME1 = "donation_requests";
let get_requests_qry1 = App.db.collection(TABLE_NAME1);
var snapshot1 = await get_requests_qry1.doc(one_item.donationId).get()
donations.push(snapshot1.data())
}));
I have a problem with this piece of code.
I import input data from a file formated like so and store it in const input:
aabcccccaaa
aaccb
shsudbud
There are no spaces or any other white characters except from '\n' newline.
I get inputs in this way: (LiveServer inside VS Code)
const getData = async () => {
const resp = await fetch("./inputs.txt");
const data = await resp.text();
return data;
};
Then I call:
const myFunc = async () => {
const input = await getData();
const rows = input.split("\n").map(row => row);
rows.forEach(row => {
const charArr = [...row];
console.log(charArr);
});
};
After logging to console first and second row it seems like there is "" (empty string) attached to the end of each of them. The third element is fine so I guess its somehow connected with newline character.
I have also tried creating charArr by doing:
const charArr = Array.from(row);
Or
const charArr = row.split("");
But the outcome was the same.
Later I found this topic: Remove empty elements from an array in Javascript
So I tried:
const charArr = [...row].filter(Boolean);
But the "" is still at the end of charArr created from 1st and 2nd row.
const input = `aabcccccaaa
aaccb
shsudbud`;
const rows = input.split("\n").map(row => row);
rows.forEach(row => {
const charArr = [...row];
console.log(charArr);
});
In this snippet everything works fine. So here is where my questions start:
Why does .filter() method not work properly in this case?
Could this problem browser specific?
Thanks in advance.
I'm trying to scrape an old website built with tr, br and iframe. Everything was going good so far before I started to want to extract data from an iframe, see iFrameScraping setTimeout, but the clicking is too fast for me to be able to get the datas. Would anyone have an idea of how to click, wait for the content to show and be scraped, then continue?
const newResult = await page.evaluate(async(resultLength) => {
const elements = document.getElementsByClassName('class');
for(i = 0; i < resultLength; i++) {
const companyArray = elements[i].innerHTML.split('<br>');
let companyStreet,
companyPostalCode;
// Get company name
const memberNumber = elements[i].getElementsByTagName('a')[0].getAttribute('href').match(/[0-9]{1,5}/)[0];
const companyName = await companyArray[0].replace(/<a[^>]*><span[^>]*><\/span>/, '').replace(/<\/a>/, '');
const companyNumber = await companyArray[0].match(/[0-9]{6,8}/) ? companyArray[0].match(/[0-9]{6,8}/)[0] : '';
// Get town name
const companyTown = await companyArray[1].replace('"', '');
// Get region name
const companyRegion = await companyArray[2].replace(/<span[^>]*>Some text:<\/span>/, '');
// Get phone number
const telNumber = await elements[i].innerHTML.substring(elements[i].innerHTML.lastIndexOf('</span>')).replace('</span>', '').replace('<br>', '');
const iFrameScraping = await setTimeout(async({elements, i}) => {
elements[i].getElementsByTagName('a')[0].click();
const iFrameContent = await document.getElementById('some-id').contentWindow.document.getElementById('lblAdresse').innerHTML.split('<br>');
companyStreet = iFrameContent[0].replace('"', '');
companyPostalCode = iFrameContent[2].replace('"', '');
}, 2000, {elements, i});
console.log(companyStreet, companyPostalCode)
};
}, pageSearchResults.length);
I fixed my issues after a while, so I'll share my solution.
I add to stop getting all the data with a loop from the evaluate because it's going to fast and creating a race condition. Instead I used a combination of page.$$ coupled with a for…of loop. Note that the forEach from es6 are causing race condition as well, since puppeteer does not wait for them to end to continue its execution.
Here is the example from my updated code:
const companies = await page.$$('.repmbr_result_item');
const companiesLinks = await page.$$('.repmbr_result_item a');
for(company of companies) {
const companyEl = await page.evaluate(el => el.innerHTML, company)
const companyElArray = companyEl.split('<br>');