Develop Reference

Grab data from website HTML table and transfer to Google Sheets using App-Script

Ok, I know there are similar questions out there to mine, but so far I have yet to find any answers that work for me. What I am trying to do is gather data from an entire HTML table on the web (https://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html) and then parse it/transfer it to a range in my Google Sheet. The code below is probably the closest thing I've found so far because at least it doesn't error out, but it will only find one string or value, not the whole table. I've found other answers where they use xmlservice.parse, however that doesn't work for me, I believe because the HTML format has issues that it can't parse. Does anyone have an idea of how to edit what I have below, or a whole new idea that may work for this website?
function SAMPLE() {
const url="http://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html#sgl-basic?"
// Get all the static HTML text of the website
const res = UrlFetchApp.fetch(url, {muteHttpExceptions: true}).getContentText();
// Find the index of the string of the parameter we are searching for
index = res.search("td class");
// create a substring to only get the right number values ignoring all the HTML tags and classes
sub = res.substring(index+92,index+102);
Logger.log(sub);
return sub;
}
I understand that I can use importHTML natively in a Google Sheet, and that's what I'm currently doing. However I am doing this for over 350 webpage tables, and iterating through each one to load it and then copy the value to another sheet. App Script bogs down quite a bit when it is repeatedly waiting on Sheets to load an importHTMl and then grab some data and do it all over again on another url. I apologize for any formatting issues in this post or things I've done wrong, this is my first time posting here.
Edit: ok, I've found a method that works, but it's still much slower than I would like, because it is using Drive API to create a document with the HTML data and then parse and create an array from there. The Drive.Files.Insert line is the most time consuming part. Anyone have an idea of how to make this quicker? It may not seem that slow to you right now, but when I need to do this 350 times, it adds up.
function parseTablesFromHTML() {
var html = UrlFetchApp.fetch("https://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html");
var docId = Drive.Files.insert(
{ title: "temporalDocument", mimeType: MimeType.GOOGLE_DOCS },
html.getBlob()
).id;
var tables = DocumentApp.openById(docId)
.getBody()
.getTables();
var res = tables.map(function(table) {
var values = [];
for (var row = 0; row < table.getNumRows(); row++) {
var temp = [];
var cols = table.getRow(row);
for (var col = 0; col < cols.getNumCells(); col++) {
temp.push(cols.getCell(col).getText());
}
values.push(temp);
}
return values;
});
Drive.Files.remove(docId);
var range=SpreadsheetApp.getActive().getSheetByName("Test").getRange(3,6,res[0].length,res[0][0].length);
range.setValues(res[0]);
SpreadsheetApp.flush();
}

Solution by formula
Try
=importhtml(url,"table",1)
Other solution by script
function importTableHTML() {
var url = 'https://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html'
var html = '<table' + UrlFetchApp.fetch(url, {muteHttpExceptions: true}).getContentText().replace(/(\r\n|\n|\r|\t| )/gm,"").match(/(?<=\<table).*(?=\<\/table)/g) + '</table>';
var trs = [...html.matchAll(/<tr[\s\S\w]+?<\/tr>/g)];
var data = [];
for (var i=0;i<trs.length;i++){
var tds = [...trs[i][0].matchAll(/<(td|th)[\s\S\w]+?<\/(td|th)>/g)];
var prov = [];
for (var j=0;j<tds.length;j++){
donnee=tds[j][0].match(/(?<=\>).*(?=\<\/)/g)[0];
prov.push(stripTags(donnee));
}
data.push(prov);
}
return(data);
}
function stripTags(body) {
var regex = /(<([^>]+)>)/ig;
return body.replace(regex,"");
}

Javascript performance optimization

I created the following js function
function csvDecode(csvRecordsList)
{
var cel;
var chk;
var chkACB;
var chkAF;
var chkAMR;
var chkAN;
var csvField;
var csvFieldLen;
var csvFieldsList;
var csvRow;
var csvRowLen = csvRecordsList.length;
var frag = document.createDocumentFragment();
var injectFragInTbody = function () {tblbody.replaceChild(frag, tblbody.firstElementChild);};
var isFirstRec;
var len;
var newEmbtyRow;
var objCells;
var parReEx = new RegExp(myCsvParag, 'ig');
var tblbody;
var tblCount = 0;
var tgtTblBodyID;
for (csvRow = 0; csvRow < csvRowLen; csvRow++)
{
if (csvRecordsList[csvRow].startsWith(myTBodySep))
{
if (frag.childElementCount > 0)
{
injectFragInTbody();
}
tgtTblBodyID = csvRecordsList[csvRow].split(myTBodySep)[1];
newEmbtyRow = getNewEmptyRow(tgtTblBodyID);
objCells = newEmbtyRow.cells;
len = newEmbtyRow.querySelectorAll('input')[0].parentNode.cellIndex; // Finds the cell index where is placed the first input (Check-box or button)
tblbody = getElById(tgtTblBodyID);
chkAF = toBool(tblbody.dataset.acceptfiles);
chkACB = toBool(tblbody.dataset.acceptcheckboxes) ;
chkAN = toBool(tblbody.dataset.acceptmultiplerows) ;
tblCount++;
continue;
}
csvRecordsList[csvRow] = csvRecordsList[csvRow].replace(parReEx, myInnerHTMLParag); // Replaces all the paragraph symbols ¶ used into the db.csv file with the tag <br> needed into the HTML content of table cells, this way will be possible to use line breaks into table cells
csvFieldsList = csvRecordsList[csvRow].split(myEndOfFld);
csvFieldLen = csvFieldsList.length;
for (csvField = 0; csvField < csvFieldLen; csvField++)
{
cel = chkAN ? csvField + 1 : csvField;
if (chkAF && cel === 1) {objCells[cel].innerHTML = makeFileLink(csvFieldsList[csvField]);}
else if (chkACB && cel === len) {objCells[cel].firstChild.checked = toBool(csvFieldsList[csvField]);}
else {objCells[cel].innerHTML = csvFieldsList[csvField];}
}
frag.appendChild(newEmbtyRow.cloneNode(true));
}
injectFragInTbody();
var recNum = getElById(tgtTblBodyID).childElementCount;
customizeHtmlTitle();
return csvRow - tblCount + ' (di cui '+ recNum + ' record di documenti)';
}
More than 90% of records could contain file names that have to be processed by the following makeFileLink function:
function makeFileLink(fname)
{
return ['<a href="', dirDocSan, fname, '" target="', previewWinName, '" title="Apri il file allegato: ', fname, '" >', fname, '</a>'].join('');
}
It aims to decode a record list from a special type of *.db.csv file (= a comma-separated values where commas are replaced by another symbol I hard-coded into the var myEndOfFld). (This special type of *.db.csv is created by another function I wrote and it is just a "text" file).
The record list to decode and append to HTML tables is passed to the function with its lone parameter: (csvRecordsList).
Into the csv file is hosted data coming from more HTML tables.
Tables are different for number of rows and columns and for some other contained data type (which could be filenames, numbers, string, dates, checkbox values).
Some tables could be just 1 row, others accept more rows.
A row of data has the following basic structure:
data field content 1|data field content 2|data field content 3|etc...
Once decoded by my algorithm it will be rendered correctly into the HTML td element even if into a field there are more paragraphs. In fact the tag will be added where is needed by the code:
csvRecordsList[csvRow].replace(par, myInnerHTMLParag)
that replaces all the char I choose to represent the paragraph symbol I have hard-coded into the variable myCsvParag.
Isn't possible to know at programming time the number of records to load in each table nor the number of records loaded from the CSV file, nor the number of fields of each record or what table field is going to contain data or will be empty: in the same record some fields could contain data others could be empty. Everything has to be discovered at runtime.
Into the special csv file each table is separated from the next by a row witch contains just a string with the following pattern: myTBodySep = tablebodyid where myTBodySep = "targettbodydatatable" that is just a hard coded string of my choice.
tablebodyid is just a placeholder that contains a string representing the id of the target table tbody element to insert new record in, for example: tBodyDataCars, tBodyDataAnimals... etc.
So when the first for loop finds into the csvRecordsList a string staring with the string into the variable myTBodySep it gets the tablebodyid from the same row: this will be the new tbodyid that has to be targeted for injecting next records in it
Each table is archived into the CSV file
The first for loop scan the csv record list from the file and the second for loop prepare what is needed to compile the targeted table with data.
The above code works well but it is a little bit slow: in fact to load into the HTML tables about 300 records from the CSV file it takes a bit more of 2.5 seconds on a computer with 2 GB ram and Pentium core 2 4300 dual-core at 1800 MHz but if I comment the row that update the DOM the function needs less than 0.1 sec. So IMHO the bottle neck is the fragment and DOM manipulating part of the code.
My aim and hope is to optimize the speed of the above code without losing functionalities.
Notice that I'm targeting just modern browsers and I don't care about others and non standards-compliant browsers... I feel sorry for them...
Any suggestions?
Thanks in advance.
Edit 16-02.2018
I don't know if it is useful but lastly I've noticed that if data is loaded from browser sessionstorage the load and rendering time is more or less halved. But strangely it is the exact same function that loads data from both file and sessionstorage.
I don't understand why of this different behavior considering that the data is exactly the same and in both cases is passed to a variable handled by the function itself before starting checking performance timing.
Edit 18.02.2018
Number of rows is variable depending on the target table: from 1 to 1000 (could be even more in particular cases)
Number of columns depending on the target table: from 10 to 18-20

In fact, building the table using DOM manipulations are way slower than simple innerHTML update of the table element.
And if you tried to rewrite your code to prepare a html string and put it into the table's innerHTML you would see a significant performance boost.
Browsers are optimized to parse the text/html which they receive from the server as it's their main purpose. DOM manipulations via JS are secondary, so they are not so optimized.
I've made a simple benchmark for you.
Lets make a table 300x300 and fill 90000 cells with 'A'.
There are two functions.
The first one is a simplified variant of your code which uses DOM methods:
var table = document.querySelector('table tbody');
var cells_in_row = 300, rows_total = 300;
var start = performance.now();
fill_table_1();
console.log('using DOM methods: ' + (performance.now() - start).toFixed(2) + 'ms');
table.innerHTML = '<tbody></tbody>';
function fill_table_1() {
var frag = document.createDocumentFragment();
var injectFragInTbody = function() {
table.replaceChild(frag, table.firstElementChild)
}
var getNewEmptyRow = function() {
var row = table.firstElementChild;
if (!row) {
row = table.insertRow(0);
for (var c = 0; c < cells_in_row; c++) row.insertCell(c);
}
return row.cloneNode(true);
}
for (var r = 0; r < rows_total; r++) {
var new_row = getNewEmptyRow();
var cells = new_row.cells;
for (var c = 0; c < cells_in_row; c++) cells[c].innerHTML = 'A';
frag.appendChild(new_row.cloneNode(true));
}
injectFragInTbody();
return false;
}
<table><tbody></tbody></table>
The second one prepares html string and put it into the table's innerHTML:
var table = document.querySelector('table tbody');
var cells_in_row = 300, rows_total = 300;
var start = performance.now();
fill_table_2();
console.log('setting innerHTML: ' + (performance.now() - start).toFixed(2) + 'ms');
table.innerHTML = '<tbody></tbody>';
function fill_table_2() {// setting innerHTML
var html = '';
for (var r = 0; r < rows_total; r++) {
html += '<tr>';
for (var c = 0; c < cells_in_row; c++) html += '<td>A</td>';
html += '</tr>';
}
table.innerHTML = html;
return false;
}
<table><tbody></tbody></table>
I believe you'll come to some conclusions.

I've got two thoughts for you.
1: If you want to know which parts of your code are (relatively) slow you can do very simple performance testing using the technique described here. I didn't read all of the code sample you gave but you can add those performance tests yourself and check out which operations take more time.
2: What I know of JavaScript and the browser is that changing the DOM is an expensive operation, you don't want to change the DOM too many times. What you can do instead is build up a set of changes and then apply all those changes with one DOM change. This may make your code less nice, but that's often the tradeoff you have when you want to have high performance.
Let me know how this works out for you.

You should start by refactoring your code in multiples functions to make it a bit more readable. Make sure that you are separating DOM manipulation functions from data processing functions. Ideally, create a class and get those variables out of your function, this way you can access them with this.
Then, you should execute each function processing data in a web worker, so you're sure that your UI won't get blocked by the process. You won't be able to access this in a web worker so you will have to limit it to pure "input/output" operations.
You can also use promises instead of homemade callbacks. It makes the code a bit more readable, and honestly easier to debug. You can do some cool stuff like :
this.processThis('hello').then((resultThis) => {
this.processThat(resultThis).then((resultThat) => {
this.displayUI(resultThat);
}, (error) => {
this.errorController.show(error); //processThat error
});
}, (error) => {
this.errorController.show(error); //processThis error
});
Good luck!

How to place images on top of table in html?

Here I'm developing snakes and ladder game.
First I have developed game board and placed numbers on that, but I don't know how to put snakes and ladder on some positions.
Could anybody guide me please?
function loadgameboard() {
var numbers = 64;
var tablestart = "<table id='contentTable' width='35%' background='' height='74%' bgcolor='#EFE8BB' border=1>";
var tableend = "</table>";
var trstart = "<tr>";
var trend = "</tr>";
var tdstart = "<td align='middle' >";
var tdend = "</td>";
var fontsizeStart = "<font size='5'>";
var fontSizeEnd = "</font>";
var numberOfRowColumns = Math.sqrt(numbers);
document.write(tablestart);
for (var i = numberOfRowColumns; i > 0; i--) {
document.write(trstart);
for (var j = numberOfRowColumns; j > 0; j--) {
document.write(tdstart);
document.write(fontsizeStart);
document.write(numbers);
document.write(fontSizeEnd);
document.write(tdend);
numbers--;
}
document.write(trend);
}
document.write(tableend);
}
window.onload = loadgameboard;
Suppose if I want to place one snake and one image, how can I do it?
Your help will be appreciated.

Your tdstart variable should include an id field, such as: var tdstart = "<td align='middle' id="; Don't set the id here, or close the tag yet. In the loop where you create the table, do something like this: document.write(tdstart+ "'"+i+j+"'>"); That gives every data-cell a unique ID and also closes the tag. After that, at any time you like, you can set some other variable, perhaps one named tcell to any of the cells in the table: tcell=document.getElementById('15'); where the 15 specifies a row and a column; if you have more than 10 of each, you might use letters of the alphabet instead of digits--just remember to assign letters instead of numbers i and j to the id= part of your main loop. That could be done by
document.write(tdstart+ "'"+"ABCDEFGHIJKLMNOP".substr(i,1)+"ABCDEFGHIJKLMNOP".substr(j,1)+"'>");
Use more letters as needed, and remember lowercase is different from uppercase, you can handle a 52x52 table this way.
Once you have access to the cell, you can edit its content, such as
tcell.innerHTML="64"+"<img src='ladder.png' />"
The 64 is the original number you put in the cell, in your loop. You can change the number, or put it after the image, or both. And of course you can remove the image: tcell.innerHTML="64";

Add numbers from dynamic site with new page load

How do you write a function/ listener in javascript that can fire when html updates?
html page is limited to the last 100 records
html page continuously adds new records (nodes)
I need to...
push the values into an array or increment a variable to sum the values
display the running total in html
Trying to create a counter that adds new values to the sum.
I believe the code below only executes once.
window.onload = function() {
var data = document.getElementsByTagName('span');
var len = data.length;
var total = 0;
var searchValue = "value";
for (var i = 0; i < len; ++i) {
var styles = data[i].getAttribute('style');
if (styles == searchValue) {
var txt = data[i].innerHTML;
split = txt.split(" ");
total += parseInt(split[2]);
}
}
console.log(total);
};

Yes you can do this with just JavaScript, and a little HTML/CSS for displaying the number.
Note that it requires a fair deal of experience to manage an addon with cross-origin requests, and to show it to the user.

Object - An Object in the Object - An array of those Objects

I'm new to javascript so let me just say that right up front.
A web site I frequent has 50 or so items, with details about that item, in a table. Each table row contains several td cells. Some rows have types of things that are similar, like USB drives or whatever. I want to capture each row so that I can group and reorder them to suit my tastes.
I have this object:
function vnlItemOnPage(){
this.Category = "unknown";
this.ItemClass = "vnlDefaultClass";
this.ItemBlock = {};
}
This represents one row.
What I've been trying to figure out is how to capture the block of html < tr>stuff< /tr> and save it into this.ItemBlock.
That part is pretty easy:
vnlItemOnPage.ItemBlock = element.getElementByClassName('className')[0]
?
That seems pretty straight forward. Am I missing something?
This part I am stuck:
There'll be 50 of them so I need an array of vnlItemOnPage?
vnlAllItems = ???
var vnlAllItems = [vnlItemOnPage]?
And, how would I add to the array and delete from the array? I probably wont delete from the array if that is complicated don't bother with it.
Once I capture the < tr> html, I can just append it to a table element like so:
myTable.appendChild(vnlAllItems[0].ItemBlock);
Correct?
I'm open to any suggestions if you think I'm approaching this from the wrong direction. Performance is not a big issue - at least right now. Later I may try to conflate several pages for a couple hundred items.
Thanks for your assistance!
[edit]
Perhaps the second part of the question is so basic it's hard to believe I don't know the answer.
The array could be: var vnlAllItems = []
And then it is just:
var row1 = new vnlItemOnPage;
vnlAllItems.push(row1);
var row2 = new vnlItemOnPage;
row2.ItemBlock = element.getElementByClassName('className')[0];
I'd like to close the question but I hate to do that without something about handling the array.

JQuery is your friend here.
This will give you the inner HTML for the first row in the body of your desired table:
var rowHtml = $('table#id-of-desired-table tbody tr:first').html() ;
To get the outer HTML, you need a jQuery extension method:
jQuery.fn.outerHTML = function() {
return $('<div>').append( this.eq(0).clone() ).html();
};
Usage is simple:
var rowHtml = $('table#id-of-desired-table tbody tr:first').outerHtml() ;
Enjoy!

Not sure if it is what you are looking for, but if I wanted to manipulate table rows I would store:
Row's whole html <td>1</td>...<td>n</td> as string so I can quickly reconstruct the row
For each row store actual cell values [1, ..., n], so I can do some manipulations with values (sort)
To get row as html you can use:
var rowHtml = element.getElementByClassName('className')[0].innerHTML;
To get array of cell values you can use:
var cells = [];
var cellElements = element.getElementByClassName('className')[0].cells;
for(var i=0;i<cellElements.length;i++) {
cells.push(cellElements[i].innerText);
}
So the object to store all this would look something like:
function vnlItemOnPage(){
this.Category = "unknown";
this.ItemClass = "vnlDefaultClass";
this.RowHtml = "";
this.RowCells = [];
}