Develop Reference

Grab data from website HTML table and transfer to Google Sheets using App-Script

Ok, I know there are similar questions out there to mine, but so far I have yet to find any answers that work for me. What I am trying to do is gather data from an entire HTML table on the web (https://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html) and then parse it/transfer it to a range in my Google Sheet. The code below is probably the closest thing I've found so far because at least it doesn't error out, but it will only find one string or value, not the whole table. I've found other answers where they use xmlservice.parse, however that doesn't work for me, I believe because the HTML format has issues that it can't parse. Does anyone have an idea of how to edit what I have below, or a whole new idea that may work for this website?
function SAMPLE() {
const url="http://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html#sgl-basic?"
// Get all the static HTML text of the website
const res = UrlFetchApp.fetch(url, {muteHttpExceptions: true}).getContentText();
// Find the index of the string of the parameter we are searching for
index = res.search("td class");
// create a substring to only get the right number values ignoring all the HTML tags and classes
sub = res.substring(index+92,index+102);
Logger.log(sub);
return sub;
}
I understand that I can use importHTML natively in a Google Sheet, and that's what I'm currently doing. However I am doing this for over 350 webpage tables, and iterating through each one to load it and then copy the value to another sheet. App Script bogs down quite a bit when it is repeatedly waiting on Sheets to load an importHTMl and then grab some data and do it all over again on another url. I apologize for any formatting issues in this post or things I've done wrong, this is my first time posting here.
Edit: ok, I've found a method that works, but it's still much slower than I would like, because it is using Drive API to create a document with the HTML data and then parse and create an array from there. The Drive.Files.Insert line is the most time consuming part. Anyone have an idea of how to make this quicker? It may not seem that slow to you right now, but when I need to do this 350 times, it adds up.
function parseTablesFromHTML() {
var html = UrlFetchApp.fetch("https://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html");
var docId = Drive.Files.insert(
{ title: "temporalDocument", mimeType: MimeType.GOOGLE_DOCS },
html.getBlob()
).id;
var tables = DocumentApp.openById(docId)
.getBody()
.getTables();
var res = tables.map(function(table) {
var values = [];
for (var row = 0; row < table.getNumRows(); row++) {
var temp = [];
var cols = table.getRow(row);
for (var col = 0; col < cols.getNumCells(); col++) {
temp.push(cols.getCell(col).getText());
}
values.push(temp);
}
return values;
});
Drive.Files.remove(docId);
var range=SpreadsheetApp.getActive().getSheetByName("Test").getRange(3,6,res[0].length,res[0][0].length);
range.setValues(res[0]);
SpreadsheetApp.flush();
}

Solution by formula
Try
=importhtml(url,"table",1)
Other solution by script
function importTableHTML() {
var url = 'https://www.sports-reference.com/cbb/schools/indiana/2022-gamelogs.html'
var html = '<table' + UrlFetchApp.fetch(url, {muteHttpExceptions: true}).getContentText().replace(/(\r\n|\n|\r|\t| )/gm,"").match(/(?<=\<table).*(?=\<\/table)/g) + '</table>';
var trs = [...html.matchAll(/<tr[\s\S\w]+?<\/tr>/g)];
var data = [];
for (var i=0;i<trs.length;i++){
var tds = [...trs[i][0].matchAll(/<(td|th)[\s\S\w]+?<\/(td|th)>/g)];
var prov = [];
for (var j=0;j<tds.length;j++){
donnee=tds[j][0].match(/(?<=\>).*(?=\<\/)/g)[0];
prov.push(stripTags(donnee));
}
data.push(prov);
}
return(data);
}
function stripTags(body) {
var regex = /(<([^>]+)>)/ig;
return body.replace(regex,"");
}

restructure CSV data to create correct format in JSON

I'm working with some CSV data. Right now the CSV has a column called 'characteristic' which is one of three types, and a column called 'value', which contains the numerical value for the characteristic.
I'd like to change the structure of the data so that the columns are the characteristics themselves, and the values fall directly under those columns.
Here are screenshots of the tables, for clarity:
Currently:
What I'd like:
I changed things manually to give an example. The actual table I'll need to change is thousands of lines, so I'm hoping I can do this programmatically in some way.
The reason I need to restructure is that I need to transform the CSV to JSON, and the JSON needs to look like this:
[
{
"country":"afghanistan",
"iso3":"afg",
"first_indicator":3,
"second_indicator":5,
"third_indicator":3
},
{
"country":"united states",
"iso3":"usa",
"first_indicator":8,
"second_indicator":6,
"third_indicator":7
},
{
"country":"china",
"iso3":"chn",
"first_indicator":6,
"second_indicator":0.7,
"third_indicator":2
}
]
So - is there any way to take my CSV as it is now (first screenshot), and transform it to the JSON I want, without doing it all manually?
I've done a lot of searching, and I think maybe I just don't know what to search for. Ideally I would use javascript for this, but any suggestions welcome.
Thank you.

I made a JSFiddle for you, something like this should be what you want.
JavaScript
function Country(name, short){
this["country"] = name;
this["iso3"] = short;
}
function getCountryByName(name) {
for(var i = 0; i < countries.length; i++){
var country = countries[i];
if(country["country"] == name){
return country;
}
}
return null;
}
var csv = "country,shortname,characteristics,value\nafghanistan,afg,first_characteristic,3\nunited states,usa,first_characteristic,8\nchina,chn,first_characteristic,6\nafghanistan,afg,second_characteristic,5\nunited states,usa,second_characteristic,6\nchina,chn,second_characteristic,0.7\nafghanistan,afg,third_characteristic,3\nunited states,usa,third_characteristic,7\nchina,chn,third_characteristic,2"
var rows = csv.split("\n");
var countries = [];
if(rows.length > 0){
var header = rows[0];
var columns = header.split(",");
var countryIndex = columns.indexOf("country");
var shortnameIndex = columns.indexOf("shortname");
var characteristicsIndex = columns.indexOf("characteristics");
var valueIndex = columns.indexOf("value");
for(var i = 1; i < rows.length; i++) {
var row = rows[i];
var columns = row.split(",");
var name = columns[countryIndex];
var short = columns[shortnameIndex];
var characteristic = columns[characteristicsIndex];
var value = columns[valueIndex];
var country = getCountryByName(name);
if(!country){
country = new Country(name, short);
countries.push(country);
}
country[characteristic.replace("characteristic", "indicator")] = +value;
}
}
console.log(countries);
console.log(JSON.stringify(countries));
Output from the last line is this:
[{"country":"afghanistan","iso3":"afg","first_indicator":"3","second_indicator":"5","third_indicator":"3"},
{"country":"united states","iso3":"usa","first_indicator":"8","second_indicator":"6","third_indicator":"7"},
{"country":"china","iso3":"chn","first_indicator":"6","second_indicator":"0.7","third_indicator":"2"}]

My suggestion is to convert the CSV to JSON first. You can use an online tool.
When you have the JSON you can write a Javascript code to modify the JSON in the format you want.

How to find data type of each column from csv file using Java Script?

I have tried to find data-type of each column using Java Script, It prints string data-type for number,string and date.
I need each column data-type whether it is date or string or number or float.
Without CSV it prints correct data-type.
Working Example without CSV file
var num=15;
var str="xyz";
console.log(typeof num);
console.log(typeof str);
Output
number
string
But using csv file it prints string for all columns.
My CSV file is:
01/01/1991,12,xyz,14.4
01/01/1992,20,abc,20.5
01/02/1980,78,xy,60.8
I am using Papa Parse javascript plug-in for getting each column and then checking data-type.
How do I get each column's data-type from CSV file?
Is there any other way to parse CSV to array where we can traverse on particular row or column?

While you are loading your CSV in your javascript variable, that variable is a string.
You have to extract your csv into a JSON array and you have to cast each variable accordingly.
Code:
$.ajax({
type: "GET",
url: "sample.csv",
success: function (data) { loadData(data); }
});
/*var data = 'date,name,value\n\
01/01/1991,xyz,14.4\n\
01/01/1992,abc,20.5\n\
01/02/1980,xy,60.8';
loadData(data);*/
function loadData(data1){
console.log(data1);
var dataPoints = data1.split(/\r\n|\n/);
//console.log(lines);
var headers = dataPoints[0].split(','); // if you have header
var lines = [];
console.log(headers);
console.log(dataPoints.length);
for (var i = 1; i < dataPoints.length; i++) {
//console.log(i);
var point = dataPoints[i].split(',');
//console.log(point);
if (point.length == headers.length) {
var json= {};
for (var j = 0; j < headers.length; j++) {
if(headers[j]=='date'){
json[headers[j]] = new Date(point[j]);
}else if(headers[j]=='value'){
//console.log(point[j]);
json[headers[j]] = Number(point[j]);
}else{
json[headers[j]] = point[j];
}
}
lines.push(json);
console.log(json);
}
}
console.log(lines);
}
Sample CSV:
date,name,value
01/01/1991,xyz,14.4
01/01/1992,abc,20.5
01/02/1980,xy,60.8
Modified Code

Use Google Apps Script functions in project

I am very new to Google Apps Scripts and am curious how I can use functions created in my own project. For example, I have a script bound to a spreadsheet with just one function:
function addOrder(title, content) {
var sheet = SpreadsheetApp.getActiveSheet();
sheet.appendRow([ Date(), title, content]);
}
It simply takes 2 arguments and adds a row to the spreadsheet with that data. I have deployed it as a web app, but I'm not sure how to use this function in an environment like JSFiddle. Any help is appreciated.
Thanks

Spreadsheet bound scripts run server side and the SpreadsheetApp.getActiveSheet() method you are using will only work in the context of a spreadsheet bound script since it's the only case where the script actually "sees" an active spreadsheet. When you deploy this as a webapp you will have to tell the script which spreadsheet it must look at using for example the SpreadsheetApp.openById('spreadsheet ID') method.
But even doing so will not allow for using such a code outside of Google environment (as in JS fiddle for example) since SpreadsheetApp is specific to Google Apps service.
You have to remember that Google Apps Script is based on JavaScript but is not "plain" JavaScript , it uses a lot of specific services that work only in relation with Google Apps.
edit to answer your comment below :
the code used in the spreadsheet to work as a data server goes like this : (this is deployed as a webapp without user interface. It runs as a service
function doGet(e) {
if(e.parameter.mode==null){return ContentService.createTextOutput("error, wrong request").setMimeType(ContentService.MimeType.TEXT)};
var mode = e.parameter.mode;
var value = e.parameter.value;
var ss = SpreadsheetApp.openById('1yad5sZZt-X6bIftpR--OSyf3VZWf3Jxx8UJBhh7Arwg');
var sh = ss.getSheets()[0];
if(mode=='read'){
var sheetValues = sh.getDataRange().getValues();// get data from sheet
var valToReturn = ContentService.createTextOutput(JSON.stringify(sheetValues)).setMimeType(ContentService.MimeType.JSON);
return valToReturn;// send it as JSon string
}
if(mode=='write'){
var val = Utilities.base64Decode(value,Utilities.Charset.UTF_8);// decode base64 and get an array of numbers
Logger.log(val);// see it !
var stringVal = ''; // create an empty string
for(var n in val){
stringVal += String.fromCharCode(val[n]);// add each character in turn
}
var sheetValues = JSON.parse(stringVal);// convert the string into an object (2D array)
Logger.log(sheetValues);// check result
sh.getRange(1,1,sheetValues.length,sheetValues[0].length).setValues(sheetValues);// update the sheet
return ContentService.createTextOutput(JSON.stringify(sheetValues)).setMimeType(ContentService.MimeType.JSON);// send back the result as a string
}
return ContentService.createTextOutput('error').setMimeType(ContentService.MimeType.TEXT);// in case mode is not 'read' nor 'write'... should not happen !
}
you can call this service by its url + parameters and it will get / set values in the spreadsheet. This is a basic example but it works nicely.
below it the webapp code of the Ui that uses this service in this spreadsheet
var stylePanel = {'padding':'50px', 'background':'#FFA'};
var styleButton = {'padding':'5px', 'border-radius':'5px', 'borderWidth':'1px', 'borderColor':'#DDD','fontSize':'12pt'};
var styleTextItalic = {'fontSize':'12pt','fontStyle':'italic','fontFamily':'arial,sans-serif','color':'#F00'};
var styleTextNormal = {'fontSize':'12pt','fontStyle':'normal','fontFamily':'arial,sans-serif','color':'#00F'};
var styleLabel = {'fontSize':'12pt','color':'#F00'};
var url = 'https://script.google.com/macros/s/AKfycbwPioVjYMSrmhKnJOaF2GG83dnstLWI7isU9SF1vxPV8td-g9E7/exec';
var numRow = 21;// the number of rows in the grid = number of rows in the SS + 1
;
function doGet() {
var app = UiApp.createApplication().setTitle('url_fetch_demo');
var panel = app.createVerticalPanel().setStyleAttributes(stylePanel);
var headers = ['Field Name','Your answer'];// grid title
var grid = app.createGrid(numRow+2,2);// create the grid with right size
var wait = app.createImage('https://dl.dropboxusercontent.com/u/211279/loading3T.gif').setId('wait').setVisible(false);// get a spinner image in animated gif
var handlerWrite = app.createServerHandler('writeSheet').addCallbackElement(grid);// 2 handlers for the buttons
var handlerRead = app.createServerHandler('readSheet').addCallbackElement(grid);
var Chandler = app.createClientHandler().forTargets(wait).setVisible(true);// a client handler for the spinner
var buttonWrite = app.createButton('Write to Sheet',handlerWrite).addClickHandler(Chandler).setStyleAttributes(styleButton);
var buttonRead = app.createButton('Read from Sheet',handlerRead).addClickHandler(Chandler).setStyleAttributes(styleButton);
for(var n=1 ; n < numRow ; n++){
for(var m=0 ; m < 2 ; m++){ // create all the textBoxes with names & IDs
var textBox = app.createTextBox().setText('no value').setName('text'+n+'-'+m).setId('text'+n+'-'+m).setStyleAttributes(styleTextNormal);
//if(m==0){textBox.setEnabled(false)};// prevent writing to left column (optional)
grid.setWidget(n,m,textBox);// place widgets
}
}
grid.setWidget(numRow,0,buttonRead).setWidget(numRow,1,buttonWrite).setWidget(numRow+1,1,wait) // place buttons
.setWidget(0,0,app.createLabel(headers[0]).setStyleAttributes(styleLabel)) // and headers
.setWidget(0,1,app.createLabel(headers[1]).setStyleAttributes(styleLabel));
app.add(panel.add(grid));
return app; // show Ui
}
function writeSheet(e){
var app = UiApp.getActiveApplication();
app.getElementById('wait').setVisible(false);// spinner will be hidden when fct returns
var dataArrayImage = [];// an array to get typed values
for(var n=1 ; n < numRow ; n++){
var row=[];
for(var m=0 ; m < 2 ; m++){
row.push(e.parameter['text'+n+'-'+m]); // get every value in every "cell"
var textBox = app.getElementById('text'+n+'-'+m).setStyleAttributes(styleTextItalic);// update "cells" style
//textBox.setText('written value = '+e.parameter['text'+n+'-'+m]);// rewrite to the cells - not usefull but serves to check while debugging
}
dataArrayImage.push(row);// store one row(=2cells)
}
var UiValues = JSON.stringify(dataArrayImage);// stringfy the array
var newValues = url+'?mode=write&value='+Utilities.base64Encode(UiValues,Utilities.Charset.UTF_8);// add to url & parameters+ encode in pure ASCII characters
Logger.log(newValues);// check in logger
var check = UrlFetchApp.fetch(newValues).getContent();// get back the result
Logger.log(check);// check result = newValues sent back in bytes format
return app;//update Ui
}
function readSheet(e){
var app = UiApp.getActiveApplication();
app.getElementById('wait').setVisible(false);
var returnedValue = UrlFetchApp.fetch(url+'?mode=read').getContentText();// get data from server
Logger.log(returnedValue);// check values
var sheetValues = JSON.parse(returnedValue);
for(var n=1 ; n < numRow ; n++){
for(var m=0 ; m < 2 ; m++){
var textBox = app.getElementById('text'+n+'-'+m).setStyleAttributes(styleTextNormal);
textBox.setText(sheetValues[n-1][m]);// iterate and update cells values
}
}
return app;// update Ui
}

javascript internet explorer byte order mark

I am working on a way to save an HTML table to a csv file. Ideally, this should be cross-browser, and I have gotten this to work on everything but Internet Explorer. However, I have gotten the obvious parts working. What remains is that I am unable to get a working csv file from my JavaScript because a byte order mark is prepended to the data I wish to download.
I have confirmed that this is the case by downloading the csv file in IE and everything else and used a hex editor to view the raw file, I can confirm the file that Internet Explorer downloaded prepends the unicode character "FFFE".
Please see the code below this does this. saveTable takes an "<a>" node that is located inside a table.
If anyone can help me disgnose the issue and offer some solution I'd be grateful. Please forgive any faux pas on my part, I don't think I've ever used a site of this nature before. So if you need me to provide any further information please do just let me know and I shall do my best to get it on here.
function findTable(node) { // Finds a nodes parent table.
return (node.nodeName !== "TABLE") ? findTable(node.parentNode) : node;
}
function saveTable(node) {
var csv = [];
var table = findTable(node);
var rows = table.getElementsByTagName("tr");
var header = [];
var csv = [];
for (var i = 0; i < rows.length; i++) {
if (i == 0) {
// Do csv stuff.
var dates = rows[i].getElementsByTagName("th");
for (var j = 0; j < dates.length; j++)
(j == 0) ? header.push("") : header.push(dates[j].innerHTML);
csv.push(header.join(","));
}
else {
var rowArray = [];
var jobName = rows[i].getElementsByTagName("th")[0].innerHTML;
var times = rows[i].getElementsByTagName("td");
rowArray.push(jobName);
for (var k = 0; k < times.length; k++)
rowArray.push(times[k].innerHTML);
csv.push(rowArray.join(","));
}
}
node.setAttribute("href", "data:text/csv;charset=utf-8," + csv.join("%0A"));
var fileName = "spreadsheet_data-" + (new Date).getTime() + ".csv";
if (node.download == "")
node.setAttribute("download", fileName);
else {
alert("Handle IE here!");
var bom = "\uFFFE";
var doc = document.open("application/octet-stream", "_blank");
var data = csv.join("\r\n");
doc.charset = "UTF-8";
doc.write(data.replace(bom, ""));
doc.focus();
doc.execCommand('SaveAs', false, fileName);
doc.close();
}
}
Table example, it's not the way I would have chosen to do it myself, but it's how the table is generated by another piece of software.
<table id='results' border='1'>
<tr><th><a href='#' onClick='saveTable(this);' id='download_link'>Download data</a></th><th>2013/05/09</th><th>2013/05/10</th><th>2013/05/10</th><th>2013/05/10</th><th>2013/05/10</th></tr>
<tr>
<th>\PDF\EXOVIGN.PDF</th><td>8.853</td><td>9.050</td><td>8.807</td><td>8.827</td><td>8.835</td></tr>
</table>

If you have no absolute requirement to do this client-side, it might save you a lot of hassle to send the file from the server instead.