Develop Reference

Google Apps Script - Improve Script to Execute Faster

I have a sheet with over 3000 points (lat, long). I am running the following code to calculate distance of each point from 9 reference points and writing the minimum distance next to each point in the sheet. The code runs fine but is slow and times out before running for all the points. How can I make the following code more efficient - so that it reads, computes and writes back the data faster?
/*
******************************
Reference Points
Point_A 41.166866 -76.151926
Point_B 41.087500 -76.204694
Point_C 40.540960 -75.704900
Point_D 40.401080 -75.589600
Point_E 40.326130 -75.642500
Point_F 40.167480 -75.921500
Point_G 40.093370 -76.084700
Point_H 39.974450 -76.063000
Point_I 39.722350 -76.156000
********************************
*/
var SpreadsheetID = "ABC";
var SheetName = "XYZ";
function DistanceCalculator() {
var ss = SpreadsheetApp.openById(SpreadsheetID)
var sheet = ss.getSheetByName(SheetName);
//Select the column
var columnToCheck = sheet.getRange("A:A").getValues();
// Get the last row in the column
var lastRow = getLastRowSpecial(columnToCheck);
// Ref point latitude
var Latitude_Point_A = 41.166866;
var Latitude_Point_B = 41.087500;
var Latitude_Point_C = 40.540960;
var Latitude_Point_D = 40.401080;
var Latitude_Point_E = 40.326130;
var Latitude_Point_F = 40.167480;
var Latitude_Point_G = 40.093370;
var Latitude_Point_H = 39.974450;
var Latitude_Point_I = 39.722350;
// Ref point longitude
var Longitude_Point_A = -76.151926;
var Longitude_Point_B = -76.204694;
var Longitude_Point_C = -75.704900;
var Longitude_Point_D = -75.589600;
var Longitude_Point_E = -75.642500;
var Longitude_Point_F = -75.921500;
var Longitude_Point_G = -76.084700;
var Longitude_Point_H = -76.084700;
var Longitude_Point_I = -76.156000;
for( var i=1; i<=lastRow;i++){
//Reading Lat_Long from the sheet
var Lat_Check = sheet.getRange(i,2).getValue();
var Long_Check = sheet.getRange(i,3).getValue();
//Calculating distance between each point and reference points
var distance01 = calculateDistance(Latitude_Point_A,Longitude_Point_A,Lat_Check,Long_Check);
var distance02 = calculateDistance(Latitude_Point_B,Longitude_Point_B,Lat_Check,Long_Check);
var distance03 = calculateDistance(Latitude_Point_C,Longitude_Point_C,Lat_Check,Long_Check);
var distance04 = calculateDistance(Latitude_Point_D,Longitude_Point_D,Lat_Check,Long_Check);
var distance05 = calculateDistance(Latitude_Point_E,Longitude_Point_E,Lat_Check,Long_Check);
var distance06 = calculateDistance(Latitude_Point_F,Longitude_Point_F,Lat_Check,Long_Check);
var distance07 = calculateDistance(Latitude_Point_G,Longitude_Point_G,Lat_Check,Long_Check);
var distance08 = calculateDistance(Latitude_Point_H,Longitude_Point_H,Lat_Check,Long_Check);
var distance09 = calculateDistance(Latitude_Point_I,Longitude_Point_I,Lat_Check,Long_Check);
//measuring minimum distance
var minimumDistance = Math.round(Math.min(distance01,distance02,distance03,distance04,distance05,distance06,distance07,distance08,distance09))
sheet.getRange(i,4).setValue(minimumDistance);
}
}
function calculateDistance(Ref_Lat, Ref_Long, Looped_Lat, Looped_Long){
var p = 0.017453292519943295; // Math.PI / 180
var c = Math.cos;
var a = 0.5 - c((Looped_Lat - Ref_Lat) * p)/2 +
c(Ref_Lat * p) * c(Looped_Lat * p) *
(1 - c((Looped_Long - Ref_Long) * p))/2;
var result = 7926 * Math.asin(Math.sqrt(a)); // 2 * R; R = 3963 miles
return result
};
function getLastRowSpecial(range){
var rowNum = 0;
var blank = false;
for(var row = 0; row < range.length; row++){
if(range[row][0] === "" && !blank){
rowNum = row;
blank = true;
}else if(range[row][0] !== ""){
blank = false;
};
};
return rowNum;
};
Sample Spreadsheet Screenshot

I believe your goal is as follows.
You want to reduce the process cost of your script.
In your script, getValue and setValue are used in a loop. I thought that this might be the reason for your issue. In this case, at first, the values are retrieved from the sheet and the values are processed, then, the values are put to the sheet. This flow is suitable for reducing the process cost. I think that "Best Practice" will be useful for understanding your issue. Ref
Also, I thought that when the process cost of your script is reduced, a little ingenuity might be required to be used for modifying your script. So I propose one of several modified scripts as a reference. Please modify your script as follows.
From:
for( var i=0; i<lastRow;i++){
//Reading Lat_Long from the sheet
var Lat_Check = sheet.getRange(i,2).getValue();
var Long_Check = sheet.getRange(i,3).getValue();
//Calculating distance between each point and reference points
var distance01 = calculateDistance(Latitude_Point_A,Longitude_Point_A,Lat_Check,Long_Check);
var distance02 = calculateDistance(Latitude_Point_B,Longitude_Point_B,Lat_Check,Long_Check);
var distance03 = calculateDistance(Latitude_Point_C,Longitude_Point_C,Lat_Check,Long_Check);
var distance04 = calculateDistance(Latitude_Point_D,Longitude_Point_D,Lat_Check,Long_Check);
var distance05 = calculateDistance(Latitude_Point_E,Longitude_Point_E,Lat_Check,Long_Check);
var distance06 = calculateDistance(Latitude_Point_F,Longitude_Point_F,Lat_Check,Long_Check);
var distance07 = calculateDistance(Latitude_Point_G,Longitude_Point_G,Lat_Check,Long_Check);
var distance08 = calculateDistance(Latitude_Point_H,Longitude_Point_H,Lat_Check,Long_Check);
var distance09 = calculateDistance(Latitude_Point_I,Longitude_Point_I,Lat_Check,Long_Check);
//measuring minimum distance
var minimumDistance = Math.round(Math.min(distance01,distance02,distance03,distance04,distance05,distance06,distance07,distance08,distance09))
sheet.getRange(i,4).setValue(minimumDistance);
}
To:
var values = sheet.getRange("B1:C" + lastRow).getValues();
var ar = [[Latitude_Point_A, Longitude_Point_A], [Latitude_Point_B, Longitude_Point_B], [Latitude_Point_C, Longitude_Point_C], [Latitude_Point_D, Longitude_Point_D], [Latitude_Point_E, Longitude_Point_E], [Latitude_Point_F, Longitude_Point_F], [Latitude_Point_G, Longitude_Point_G], [Latitude_Point_H, Longitude_Point_H], [Latitude_Point_I, Longitude_Point_I]];
var res = values.map(([b, c]) => [Math.round(Math.min(...ar.map(e => calculateDistance(...e, b, c))))]);
sheet.getRange(1, 4, res.length).setValues(res);
In this modification, from your script, it supposes that the 1st row is not the header row. Please be careful about this.
In this modification, the values are retrieved from the sheet by getValues, and then, the retrieved values are calculated and put to the sheet by setValues.
Note:
When I saw your script, I thought that an error occurs at var Lat_Check = sheet.getRange(i,2).getValue(); because the 1st value of i is 0. And in the function calculateDistance, Ref_Lat, Ref_Long are not used. So I'm worried that your showing script might be different from the script that you have the issue with. So, when you test the above modification, when an error occurs, can you confirm whether your showing script is the correct script for your situation. And, please provide the sample Spreadsheet for replicating the issue as the image. By this, I would like to confirm it.
References:
map()
getValues()
setValues(values)

Google scripts: How to put a range in setValue

I have used the example code from this link, to make the code below.
https://yagisanatode.com/2017/12/13/google-apps-script-iterating-through-ranges-in-sheets-the-right-and-wrong-way/
Most of the code is working as expected, except for the last row.
In Column E, I want to place the custom function =apiav() with the data from cell A.
However the code is returning =apiav(Range) in the Google sheet cells. (to be clear it should be something like =apiav(a1))
I tried everything i could think of and of course googled for hours, but i am really lost and can't find the right solution for this.
function energy(){
var sector = "Energy";
var ss=SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getActiveSheet();
var rangeData = sheet.getDataRange();
var lastColumn = 2;
var lastRow = 999 ;
var sheet = ss.getSheets()[0];
var searchRange = sheet.getRange(2,2, lastRow-1 ,1 );
var ouputrange = sheet.getRange(2,4, lastRow-1 ,1 );
//clear range first
ouputrange.clear("D:D");
ouputrange.clear("E:E");
/*
GOOD - Create a client-side array of the relevant data
*/
// Get array of values in the search Range
var rangeValues = searchRange.getValues();
// Loop through array and if condition met, add relevant
// background color.
for ( i = 0; i < lastColumn - 1; i++){
for ( j = 0 ; j < lastRow - 1; j++){
if(rangeValues[j][i] === sector){
sheet.getRange(j+2,i+4).setValue("yes");
var formularange = sheet.getRange (j+2,i+1);
sheet.getRange(j+2,i+5).setValue('=apiav(' + formularange + ')');
}
};
};
};

Replace:
var formularange = sheet.getRange(j+2,i+1);
with:
var formularange = sheet.getRange(j+2,i+1).getA1Notation();
So you will be able to pass the cell reference instead of the range object.

ReDash to Google Sheets using Google Script - How do I add the date at the end?

I'm trying to use the following Google script to add data from Redash, then add the current date at the end. Where could I add a var now = new Date(); to input the date on the last row?
The desired outcome is the following. A-C is the data I'm successfully already pulling from Redash, but I want to add the current date. The date will always be in F if that helps.
https://www.dropbox.com/s/lfu9jk06j0fduo8/Screenshot%202020-05-20%2018.44.26.png?dl=0
Thank you for your help!
//FUNCTION FOR GETTING DATA FROM REDASH QUERIES
function getapidata(response, sheetname) {
var array1 = response.getContentText(); //Store data from redash into this variable
var sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(sheetname);
var iLen = array1.length;
var i = 0;
var stringvalue1 = "";
var s1 = "";
var num1 = 1;
var num2 = 1;
var noheader = 1;
for (i = 0; i < iLen; i++) {
var j = i;
do {
s1 = array1[j];
if (s1 == ",") {
if (noheader == 1)
noheader = 1;
else
sheet.getRange(sheet.getLastRow() + num1, num2).setValue(stringvalue1);
num1 = 0;
num2++;
stringvalue1 = "";
} else
stringvalue1 = stringvalue1 + s1;
j++;
} while (s1 != "\n")
if (noheader == 1)
noheader = 1;
else
sheet.getRange(sheet.getLastRow() + num1, num2).setValue(stringvalue1);
noheader = 2;
num1 = 1;
num2 = 1;
stringvalue1 = "";
i = j - 1;
}
}
function getfromRedash() {
//Current SHR (Max Preps)
var redashapi = "API"; //Storing Redash API Value.
var sheetname = "SHEETTITLE"; //Assign your sheetname(where you would want data to be imported) to this variable
var response = UrlFetchApp.fetch(redashapi, {
muteHttpExceptions: true
}); //Storing Redash Cached Query result in this variable.
Logger.log(response.getContentText()); //If any error, error to be logged
getapidata(response, sheetname); //Call function for writing data in google sheets
//The 5 lines of code above can be repeated for multiple sheets in a single google spreadsheet file. For. Eg. If you have 5 different sheets in a single google worksheet, you can just copy paste the above lines of code, and just change the variable “redashapi” so as to make calls to the appropriate redash queries.
}
//THIS FUNCTION IS TO MAKE A NEW MENU IN GOOGLE SHEETS
function onOpen() {
var ui = SpreadsheetApp.getUi();
ui.createMenu('MENU')
.addItem('Get Data from Redash', 'getfromRedash')
.addToUi();
}

Well, you row number addressing is a bit confusing, cause sheet.getLastRow() would change every time you add values after the last row, but you can write something like
sheet.getRange(sheet.getLastRow() + num1, 6).setValue(new Date());
to insert date. You can refer other answers, like
How to format a JavaScript date
for date formatting.

Google Sheet Script - Return header value if cell value found

Thank you in advance for your help.
I have a google sheet that contains header values in the first row. I have a script that is looking through the remainder of the sheet (row by row) and if a cell is a certain color the script keeps a count. At the end, if the count number is greater than a variable I set in the sheet the script will trigger an email.
What I am looking at trying to do, is to also capture the column header value if the script finds a cell with the set color? I'm sure I need to create an array with the header values and then compare the positions, I'm just not sure how to do so efficiently.
function sendEmails() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var dataSheet = ss.getSheets()[0];
var lastRow = dataSheet.getLastRow();
var lastColumn = dataSheet.getLastColumn();
//Project Range Information
var projectRange = dataSheet.getRange(6,3,lastRow-5,lastColumn);
var projectRangeValues = projectRange.getValues()[0];
var cellColors = projectRange.getBackgrounds();
//Student Information Range
var studentRange = dataSheet.getRange(6,1,lastRow-5,lastColumn);
var studentRangeValues = studentRange.getValues();
//Pull email template information
var emailSubject = ss.getRange("Variables!B1").getValue();
var emailText = ss.getRange("Variables!B2").getValue();
var triggerValue = ss.getRange("Variables!B4").getValue();
var ccValue = ss.getRange("Variables!B5").getValue();
//Where to Start and What to Check
var colorY = ss.getRange("Variables!B6").getValue();
var count = 0;
var startRow = 6;
//Loop through sheet and pull data
for(var i = 0; i < cellColors.length; i++) {
//Pull some information from the rows to use in email
var studentName = studentRangeValues[i][0];
var studentBlogUrl = studentRangeValues[i][1];
var studentEmail = studentRangeValues[i][2];
var studentData = [studentName,studentBlogUrl];
//Loop through cell colors and count them
for(var j = 0; j < cellColors[0].length ; j++) {
if(cellColors[i][j] == colorY) {
/*This is where I feel I need to add the array comparisons to get the header values */
count = count + 1;
};//end if statement
};//end for each cell in a row
//If the count is greater than trigger, send emails
if (count >= triggerValue) {
//A call to another function that merges the information
var emailBody = fillInTemplateFromObject(emailText, studentData);
MailApp.sendEmail({
to: studentEmail,
cc: ccValue,
subject: emailSubject,
htmlBody: emailBody,
});
} else {};
//reset count to 0 before next row
count = 0;
};//end for each row
};
EDIT:
I have updated the above sections of the code to based on the responses:
//Header Information
var headers = dataSheet.getRange(4,4,1,lastColumn);
var headerValues = headers.getValues();
var missingAssignments = new Array();
In the for loop I added:
//Loop through cell colors and count them
for(var j = 0; j < cellColors[0].length ; j++) {
if(cellColors[i][j] == colorY) {
//This pushes the correct information into the array that matches up with the columns with a color.
missingAssignments.push(headervalues[i][j]);
count = count + 1;
};//end if statement
};//end for each cell in a row
The issue I am running into is that I am getting an error - TypeError: Cannot read property "2" from undefined. This is being caused by the push in the for loop as the script moves to the next row. I am unsure why I am getting this error. From other things I have read, the array is set as undefined. I have tried to set the array to empty and set it's length to 0, but it does not help. I don't think I understand the scoping of the array as it runs through.
EDIT:
Figured it out, the "i" should not iterate. It should read:
missingAssignments.push(headervalues[0][j]);
The end of the first for loop I clear the array for the next row.
missingAssignments.length = 0;

You should get the values of the entire sheet. Then use the shift method to get just the headers. It is hard for me to completely understand your intent without more information about your sheet. Let me know if I can provide more information.
function sendEmails() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var dataSheet = ss.getSheets()[0];
var lastRow = dataSheet.getLastRow();
var lastColumn = dataSheet.getLastColumn();
//below gets the whole sheet and shifts off the first row as headers
var fullSheet = dataSheet.getDataRange().getValues();
var headers = fullSheet.shift();
//then in your loops you can check against the index of the headers array

Spreadsheets with Apps Scripts is really slow especially if you have lots of data to read.
Check these tips from Apps doc:
Use batch operations
Scripts commonly need to read in data from a spreadsheet, perform
calculations, and then write out the results of the data to a
spreadsheet. Google Apps Script already has some built-in
optimization, such as using look-ahead caching to retrieve what a
script is likely to get and write caching to save what is likely to be
set.
You can write scripts to take maximum advantage of the built-in
caching, by minimizing the number of reads and writes. Alternating
read and write commands is slow. To speed up a script, read all data
into an array with one command, perform any operations on the data in
the array, and write the data out with one command.
Here's an example — an example you should not follow or use. The
Spreadsheet Fractal Art script in the Gallery (only available in the
older version of Google Sheets) uses the following code to set the
background colors of every cell in a 100 x 100 spreadsheet grid:
// DO NOT USE THIS CODE. It is an example of SLOW, INEFFICIENT code.
// FOR DEMONSTRATION ONLY
var cell = sheet.getRange('a1');
for (var y = 0; y < 100; y++) {
xcoord = xmin;
for (var x = 0; x < 100; x++) {
var c = getColor_(xcoord, ycoord);
cell.offset(y, x).setBackgroundColor(c);
xcoord += xincrement;
}
ycoord -= yincrement;
SpreadsheetApp.flush();
}
The script is inefficient: it loops through 100 rows and 100 columns,
writing consecutively to 10,000 cells. The Google Apps Script
write-back cache helps, because it forces a write-back using flush at
the end of every line. Because of the caching, there are only 100
calls to the Spreadsheet.
But the code can be made much more efficient by batching the calls.
Here's a rewrite in which the cell range is read into an array called
colors, the color assignment operation is performed on the data in the
array, and the values in the array are written out to the spreadsheet:
// OKAY TO USE THIS EXAMPLE or code based on it.
var cell = sheet.getRange('a1');
var colors = new Array(100);
for (var y = 0; y < 100; y++) {
xcoord = xmin;
colors[y] = new Array(100);
for (var x = 0; x < 100; x++) {
colors[y][x] = getColor_(xcoord, ycoord);
xcoord += xincrement;
}
ycoord -= yincrement;
}
sheet.getRange(1, 1, 100, 100).setBackgroundColors(colors);
The
inefficient code takes about 70 seconds to run. The efficient code
runs in just 1 second!
If you're looking at the Spreadsheet Fractal Art script (only
available in the older version of Google Sheets), please be aware that
we made a minor change to it to make this example easier to follow.
The script as published uses the setBackgroundRGB call, rather than
setBackgroundColor, which you see above. The getColor_ function was
changed as follows:
if (iteration == max_iteration) {
return '#000000';
} else {
var c = 255 - (iteration * 5);
c = Math.min(255, Math.max(0, c));
var hex = Number(c).toString(16);
while (hex.length < 2)
hex = '0' + hex;
return ('#' + hex + '3280');
}

Declaring Global Array in Google Spreadsheet

I am trying speedup the processing/calculation in Google spreadsheet by putting my bonus setting into global array using PropertiesServices instead of loading the bonus % from the Setting sheet whenever user update their Daily sheet.
The method I used seem not working as I expected, the FOR loop is unable to look for "ALL". Hope someone able to give some advice.
I suspected that JSON structure is different from my 2D array, but I don't know how to solve it, I am new to javascripts.
Setting sheet only containing 4 columns: Game, StartDate, EndDate, Bonus %
p/s: This is a simplified scripts.
function onOpen(event) {
var bonusSheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Setting");
var bonusCompany = bonusSheet.getRange("A46:D503").getValues();
PropertiesService.getDocumentProperties().setProperty("bonusCompany", JSON.stringify(bonusCompany));
}
function onEdit(event) {
var colGame = "H";
var colBonus = "K";
var numColBonus = 11;
var nPoint;
var sheet = event.source.getActiveSheet();
var row = sheet.getActiveCell().getRow();
var col = sheet.getActiveCell().getColumn();
var cGame = sheet.getRange(colGame+row).getValue();
var dDate = new Date();
for(var k=0, kLen=bonusCompany.length; k<kLen; k++)
{
if((bonusCompany[k][0] == cGame || bonusCompany[k][0] == "ALL") && dDate.valueOf() >= bonusCompany[k][1] && dDate.valueOf() <= bonusCompany[k][2] ){
sheet.getRange(event.range.rowStart,numColBonus,event.range.rowEnd-event.range.rowStart+1,1).setValue(nPoint*bonusCompany[i][4]/100);
return;}
}
sheet.getRange(event.range.rowStart,numColBonus,event.range.rowEnd-event.range.rowStart+1,1).setValue(0);
}

If you want to use PropertiesService.getDocumentProperties(), how about following sample script?
Your script is almost correct. When the data is set by PropertiesService.getDocumentProperties().setProperty(), the data can be retrieved by PropertiesService.getDocumentProperties().getProperty(). The detail information is https://developers.google.com/apps-script/reference/properties/properties-service.
And when the array data is set by setProperty(), the array data is stored as strings by JSON.stringify(). So it is necessary to convert from the string data to the array data. Fortunately, it has already known that the values retrieved by getValues() from spreadsheet is 2 dimensional array. So the string data can be converted to an array data using the regular expression. The sample script is as follows. I added 2 lines to your script. Please check them.
Script :
function onOpen(event) {
var bonusSheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Setting");
var bonusCompany = bonusSheet.getRange("A46:D503").getValues();
PropertiesService.getDocumentProperties().setProperty("bonusCompany", JSON.stringify(bonusCompany));
}
function onEdit(event) {
var colGame = "H";
var colBonus = "K";
var numColBonus = 11;
var nPoint;
var sheet = event.source.getActiveSheet();
var row = sheet.getActiveCell().getRow();
var col = sheet.getActiveCell().getColumn();
var cGame = sheet.getRange(colGame+row).getValue();
var dDate = new Date();
var d = PropertiesService.getDocumentProperties().getProperty("bonusCompany").split(/^\[|\]$/)[1].split(/\[(.*?)\]/); // I added this line.
var bonusCompany = [[(e=="\"\"" ? "" : e) for each (e in f)] for each (f in [d[j].split(",") for (j in d) if(j%2==1)])]; // I added this line.
for(var k=0, kLen=bonusCompany.length; k<kLen; k++) {
if((bonusCompany[k][0] == cGame || bonusCompany[k][0] == "ALL") && dDate.valueOf() >= bonusCompany[k][1] && dDate.valueOf() <= bonusCompany[k][2] ){
sheet.getRange(event.range.rowStart,numColBonus,event.range.rowEnd-event.range.rowStart+1,1).setValue(nPoint*bonusCompany[i][4]/100);
return;
}
}
sheet.getRange(event.range.rowStart,numColBonus,event.range.rowEnd-event.range.rowStart+1,1).setValue(0);
}
I don't know data of your spreadsheet. So if this modification doesn't work, can I ask you about the data of your spreadsheet?