Develop Reference

Google Apps Script - Improve Script to Execute Faster

I have a sheet with over 3000 points (lat, long). I am running the following code to calculate distance of each point from 9 reference points and writing the minimum distance next to each point in the sheet. The code runs fine but is slow and times out before running for all the points. How can I make the following code more efficient - so that it reads, computes and writes back the data faster?
/*
******************************
Reference Points
Point_A 41.166866 -76.151926
Point_B 41.087500 -76.204694
Point_C 40.540960 -75.704900
Point_D 40.401080 -75.589600
Point_E 40.326130 -75.642500
Point_F 40.167480 -75.921500
Point_G 40.093370 -76.084700
Point_H 39.974450 -76.063000
Point_I 39.722350 -76.156000
********************************
*/
var SpreadsheetID = "ABC";
var SheetName = "XYZ";
function DistanceCalculator() {
var ss = SpreadsheetApp.openById(SpreadsheetID)
var sheet = ss.getSheetByName(SheetName);
//Select the column
var columnToCheck = sheet.getRange("A:A").getValues();
// Get the last row in the column
var lastRow = getLastRowSpecial(columnToCheck);
// Ref point latitude
var Latitude_Point_A = 41.166866;
var Latitude_Point_B = 41.087500;
var Latitude_Point_C = 40.540960;
var Latitude_Point_D = 40.401080;
var Latitude_Point_E = 40.326130;
var Latitude_Point_F = 40.167480;
var Latitude_Point_G = 40.093370;
var Latitude_Point_H = 39.974450;
var Latitude_Point_I = 39.722350;
// Ref point longitude
var Longitude_Point_A = -76.151926;
var Longitude_Point_B = -76.204694;
var Longitude_Point_C = -75.704900;
var Longitude_Point_D = -75.589600;
var Longitude_Point_E = -75.642500;
var Longitude_Point_F = -75.921500;
var Longitude_Point_G = -76.084700;
var Longitude_Point_H = -76.084700;
var Longitude_Point_I = -76.156000;
for( var i=1; i<=lastRow;i++){
//Reading Lat_Long from the sheet
var Lat_Check = sheet.getRange(i,2).getValue();
var Long_Check = sheet.getRange(i,3).getValue();
//Calculating distance between each point and reference points
var distance01 = calculateDistance(Latitude_Point_A,Longitude_Point_A,Lat_Check,Long_Check);
var distance02 = calculateDistance(Latitude_Point_B,Longitude_Point_B,Lat_Check,Long_Check);
var distance03 = calculateDistance(Latitude_Point_C,Longitude_Point_C,Lat_Check,Long_Check);
var distance04 = calculateDistance(Latitude_Point_D,Longitude_Point_D,Lat_Check,Long_Check);
var distance05 = calculateDistance(Latitude_Point_E,Longitude_Point_E,Lat_Check,Long_Check);
var distance06 = calculateDistance(Latitude_Point_F,Longitude_Point_F,Lat_Check,Long_Check);
var distance07 = calculateDistance(Latitude_Point_G,Longitude_Point_G,Lat_Check,Long_Check);
var distance08 = calculateDistance(Latitude_Point_H,Longitude_Point_H,Lat_Check,Long_Check);
var distance09 = calculateDistance(Latitude_Point_I,Longitude_Point_I,Lat_Check,Long_Check);
//measuring minimum distance
var minimumDistance = Math.round(Math.min(distance01,distance02,distance03,distance04,distance05,distance06,distance07,distance08,distance09))
sheet.getRange(i,4).setValue(minimumDistance);
}
}
function calculateDistance(Ref_Lat, Ref_Long, Looped_Lat, Looped_Long){
var p = 0.017453292519943295; // Math.PI / 180
var c = Math.cos;
var a = 0.5 - c((Looped_Lat - Ref_Lat) * p)/2 +
c(Ref_Lat * p) * c(Looped_Lat * p) *
(1 - c((Looped_Long - Ref_Long) * p))/2;
var result = 7926 * Math.asin(Math.sqrt(a)); // 2 * R; R = 3963 miles
return result
};
function getLastRowSpecial(range){
var rowNum = 0;
var blank = false;
for(var row = 0; row < range.length; row++){
if(range[row][0] === "" && !blank){
rowNum = row;
blank = true;
}else if(range[row][0] !== ""){
blank = false;
};
};
return rowNum;
};
Sample Spreadsheet Screenshot

I believe your goal is as follows.
You want to reduce the process cost of your script.
In your script, getValue and setValue are used in a loop. I thought that this might be the reason for your issue. In this case, at first, the values are retrieved from the sheet and the values are processed, then, the values are put to the sheet. This flow is suitable for reducing the process cost. I think that "Best Practice" will be useful for understanding your issue. Ref
Also, I thought that when the process cost of your script is reduced, a little ingenuity might be required to be used for modifying your script. So I propose one of several modified scripts as a reference. Please modify your script as follows.
From:
for( var i=0; i<lastRow;i++){
//Reading Lat_Long from the sheet
var Lat_Check = sheet.getRange(i,2).getValue();
var Long_Check = sheet.getRange(i,3).getValue();
//Calculating distance between each point and reference points
var distance01 = calculateDistance(Latitude_Point_A,Longitude_Point_A,Lat_Check,Long_Check);
var distance02 = calculateDistance(Latitude_Point_B,Longitude_Point_B,Lat_Check,Long_Check);
var distance03 = calculateDistance(Latitude_Point_C,Longitude_Point_C,Lat_Check,Long_Check);
var distance04 = calculateDistance(Latitude_Point_D,Longitude_Point_D,Lat_Check,Long_Check);
var distance05 = calculateDistance(Latitude_Point_E,Longitude_Point_E,Lat_Check,Long_Check);
var distance06 = calculateDistance(Latitude_Point_F,Longitude_Point_F,Lat_Check,Long_Check);
var distance07 = calculateDistance(Latitude_Point_G,Longitude_Point_G,Lat_Check,Long_Check);
var distance08 = calculateDistance(Latitude_Point_H,Longitude_Point_H,Lat_Check,Long_Check);
var distance09 = calculateDistance(Latitude_Point_I,Longitude_Point_I,Lat_Check,Long_Check);
//measuring minimum distance
var minimumDistance = Math.round(Math.min(distance01,distance02,distance03,distance04,distance05,distance06,distance07,distance08,distance09))
sheet.getRange(i,4).setValue(minimumDistance);
}
To:
var values = sheet.getRange("B1:C" + lastRow).getValues();
var ar = [[Latitude_Point_A, Longitude_Point_A], [Latitude_Point_B, Longitude_Point_B], [Latitude_Point_C, Longitude_Point_C], [Latitude_Point_D, Longitude_Point_D], [Latitude_Point_E, Longitude_Point_E], [Latitude_Point_F, Longitude_Point_F], [Latitude_Point_G, Longitude_Point_G], [Latitude_Point_H, Longitude_Point_H], [Latitude_Point_I, Longitude_Point_I]];
var res = values.map(([b, c]) => [Math.round(Math.min(...ar.map(e => calculateDistance(...e, b, c))))]);
sheet.getRange(1, 4, res.length).setValues(res);
In this modification, from your script, it supposes that the 1st row is not the header row. Please be careful about this.
In this modification, the values are retrieved from the sheet by getValues, and then, the retrieved values are calculated and put to the sheet by setValues.
Note:
When I saw your script, I thought that an error occurs at var Lat_Check = sheet.getRange(i,2).getValue(); because the 1st value of i is 0. And in the function calculateDistance, Ref_Lat, Ref_Long are not used. So I'm worried that your showing script might be different from the script that you have the issue with. So, when you test the above modification, when an error occurs, can you confirm whether your showing script is the correct script for your situation. And, please provide the sample Spreadsheet for replicating the issue as the image. By this, I would like to confirm it.
References:
map()
getValues()
setValues(values)

Google scripts: How to put a range in setValue

I have used the example code from this link, to make the code below.
https://yagisanatode.com/2017/12/13/google-apps-script-iterating-through-ranges-in-sheets-the-right-and-wrong-way/
Most of the code is working as expected, except for the last row.
In Column E, I want to place the custom function =apiav() with the data from cell A.
However the code is returning =apiav(Range) in the Google sheet cells. (to be clear it should be something like =apiav(a1))
I tried everything i could think of and of course googled for hours, but i am really lost and can't find the right solution for this.
function energy(){
var sector = "Energy";
var ss=SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getActiveSheet();
var rangeData = sheet.getDataRange();
var lastColumn = 2;
var lastRow = 999 ;
var sheet = ss.getSheets()[0];
var searchRange = sheet.getRange(2,2, lastRow-1 ,1 );
var ouputrange = sheet.getRange(2,4, lastRow-1 ,1 );
//clear range first
ouputrange.clear("D:D");
ouputrange.clear("E:E");
/*
GOOD - Create a client-side array of the relevant data
*/
// Get array of values in the search Range
var rangeValues = searchRange.getValues();
// Loop through array and if condition met, add relevant
// background color.
for ( i = 0; i < lastColumn - 1; i++){
for ( j = 0 ; j < lastRow - 1; j++){
if(rangeValues[j][i] === sector){
sheet.getRange(j+2,i+4).setValue("yes");
var formularange = sheet.getRange (j+2,i+1);
sheet.getRange(j+2,i+5).setValue('=apiav(' + formularange + ')');
}
};
};
};

Replace:
var formularange = sheet.getRange(j+2,i+1);
with:
var formularange = sheet.getRange(j+2,i+1).getA1Notation();
So you will be able to pass the cell reference instead of the range object.

How should I add the rows (without the headers) after the last row with data (without overwriting anything in the "Master_db")

The code below is overwriting on the existing data.
#OMila helped me with the original code, I could not articulate exactly what I needed hence starting a new question.
function Dom() {
var origin_sheet = SpreadsheetApp.getActive().getSheetByName('Dom_Sum');
var firstRow = 1;
var firstCol = 1;
var numRows = origin_sheet.getLastRow();
var numCols = 22;
var origin_values = origin_sheet.getRange(firstRow, firstCol, numRows, numCols).getValues();
var dest_values = [];
for(var i = 0; i < origin_values.length; i++) {
if(origin_values[i][0] != '') {
dest_values.push(origin_values[i]);
}
}
var dest_id = "1ZGq7L7bvF1INuDgZxhHnVsihkYkYubmncSAE5uC-Pq4";
var dest_sheet = SpreadsheetApp.openById(dest_id).getSheetByName("Master_Db");
var numRowsDest = dest_values.length;
var dest_range = dest_sheet.getRange(1, 1, numRowsDest, 22);
dest_range.setValues(dest_values);
}
I would like to add the data created in the "Dom_Sum" worksheet below the last row of data in the other workbook with the worksheet name "Master_Db"
#OMila I'm really grateful to you, and if you like we could offer you a consultation fee for future projects. (boseav#gmail.com)

Instead of writing your value into the range dest_sheet.getRange(1, 1, numRowsDest, numCols)
retrieve the last row of your destination sheet and write starting with the next row
var destLastRow=dest_sheet.getLastRow();
var dest_range = dest_sheet.getRange(destLastRow+1, 1, numRowsDest, numCols);
dest_range.setValues(dest_values);

How to retrieve data from columns dependent upon date

I have a rota (a fixed order of rotation (as of persons or duties)) that I've already had help with this week. It's up & running as is, but for simpler reading I'd like to transpose it.
You can see the transposed sheet as I'd like it here
The current script is for the pre-transposed table.
It would search Column 0 for the date. If it was was 7 days away it would retrieve the name from Column 1 & match it with e-mail address in separate sheet etc.
What I'd like to do is instead have the Date in Row 0 & then subsequent names in Row 1 etc etc
I've tried various things. I've stepped through the code & can see what it's doing, & I've done some reading through about 2 dimensional arrays, but I can't seem to find a way of getting the code to work down through columns, instead of across the rows.
Here's the code:
function sendEmails() {
var ss1 = SpreadsheetApp.getActiveSpreadsheet();
var sh1 = ss1.getSheetByName("Rota")
ss1.setActiveSheet(sh1);
var rotalink = "https://docs.google.com/spreadsheets/d/1LgzUWSAGA2kbpar8r5nosU1bSHF7nrtvtUiHS3nB_e8";
var sheet = SpreadsheetApp.getActiveSheet();
// Fetch the range
var dataRange = sheet.getRange("B3:G50")
// Fetch values for each row in the Range.
var data = dataRange.getValues();
for (i in data) {
var row = data[i];
var today=new Date();
var timecell = new Date(row[0]);
var timediff = new Date();
var one_day=1000*60*60*24;
var daystogo = Math.ceil((timecell.getTime()-today.getTime())/(one_day));
if (daystogo==7) {//only e-mail people with one week to go. To change that alter the "7" to the number of days you want
var subject = "Rota reminder!";
var emailAddress = [];
var message;
message = "Hello \n\n"+
"You are down to help at Youth Café this week. \n\n" +
"Please see the below rota for your role \n\n" +
"If you have any questions or problems let us know at thameyouthcafe#gmail.com \n\n" +
"Remember, you can check the rota anytime by clicking on the link below: \n\n"+
rotalink
for (var x = 1; x < 5; x++) { // 5 because emails are till col4
// var emailAddress = []; // Start by collecting the non-blank emails in an array
if (getEmailFromName(row[x]) != "") {
emailAddress.push(getEmailFromName(row[x]))
}
}
emailAddress = emailAddress.join(); // Join the array to get a comma separated string
MailApp.sendEmail(emailAddress, subject, message);
}
}
}
and here's the getEmailFromName function that matches with SKey (which I presume comes from the "i" variable in the first function?
function getEmailFromName(sKey) {
// to use this function, don’t put anything in the first column (A) or row (1).
// Put the name (i.e. the key, or what we’re looking for) in column B.
// Put what we want to return in column C.
var columnToSearch = 1; //column B
// Set the active sheet to our email lookup
var ss1 = SpreadsheetApp.getActiveSpreadsheet();
var sh1 = ss1.getSheetByName("EmailContactList")
ss1.setActiveSheet(sh1);
var data = SpreadsheetApp.getActiveSheet().getDataRange().getValues();
var line = -1;
for( var i = 0; i < data.length; i++ ) {
if( data[i][columnToSearch] == sKey ) {
line = i;
break;
}
}
if( line != -1 ) {
//do what you want with the data on "line"
return data[line][2]; //value on column C of the matched line
}
else {
return "";
// if criteria is not found
}
}

Try it this way:
function sendEmails() {
var ss1 = SpreadsheetApp.getActive();
var sh1 = ss1.getSheetByName("Rota")
ss1.setActiveSheet(sh1);
var rotalink = "https://docs.google.com/spreadsheets/d/1LgzUWSAGA2kbpar8r5nosU1bSHF7nrtvtUiHS3nB_e8";
var sheet = SpreadsheetApp.getActiveSheet();
var dataRange = sheet.getRange("B3:G50")
var data = dataRange.getValues();
for (var i=0;i<dataRange.length;i++) {
var row = data[i];
var today=new Date();
var timecell = new Date(row[0]);
var timediff = new Date();
var one_day=1000*60*60*24;
var daystogo = Math.ceil((timecell.getTime()-today.getTime())/(one_day));
if (daystogo==7) {
var subject = "Rota reminder!";
var emailAddress = [];
var message = Utilities.formatString('Hello\n\nYou are down to help at Youth Café this week.\n\n Please see the below rota for your role \n\nIf you have any questions or problems let us know at thameyouthcafe#gmail.com \n\nRemember, you can check the rota anytime by clicking on the link below: \n\n%s',rotalink);
for (var x=1;x<5;x++) {
if(data[i][x]) {
emailAddress.push(data[i][x]);
}
}
MailApp.sendEmail(emailAddress.join(), subject, message);
}
}
}

Managed to solve it - thank you for your contributions. Turned out it was incredibly simple.
Just had to change this line:
var timecell = new Date(data[0])
to this:
var timecell = new Date(data[0][i])
so it iterates through the first row of each column.

Google sheets script - Strip extra data from a second line and add it to the first line

I'm trying to build a script in google sheets script that will iterate down a sheet, row by row, and if it encounters an ID number in the first cell of that row that is identical to the one above it, it strips out the data from every cell EXCEPT Column A and B, and appends it to the line above. Ideally, this would work with an indeterminate number of duplicate row IDs, might be 2, might be 3, might be 4.
After stripping out the data I want to keep (eg colums C and onward), I then want to delete the entire contents of the processed duplicate ID row, but I just haven't put that in my script until after it copies the data correctly.
In this example, sheet rows 6, 7 and 8 have identical ID numbers (Column A)
Here is the result I'm trying to get:
And here is the result I'm getting:
I've tried a number of different ways, and torn down and rebuilt my script a couple of times without getting the result I want:
function stripMiner() {
var ss = SpreadsheetApp.openById("1WDPoTICQvdruxfhAwHLtA51fz05DqyZ-NhNfpAyPO6Y");
var mainSheet = ss.getSheetByName("Main");
var startRow = 5;
var numRows = mainSheet.getLastRow();//obtains the last row in the sheet
var setrgh = mainSheet
var dataRange = mainSheet.getRange(startRow, 1,4,120); //rowStart, columnStart, row count, column count, the columncount needs to be large enough to encompass all your ancillary data
var data = dataRange.getValues();
var iter = 0;
var maxItRow = 4;
var prevIdNum = 0;
var dupCount = 1;
var cc1 = "P5"; //Cells to dump check values into
var cc2 = "P6";
var dumpRow = startRow;
//if (numRows >= maxItRow){var maxIter = maxItRow;}
for (i in data){
if (iter != maxItRow){ //making sure we haven't gone over the iteration limit
var row = data[i];
var idNum = (row[0]);
var jCount = 0; //resets icount if the id number is different icount is used to skip some cells in a row
if (idNum == prevIdNum){//only proceed if we've hit another line with the same ID number
dupCount = +1; //increment the dupcount value
mainSheet.getRange(cc2).setValue("dupCount"+dupCount); //dupcount check value
var rowIterStart = 5; //RowIterStart is used to add to rowiter, EG if your data is 20 columns wide, and you start transposing from column 4, then this will want to be about 17
var rowIter = 1;
for (j in row){
if (jCount >= 2){ //the integer here is the column where it will begin to transpose data
mainSheet.getRange(dumpRow-1,(rowIterStart*dupCount)+(rowIter)).setValue(row[j]); //startRow+(iter-dupCount)
mainSheet.getRange(cc1).setValue("dumprow"+dumpRow);
}
rowIter+=1;
jCount +=1;
}
}
else{
var dupCount = 1;
dumpRow +=1;
}
prevIdNum = (row[0]); //sets the most recently processed rows ID number
}
iter +=1;
}
}
I'm not quite sure where I'm going wrong. Does anyone have any suggestions? Thanks!
(Also I'm still just a beginner with this so if I've overlooked anything obvious or taken the wrong approach to do this, I apologize!)

The results for the questioner's code in the case of copied data arise from a convoluted loop. In essence, though duplicates were identified, there was a mis-counting to assign the copied data to the correct rowID. So far as clearing data, no provision was included.
The following code works to meet the questioner's goals, though it is far from perfect.
At present, the recalculation of the "last column" after copy each duplicate is an absolute rather than than a row-based figure. So, if a duplicate was detected for, say, ID=3, the data would be copied to column 12 rather than column 6. This requires the addition of a simple dupID row counter.
The second factor is the calculation of the last column in the spreadsheet.
var dataRange = mainSheet.getRange(startRow, 1,Rowlast+1,120);
The questioner used 120 columns; and I have retained that number simply for the sake of consistency. The questioner should re-assess whether this is excessive.
function ejb_so_5284922701() {
var ss = SpreadsheetApp.openById("<< insert questioners spreadsheet ID>>");
var mainSheet = ss.getSheetByName("<< insert questioner's sheet name >>");
var startRow = 5;
// calculate the last row containing data
var Rowvals = ss.getRange("A5:A").getValues();
var Rowlast = Rowvals.filter(String).length; //6
Logger.log("last row = " + Rowlast); // DEBUG
// calculate the last column containing data
var cell = mainSheet.getRange("A5"); //or however you determine "cell"
var drCol = mainSheet.getDataRange().getLastColumn();
Logger.log('getLastColumn = ' + drCol); //DEBUG
for (var i = drCol; i >= 1; i--) {
if (mainSheet.getRange(cell.getRow(), i).getValue() != "") {
break;
}
}
var lastColumn = i;
Logger.log("Last column with data = " + lastColumn); //DEBUG
var setrgh = mainSheet
// numColumns neds to be reviewed
var dataRange = mainSheet.getRange(startRow, 1, Rowlast + 1, 120); //rowStart, columnStart, row count, column count, the column count needs to be large enough to encompass all your ancillary data
// start row = 5, 1= column A, 4, rows, 120, columns
Logger.log("startRow = " + startRow + ", and the datarange = " + dataRange.getA1Notation()); //DEBUG
var data = dataRange.getValues();
Logger.log("length of data =" + data.length); //DEBUG
var lastid = 0;
for (i = 0; i < data.length; i++) {
if (i == 0) {
// if this is the first row, then assign anything but zero to last id
lastid = 100;
Logger.log(" this is the first row; set last id to 100");
}
var thisid = data[i][0];
// evaluate whether this is a duplicate ID
if (thisid == lastid) {
// this is a dup
Logger.log("i=" + i + ". This is a dup" + ", name is " + data[i][2]); //DEBUG
var stufftocopyrange = mainSheet.getRange(startRow + i, 3, 1, 3);
var stufftocopy = stufftocopyrange.getValues();
Logger.log("the range to copy is " + stufftocopyrange.getA1Notation()); //DEBUG
var targetrange = mainSheet.getRange(startRow + lastid - 1, lastColumn + 1, 1, 3);
targetrange.setValues(stufftocopy);
lastColumn = lastColumn + 3;
var duprange = mainSheet.getRange(startRow + i, 1, 1, 5);
Logger.log("the range to clear is " + duprange.getA1Notation()); //DEBUG
duprange.clearContent();
} else {
// no dup
//assign lastid value
var lastid = thisid;
Logger.log("i=" + i + ". No dup. Last id set to " + lastid); // DEBUG
} // if
} // end for loop
}
BEFORE
AFTER

The solutions previously posted didn't quite get the result I needed, however I managed to cobble together something that works for my purposes. It expects to see data in the format like:
And turn it into something like this:
Where it uses duplicate ID numbers (with an indeterminite number of duplicates) to pull only certain columns of data from the duplicate lines and append it to the first line.
function stripMiner() {
var ss = SpreadsheetApp.openById("1WDPoTICQvdruxfhAwHLtA51fz05DqyZ-NhNfpAyPO6Y");
var mainSheet = ss.getSheetByName("Main");
var startRow = 5;
var numRows = mainSheet.getLastRow();//obtains the last row in the sheet
var setrgh = mainSheet
var dataRange = mainSheet.getRange(startRow, 1,3,120); //rowStart, columnStart, row count, column count, the columncount needs to be large enough to encompass all your ancillary data
var data = dataRange.getValues();
var iter = 0;
var maxItRow = 4;
var prevIdNum = 0;
var dupCount = 1;
var cc1 = "P5"; //Cells to dump check values into
var cc2 = "P6";
var dumpRow = startRow;
//if (numRows >= maxItRow){var maxIter = maxItRow;}
for (i in data){
if (iter != maxItRow){ //making sure we haven't gone over the iteration limit
var row = data[i];
var idNum = (row[0]);
var jCount = 0; //resets icount if the id number is different icount is used to skip some cells in a row
if (idNum == prevIdNum){//only proceed if we've hit another line with the same ID number
dupCount = +1; //increment the dupcount value
mainSheet.getRange(cc2).setValue("dupCount"+dupCount); //dupcount check value
var rowIterStart = 5; //RowIterStart is used to add to rowiter, EG if your data is 20 columns wide, and you start transposing from column 4, then this will want to be about 17
var rowIter = 1;
for (j in row){
if (jCount >= 2){ //the integer here is the column where it will begin to transpose data
mainSheet.getRange(dumpRow-2,(rowIterStart*dupCount)+(rowIter)).setValue(row[j]); //startRow+(iter-dupCount)
mainSheet.getRange(cc1).setValue("dumprow"+dumpRow);
}
rowIter+=1;
jCount +=1;
}
}
else{
var dupCount = 1;
dumpRow +=1;
}
prevIdNum = (row[0]); //sets the most recently processed rows ID number
}
iter +=1;
}
}
Hopefully someone else who wants to do a similar thing can make use of this too.