Develop Reference

Improve performance in Google App Script Search functions

I have written a script using Google App Script (in javascript) and I am looking for a way to be best optimize function that return objects, based on one or more search fields. The data is stored in a Google Sheet. The UI I have passes parameters to my function, then I iterate over a given sheet to find rows that meet a criteria, and add cells to an object, to be returned. The return could be just one object or a list of objects. For the most part, this works fine, but if I have this type of function nested in a loop it can really drag the performance. Any advise on how to improve performance would be greatly appreciated. Here is an example of my code:
function GetAllReportByOrgID_DataLayer_(org_id, reporting_periods) {
//get all reporting period for program
var rows = GetDataRows_(DATA_SPREAD_SHEET_ID, RESPONSE_PAGE);
var surveys = [];
for (var i = 1; i < rows.length; i++) {
var row = rows[i];
var found_org_id = row[2];
var found_is_active = row[13];
if (found_org_id == org_id && found_is_active == true ) {
var survey = {};
survey.indicator_id = row[0];
survey.program_id = row[1];
survey.org_guid = row[2];
survey.survey_response = row[3];
survey.reporting_period = row[5];
survey.reporting_period_name = GetReportingPeriodNameById_(row[5], reporting_periods);
survey.is_final_report = row[6];
survey.is_submitted = row[7];
survey.submitted_by = row[8];
survey.submitted_by_email = row[9];
survey.date_created = ConvertUnixTimeStampToDateTime_(row[10]);
survey.date_updated = ConvertUnixTimeStampToDateTime_(row[11]);
survey.fiscal_year = row[12];
survey.documents = GetDocumentsById_DataLayer_({
program_id: row[13]
});
surveys.push(survey);
}
}
surveys.success = true;
return surveys;
}
function GetDataRows_(Sheet_Id, SheetName) {
var sheet = GetSheet_(Sheet_Id, SheetName);
var rows = [];
if (sheet) {
rows = sheet.getDataRange().getValues();
}
return rows;
}
function GetSheet_(Sheet_Id, SheetName) {
var ss = SpreadsheetApp.openById(Sheet_Id);
var sheet = ss.getSheetByName(SheetName);
return sheet;
}
function GetReportingPeriodNameById_(id, reporting_periods) {
if (id) {
for (var i = 0; i < reporting_periods.length; i++) {
if (reporting_periods[i].id == id) {
return reporting_periods[i].value
}
}
return "Reporting Period Not Found"
} else {
return "Reporting Period Not Found"
}
}
function GetDocumentsById_DataLayer_(data) {
var rows = GetDataRows_(DATA_SPREAD_SHEET_ID, PROGAM_DOCUMENTS_PAGE);
var documents = [];
var program_id = data.program_id.trim();
for (var i = 1; i < rows.length; i++) {
var row = rows[i];
var found_program_id = row[1];
var is_active = row[6];
if(is_active === true){
if (found_program_id === program_id) {
var document = {};
document.document_id = row[0];
document.program_id = row[1];
document.document_name = row[2];
document.file_id = row[3];
document.file_name = row[4];
document.file_url = row[5]
document.date_created = ConvertUnixTimeStampToDateTime_(row[7]);
document.date_updated = ConvertUnixTimeStampToDateTime_(row[8]);
documents.push(document);
}
}
}
documents.success = true;
return documents;
}
function ConvertUnixTimeStampToDateTime_(unix_timestamp) {
if (!unix_timestamp) {
return "";
}
var a = new Date(unix_timestamp * 1000);
var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
var year = a.getFullYear();
var month = months[a.getMonth()];
var date = a.getDate();
var hour = a.getHours();
var min = a.getMinutes();
var sec = a.getSeconds();
var time = a.getMonth() + "/" + date + "/" + year + " " + hour + ":" + min + ":" + sec;
return time;
}
This code mostly works fine, except when it is in a loop that gets called 100 times or so, then things lag and can take a minute or more to process. The sheets aren't that big, less that 200 rows and 15 columns.
thanks

The reason for performance depreciation is the GetDataRows_ function, because it makes repeated calls to Spreadsheet.openById(), ss.getSheetByName(SheetName);. You could try using a global map to cache these objects. For example, Use a Spreadsheet map
const globalSSMap = new Map();//global SS map
Then your getter functions can be rewritten. For eg, to get ss, use
if(!globalSSMap.has(Sheet_id)) globalSSMap.set(Sheet_id,SpreadsheetApp.openById(Sheet_Id));
var ss = globalSSMap.get(Sheet_id);

You can refer to this sample code:
Data_Layer.gs
function GetAllDataByManager_DataLayer_(loggedin_user) {
var ss = SpreadsheetApp.openById(DATA_SPREAD_SHEET_ID);
var sheets = ss.getSheets();
// create sheet objects
var sheetData = {};
sheets.forEach(sheet => {
sheetData[sheet.getName()] = sheet.getDataRange().getValues();
})
var program_rows = sheetData[PROGRAM_MANAGERS];
var ip_ids = [];
var Company_data = [];
var runtimeCountStart = new Date();
//first search to make sure logged in user exists, and is marked as Active == True and is marked as an Admin
//if they exist, add their ID to the list
//Should only see records for the logged in user if
//they are either listed as the primary manager (company tab)
//or they are listed as an additional manager (program managers tab)
for (var i = 1; i < program_rows.length; i++) {
var row = program_rows[i];
var found_admin_email = row[2];
var found_is_active = row[10];
if (found_admin_email == loggedin_user && found_is_active == true) {
ip_ids.push(row[1])
}
}
var partner_rows = sheetData[PARTNER_PAGE];
for (var i = 1; i < partner_rows.length; i++) {
var partner_row = partner_rows[i]
var found_partner_id = partner_row[0];
var add_record = false;
if(ip_ids.includes(found_partner_id)){
add_record = true;
}else{
var found_cor_email = partner_row[5]
if(loggedin_user.toUpperCase() == found_cor_email.toUpperCase()){
add_record = true;
}
}
if(add_record == true){
var partner = {
ip_id: partner_row[0],
ip_name: partner_row[1],
ip_poc_name: partner_row[2],
ip_poc_email: partner_row[3],
manager_name: partner_row[4],
manager_email: partner_row[5],
is_active: partner_row[6],
date_created: partner_row[7],
created_by: partner_row[8],
partner_programs: GetAllProgramDataByIP_DataLayer_(sheetData, found_partner_id),
partner_notes: GetProgramNarrativesByPartnerID_DataLayer_(sheetData, found_partner_id),
partner_reports: GetAllReportByPartnerID_DataLayer_(sheetData, found_partner_id)
};
Company_data.push(partner)
}
}
stop = new Date();
newRuntime = Number(stop) - Number(runtimeCountStart);
newRuntime = (newRuntime /1000)/60
return Company_data;
}
function GetAllProgramDataByIP_DataLayer_(sheetData, ip_id) {
var rows = sheetData[PROGRAM_PAGE];
var programs = [];
for (var i = 1; i < rows.length; i++) {
var row = rows[i];
var found_partner_id = row[1];
var program = {}
if (found_partner_id === ip_id) {
program.program_id = row[0].toString();
program.partner_id = row[1].toString();
program.program_name = row[2].toString();
program.program_country = row[3].toString();
program.program_background = row[4].toString();
program.program_objectives = row[5].toString();
program.program_justification = row[6].toString();
program.program_start_date = row[7].toString();
program.program_status = row[8].toString();
program.program_contract_number = row[9].toString();
program.is_active = row[10]
program.date_created = ConvertUnixTimeStampToDateTime_(row[11].toString());
program.date_updated = ConvertUnixTimeStampToDateTime_(row[12].toString());
program.success = true;
programs.push(program)
}
}
return programs;
}
function GetProgramNarrativesByPartnerID_DataLayer_(sheetData, partner_id) {
var rows = sheetData[PROGRAM_NARRATIVE_NOTE];
var programs_notes = [];
var response = {};
for (var i = 1; i < rows.length; i++) {
var row = rows[i];
var found_partner_id = row[2];
var is_active = row[7];
if(is_active === true){
if (found_partner_id === partner_id) {
var note = {};
note.note_id = row[0];
note.program_id = row[1];
note.company_guid = row[2];
note.note_title = htmlEscape_(row[3]);
note.note_desc = htmlEscape_(row[4]);
note.note_reportingPeriod = row[5];
note.activity_theme = row[6];
note.date_created = ConvertUnixTimeStampToDateTime_(row[8]);
note.date_updated = ConvertUnixTimeStampToDateTime_(row[9]);
programs_notes.push(note);
}
}
}
response.success = true;
response.programs_notes = programs_notes
return response;
}
function GetAllReportByPartnerID_DataLayer_(sheetData, partner_id) {
//get all reporting period for program
var rows = sheetData[RESPONSE_PAGE];
var surveys = [];
for (var i = 1; i < rows.length; i++) {
var row = rows[i];
var found_partner_id = row[2];
if (found_partner_id == partner_id) {
var survey = {};
survey.indicator_id = row[0];
survey.program_id = row[1];
survey.company_guid = row[2];
survey.survey_response = row[3];
survey.reporting_period = row[5];
survey.is_final_report = row[6];
survey.is_submitted = row[7];
survey.submitted_by = row[8];
survey.submitted_by_email = row[9];
survey.date_created = ConvertUnixTimeStampToDateTime_(row[10]);
survey.date_updated = ConvertUnixTimeStampToDateTime_(row[11]);
survey.fiscal_year = row[12];
surveys.push(survey);
}
}
surveys.success = true;
return surveys;
}
Helper_Functions.gs
function ConvertUnixTimeStampToDateTime_(unix_timestamp) {
if (!unix_timestamp) {
return "";
}
var a = new Date(unix_timestamp * 1000);
var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
var year = a.getFullYear();
var month = months[a.getMonth()];
var date = a.getDate();
var hour = a.getHours();
var min = a.getMinutes();
var sec = a.getSeconds();
var time = a.getMonth() + "/" + date + "/" + year + " " + hour + ":" + min + ":" + sec;
return time;
}
function CreateGUID_() {
return Utilities.getUuid();
}
function htmlEscape_(str) {
str = str.toString();
if (str && str.length > 0) {
return str.replace(/<[^>]+>/g, "")
} else {
return "";
}
}
Note:
I didn't see GetDocumentsById_DataLayer_ and survey.documents in the shared script.
I just looked for something to improve in your main function GetAllDataByManager_DataLayer_() where nested loop occur when you are checking your partner_rows variable.
I don't think there's any more we can do regarding your nested loop check.
Modifications Done:
Although you prevented repeating the calls for Spreadsheet.openById(), you still make individual calls when you map your sheet objects since you use ss.getSheetByName(SheetName); to get individual sheet object based on the SheetName and even reading the sheet values during every loop for 3 different sheets
What I did is to get all your sheet object available in your spreadsheet using getSheets(), get its data range values then map them based on their sheet name. As what was suggested by #TheMaster in the other answer
Remove unnecessary functions in Helper_Functions.gs
Execution Log:
For the code changes, refer to the last 2 recent Web Apps execution logs. Compare it to the first execution

Save import data in google sheet

I created two google sheet with an exchange of data from sheet 1 to 2 with the IMPORTRANGE function.
I would like to save my data on the second sheet while importing. But I don't know how.
Below is my code which allows me to delete the lines as soon as 30 days have passed.
function onEdit() {
// Sheet the data is on.
var SHEET = "Copie de TF VBX";
// The value that will cause the row to hide.
var VALUE = "date";
// The column we will be using
var COLUMN_NUMBER = 12
var ss = SpreadsheetApp.getActiveSpreadsheet();
var activeSheet = ss.getActiveSheet();
var cell = ss.getActiveCell()
var cellValue = cell.getValue();
var colonne = cell.getColumn();
var ligne = cell.getRow();
var date2 = new Date(Date());
var date1 = new Date(cellValue);
// différence des heures
var time_diff = date2.getTime() - date1.getTime();
// différence de jours
var days_Diff = time_diff / (1000 * 3600 * 24);
//Ensure on correct sheet.
if(SHEET == activeSheet.getName()){
//Ensure we are looking at the correct column.
if(colonne == COLUMN_NUMBER){
//If the cell matched the value we require,hide the row.
if(days_Diff>31){
activeSheet.deleteRow(ligne)
//Browser.msgBox(days_Diff)
};
};
};
}
Thank you for your help.

function onEdit(e) {
const sh = e.range.getSheet();
if (sh.getName() = "Copie de TF VBX" && e.range.columnStart == 12 && e.value) {
const td = new Date()
const thd = new Date(td.getFullYear(), td.getMonth(), td.getDate() - 31).valueOf();
const date1v = new Date(e.value).valueOf();
if (date1v > thd) {
sh.deleteRow(e.range.rowStart);
e.source.toast(DiffInDays(td,new Date(e.value));
}
}
}
function DiffInDays(Day1,Day2) {
if(Day1 && Day2 && (Object.prototype.toString.call(Day1) === '[object Date]') && (Object.prototype.toString.call(Day2) === '[object Date]')) {
var day=86400000;
var t1=new Date(Day1).valueOf();
var t2=new Date(Day2).valueOf();
var d=Math.abs(t2-t1);
var days=Math.floor(d/day);
//Logger.log(days);
return days;
} else {
throw 'Invalid Inputs';
}
}

Translating PHP strtotime function into Javascript

I have a PHP function which calculates the time difference (hr/min/sec) of two values:
$mf_start_time = '10:00:00';
$mf_end_time = '11:00:00';
$st_start_time = '10:00:00';
$st_end_time = '12:00:00';
$sn_start_time = '10:00:00';
$sn_end_time = '13:00:00';
$ph_start_time = '10:00:00';
$ph_end_time = '14:00:00';
function time_interval($start,$end) {
$s = strtotime($start);
$e = strtotime($end);
if ($s < $e)
{
$a = $e - $s;
}
else
{
$e = strtotime('+1 day',$e);
$a = $e - $s;
}
$h = floor($a/3600);
$m = floor(($a%3600)/60);
$s = $a%60;
return trim(($h?$h.' hour ':'').($m?$m.' minute ':'').($s?$s.' second ':''));
}
$mf_duration = time_interval($mf_start_time,$mf_end_time);
$st_duration = time_interval($st_start_time,$st_end_time);
$sn_duration = time_interval($sn_start_time,$sn_end_time);
$ph_duration = time_interval($ph_start_time,$ph_end_time);
echo $mf_duration;
echo $st_duration;
echo $sn_duration;
echo $ph_duration;
My output for this is:
1 hour
2 hour
3 hour
4 hour
Now I am trying to translate this into Javascript, my problem is that I need to get strtotime() to work the same.
Here is what I tried in Javascript:
var mf_start_time = '10:00:00';
var mf_end_time = '11:00:00';
var st_start_time = '10:00:00';
var st_end_time = '12:00:00';
var sn_start_time = '10:00:00';
var sn_end_time = '13:00:00';
var ph_start_time = '10:00:00';
var ph_end_time = '14:00:00';
function time_interval(start,end)
{
var s = strtotime(start);
var e = strtotime(end);
if (s < e)
{
a = e - s;
}
else
{
var e = strtotime('+1 day',e);
var a = e - s;
}
var h = floor(a/3600);
var m = floor((a%3600)/60);
var s = a%60;
return trim((h?h+' hour ':'')+(m?m+' minute ':'')+(s?s+' second ':''));
}
var mf_duration = time_interval(mf_start_time,mf_end_time);
var st_duration = time_interval(st_start_time,st_end_time);
var sn_duration = time_interval(sn_start_time,sn_end_time);
var ph_duration = time_interval(ph_start_time,ph_end_time);
console.log(mf_duration);
console.log(st_duration);
console.log(sn_duration);
console.log(ph_duration);
This does not work because strtotime does not work for Javascript (error not defined). What can I use for that?

I did my best to replicate what your php does, however I don't know if this would be best practice in js, nor if it would be of the same use as you have in your php app. Hopefully this is a good starting point for you to see how js works 👍
var mf_start_time = '10:00:00';
var mf_end_time = '11:00:00';
var st_start_time = '10:00:00';
var st_end_time = '12:00:00';
var sn_start_time = '10:00:00';
var sn_end_time = '13:00:00';
var ph_start_time = '10:00:00';
var ph_end_time = '14:00:00';
function time_interval(start, end) {
var [sHour, sMinute, sSecond] = start.split(":");
var [eHour, eMinute, eSecond] = end.split(":");
var s = new Date();
s.setHours(sHour, sMinute, sSecond);
var e = new Date();
e.setHours(eHour, eMinute, eSecond);
var a;
if (s.getTime() < e.getTime()) {
a = e.getTime() - s.getTime();
} else {
e.setDate(e.getDate() + 1);
a = e.getTime() - s.getTime();
}
a = a / 1000;
var h = Math.floor(a/3600);
var m = Math.floor((a%3600)/60);
var s = a%60;
return (
(h ? h + ' hour ' : '') +
(m ? m +' minute ':'') +
(s ? s +' second ':'')
).trim();
}
var mf_duration = time_interval(mf_start_time, mf_end_time);
var st_duration = time_interval(st_start_time, st_end_time);
var sn_duration = time_interval(sn_start_time, sn_end_time);
var ph_duration = time_interval(ph_start_time, ph_end_time);
console.log(mf_duration);
console.log(st_duration);
console.log(sn_duration);
console.log(ph_duration);

javascript google script appendRow fails Service Error

I have a Google Apps Script that has been running for 3 months and starting a few weeks ago I'm getting a "Service Error" on the Appendrow function which I've bolded below. Any ideas on how to fix this?
function updateACOPS2(){
var ss = SpreadsheetApp.openById(".....")
var sheetSubmission = ss.getSheetByName("Sheet8");
var dataSubmission = sheetSubmission.getDataRange().getValues();
var lastColSubmission = sheetSubmission.getLastColumn();
var ssActive = SpreadsheetApp.openById("....")
var sheetActive = ssActive.getSheetByName("AcopsAll");
var sheetMain = ssActive.getSheetByName("Sheet1");
var dataActive = sheetActive.getDataRange().getValues();
var lastrow = sheetActive.getLastRow();
for(var i = 1; i < dataSubmission.length && dataSubmission[i][2] != ""; i++){
var currentIDSubmission = dataSubmission[i][2] + dataSubmission[i][3];
var checkGotMatch = false;
var toCopyRow = sheetSubmission.getRange(i+1,1,1,71);
// copy entire row for new record
Logger.log(currentIDSubmission);
// if there is a matching record flag as matched
for(var j = 1; j<dataActive.length; j++){
var currentIDActive = dataActive[j][2] + dataActive[j][3];
var currentIDSub = dataSubmission[i][2];
if(currentIDSub != '' && currentIDSubmission == currentIDActive){
checkGotMatch = true;
Logger.log(currentIDActive);
break;
}
}
// if it is a new record Append entire row
if(currentIDSub != '' && checkGotMatch == false){
**sheetMain.appendRow(toCopyRow.getValues()[0]);**
}
}
SpreadsheetApp.flush();
ss.toast("ACOPS Active has been updated.", "Complete");
}

In appendRow you need to pass an array so update your appendRow and try
sheetMain.appendRow([toCopyRow.getValues()[0]])

Adobe AIR readLine

I need to process a text file one line at a time. In BASIC, I could use a readline command, which would read until the next carriage return/line feed.
How would you write a function for looping through a file one line at a time in AIR?
var myDir = air.File.documentsDirectory;
var myFile = myDir.resolvePath("Test.txt");
if (myFile.exists) {
var myFileStream = new air.FileStream();
myFileStream.open(myFile, air.FileMode.READ);
var myByteArray = new air.ByteArray();
myFileStream.readBytes(myByteArray,0,myFileStream.bytesAvailable);
air.Introspector.Console.log(myByteArray.length);
} else {
alert ('File not found.');
}

var LineNumber;
var ItemCode;
var OrderCode;
var Qty;
var myDir = air.File.documentsDirectory;
var myFile = myDir.resolvePath("Test.txt");
if (myFile.exists) {
var myFileStream = new air.FileStream();
myFileStream.open(myFile, air.FileMode.READ);
var myData = new air.ByteArray();
myFileStream.readBytes(myData,0,myFileStream.bytesAvailable);
var str = myData.toString();
var Pos = 0;
var Tab = 0;
var CRLF = 0;
EOL = str.indexOf("\r",Pos);
while (EOL > 0) {
Tab = str.indexOf('\t',Pos);
LineNumber = str.substring(Pos,Tab);
Pos = Tab + 1;
Tab = str.indexOf('\t',Pos);
ItemCode = str.substring(Pos,Tab);
Pos = Tab + 1;
Tab = str.indexOf('\t',Pos);
OrderCode = str.substring(Pos,Tab);
Pos = Tab + 1;
CRLF = str.indexOf('\r',Pos);
Qty = str.substring(Pos,CRLF);
Pos = EOL+1;
EOL = str.indexOf("\r",Pos);
}
} else {
alert ('File not found.');
}