Goal: To stack data from 90+ google workbooks, all with the same sheet name, into the one master sheet for reporting
Info:
All worksheets have the same number of columns.
I have the following script but it does not run properly, I think the issue is with how I am caching / Pushing the data to the array before pasting to the output sheet.
I am trying to build an array then paste it in one go.
The tables I am stacking have 47 columns, unknown number of rows.
The part that opens the sheets is all working perfectly.
// Get the data from the worksheets
var indexsheet = SpreadsheetApp.getActive().getSheetByName("Index");
var outputsheet = SpreadsheetApp.getActive().getSheetByName("Output");
var response = SpreadsheetApp.getUi().prompt('Current Cycle', 'Enter Cycle Name Exactly in YY-MMM-Cycle# format', SpreadsheetApp.getUi().ButtonSet.OK_CANCEL)
var CurrentCycleName = response.getResponseText()
// Assign datasets to variables
var indexdata = indexsheet.getDataRange().getValues();
// For each workbook in the index sheet, open it and copy the data to a cache
indexdata.forEach(function(row, r) {
try {
//open Entity specific workbook
var workbookid = indexsheet.getRange(r + 1, 7, 1, 1).getValues();
var Entityworkbook = SpreadsheetApp.openById(workbookid)
// Open workhseet
Entitysheet.getSheetByName(CurrentCycleName)
// Add PR Data to cache - stacking for all countrys
var PRDataCache = Entitysheet.getDataRange().push()
} catch {}
})
// Set the all values of the sheet at once
outputsheet.getRange(r + 1, 14).setValue('Issue Splitting Data')
Entitysheet.getRange(2, 1, PRDataCache.length || 1, 47).setValues(PRDataCache)
};
This is the index tab where we are getting the workbookid from to open each file
This is the output file, we are stacking all data from each country
I believe your goal is as follows.
You want to retrieve the Spreadsheet IDs from the column "G" of "Index" sheet.
You want to give the specific sheet name using a dialog.
You want to retrieve all values from the specification sheet in all Spreadsheets. In this case, you want to remove the header row.
You want to put the retrieved values on "Output" sheet.
In this case, how about the following sample script?
Sample script:
function myFunction() {
var ss = SpreadsheetApp.getActive();
var indexsheet = ss.getSheetByName("Index");
var outputsheet = ss.getSheetByName("Output");
var response = SpreadsheetApp.getUi().prompt('Current Cycle', 'Enter Cycle Name Exactly in YY-MMM-Cycle# format', SpreadsheetApp.getUi().ButtonSet.OK_CANCEL);
var CurrentCycleName = response.getResponseText();
var ids = indexsheet.getRange("G1:G" + indexsheet.getLastRow()).getValues();
var values = ids.reduce((ar, [id]) => {
try {
var [, ...values] = SpreadsheetApp.openById(id).getSheetByName(CurrentCycleName).getDataRange().getValues();
ar = [...ar, ...values];
} catch (e) {
console.log(`"${id}" was not found.`);
}
return ar;
}, []);
if (values.length == 0) return;
// If the number of columns is different in all Spreadsheets, please use the following script.
// var maxLen = Math.max(...values.map(r => r.length));
// values = values.map(r => r.length < maxLen ? [...r, ...Array(maxLen - r.length).fill("")] : r);
outputsheet.getRange(outputsheet.getLastRow() + 1, 1, values.length, values[1].length).setValues(values);
}
Note:
When the number of Spreadsheet IDs is large, the processing time might be over 6 minutes. I'm worried about this. At that time, how about separating the Spreadsheet IDs?
Reference:
reduce()
Related
I am trying to write an App Script that takes string data from multiple different spreadsheets (completely separate documents) and puts them all in a new spreadsheet. When I run the logger, it shows me all the data I want. I want each piece of data to show up in Column A, but when I run my script, it only puts 1 data point in the spreadsheet instead of all of them. Can someone give me some guidance? Here is my code:
function pullTogether() {
var files = DriveApp.getFolderById('Folder ID').searchFiles('title != "nothing"');
const rangeName = 'Sheet1!B2:C';
while(files.hasNext()){
var xFile = files.next();
var name = xFile.getId();
const values = Sheets.Spreadsheets.Values.get(name, rangeName).values;
for (const row in values) {
var a1 = (values[row][0]);
Logger.log(a1);
var ss = SpreadsheetApp.openById("ID of new spreadsheet"); //I have the real ID in my code
var cell = ss.getRange("A2");
cell.setValue(a1);
}
}
}
I believe your goal is as follows.
You want to retrieve the values from the column "B" of each Spreadsheet under the specific folder.
You want to put the retrieved values to the column "A" of the destination sheet.
Modification points:
About but when I run my script, it only puts 1 data point in the spreadsheet instead of all of them., when I saw your script, the retrieved value is always put to the cell "A2" of the destination sheet. I think that this might be the reason for your issue.
In your script, I thought that when the following flow is used, the process cost will become low. By this flow, your issue can be also removed.
In your situation, even when Sheets API is not used, the script might work using getValues().
When these points are reflected in your script, it becomes as follows.
Modified script:
Please set the folder ID and the destination Spreadsheet ID.
function pullTogether() {
// Retrieve values from each Spreadsheet.
var values = [];
var files = DriveApp.getFolderById('Folder ID').searchFiles(`title != 'nothing' and mimeType='${MimeType.GOOGLE_SHEETS}'`);
var sheetName = 'Sheet1'
while (files.hasNext()) {
var xFile = files.next();
var sheet = SpreadsheetApp.open(xFile).getSheetByName(sheetName);
if (sheet) {
var v = sheet.getRange("B2:B" + sheet.getLastRow()).getValues();
values = [...values, ...v];
}
}
// Put values to the destination sheet.
var ss = SpreadsheetApp.openById("ID of new spreadsheet"); //I have the real ID in my code
var dstSheet = ss.getSheets()[0];
dstSheet.getRange(2, 1, values.length, values[0].length).setValues(values);
}
Note:
Although I'm not sure about your actual situation, when the above script didn't work by the large data, please modify as follows.
From
dstSheet.getRange(2, 1, values.length, values[0].length).setValues(values);
To
Sheets.Spreadsheets.Values.update({ values }, ss.getId(), `'${dstSheet.getSheetName()}'!A2`, { valueInputOption: "USER_ENTERED" });
References:
getValues()
setValues(values)
My question is similar to Need script to loop through stock ticker list, copy resulting output, and paste output to separate sheet within workbook. However, instead of copying and pasting a single row (for each ticker) below the last non-empty row, I need to copy and paste 100 rows (for each ticker) below the last non-empty row.
I have three sheets named Tickers , Data, and Results within a single workbook.
If done manually, the job would be the following: Enter a stock symbol into cell A2 in the Data sheet. The sheet then retrieves the historical time series data from google finance, runs formulas and returns 100 rows and 7 columns of data (Ticker, Date, Open, High, Low, Close, Volume) into range A5:G254 within the same Data sheet. Copy the 100 rows of returned data and paste the data on the Results sheet below the last non-empty row. Then repeat the process with each ticker.
Request: Create a script to loop through a list of 50 stock symbols from the Ticker sheet (range B2:B51), paste 1 symbol at a time into cell A2 in the Data sheet, wait for google finance API to run, and then copy all the resulting rows generated in range A5:G254, and pasting those results into the Results sheet, below the data generated from previous ticker, until there is a "long format" table of time series data of each of the 50 tickers.
The issue with my current script is that it pastes only the first row of the wanted range A5:G254.
function getNewPrices() {
const ss = SpreadsheetApp.getActiveSpreadsheet();
const tickerSheet = ss.getSheetByName('Tickers');
const dataSheet = ss.getSheetByName('Data');
const resultsSheet = ss.getSheetByName('Results');
let data;
let myListOfTickers = tickerSheet.getRange('B2:B51').getValues().filter(String).sort();
myListOfTickers.forEach(ticker => {
dataSheet.getRange('A2').setValue(ticker[0]);
data = dataSheet.getRange('A5:G254').getValues()[0];
pasteData(resultsSheet, data);
});
};
function pasteData(resultsSheet,data){
let nextRow = resultsSheet.getLastRow() + 1;
data.forEach((datum,index) => {
resultsSheet.getRange(nextRow,index + 1,1,1).setValue(datum);
});
};
I'm not completely sure but I think this is what you want
function getNewPrices() {
const ss = SpreadsheetApp.getActive();
const tsh = ss.getSheetByName('Tickers');
let tvs = tsh.getRange('B2:B51').getValues().filter(String).sort();
const dsh = ss.getSheetByName('Data');
const rsh = ss.getSheetByName('Results');
dsh.getRange(2,1,tvs.length,tvs[0].length).setValues(tvs);
let dvs = dsh.getRange('A5:G254').getValues();
rsh.getRange(rsh.getLastRow() + 1, 1, dvs.length, dvs[0].length).setValues(dvs);
}
You can goto to Google Apps Script Reference and using the search box find any function that you don't understand. If it's a pure JavaScript function the go here
Problem solved. Here is the final code if anyone wants to use it later on. Thank you for the initial code Cooper.
function getNewPrices() {
const ss = SpreadsheetApp.getActive();
const tsh = ss.getSheetByName('Tickers');
let tvs = tsh.getRange('B2:B51').getValues().filter(String).sort();
const dsh = ss.getSheetByName('Data');
const rsh = ss.getSheetByName('Results');
tvs.forEach(ticker =>{
dsh.getRange('A2').setValue(ticker[0]);
let dvs = dsh.getRange('A5:G254').getValues();
rsh.getRange(rsh.getLastRow() + 1, 1, dvs.length, dvs[0].length).setValues(dvs);
});
}
I have a function that allows me to import spreadsheet data from my gmail to Google Sheets. Previously, the spreadsheet data had only had 6 columns to import. Now, some new changes are made and a 7th column was added. After this change was implemented, my function no longer works and Google Sheets throws this error. May I please have some assistance?
So, as I'm looking at this, the intended functionality looks right to me. Skip the first 3 rows (netdata) and take everything below. Could it be the + 1, 1 ?
The Error:
TypeError: Cannot read property 'getRange' of null
My import function:
function importCSVFromGmail() {
var sheetName = "SHEET_NAME"; // Name of sheet tab.
var threads = GmailApp.search("from:EMAIL HERE label:LABEL HERE"); // "from:recipient email here label:name of your filter/label here"
var messages = threads[0].getMessages();
var message = messages[messages.length - 1];
var attachment = message.getAttachments()[0]; // [0] will by default look for only 1 attachment. If there are more than two attachment increase value. ex: [1] , [2]
var data = [];
if (attachment.getContentType() == MimeType.CSV) { // This will look for a CSV file type first
data = Utilities.parseCsv(attachment.getDataAsString(), ",");
} else if (attachment.getContentType() == MimeType.MICROSOFT_EXCEL || attachment.getContentType() == MimeType.MICROSOFT_EXCEL_LEGACY) { // If attachment is an xls, this line will look at the content to determine and convert accordingly.
var tempFile = Drive.Files.insert({title: "temp", mimeType: MimeType.GOOGLE_SHEETS}, attachment).id;
data = SpreadsheetApp.openById(tempFile).getSheets()[0].getDataRange().getValues();
Drive.Files.trash(tempFile);
}
if (data.length > 0) {
var sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(sheetName);
var netdata = data.slice(3); // This will skip the number of rows starting from the top.
sheet.getRange(sheet.getLastRow() + 1, 1, netdata.length, netdata[0].length).setValues(netdata );
}
}
I'm think the issue is here:
sheet.getRange(sheet.getLastRow() + 1, 1, netdata.length, netdata[0].length).setValues(netdata );
The error message indicates that the issue is with the sheet variable being null. That happens when there is no sheet by the name SHEET_NAME in the spreadsheet.
To fix the error, replace SHEET_NAME with the name of the sheet you want the function to work with. Check for things like leading and trailing whitespace in the sheet's name.
I am trying to avoid the following code because it is too slow:
for (var c = 25; c>2; c--){
if (sheet2.getRange(1,c).getValue() == 0)
{sheet2.deleteColumn(c)}
}
Instead I tried to find a list of columns I want to delete from the array and then set the array. (I recently figure out that deleting rows/columns in a loop is very expensive: google script loop performance)
I found this Removing columns of data in javascript array and try to apply it to my code, but it is not working.
Here is the code.
var ary = sheet2.getRange(2,1,outData.length+1,outData[0].length).getValues();
var indexesToRemove = [];
for (var c = 25; c>2; c--){
if (sheet2.getRange(1,c).getValue() == 0)
{
indexesToRemove.push(c);
}
}
The part above works well. What is not working is the function to remove the columns from the array once I found the indexes to remove. The array _row is not what I am looking for. What am I doing wrong?
removeColumns(ary, indexesToRemove);}
function removeColumns(data, indexes) {
return data.map(function (row) {
// when we remove columns, the indexing gets off by 1 each time, keep track of how many to adjust
var indexAdjustment = 0;
// copy row w/ .slice so we do not modify the original array
var _row = row.slice();
indexes.forEach(function (colIndex) {
// remove column
_row.splice(colIndex - indexAdjustment, 1);
// add 1 to adjustment to account for the column we just removed
indexAdjustment++
});
return _row;
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet2 = ss.getSheetByName('Cache');
sheet2.clear();
sheet2.getRange(2,1,_row.length,_row[0].length).setValues(_row);
});
}
BTW, I have also tried this before, but still not working:
var ary = sheet2.getRange(2,1,outData.length+1,outData[0].length).getValues();
for (var c = 25; c>2; c--){
if (sheet2.getRange(1,c).getValue() == 0)
{ ary = ary.map(function(item){
return item.splice(0,c)});
}
}
You want to delete the columns that the value of is 0 in the cells C1:Y1.
You want to reduce the process cost of the script.
You want to achieve this without using Sheets API.
Pattern 1:
In this pattern, at first, the cells which have the value of 0 from the cells C1:Y1 using TextFinder, and the columns are deleted from the retrieved cells using deleteColumn().
Sample script:
const sheet = SpreadsheetApp.getActiveSheet();
sheet.getRange("C1:Y1")
.createTextFinder(0)
.matchEntireCell(true)
.findAll()
.reverse()
.forEach(e => sheet.deleteColumn(e.getColumn()));
Pattern 2:
In this pattern, at first, all values are retrieved from "C1" to the last column for the all data rows, and delete the columns in the array and clear the range, and then, the values are put to the sheet. The method for directly processing the retrieved values has already been proposed. So as other pattern, I proposed the method which uses the transpose.
Sample script:
const sheet = SpreadsheetApp.getActiveSheet();
const range = sheet.getRange(1, 3, sheet.getLastRow(), sheet.getLastColumn() - 2);
const values = range.getValues();
const t = values[0].reduce((ar, r, i) => {
if (r != 0) ar.push(values.map(c => c[i]));
return ar;
}, []);
const v = t[0].map((_, i) => t.map(c => c[i]));
range.clearContent();
sheet.getRange(1, 3, v.length, v[0].length).setValues(v);
Pattern 3:
In this pattern, the request body for the batchUpdate method of Sheets API is created using the 1st row values, and the request body is used for requesting to Sheets API. By this, several columns can be deleted by one API call.
Before you run the script, please enable Sheets API at Advanced Google services.
Sample script:
const spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
const sheet = spreadsheet.getActiveSheet();
const sheetId = sheet.getSheetId();
// Create rerequests for DeleteDimensionRequest.
const requests = sheet.getRange("C1:Y1")
.createTextFinder(0)
.matchEntireCell(true)
.findAll()
.reverse()
.map(e => {
const col = e.getColumn();
return {deleteDimension: {range: {sheetId: sheetId, dimension: "COLUMNS", startIndex: col - 1, endIndex: col}}}
});
// Request to the batchUpdate method using the request body.
Sheets.Spreadsheets.batchUpdate({requests: requests}, spreadsheet.getId());
In this case, requests is created using the method of pattern 1. Each request is as follows. You can see about this structure at the document.
{
"deleteDimension": {
"range": {
"sheetId": "###",
"dimension": "COLUMNS",
"startIndex": ##,
"endIndex": ##
}
}
}
References:
Class TextFinder
Advanced Google services
Method: spreadsheets.batchUpdate
DeleteDimensionRequest
function runOne() {
var d=0;
var ss=SpreadsheetApp.getActive();
var sh=ss.getActiveSheet();
var hA=sh.getRange(1,1,1,sh.getLastColumn()).getValues()[0];//header array
var vs=sh.getRange(2,1,sh.getLastRow()-1,sh.getLastColumn()).getValues();//data array
vs.forEach(function(r,j){
var d=0;
hA.forEach(function(h,i){
if(h==0)r.splice(i-d++,1);//removes elements in columns whose headers are == 0
});
});
Logger.log(vs);
}
Try using Sheets Advanced Service and batchUpdate.
Related
Remove all grouped rows / columns in a spreadsheet
Reference
https://developers.google.com/sheets/api/samples/rowcolumn#delete_rows_or_columns
I am currently trying to get grab values from another spreadsheet and then paste it into a destination spreadsheet. The problem I am running into is that I am getting incorrect range height and incorrect range widths when I run this code. I read something about 2d arrays but I believe I already have a 2d array here to paste to the spreadsheet. Thank you for your time.
function GmailToDrive_StaticTest(gmailSubject, importFileID){
var threads = GmailApp.search('subject:' + gmailSubject + ' -label:uploaded has:attachment'); // performs Gmail query for email threads
for (var i in threads){
var messages = threads[i].getMessages(); // finds all messages of threads returned by the query
for (var j in messages){
var attachments = messages[j].getAttachments(); // finds all attachments of found messages
var timestamp = messages[j].getDate(); // receives timestamp of each found message
var timestampMinusOne = new Date(timestamp.getTime() - (86400000)); // sets the received timestamp to exactly one day prior (# in milliseconds)
var date = Utilities.formatDate(timestampMinusOne, "MST", "yyyy-MM-dd"); // cleans the timestamp string
for (var k in attachments){
var blobs = {
dataType: attachments[k].getContentType(), // retrives the file types of the attachments
data: attachments[k].copyBlob(), // creates blob files for every attachment
fileName: attachments[k].getName()
};
var tempFile = DriveApp.createFile(blobs.data.setContentType('text/csv')).setName(blobs.fileName.split("-", 1).toString() + date); // creates csv files in drive's root per blob file
var tempFileConverted = Drive.Files.copy( {}, tempFile.getId(), {convert: true} ); // converts created files to gsheets
var importData = {
file: tempFileConverted,
ID: tempFileConverted.getId(),
Sheet1: SpreadsheetApp.openById(tempFileConverted.getId() ).getActiveSheet(),
Sheet1_Values: SpreadsheetApp.openById(tempFileConverted.getId() ).getActiveSheet().getDataRange().getValues()
};
tempFile.setTrashed(true);
var importData_Sheet1_Rows = importData.Sheet1.getMaxRows(); - 2;
var importData_Sheet1_Columns = importData.Sheet1.getMaxColumns(); - 2;
var destSheet = SpreadsheetApp.openById(importFileID).getSheets()[0];
destSheet.clearContents();
Logger.log(importData.Sheet1_Values)
destSheet.getRange(1, 1, importData_Sheet1_Rows, importData_Sheet1_Columns).setValues(importData.Sheet1_Values);
DriveApp.getFileById(importData.ID).setTrashed(true);
}
}
}
}
getMaxRows() and getMaxColumns() return the maximum number of column and rows in a sheet, while getDataRange().getValues() return all the values in a sheet that contain data .
So, unless all the cells in a sheet have data the dimensions won't match !
The best you could do is to get the actual size of the data array and use that to set the range for the values in the destination sheet.
It goes (more) simply like this :
destSheet.getRange(1, 1, importData.Sheet1_Values.length, importData.Sheet1_Values[0].length).setValues(importData.Sheet1_Values);
you don't need the other values for rows and columns, just ignore that in your script.