I need your help. I have written a small javascript that reads a Google spreadsheet. This spreadsheet is maintained by another employee. There are the following columns named last name (column A, as example "Doe"), first name (column B, as example "John"), a Chayns ID which is used for matching (column C, as example 112-73302) and a score which the person has achieved so far.
In the Chayns environment (A platform for various apps) I query the current Chayns ID, which is assigned when logged in. Matching the JSON file provided by Google is read and the score matching the Chayns ID is output accordingly. Now unfortunately something has changed in the Chayns system. The Chayns IDs used to consist of pure numbers, but now this has been replaced by a combination of numbers and letters. The script still works with pure number-based IDs, but no longer with the new IDs in combination with the letters. Now I'm wondering where I just got the thinking error. I tried to examine the JSON file provided by Google. Where the letter number combination should be output, there is only a "NULL". How can I work around this? I am not so fit in the topic JSON!
Thanks a lot for your help!
chayns.ready.then(() => {
if (chayns.env.user.isAuthenticated) { // check if person is logged-in
const chaynsid = chayns.env.user.personId; // Chayns-ID to variable
const chaynsperson = chayns.env.user.name; // Chayns-Username to variable
let chaynsBereinigt = chaynsid; // rename
chaynsBereinigt = chaynsBereinigt.replace(/-/g, ''); // replace all dashes in Chayns-ID with nothing
document.getElementById('idausspielung').innerHTML = chaynsid;
const base = 'https://docs.google.com/spreadsheets/d/XXXXXXXXXXXXXXXXXXXXXXXXXX/gviz/tq?'; // Google Sheets Info
const output = document.querySelector('.output'); // Target is Html table with class output
const query = encodeURI('Select D where C = ' + chaynsBereinigt); // Which columns should be output with
const url = base + '&tq=' + query;
fetch(url)
.then(res => res.text())
.then(rep => {
const data = JSON.parse(rep.substr(47).slice(0, -2));
const row = document.createElement('tr');
output.append(row);
data.table.cols.forEach((heading) => {
const cell = document.createElement('td');
cell.textContent = heading.label;
row.append(cell);
})
data.table.rows.forEach((main) => {
const container = document.createElement('tr');
output.append(container);
main.c.forEach((ele) => {
const cell = document.createElement('td');
cell.textContent = ele.v;
container.append(cell);
})
})
})
console.log("Hallo " + chaynsperson + " , deine Chayns-ID lautet: " + chaynsBereinigt); // Play out values to console
}
})
JSON OUTPUT
// Correctly output values number-based
{"c":[{"v":"Doe"},{"v":"Francis"},{"v":5.0928467E7,"f":"50928467"},{"v":1028.0,"f":"1028"}]}
// Incorrectly output values number and letter based (example: 75VBUA3I)
{"c":[{"v":"Doe"},{"v":"Francis"},null,{"v":100.0,"f":"100"}]}
Related
Okay, I know this might be a weird one.
I have this block of code in GAS:
function setHeroData(ss_id, column, row, valueInputOption) {
try {
var sheet = SpreadsheetApp.getActiveSheet();
let range = sheet.getName() + "!" + column + row;
sheet.setColumnWidth(1, 50);
let domain = "https://gamepress.gg";
let response = UrlFetchApp.fetch(`${domain}/feheroes/heroes`);
let page = Cheerio.load(response.getContentText());
const heroes = new Array(page(".icon\-cell").length+1);
const profiles = new Array(page(".icon\-cell").length);
const vas = new Array(page(".icon\-cell").length);
const illus = new Array(page(".icon\-cell").length);
let profile;
let va;
let ill;
heroes[0] = [];
heroes[0].push("Icon");
heroes[0].push("Hero");
heroes[0].push("Hero Epithet");
heroes[0].push("");
heroes[0].push("Weapon");
heroes[0].push("")
heroes[0].push("Movement");
heroes[0].push("Rarity");
heroes[0].push("Origin");
heroes[0].push("VA (EN)");
heroes[0].push("VA (JP)");
heroes[0].push("Illustrator");
page(".icon\-cell").each(function (i, elem) {
let img = domain + page(elem).find(".hero\-icon").children("img").attr("src");
heroes[i+1] = [];
heroes[i+1].push(`=image("${img}", 3)`);
heroes[i+1].push(page(elem).find(".adventurer\-title").text());
profile = domain + page(this).children("a").attr("href");
profiles[i] = profile;
va = page(elem).parent().attr("data-va");
vas[i] = va;
ill = page(elem).parent().attr("data-ill");
illus[i] = ill;
});
let prof_pages = UrlFetchApp.fetchAll(profiles);
// Get epithets from profile pages
for (let i = 0; i<heroes.length-1; ++i) {
let prof_page = Cheerio.load(prof_pages[i].getContentText());
let attrib = prof_page(".vocabulary-attribute").find(".field--name-name").text();
let attrib_img = domain + prof_page(".vocabulary-attribute").find("img").attr("src");
let move_type = prof_page(".vocabulary-movement").find(".field--name-name").text();
let move_type_img = domain + prof_page(".vocabulary-movement").find("img").attr("src");
let stars = prof_page(".vocabulary-obtainable-stars").find(".field--name-name").text()[0];
let origin = prof_page(".field--name-field-origin").text().trim();
// Populate hero data
heroes[i+1].push(prof_page(".field--name-title").siblings("span").text().replace(" - ", ""));
heroes[i+1].push(`=image("${attrib_img}")`);
heroes[i+1].push(attrib);
heroes[i+1].push(`=image("${move_type_img}")`);
heroes[i+1].push(move_type);
heroes[i+1].push(`=image("https://gamepress.gg/sites/fireemblem/files/2017-06/stars${stars}.png", 3)`)
heroes[i+1].push(origin);
// https://stackoverflow.com/questions/36342430/get-substring-before-and-after-second-space-in-a-string-via-javascript
// Separate the EN and JP voice actors names
let index = vas[i].includes(".") ? vas[i].indexOf(' ', vas[i].indexOf('.') + 2) : vas[i].indexOf(' ', vas[i].indexOf(' ') + 1);
let en_va = index >= 0 ? vas[i].substr(0, index) : vas[i].substr(index + 1);
let jp_va = index >= 0 ? vas[i].substr(index + 1) : "";
if (en_va.toLowerCase() === "Cassandra Lee".toLowerCase()) {
en_va = en_va.concat(" Morris");
jp_va = jp_va.replace("Morris", "");
// Logger.log(en_va);
// Logger.log(jp_va);
}
heroes[i+1].push(en_va.trim());
heroes[i+1].push(jp_va.trim());
heroes[i+1].push(illus[i]);
Logger.log((i*100)/(heroes.length-1));
}
let first_col = column.charCodeAt(0) - 64;
Sheets.Spreadsheets.Values.update({values: heroes}, ss_id, range, valueInputOption);
sheet.autoResizeColumns(first_col, heroes[0].length).autoResizeRows(row, heroes.length);
sheet.setRowHeights(row+1, (heroes.length-1)+row, 50);
sheet.setColumnWidth(first_col, 50);
sheet.setColumnWidth(first_col + 3, 30);
sheet.setColumnWidth(first_col + 5, 30);
sheet.setColumnWidth(first_col + 7, 100);
sheet.setColumnWidth(first_col + 8, 319);
}
catch (err) {
Logger.log(err);
}
}
Now, this code works fine for now, which means it populates a sheet with data scraped from a website. The problem is that there's a group of strings (VA names) that are not entirely consistent in how they're formatted. There's supposed to be two VA's, an English and a Japanese VA, and both are stored in the same html attribute (meaning they're both in the same string), but as names tend to be, there's a lot of variation. Some names are shorter, some longer; some have more words, some less and sometimes it's hard to tell when one name ends and another starts. So far, I've only managed to solve this issue for names with dots (ex: "Joe J. Thomas"), and for names like "Cassandra Lee Morris" I need to specify and edge case, which (from my understanding) is less than ideal. I have also tried to scrape a tag within the webpage that contains all names so that I can maybe validate the names, or take the names directly from that list, but I haven't had any luck.
EDIT:
For example, these are strings with different sets of names:
a) "Julie Kliewer Uchida Maaya (内田真礼)"
b) "Joe J. Thomas Suzuki Tatsuhisa (鈴木達央)"
c) "Cassandra Lee Morris Fujiwara Natsumi (藤原夏海)"
I want to be able to extract the english names ("Julie Kliewer", "Joe J. Thomas", "Cassandra Lee Morris") and the japanese names ("Uchida Maaya (内田真礼)", "Suzuki Tatsuhisa (鈴木達央)", "Fujiwara Natsumi (藤原夏海)"), and store each set of names separate from each other.
After looking for a while, I found this json file that contains all the information I need, and also has separate entries for the different Voice Actors of each character. This works great for my purposes, and also makes it easier to associate the correct data to each character.
complete noob, and my first ever post,so sorry in advance for the eventual poor choice of words.
I am working on a mail merge script, that will fill a GDoc template with data from a GSheet, creating a separate GDoc for each row in GSheet.
Script is working well, I'm using the .replacetext method on the template's body, like below:
function createNewGoogleDocs() {
const documentLink_Col = ("Document Link");
const template = DriveApp.getFileById('1gZG-NR8CcOpnBTZfTy8gEsGDOLXa9Ba9Ks5zXJbujY4');
const destinationFolder = DriveApp.getFolderById('1DcpZGeyoCJxAQu1vMbSj31amzpwfr_JB');
const sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('data');
const data = sheet.getDataRange().getDisplayValues();
const heads = data[0]; // Assumes row 1 contains the column headings
const documentLink_ColIndex = heads.indexOf(documentLink_Col);
data.forEach(function(row, index){
if(index === 0) return;
const templateCopy = template.makeCopy(`${row[0]} ${row[1]} Report`, destinationFolder); //create a copy of the template document
const templateCopyId = DocumentApp.openById(templateCopy.getId());
const templateCopyBody = templateCopyId.getBody();
templateCopyBody.replaceText('{{Name}}', row[0]);
templateCopyBody.replaceText('{{Address}}', row[1]);
templateCopyBody.replaceText('{{City}}', row[2]);
templateCopyId.saveAndClose();
const url = templateCopyId.getUrl();
sheet.getRange(index +1 , documentLink_ColIndex + 1).setValue(url);
})
}
What I want to change:
Have freedom to add/remove columns in the sheet without having to hard code every header column with a .replacetext method
I have found a kind of similar script that achieves that for sending emails based on GmailApp, and I extracted 2 functions that do a token replacement, but I don't know how to call the function fillInTemplateFromObject_ in my function createNewGoogleDocs
here is the code for the functions I found in the other script:
function fillInTemplateFromObject_(template, data) {
// We have two templates one for plain text and the html body
// Stringifing the object means we can do a global replace
let template_string = JSON.stringify(template);
// Token replacement
template_string = template_string.replace(/{{[^{}]+}}/g, key => {
return escapeData_(data[key.replace(/[{}]+/g, "")] || "");
});
return JSON.parse(template_string);
}
/**
* Escape cell data to make JSON safe
* #see https://stackoverflow.com/a/9204218/1027723
* #param {string} str to escape JSON special characters from
* #return {string} escaped string
*/
function escapeData_(str) {
return str
.replace(/[\\]/g, '\\\\')
.replace(/[\"]/g, '\\\"')
.replace(/[\/]/g, '\\/')
.replace(/[\b]/g, '\\b')
.replace(/[\f]/g, '\\f')
.replace(/[\n]/g, '\\n')
.replace(/[\r]/g, '\\r')
.replace(/[\t]/g, '\\t');
};
Thanks everyone in advance for your support.
Using column headers to make programmatic assignments
function myfunction() {
const ss = SpreadsheetApp.getActive();
const sh = ss.getSheetByName("Sheet0");
const [hA, ...vs] = sh.getDataRange();// hA is Name Address City
const idx = {};
const body = DocumentApp.getActiveDocument().getBody();
hA.forEach((h, i) => { idx[h] = i; })
vs.forEach(row => {
body.replaceText("{{Name}}", row[idx["Name"]]);
body.replaceText("{{Address}}", row[idx["Address"]]);
body.replaceText("{{City}}", row[idx["City"]]);
});
}
As long as you keep the column titles the same you can move them around anywhere you wish
Goal: To stack data from 90+ google workbooks, all with the same sheet name, into the one master sheet for reporting
Info:
All worksheets have the same number of columns.
I have the following script but it does not run properly, I think the issue is with how I am caching / Pushing the data to the array before pasting to the output sheet.
I am trying to build an array then paste it in one go.
The tables I am stacking have 47 columns, unknown number of rows.
The part that opens the sheets is all working perfectly.
// Get the data from the worksheets
var indexsheet = SpreadsheetApp.getActive().getSheetByName("Index");
var outputsheet = SpreadsheetApp.getActive().getSheetByName("Output");
var response = SpreadsheetApp.getUi().prompt('Current Cycle', 'Enter Cycle Name Exactly in YY-MMM-Cycle# format', SpreadsheetApp.getUi().ButtonSet.OK_CANCEL)
var CurrentCycleName = response.getResponseText()
// Assign datasets to variables
var indexdata = indexsheet.getDataRange().getValues();
// For each workbook in the index sheet, open it and copy the data to a cache
indexdata.forEach(function(row, r) {
try {
//open Entity specific workbook
var workbookid = indexsheet.getRange(r + 1, 7, 1, 1).getValues();
var Entityworkbook = SpreadsheetApp.openById(workbookid)
// Open workhseet
Entitysheet.getSheetByName(CurrentCycleName)
// Add PR Data to cache - stacking for all countrys
var PRDataCache = Entitysheet.getDataRange().push()
} catch {}
})
// Set the all values of the sheet at once
outputsheet.getRange(r + 1, 14).setValue('Issue Splitting Data')
Entitysheet.getRange(2, 1, PRDataCache.length || 1, 47).setValues(PRDataCache)
};
This is the index tab where we are getting the workbookid from to open each file
This is the output file, we are stacking all data from each country
I believe your goal is as follows.
You want to retrieve the Spreadsheet IDs from the column "G" of "Index" sheet.
You want to give the specific sheet name using a dialog.
You want to retrieve all values from the specification sheet in all Spreadsheets. In this case, you want to remove the header row.
You want to put the retrieved values on "Output" sheet.
In this case, how about the following sample script?
Sample script:
function myFunction() {
var ss = SpreadsheetApp.getActive();
var indexsheet = ss.getSheetByName("Index");
var outputsheet = ss.getSheetByName("Output");
var response = SpreadsheetApp.getUi().prompt('Current Cycle', 'Enter Cycle Name Exactly in YY-MMM-Cycle# format', SpreadsheetApp.getUi().ButtonSet.OK_CANCEL);
var CurrentCycleName = response.getResponseText();
var ids = indexsheet.getRange("G1:G" + indexsheet.getLastRow()).getValues();
var values = ids.reduce((ar, [id]) => {
try {
var [, ...values] = SpreadsheetApp.openById(id).getSheetByName(CurrentCycleName).getDataRange().getValues();
ar = [...ar, ...values];
} catch (e) {
console.log(`"${id}" was not found.`);
}
return ar;
}, []);
if (values.length == 0) return;
// If the number of columns is different in all Spreadsheets, please use the following script.
// var maxLen = Math.max(...values.map(r => r.length));
// values = values.map(r => r.length < maxLen ? [...r, ...Array(maxLen - r.length).fill("")] : r);
outputsheet.getRange(outputsheet.getLastRow() + 1, 1, values.length, values[1].length).setValues(values);
}
Note:
When the number of Spreadsheet IDs is large, the processing time might be over 6 minutes. I'm worried about this. At that time, how about separating the Spreadsheet IDs?
Reference:
reduce()
Objective:
I would like to eliminate the use of formulas (array formulas, importranges, and vlookups). Instead, I would like to use Google App Script to populate columns in the Child Database Spreadsheet. This is because of the current performance issues every time I open the sheet, and issues with Google Data Studio timing out when pulling the data.
I have 2 spreadsheets.
#1 - Master Database (~1,000,000 rows) - 100% Manual Input
A (Manual Input)
B (Manual Input)
C (Manual Input)
1
X123456
John Doe
JohnDoe#examplecom
2
X987654
Jane Smith
JaneSmith#examplecom
3
X543210
Sarah Smith
SarahSmith#examplecom
#2 - Child Database (~10,000 rows)
Its purpose: Manually enter ID's in Col A, and the formula will auto-populate Col B:C (Name & Email)
This is the expected results with GAS instead of the current formula.
A (Manual Input)
B (Auto-Populate)
C (Auto-Populate)
1
X543210
Sarah Smith
SarahSmith#examplecom
2
X123456
John Doe
JohnDoe#examplecom
Col A - Manual Input of ID.
Col B1 contains formula =ARRAYFORMULA(VLOOKUP(A2:A,IMPORTRANGE("URL","MasterDB!A2:C"),{2,3},FALSE)) which get's the ID's from Col A, searches the Master Database spreadsheet, and returns the Name, and Email.
What is the best solution?
Here is what I came up with, so far...
function myFunction() {
//Source Info.
const sss = SpreadsheetApp.openById('ABC');
const ssh = sss.getSheetByName("MasterDB");
const mDB = ssh.getRange("A2:A").getValues; //Get's ID's from Master Spreadsheet
//Destination Info.
const dss = SpreadsheetApp.openById('XYZ');
const dsh = dss.getSheetByName("ChildDB");
const cDB = dsh.getRange("A2:A").getValues; //Get's ID's from Child Spreadsheet
[Some Code Here]
- Return Col B,C from Master Sheet, if Col A matches in both Master & Child Sheet.
}
Thanks for any of your input, guidance, and help :)
Modification points:
In your script, const mDB = ssh.getRange("A2:A").getValues; and const cDB = dsh.getRange("A2:A").getValues; are required to be added () for executing the function of getValues.
It seems that import of the function name is the reserved name. So please modify the function name. When V8 runtime is used.
When these points are reflected to the script, it becomes as follows.
Modified script:
function myFunction() {
const sss = SpreadsheetApp.openById('ABC');
const ssh = sss.getSheetByName("MasterDB");
const mDB = ssh.getRange("A2:C" + ssh.getLastRow()).getValues(); //Get's ID's from Master Spreadsheet
const dss = SpreadsheetApp.openById('XYZ');
const dsh = dss.getSheetByName("ChildDB");
const cDB = dsh.getRange("A2:A" + dsh.getLastRow()).getValues(); //Get's ID's from Child Spreadsheet
// Create an object for searching the values of column "A".
const obj = mDB.reduce((o, [a, ...bc]) => ((o[a] = bc), o), {});
// Create an array for putting to the Spreadsheet.
const values = cDB.map(([b]) => obj[b] || ["", ""]);
// Put the array to the Spreadsheet.
dsh.getRange(2, 2, values.length, 2).setValues(values);
}
In order to achieve your goal, I modified the sample script at this thread.
Note:
This script is used with V8 runtime. So when you disable V8 runtime, please enable it.
If this was not the result you expect, can you provide the sample Spreadsheet? By this, I would like to modify the script.
References:
reduce()
map()
Added:
About your new 3 questions, I answered as follows.
[Question #1] I assume o is just a placeholder and can be any letter I want. Is that true? or does the letter o have some significant?
Yes. You can use other variable name except for o. In this script, the initial value of o is {}. Ref
[Question #2] What do the 3 dots do? [a, ...bc] ?
... is spread syntax. Ref
[Question #3] How would I skip a returned column? Currently it returns b,c. How would I return c,d instead?
In this case, the sample script is as follows.
function Q69818704_myFunction() {
const sss = SpreadsheetApp.openById('ABC');
const ssh = sss.getSheetByName("MasterDB");
const mDB = ssh.getRange("A2:D" + ssh.getLastRow()).getValues();
const dss = SpreadsheetApp.openById('XYZ');
const dsh = dss.getSheetByName("ChildDB");
const cDB = dsh.getRange("A2:A" + dsh.getLastRow()).getValues();
const obj = mDB.reduce((o, [a,, ...cd]) => ((o[a] = cd), o), {});
const values = cDB.map(([b]) => obj[b] || ["", ""]);
dsh.getRange(2, 2, values.length, 2).setValues(values);
}
I've got an array of rows that I've parsed out of a table from html, stored in a list. Each of the rows in the list is a string that looks (something) like this:
["<td headers="DOCUMENT" class="t14data"><a target="6690-Exhibit-C-20190611-1" href="http://www.fara.gov/docs/6690-Exhibit-C-20190611-1.pdf" class="doj-analytics-processed"><span style="color:blue">Click Here </span></a></td><td headers="REGISTRATIONNUMBER" class="t14data">6690</td><td headers="REGISTRANTNAME" class="t14data">SKDKnickerbocker LLC</td><td headers="DOCUMENTTYPE" class="t14data">Exhibit C</td><td headers="STAMPED/RECEIVEDDATE" class="t14data">06/11/2019</td>","<td headers="DOCUMENT" class="t14data"><a target="5334-Supplemental-Statement-20190611-30" href="http://www.fara.gov/docs/5334-Supplemental-Statement-20190611-30.pdf" class="doj-analytics-processed"><span style="color:blue">Click Here </span></a></td><td headers="REGISTRATIONNUMBER" class="t14data">5334</td><td headers="REGISTRANTNAME" class="t14data">Commonwealth of Dominica Maritime Registry, Inc.</td><td headers="DOCUMENTTYPE" class="t14data">Supplemental Statement</td><td headers="STAMPED/RECEIVEDDATE" class="t14data">06/11/2019</td>"]
The code is pulled from the page with the following page.evaluate function using puppeteer.
I'd like to then parse this code with cheerio, which I find to be simpler and more understandable. However, when I pass each of the strings of html into cheerio, it fails to parse them correctly. Here's the current function I'm using:
let data = res.map((tr) => {
let $ = cheerio.load(tr);
const link = $("a").attr("href");
const number = $("td[headers='REGISTRATIONNUMBER']").text();
const name = $("td[headers='REGISTRANTNAME']").text();
const type = $("td[headers='DOCUMENTTYPE']").text();
const date = $("td[headers='STAMPED/RECEIVEDDATE']").text();
return { link, number, name, type, date };
});
For some reason, only the "a" tag is working correctly for each row. Meaning, the "link" variable is correctly defined, but none of the other ones are. When I use $("*") to return a list of what should be all of the td's, it returns an unusual node list:
What am I doing wrong, and how can I gain access to the td's with the various headers, and their text content? Thanks!
It usually looks more like this:
let data = res.map((i, tr) => {
const link = $(tr).find("a").attr("href");
const number = $(tr).find("td[headers='REGISTRATIONNUMBER']").text();
const name = $(tr).find("td[headers='REGISTRANTNAME']").text();
const type = $(tr).find("td[headers='DOCUMENTTYPE']").text();
const date = $(tr).find("td[headers='STAMPED/RECEIVEDDATE']").text();
return { link, number, name, type, date };
}).get();
Keep in mind that cheerio map has the arguments reversed from js map.
I found the solution. I'm simply returning the full html through puppeteer instead of trying to get individual rows, and then using the above suggestion (from #pguardiario) to parse the text:
const res = await page.evaluate(() => {
return document.body.innerHTML;
});
let $ = cheerio.load(res);
let trs = $(".t14Standard tbody tr.highlight-row");
let data = trs.map((i, tr) => {
const link = $(tr).find("a").attr("href");
const number = $(tr).find("td[headers='REGISTRATIONNUMBER']").text();
const registrant = $(tr).find("td[headers='REGISTRANTNAME']").text();
const type = $(tr).find("td[headers='DOCUMENTTYPE']").text();
const date = moment($(tr).find("td[headers='STAMPED/RECEIVEDDATE']").text()).valueOf().toString();
return { link, number, registrant, type, date };
});