Related
i use a script to take all my files from drive to a google spreadsheet with name, url...
My problem is there are a lot of files and the script run during ~30 min and exceed during time permission for apps script.
How can i speed this script please ?
function listFilesAndFolders() {
var folderid = " "; // change FolderID
var sh = SpreadsheetApp.getActiveSheet();
sh.clear();
sh.appendRow(["parent","folder", "name", "update", "Size", "URL", "ID", "description", "type"]);
try {
var parentFolder =DriveApp.getFolderById(folderid);
listFiles(parentFolder,parentFolder.getName())
listSubFolders(parentFolder,parentFolder.getName());
} catch (e) {
Logger.log(e.toString());
}
}
function listSubFolders(parentFolder,parent) {
var childFolders = parentFolder.getFolders();
while (childFolders.hasNext()) {
var childFolder = childFolders.next();
Logger.log("Fold : " + childFolder.getName());
listFiles(childFolder,parent)
listSubFolders(childFolder,parent + "|" + childFolder.getName());
}
}
function listFiles(fold,parent){
var sh = SpreadsheetApp.getActiveSheet();
var data = [];
var files = fold.getFiles();
while (files.hasNext()) {
var file = files.next();
data = [
parent,
fold.getName(),
file.getName(),
file.getLastUpdated(),
file.getSize(),
file.getUrl(),
file.getId(),
file.getDescription(),
file.getMimeType()
];
sh.appendRow(data);
}
}
Rather then use Sheet.appendRow() collect each row in an array rows and when all done looping use .getRange().setValues().
function listFilesAndFolders() {
var folderid = " "; // change FolderID
var sh = SpreadsheetApp.getActiveSheet();
sh.clear();
let rows = [["parent","folder", "name", "update", "Size", "URL", "ID", "description", "type"]];
try {
var parentFolder =DriveApp.getFolderById(folderid);
listFiles(parentFolder,parentFolder.getName(), rows )
listSubFolders(parentFolder,parentFolder.getName(), rows );
sh.getRange(sh.getLastRow()+1,1,rows.length,rows[0].length).setValues(rows);
} catch (e) {
Logger.log(e.toString());
}
}
function listSubFolders(parentFolder,parent, rows ) {
var childFolders = parentFolder.getFolders();
while (childFolders.hasNext()) {
var childFolder = childFolders.next();
Logger.log("Fold : " + childFolder.getName());
listFiles(childFolder,parent, rows )
listSubFolders(childFolder,parent + "|" + childFolder.getName(), rows );
}
}
function listFiles(fold,parent, rows ){
var sh = SpreadsheetApp.getActiveSheet();
var files = fold.getFiles();
while (files.hasNext()) {
var file = files.next();
rows.push([
parent,
fold.getName(),
file.getName(),
file.getLastUpdated(),
file.getSize(),
file.getUrl(),
file.getId(),
file.getDescription(),
file.getMimeType()
]);
}
}
I believe your goal is as follows.
You want to reduce the process cost of your script.
Modification points:
In your script, appendRow is used in a loop. In this case, the process cost becomes high. Ref: Author me
I thought that when Drive API is used, the process cost might be able to be reduced a little.
When these points are reflected in your script, how about the following modification? In this modification, a Google Apps Script library is used. The author of this Google Apps Script library of FilesApp is me.
Usage:
1. Install Google Apps Script library.
You can see how to install Google Apps Script library of [FilesApp] at here.
2. Enable Drive API.
This modified script uses Drive API. So, please enable Drive API at Advanced Google services.
3. Sample script.
Please copy and paste the following script to the script editor of Spreadsheet. And, please set the top folder ID to folderId. If you use var folderId = "root";, all files in your Google Drive are retrieved.
function myFunction() {
var folderId = "###"; // Please set the top folder ID.
var header = ["parent", "folder", "name", "update", "Size", "URL", "ID", "description", "type"]; // This is from your script.
var obj = FilesApp.createTree(folderId, null, "files(name,modifiedTime,size,webViewLink,id,description,mimeType)");
var values = [header, ...obj.files.flatMap(({ folderTreeByName, filesInFolder }) => {
const f = [folderTreeByName.join("|"), folderTreeByName.pop()];
return filesInFolder.length == 0 ? [[...f, ...Array(7).fill(null)]] : filesInFolder.filter(({ mimeType }) => mimeType != MimeType.FOLDER).map(({ name, modifiedTime, size, webViewLink, id, description, mimeType }) => [...f, name || null, new Date(modifiedTime), size || 0, webViewLink, id, description || null, mimeType]);
})];
SpreadsheetApp.getActiveSheet().clear().getRange(1, 1, values.length, values[0].length).setValues(values);
}
References:
FilesApp of Google Apps Script library (Author me)
Files: list of Drive API v3
This will list all of the files and folders in your drive into Sheet1
var level = 1;
function getFnF1(folder = DriveApp.getRootFolder()) {
let tree = JSON.parse(PropertiesService.getScriptProperties().getProperty('FnF'));
//Logger.log(JSON.stringify(tree));
if (tree.level < level) {
tree.level = level;
PropertiesService.getScriptProperties().setProperty('FnF', JSON.stringify(tree));
}
const files = folder.getFiles();
let row = Array.from([...Array(level).keys()], ((x, i) => { if (i == level - 1) { x = folder.getName(); } else { x = ''; } return x; }));
tree.txt.push(row);
row = Array.from([...Array(level).keys()], ((x, i) => { if (i == level - 1) { x = 'bold'; } else { x = 'normal'; } return x; }));
tree.fwt.push(row);
PropertiesService.getScriptProperties().setProperty('FnF', JSON.stringify(tree));
if (files.hasNext()) {
let row = Array.from([...Array(level).keys()], ((x, i) => { if (i == level - 1) { x = 'Files:'; } else { x = ''; } return x; }));
tree.txt.push(row);
tree.fwt.push(['normal']);
PropertiesService.getScriptProperties().setProperty('FnF', JSON.stringify(tree));
}
while (files.hasNext()) {
let file = files.next();
let row = Array.from([...Array(level + 1).keys()], ((x, i) => { if (i == level) { x = file.getName(); } else { x = ''; } return x; }));
tree.txt.push(row);
tree.fwt.push(['normal']);
PropertiesService.getScriptProperties().setProperty('FnF', JSON.stringify(tree));
}
const subfolders = folder.getFolders()
while (subfolders.hasNext()) {
let subfolder = subfolders.next();
level++;
getFnF1(subfolder);
}
level--;
}
function getFilesAndFolders1() {
const fldr = null;
const ss = SpreadsheetApp.getActive();
ss.toast("Entry");
const sh = ss.getSheetByName('Sheet1');
sh.clearContents();
SpreadsheetApp.flush();
PropertiesService.getScriptProperties().setProperty('FnF', JSON.stringify({ txt: [], fwt: [], level: 0 }));
getFnF1();
//Logger.log(PropertiesService.getScriptProperties().getProperty('FnF'));
let tree = JSON.parse(PropertiesService.getScriptProperties().getProperty('FnF'));
const l = tree.level + 1
tree.txt.forEach(r => {
if (r.length < l) {
//Array.from(Array(l - r.length).keys()).forEach(e => r.push(''));
r.splice(r.length, 0, ...Array(l - r.length).fill(''));
}
});
tree.fwt.forEach(r => {
if (r.length < l) {
//Array.from(Array(l - r.length).keys()).forEach(e => r.push('normal'));
r.splice(r.length, 0, ...Array(l - r.length).fill('normal'));
}
});
//Logger.log(JSON.stringify(tree));
sh.getRange(1, 1, tree.txt.length, tree.level + 1).setValues(tree.txt);
sh.getRange(1, 1, tree.fwt.length, tree.level + 1).setFontWeights(tree.fwt);
PropertiesService.getScriptProperties().deleteProperty('FnF');
ss.toast("EOF");
}
It takes about 3 minutes on my drive. I keep my drive pretty clean and I don't list as many parameters as you desired.
I'm looking to retrieve data from a local CSV file, store it in an Object, then send it via an API. But when I try to display data[i].id; to see if I can get a value, it is undefined.
I use ya-csv to parse the CSV file.
myFile.csv
id;name;email;
1;John;john#doe.com
2;Jane;jane#doe.com
csvParser.js
const csv = require('ya-csv');
const data = [];
const file = 'myFile.csv';
function loopForPrint() {
for (var i = 0; i < data.length; i++) {
let id = data[i].id;
console.log(id);
}
}
function csvToJson() {
var reader = csv.createCsvFileReader(file, {columnsFromHeader: true, 'separator': ';'});
reader.addListener('data', function(data) {
data.push(data);
loopForPrint();
})
reader.addListener('end', function() {
console.log('end');
});
};
First of all, you can remove the tailing ; so that there won't be an empty key and undefined value
Secondly, you can replace this code with
reader.addListener('data', function(data) {
data.push(data);
loopForPrint();
})
this one
reader.addListener('row', function(row) {
data.push(row);
loopForPrint();
})
as data is defined globally and locally will create a conflict
Here is the complete code
const csv = require('ya-csv');
const data = [];
const file = 'myFile.csv';
function loopForPrint() {
for (var i = 0; i < data.length; i++) {
console.log(data[i].id);
}
}
function csvToJson() {
var reader = csv.createCsvFileReader(file, {columnsFromHeader: true, 'separator': ';'});
reader.addListener('data', function(row) {
data.push(row);
})
reader.addListener('end', function() {
loopForPrint();
console.log('end');
});
};
csvToJson()
Output
1
2
end
I have also created StackBlitz for this if you want to look at it
I have an Excel file with the following template.
Name, Surname, Class
Alex, Smith, B1,
Maria, Smith, B2
The Excel file has multiple sheets with data in different languages.
I want to read from the Excel file and to create a hash structure (key/value) because I want to add later more information in each entry.
I have achieved to set the keys names for each entry but I cannot understand how I can import the values for each entry.
You can see the required format of the structure here:
var arr = [{
"name":{default:"Alex", cellId:"B1"},
"surname":{default:"Smith", cellId:"B2"},
"class":{default: "B1", cellId:"B3"}
},
{
"name":{default:"Maria", cellId:"C1"},
"surname":{default:"Smith", cellId:"C2"},
"class":{default: "B1", cellId:"C3"}
} ]
$("#btn").on("change", function(e) {
e.preventDefault();
var files = e.target.files;
var i, f;
for (i = 0, f = files[i]; i != files.length; ++i) {
var reader = new FileReader();
reader.onload = function(e) {
var data = e.target.result;
var workbook = XLSX.read(data, {
type: "buffer",
blankRows: true,
defval: ' '
});
var sheet_name_list = workbook.SheetNames;
var theQuestionLanguage = sheet_name_list; //english
var theExcelDataArray = [];
var test;
var data = [];
var headers = {};
sheet_name_list.forEach(function(y) {
var worksheet = workbook.Sheets[y];
var headers = {};
var data = [];
for (z in worksheet) {
//parse out the column, row, and value
var col = z.substring(0, 1);
var row = parseInt(z.substring(1));
var value = worksheet[z].v;
//store header names
if (row == 1 && !headers[value]) {
headers[value] = {};
}
console.log(headers);
if ((headers[value]) && (row !== 1)) {
headers[value].default = "value"
console.log("headers");
}
}
//drop those first two rows which are empty
});
};
reader.readAsArrayBuffer(f);
}
});
First of all, Save As your Excel file in CSV format.
After that, go to this website : Let's go. In this website you can add for information in your each entry and convert them into key/value pair.
I hope this this above information will help you.
In our Freifunk project gluon, we use i18n GNU gettext internationalisation in our Lua code (for example for the package gluon-config-mode-hostname) we create separate files in a subfolder i18n. I want to use this .po files to add them in our status-page javascript code:
https://github.com/rubo77/gluon/tree/status-i18n/package/gluon-status-page/i18n
Those contain the translations created by the msginit program.
How can I use the same i18n files for the javascript based status-page (without jQuery) to translate those strings?
Here's a dirty but verbose way of accomplishing it. Is this what you're looking for?
let url = "https://raw.githubusercontent.com/rubo77/gluon/status-i18n/package/gluon-status-page/i18n/de.po"
fetch(url)
.then((res) => {
return res.body.getReader();
})
.then((reader) => {
return reader.read();
})
.then((stream) => {
let decoder = new TextDecoder();
let body = decoder.decode(stream.value || new Uint8Array);
return body
})
.then((body) => {
let text = body.replace(/\\n/g, '');
let lines = text.split('\n');
console.log(text)
let arr = []
let obj = {}
for (let i = 0; i < lines.length; i++) {
// key:value pairs
if (lines[i].indexOf(':') !== -1) {
let line = lines[i].replace(/"/g, '');
let pair = line.split(':');
if (pair.length) {
obj[pair[0]] = pair[1].trim();
}
}
// msgid
if (lines[i].indexOf('msgid') !== -1) {
let msgobj = {};
let msgid = lines[i].split(' "')[1].replace(/\"/g, '');
msgobj.msgid = msgid;
// msgstr
if (lines[i+1].indexOf('msgstr') !== -1) {
let msgstr = lines[i+1].split(' "')[1].replace(/\"/g, '');
msgobj.msgstr = msgstr;
}
arr.push(msgobj);
}
}
arr.push(obj)
document.getElementById('output-source')
.innerHTML = body
document.getElementById('output-js')
.innerHTML = JSON.stringify(arr, null, 2);
});
.output {
background-color: #fafafa;
border: 1px solid #e1e1e1;
}
<pre id="output-source" class="output"></pre>
<pre id="output-js" class="output"></pre>
NB: Above example likely only works in Chrome. Here's a JSBin that should work in FF.
I have Json data and i need convert json data to Excel file using javascript,
Reference URL : http://jsfiddle.net/hybrid13i/JXrwM/
i am using this code:
function JSONToTSVConvertor(JSONData, ReportTitle, ShowLabel, myTemplateName){
//If JSONData is not an object then JSON.parse will parse the JSON string in an Object
var arrData = typeof JSONData != 'object' ? JSON.parse(JSONData) : JSONData;
var TSV = '';
//Set Report title in first row or line
//TSV += ReportTitle + '\r\n\n';
//This condition will generate the Label/Header
if (ShowLabel) {
var row = "";
//This loop will extract the label from 1st index of on array
for (var index in arrData[0]) {
//Now convert each value to string and tab-seprated
row += index + ' ';
}
row = row.slice(0, -1);
//append Label row with line break
TSV += row + '\r\n';
}
//1st loop is to extract each row
for (var i = 0; i < arrData.length; i++) {
var row = "";
//2nd loop will extract each column and convert it in string tab-seprated
for (var index in arrData[i]) {
row += '"' + arrData[i][index] + '" ';
}
row.slice(0, row.length - 1);
//add a line break after each row
TSV += row + '\r\n';
}
if (TSV == '') {
alert("Invalid data");
return;
}
var blob = new Blob([TSV], {type: "data:text/tsv;charset=utf-8"});
//Generate a file name
var fileName = myTemplateName;
//this will remove the blank-spaces from the title and replace it with an underscore
fileName += ReportTitle.replace(/ /g,"_");
saveAs(blob, ""+fileName+".tsv");
}
this sample code work to csv and tsv format. and i need to Excel format i don't think any idea please help me.
pls suggest some example code.
Thanks...
I have used the following code Javascript JSON to Excel or CSV file download
change file extension only (reports.xlsx or reports.CSV)
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.13.1/xlsx.full.min.js"></script>
<script>
function ExportData()
{
filename='reports.xlsx';
data=[{Market: "IN", New Arrivals: "6", Upcoming Appointments: "2", Pending - 1st Attempt: "4"},
{Market: "KS/MO", New Arrivals: "4", Upcoming Appointments: "4", Pending - 1st Attempt: "2"},
{Market: "KS/MO", New Arrivals: "4", Upcoming Appointments: "4", Pending - 1st Attempt: "2"},
{Market: "KS/MO", New Arrivals: "4", Upcoming Appointments: "4", Pending - 1st Attempt: "2"}]
var ws = XLSX.utils.json_to_sheet(data);
var wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, ws, "People");
XLSX.writeFile(wb,filename);
}
</script>
CSV, as said, is excel file itself. But, in many locales, csv generated by the script above is opened incorrectly, where excel puts everything into 1 cell. Small modification of original script helps: just replace header with "sep=,".
var CSV = 'sep=,' + '\r\n\n';
Working example with change here: https://jsfiddle.net/1ecj1rtz/.
Spent some time figuring this out, and therefore answering old thread to help others save some time.
I've created a class to export json data to excel file. I'll be happy if some productive edit is made in my code.
Just add the class in your JS library and call:
var myTestXML = new myExcelXML(myJsonArray);
myTestXML.downLoad();
My myExcelXML Class:
let myExcelXML = (function() {
let Workbook, WorkbookStart = '<?xml version="1.0"?><ss:Workbook xmlns="urn:schemas-microsoft-com:office:spreadsheet" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet" xmlns:html="http://www.w3.org/TR/REC-html40">';
const WorkbookEnd = '</ss:Workbook>';
let fs, SheetName = 'SHEET 1',
styleID = 1, columnWidth = 80,
fileName = "Employee_List", uri, link;
class myExcelXML {
constructor(o) {
let respArray = JSON.parse(o);
let finalDataArray = [];
for (let i = 0; i < respArray.length; i++) {
finalDataArray.push(flatten(respArray[i]));
}
let s = JSON.stringify(finalDataArray);
fs = s.replace(/&/gi, '&');
}
downLoad() {
const Worksheet = myXMLWorkSheet(SheetName, fs);
WorkbookStart += myXMLStyles(styleID);
Workbook = WorkbookStart + Worksheet + WorkbookEnd;
uri = 'data:text/xls;charset=utf-8,' + encodeURIComponent(Workbook);
link = document.createElement("a");
link.href = uri;
link.style = "visibility:hidden";
link.download = fileName + ".xls";
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
get fileName() {
return fileName;
}
set fileName(n) {
fileName = n;
}
get SheetName() {
return SheetName;
}
set SheetName(n) {
SheetName = n;
}
get styleID() {
return styleID;
}
set styleID(n) {
styleID = n;
}
}
const myXMLStyles = function(id) {
let Styles = '<ss:Styles><ss:Style ss:ID="' + id + '"><ss:Font ss:Bold="1"/></ss:Style></ss:Styles>';
return Styles;
}
const myXMLWorkSheet = function(name, o) {
const Table = myXMLTable(o);
let WorksheetStart = '<ss:Worksheet ss:Name="' + name + '">';
const WorksheetEnd = '</ss:Worksheet>';
return WorksheetStart + Table + WorksheetEnd;
}
const myXMLTable = function(o) {
let TableStart = '<ss:Table>';
const TableEnd = '</ss:Table>';
const tableData = JSON.parse(o);
if (tableData.length > 0) {
const columnHeader = Object.keys(tableData[0]);
let rowData;
for (let i = 0; i < columnHeader.length; i++) {
TableStart += myXMLColumn(columnWidth);
}
for (let j = 0; j < tableData.length; j++) {
rowData += myXMLRow(tableData[j], columnHeader);
}
TableStart += myXMLHead(1, columnHeader);
TableStart += rowData;
}
return TableStart + TableEnd;
}
const myXMLColumn = function(w) {
return '<ss:Column ss:AutoFitWidth="0" ss:Width="' + w + '"/>';
}
const myXMLHead = function(id, h) {
let HeadStart = '<ss:Row ss:StyleID="' + id + '">';
const HeadEnd = '</ss:Row>';
for (let i = 0; i < h.length; i++) {
const Cell = myXMLCell(h[i].toUpperCase());
HeadStart += Cell;
}
return HeadStart + HeadEnd;
}
const myXMLRow = function(r, h) {
let RowStart = '<ss:Row>';
const RowEnd = '</ss:Row>';
for (let i = 0; i < h.length; i++) {
const Cell = myXMLCell(r[h[i]]);
RowStart += Cell;
}
return RowStart + RowEnd;
}
const myXMLCell = function(n) {
let CellStart = '<ss:Cell>';
const CellEnd = '</ss:Cell>';
const Data = myXMLData(n);
CellStart += Data;
return CellStart + CellEnd;
}
const myXMLData = function(d) {
let DataStart = '<ss:Data ss:Type="String">';
const DataEnd = '</ss:Data>';
return DataStart + d + DataEnd;
}
const flatten = function(obj) {
var obj1 = JSON.parse(JSON.stringify(obj));
const obj2 = JSON.parse(JSON.stringify(obj));
if (typeof obj === 'object') {
for (var k1 in obj2) {
if (obj2.hasOwnProperty(k1)) {
if (typeof obj2[k1] === 'object' && obj2[k1] !== null) {
delete obj1[k1]
for (var k2 in obj2[k1]) {
if (obj2[k1].hasOwnProperty(k2)) {
obj1[k1 + '-' + k2] = obj2[k1][k2];
}
}
}
}
}
var hasObject = false;
for (var key in obj1) {
if (obj1.hasOwnProperty(key)) {
if (typeof obj1[key] === 'object' && obj1[key] !== null) {
hasObject = true;
}
}
}
if (hasObject) {
return flatten(obj1);
} else {
return obj1;
}
} else {
return obj1;
}
}
return myExcelXML;
})();
I know its a little late to answer but I have found an nice angular library that does all the hard work it self.
GITHUB: ngJsonExportExcel
Library Direct Download : Download
Filesaver JS : Download
How to use?
Include the module in you app
var myapp = angular.module('myapp', ['ngJsonExportExcel'])
Add a export button and use the ng-json-export-excel directive and pass data into the directive
ng-json-export-excel : it is the directive name
data : it is the data that will be exported (JSON)
report-fields :
pass the column name and the keys that are present in your JSON e.g.
customer_name": "Customer Name"
HTML
<button ng-json-export-excel data="dataList" report-fields="{'uesr.username': 'Heder 1', keyjson2: 'Header 2', keyjson3: 'Head 3'}" filename =" 'export-excel' " separator="," class="css-class"></button>
Excel is a very complex format with many versions. If you really need to do this I would investigate some of the JavaScript libraries that others have written. Do a Google search for "javascript excel writer" to see some examples.
This code snippet is using node.js with the excel4node and express modules in order to convert JSON data to an Excel file and send it to the client, using Javascript.
const xl = require('excel4node');
const express = require('express');
const app = express();
var json = [{"Vehicle":"BMW","Date":"30, Jul 2013 09:24 AM","Location":"Hauz Khas, Enclave, New Delhi, Delhi, India","Speed":42},{"Vehicle":"Honda CBR","Date":"30, Jul 2013 12:00 AM","Location":"Military Road, West Bengal 734013, India","Speed":0},{"Vehicle":"Supra","Date":"30, Jul 2013 07:53 AM","Location":"Sec-45, St. Angel's School, Gurgaon, Haryana, India","Speed":58},{"Vehicle":"Land Cruiser","Date":"30, Jul 2013 09:35 AM","Location":"DLF Phase I, Marble Market, Gurgaon, Haryana, India","Speed":83},{"Vehicle":"Suzuki Swift","Date":"30, Jul 2013 12:02 AM","Location":"Behind Central Bank RO, Ram Krishna Rd by-lane, Siliguri, West Bengal, India","Speed":0},{"Vehicle":"Honda Civic","Date":"30, Jul 2013 12:00 AM","Location":"Behind Central Bank RO, Ram Krishna Rd by-lane, Siliguri, West Bengal, India","Speed":0},{"Vehicle":"Honda Accord","Date":"30, Jul 2013 11:05 AM","Location":"DLF Phase IV, Super Mart 1, Gurgaon, Haryana, India","Speed":71}]
const createSheet = () => {
return new Promise(resolve => {
// setup workbook and sheet
var wb = new xl.Workbook();
var ws = wb.addWorksheet('Sheet');
// Add a title row
ws.cell(1, 1)
.string('Vehicle')
ws.cell(1, 2)
.string('Date')
ws.cell(1, 3)
.string('Location')
ws.cell(1, 4)
.string('Speed')
// add data from json
for (let i = 0; i < json.length; i++) {
let row = i + 2
ws.cell(row, 1)
.string(json[i].Vehicle)
ws.cell(row, 2)
.date(json[i].Date)
ws.cell(row, 3)
.string(json[i].Location)
ws.cell(row, 4)
.number(json[i].Speed)
}
resolve( wb )
})
}
app.get('/excel', function (req, res) {
createSheet().then( file => {
file.write('ExcelFile.xlsx', res);
})
});
app.listen(3040, function () {
console.log('Excel app listening on port 3040');
});