Develop Reference

How to push an entire array in javascript based on conditional content of single array element?

I have created a Google script that pushes data every hour from the Capital Bikeshare API to a Google Sheet, but I have noticed that the way I am currently pulling the data doesn't maintain consistency over time. Here's the code I'm using:
function myFunction() {
// Set the active spreadsheet
var ss = SpreadsheetApp.getActiveSpreadsheet();
var currentData = ss.getSheetByName("Current");
var historicData = ss.getSheetByName("Historic");
// Fetch API
var stationInfo = UrlFetchApp.fetch('https://gbfs.capitalbikeshare.com/gbfs/en/station_information.json');
var stationStatus = UrlFetchApp.fetch('https://gbfs.capitalbikeshare.com/gbfs/en/station_status.json');
// Get the current date and time
var today = new Date();
var date = today.getFullYear()+'-'+(today.getMonth()+1)+'-'+today.getDate();
var time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
var dateTime = date+' '+time;
// Parse the JSON reply
var jsonInfo = stationInfo.getContentText();
var dataInfo = JSON.parse(jsonInfo);
var jsonStatus = stationStatus.getContentText();
var dataStatus = JSON.parse(jsonStatus);
// Create the data frame for every BID station
var stationInfo72 = dataInfo["data"]["stations"][69];
var stationStatus72 = dataStatus["data"]["stations"][69];
var stationInfo87 = dataInfo["data"]["stations"][83];
var stationStatus87 = dataStatus["data"]["stations"][83];
var stationInfo330 = dataInfo["data"]["stations"][311];
var stationStatus330 = dataStatus["data"]["stations"][311];
var stationInfo153 = dataInfo["data"]["stations"][143];
var stationStatus153 = dataStatus["data"]["stations"][143];
var stationInfo226 = dataInfo["data"]["stations"][213];
var stationStatus226 = dataStatus["data"]["stations"][213];
var stationInfo365 = dataInfo["data"]["stations"][342];
var stationStatus365 = dataStatus["data"]["stations"][342];
var stationInfo473 = dataInfo["data"]["stations"][446];
var stationStatus473 = dataStatus["data"]["stations"][446];
var outputStationsInfo = [stationInfo72, stationInfo87, stationInfo330, stationInfo153, stationInfo226, stationInfo365, stationInfo473]
var outputStationsStatus = [stationStatus72, stationStatus87, stationStatus330, stationStatus153, stationStatus226, stationStatus365, stationStatus473]
Logger.log(outputStationsInfo, outputStationsStatus)
// Create lists of each element
var outputHead = [];
var outputTail = [];
outputStationsInfo.forEach(function(elem,i) {
outputHead.push([elem["station_id"],elem["name"],elem["capacity"], elem["lat"], elem["lon"]]);
});
outputStationsStatus.forEach(function(elem,i) {
outputTail.push([elem["num_bikes_available"], elem["num_ebikes_available"], dateTime]);
});
// Publish arrays in the Current sheet
currentData.getRange(2,1,7,5).setValues(outputHead);
currentData.getRange(2,6,7,3).setValues(outputTail);
// Publish arrays in the Historic sheet
historicData.getRange(historicData.getLastRow() + 1,1,7,5).setValues(outputHead);
historicData.getRange(historicData.getLastRow() - 6,6,7,3).setValues(outputTail);
}
Essentially, I am drilling into the 69th item in the indexes of the JSONs to get the data that I need from two different APIs, and then I merge them together to create a data frame of everything I need to push to the sheet. However, sometimes the API does not report them in the normal order and I end up getting bikeshare stations that aren't in my study area. For example, 99% of the time the 69th item in the array is station_id = 72, but occasionally it's station_id = 73 or something.
Is there a way to conditionally pull a specific array based on the station_id number within the array? I feel like the answer might allow me to do a loop as well to clean this up. Any advice is helpful, as I'm super new to this.

You have to check if the element's station_id is as expected. If not, check through the surrounding parts of the array using a custom iterator.
Snippet:
/**
* #return indexes of the surrounding ``i`` in batches of 5
*/
function* checkSurroundings(i, lastIndex) {
let j = i;
function* check(ct, border, reverse = true, limit = border < 5 ? border : 5) {
const margin = reverse ? ct - limit : ct + limit;
while (ct - margin !== 0) yield reverse ? --ct : ++ct;
return ct;
}
while (i !== 0 || j < lastIndex) {
if (i !== 0) i = yield* check(i, i);
if (j < lastIndex) j = yield* check(j, lastIndex - j, false);
//console.log({ i, j });
}
}
var stations = dataInfo["data"]["stations"];
var stationInfo72 = stations[69];
const iter = checkSurroundings(69,stations.length-1)
//if station_id is not 72, loop through the surrounding indexes
while(stationInfo72["station_id"] !== 72){
const next = iter.next();
if(next.done) {
console.error("station id 72 not found");
break;
}
stationInfo72 = stations[next.value]
}
Snippet showing how checkSurroundings iterates:
/**
* #return indexes of the surrounding ``i`` in batches of 5
*/
function* checkSurroundings(i, lastIndex) {
let j = i;
function* check(ct, border, reverse = true, limit = border < 5 ? border : 5) {
const margin = reverse ? ct - limit : ct + limit;
while (ct - margin !== 0) yield reverse ? --ct : ++ct;
return ct;
}
while (i !== 0 || j < lastIndex) {
if (i !== 0) i = yield* check(i, i);
if (j < lastIndex) j = yield* check(j, lastIndex - j, false);
console.log({ i, j });
}
}
console.log("Order of iteration",[...checkSurroundings(50, 100)])

Conditionally picking elements: filter
For conditionally picking elements from an array in JavaScript, Array.prototype.filter should always be a consideration.
Create a predicate function that matches the shape of your data and checks for certain station IDs.
Here is a function that returns a predicate function. You put in the IDs you want in an array, and it returns the required function for filter.
function byStationId(stationIds) {
return function (obj) {
return stationIds.indexOf(obj.station_id) > -1;
};
}
var myStationFilter = byStationId([72, 73, 74]);
var outputStationsInfo = dataInfo.data.stations.filter(myStationFilter);
Transforming data: map
The pattern
var newArray = [];
oldArray.forEach(function (item) {
newArray.push(/* something based on item */);
});
can usually be replaced with Array.prototype.map
var newArray = oldArray.map(function (item) { return /* something based on item */});
Think of this as the "adapter" from one data shape to another.
function cleanInfo(info) {
return [info.station_id, info.name, info.capacity, info.lat, info.lon];
}
var outputHead = outputStationsInfo.map(cleanInfo);
For the dateTime injection, just do the same trick demonstrated above with the station IDs: have a function that takes a date string and returns the appropriate adapter function.
(Also note the provided date formatting utility Apps Scripts provides, Utilities.formatDate())
var dateTime = Utilities.formatDate(
new Date(),
ss.getSpreadsheetTimeZone(),
"yyyy-MM-dd HH:mm:ss"
);
function cleanStatus(dateTime) {
return function (status) {
return [status.num_bikes_available, status.num_bikes_available, dateTime];
};
}
var outputTail = outputStationsStatus.map(cleanStatus(dateTime));
Here's everything together, untested, just for inspiration. You must at the very least update the line with the station IDs to match your desired station IDs. Note that the helper functions for map and filter are at the bottom, taking advantage of JavaScript's hoisting feature.
function myFunction() {
// Set the active spreadsheet
var ss = SpreadsheetApp.getActiveSpreadsheet();
// Get the current date and time
var dateTime = Utilities.formatDate(
new Date(),
ss.getSpreadsheetTimeZone(),
"yyyy-MM-dd HH:mm:ss"
);
// Fetch API
var stationInfo = UrlFetchApp.fetch(
"https://gbfs.capitalbikeshare.com/gbfs/en/station_information.json"
);
var stationStatus = UrlFetchApp.fetch(
"https://gbfs.capitalbikeshare.com/gbfs/en/station_status.json"
);
// Parse the JSON reply
var dataInfo = JSON.parse(stationInfo.getContentText());
var dataStatus = JSON.parse(stationStatus.getContentText());
// Create the data frame for every BID station
var myStationFilter = byStationId([72, 73, 74]); //!! UPDATE THESE NUMBERS
var outputStationsInfo = dataInfo.data.stations.filter(myStationFilter);
var outputStationsStatus = dataStatus.data.station.filter(myStationFilter);
// Create lists of each element
var outputHead = outputStationsInfo.map(cleanInfo);
var outputTail = outputStationsStatus.map(cleanStatus(dateTime));
// Publish arrays in the Current sheet
var currentData = ss.getSheetByName("Current");
currentData.getRange(2, 1, 7, 5).setValues(outputHead);
currentData.getRange(2, 6, 7, 3).setValues(outputTail);
// Publish arrays in the Historic sheet
var historicData = ss.getSheetByName("Historic");
historicData
.getRange(historicData.getLastRow() + 1, 1, 7, 5)
.setValues(outputHead);
historicData
.getRange(historicData.getLastRow() - 6, 6, 7, 3)
.setValues(outputTail);
//-------- helper functions ------------
function byStationId(stationIds) {
return function (obj) {
return stationIds.indexOf(obj.station_id) > -1;
};
}
function cleanInfo(info) {
return [info.station_id, info.name, info.capacity, info.lat, info.lon];
}
function cleanStatus(dateTime) {
return function (status) {
return [status.num_bikes_available, status.num_bikes_available, dateTime];
};
}
}

How to get the count of duplicate value assigned to same key in JSON array - JavaScript/NodeJS

Hi Guys i need small help in getting this solved.
If this is a duplicate post please point me to the original question.
Here I have an JSON array of elements
Ex : var consume = [{"key":"Test1"},{"key":"Test2"},{"key":"Test3"},{"key":"Test1"},{"key":"Test3"},{"key":"Test1"}];
Expected OutPut:
var produce = [{"name":"test1","count":3},{"name":"test2","count":2},{"name":"test3","count":2}]

Finally I wrote Answer to my question in pure javascript.
Thanks for your support guys who ever tried to guide me to solve my question
var consume = [{"key":"Test1"},{"key":"Test2"},{"key":"Test3"},{"key":"Test1"},{"key":"Test3"},{"key":"Test1"}]
var temp = [];
var produce = [];
for(var i=0;i<consume.length;i++){
if(temp.indexOf(consume[i].key) == -1){
temp.push(consume[i].key);
var _data = {};
_data.name = consume[i].key;
_data.count = 1;
produce.push(_data);
}else{
for(var j=0;j<produce.length;j++){
if(produce[j].name === consume[i].key){
var _x = parseInt(produce[j].count) + 1;
produce[j].count = _x;
}
}
}
}
console.log(produce);

Hmm.. these days when it comes to object property thingies i am intrigued with ES6 iterators. This is how i would approach this job;
var consume = [{"key":"Test1"},{"key":"Test2"},{"key":"Test3"},{"key":"Test1"},{"key":"Test3"},{"key":"Test1"}],
reduced = consume.reduce((p,c) => (p[c.key] ? p[c.key]++ : p[c.key] = 1,p),{}),
produce = [];
reduced[Symbol.iterator] = function*(){
var oKeys = Object.keys(this);
for(var key of oKeys) yield {name : key, count: this[key]};
};
produce = [...reduced];
console.log(produce);

Google script says - Exceeded maximum execution time

I am using the below script to delete duplicate rows from the google spreadsheet. The script was working good but as the data in the spreadsheet is being added daily, now the script is throwing "Exceeded maximum execution time" error. As I am new to scripting I don't understand what is my problem.
Could someone help me in solving this problem of mine.
function Deleteduplicates() {
var SpreadSheetKey = "My key";
var sheetD = SpreadsheetApp.openById(SpreadSheetKey).getSheetByName("Daily");
var sheetW = SpreadsheetApp.openById(SpreadSheetKey).getSheetByName("Weekly");
var dataD = sheetD.getDataRange().getValues();
var dataW = sheetW.getDataRange().getValues();
//Daily
var newDataD = new Array();
for(i in dataD){
var row = dataD[i];
var duplicate = false;
for(j in newDataD){
if(row.join() == newDataD[j].join()){
duplicate = true;
}
}
if(!duplicate){
newDataD.push(row);
}
}
//weekly
var newDataW = new Array();
for(i in dataW){
var row = dataW[i];
var duplicate = false;
for(j in newDataW){
if(row.join() == newDataW[j].join()){
duplicate = true;
}
}
if(!duplicate){
newDataW.push(row);
}
}
sheetD.clearContents();
sheetW.clearContents();
sheetD.getRange(1, 1, newDataD.length, newDataD[0].length).setValues(newDataD);
sheetW.getRange(1, 1, newDataW.length, newDataW[0].length).setValues(newDataW);
}

Conceptually, this should be quite a bit faster. I have not tried it on a large data set. The first version will leave the rows sorted as they were originally. The second version will be faster but will leave the rows sorted according to the columns from first to last on first text.
function Deleteduplicates() {
var SpreadSheetKey = "My key";
var ss = SpreadsheetApp.openById(SpreadSheetKey);
var sheetD = ss.getSheetByName("Daily");
var sheetW = ss.getSheetByName("Weekly");
var sheets = [sheetD, sheetW];
var toSs = {};
for(s in sheets) {
var data = sheets[s].getDataRange().getValues();
for(i in data){
// EDIT: remove commas from join("") for blank test
data[i].unshift(data[i].join(""),(1000000 + i).toString());
}
data.sort();
// remove blank rows -- Edit
var blank = 0;
while(data[blank][0].trim().length == 0) {blank++};
if(blank > 0) data.splice(0, blank);
// end Edit
var len = data.length - 1;
for(var x = len; x > 0; x-- ) {
if(data[x][0] == data[x-1][0]) {
data.splice(x, 1);
};
};
for(i in data) {
data[i].splice( 0, 1);
};
data.sort();
for(i in data) {
data[i].splice(0, 1);
};
toSs[sheets[s].getSheetName()] = data;
};
for(s in sheets) {
var data = toSs[sheets[s].getSheetName()];
sheets[s].clearContents();
sheets[s].getRange(1, 1, data.length, data[0].length).setValues(data);
}
}
Faster leaving rows sorted by join() created to test for duplicates
function Deleteduplicates() {
var SpreadSheetKey = "My key";
var ss = SpreadsheetApp.openById(SpreadSheetKey);
var sheetD = ss.getSheetByName("Daily");
var sheetW = ss.getSheetByName("Weekly");
var sheets = [sheetD, sheetW];
var toSs = {};
for(s in sheets) {
var data = sheets[s].getDataRange().getValues();
for(i in data){
// EDIT: remove commas from join("") for blank test
data[i].unshift(data[i].join(""));
}
data.sort();
// remove blank rows -- Edit
var blank = 0;
while(data[blank][0].trim().length == 0) {blank++};
if(blank > 0) data.splice(0, blank);
// end Edit
var len = data.length - 1;
for(var x = len; x > 0; x-- ) {
if(data[x][0] == data[x-1][0]) {
data.splice(x, 1);
};
};
for(i in data) {
data[i].splice( 0, 1);
};
toSs[sheets[s].getSheetName()] = data;
};
for(s in sheets) {
var data = toSs[sheets[s].getSheetName()];
sheets[s].clearContents();
sheets[s].getRange(1, 1, data.length, data[0].length).setValues(data);
}
}
Edited per Henrique's comment.
Edited 5/8: Remove blank rows(2 edited areas marked)

There is no problem with your script. It is just exceeding the "maximum execution time" allowed for any script (which is currently 6 minutes).
To workaround this problem you'll have to split your problem into "less than 6 minutes" parts.
For example, in your code you're clearing duplicates from 2 sheets. Trying creating two functions, one for each, and run them separately.
Also, there could be some performance enhancements that could make the script run under 6 minutes. For example, I'm not sure joining each row is the best way (performance-wise) to do an array comparison.
Creating a new array to re-set the data might not be optimal either, I'd probably go with a map verification, which is constant-time, instead of O(n^2) double array checking you're doing.
Bottom line, this is a limitation you have to live with in Apps Script. And any solution anyone proposes is just a workaround, that will also eventually fail if your data gets overly big.

How to test if an id is present in an associative array

I'm am starting in javascript. I'm trying to do a little program that make a statistic upon the number of answer found in a text document.
The situation is this: each question has one id, e.g 8000001 and W if answer is good or R if answer is not good, e.g for an user answer is 8000001W. I have many user so many question of the same id. I want to get number of good answers per questions. E.g id: 800001 have W: 24 and "R": 5.
I have split the answer into id for 8000001 and ans for W or R. I wanted to create an associative table to get question[id]=["W": 0, "R": 0]. But I'm blocking on this. I've tried this code:
var tab = [];
tab[0] = [];
tab[0] = ['8000001W', '8000002W', '8000003W', '8000004R', '8000005W', '8000006R'];
tab[1] = [];
tab[1] = ['8000001R', '8000002W', '8000003R', '8000004W', '8000005R', '8000006W'];
var question = [];
var id;
for (var i=0;i<tab.length;i++) {
document.write("<dl><dt>tableau n° "+i+"<\/dt>");
for (var propriete in tab[i]) {
id = tab[i][propriete].slice(0,7);
var ans = tab[i][propriete].slice(7,8);
question[id] = [];
if(question[id]){
incrementResp.call(rep, ans);
}else{
var rep = initResp(ans);
question[id] = rep;
}
}
document.write("<\/dl>");
}
function incrementResp(type){
this.this++;
}
function initResp(t){
rep = [];
rep.W = (t=='W'?1:0);
rep.R = (t=='R'?1:0);
}

Based on what your want finally, the 'question' should be used as an object literal, defined as question = {} (similar to association array), what you defined here is an array literal. You can check this for more information about different types of literals in JavaScript:
https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Values,_variables,_and_literals#Literals
In terms of your code, you can simple do like this:
if (question[id]) {
question[id][ans] += 1;
}
else {
var rep = initResp(ans);
question[id] = rep;
}
Also your 'initResp' function better to return an object literal 'rep', not as an array literal:
function initResp(t){
var rep = {};
rep.W = (t=='W'?1:0);
rep.R = (t=='R'?1:0);
return rep;
}

For an "associative array" in JavaScript, use a regular object. In the code below, "results" is one of these objects. It has two keys, "W" and "R" that point to numbers starting at 0. Just iterate through your answer arrays and continuously increment the correct key.
There are two ways to access a key in an object: 1) using brackets, 2) using "dot" notation. In the loop I use brackets because 'key' is a variable--it will resolve to "W" or "R" and therefore access the "W" or "R" key in that object. In the final two lines I use dot notation because "W" and "R" are literally the keys I want to access. It would also work if I did this instead: results['W']++ and results['R']++.
var tab = [];
tab[0] = ['8000001W', '8000002W', '8000003W', '8000004R', '8000005W', '8000006R'];
tab[1] = ['8000001R', '8000002W', '8000003R', '8000004W', '8000005R', '8000006W'];
var results = {
W: 0,
R: 0
};
// go through each tab
for (var tabIdx = 0; tabIdx < tab.length; tabIdx++) {
// go through each answer and apppend to an object that keeps the results
for (var i = 0; i < tab[tabIdx].length; i++) {
var answer = tab[tabIdx][i];
// the last character in the string is the "key", (W or R)
var key = answer.slice(-1);
// append to the results object
results[key]++;
}
}
console.log(results);
console.log(results.W); // 7
console.log(results.R); // 5
Open up your development console (on Chrome it's F12) to see the output.

This is how i resolved my problem for associative array.
var tab = [];
tab[0] = ['8000001W', '8000002W', '8000003W', '8000004R', '8000005W', '8000006R'];
tab[1] = ['8000001R', '8000002W', '8000003R', '8000004W', '8000005R', '8000006W'];
tab[2] = ['8000001R', '8000002W', '8000003R', '8000004W', '8000005R', '8000006W'];
var question = {};
for (var tabIndex = 0; tabIndex < tab.length; tabIndex++) {
for (var i = 0; i < tab[tabIndex].length; i++) {
var answer = tab[tabIndex][i];
var id = answer.slice(0,7);
var ans = answer.slice(-1);
if (question[id]) {
question[id][ans] += 1;
}else {
var results = initResp(ans);
question[id] = results;
}
}
}
console.log(question);
function initResp(t) {
var results = [];
results.W = (t === 'W' ? 1 : 0);
results.R = (t === 'R' ? 1 : 0);
//console.log(results);
return results;
}

How can this Javascript be expressed more succinctly?

I have some Python code that I'm porting to Javascript:
word_groups = defaultdict(set)
for sentence in sentences:
sentence.tokens = stemmed_words(sentence.str_)
for token in sentence.tokens:
word_groups[sentence.actual_val].add(token)
I don't know a lot about Javascript, so this was the best I could do:
var word_groups = {}
for(var isent = 0; isent < sentences.length; isent++) {
var sentence = sentences[isent]
sentence.tokens = stemmed_words(sentence.str_)
for(var itoken = 0; itoken < sentence.tokens.length; itoken++) {
var token = sentence.tokens[itoken]
if(!(sentence.actual_val in word_groups))
word_groups[sentence.actual_val] = []
var group = word_groups[sentence.actual_val]
if(!(token in group))
group.push(token)
}
}
Can anyone suggest ways to make the javascript code look more like the python?

I'm going to assume that if you're using an environment where forEach is available, reduce and Object.keys are available as well. (e.g. ECMAScript >= 1.8.5):
var word_groups = sentences.reduce(function (groups, sentence) {
var val = sentence.actual_val
var group = groups[val] = groups[val] || []
stemmed_words(sentence.str_).forEach(function (t) {
if (!(t in group)) group.push(t)
})
return groups
}, {})

It's quite possible that I've misinterpreted what your Python code does, but assuming you're after word counts, I'd write it as follows:
var word_groups = {}
sentences.forEach(function (sentence) {
sentence.tokens = stemmed_words(sentence.str_)
sentence.tokens.forEach(function (token) {
var val = sentence.actual_val
word_groups[val] = (word_groups[val] || 0) + 1
})
})
The above will fail should the word "constructor" appear in the input. It's possible to work around this JavaScript quirk:
var word_groups = {}
sentences.forEach(function (sentence) {
sentence.tokens = stemmed_words(sentence.str_)
sentence.tokens.forEach(function (token) {
var val = sentence.actual_val
if (!word_groups.hasOwnProperty(val)) word_groups[val] = 0
word_groups[val] += 1
})
})

If you're not definitely in Javascript 1.6 or higher (notable IE 8 has Javascript 1.5) you may want jQuery as a compatibility layer. For example $.each(a, f) is compatible with a.forEach(f).