Develop Reference

Firebase cloud functions is not deleting old nodes as expected

I'm using Firebase Cloud Functions to delete old nodes inside the Firebase Database. Almost 1 month ago I asked a similar question in which the problem was that the values ​​were in seconds instead of milliseconds, but the problem persists. How to fix this problem?
My index.js:
'use strict';
var functions = require('firebase-functions');
var admin = require('firebase-admin');
admin.initializeApp();
// Cut off time. Child nodes older than this will be deleted.
const CUT_OFF_TIME = 30000; // 30 seconds in milliseconds.
/**
* This database triggered function will check for child nodes that are older than the
* cut-off time. Each child needs to have a `timestamp` attribute.
*/
exports.deleteOldItems = functions.database.ref('/posts/{id1}/{id2}/timestamp').onWrite(async (change) => {
var ref = change.after.ref.parent; // reference to the parent
var now = Date.now();
var cutoff = now - CUT_OFF_TIME;
var oldItemsQuery = ref.orderByChild('timestamp').endAt(cutoff);
var snapshot = await oldItemsQuery.once('value');
// create a map with all children that need to be removed
var updates = {};
snapshot.forEach(child => {
updates[child.key] = null;
});
// execute all updates in one go and return the result to end the function
return ref.update(updates);
});
and my database struture
"posts" : {
"38d1d0aa-d774-4a69-a78e-44d02b285669" : {
"-LggzAxeAYXF7tOav_vF" : {
"timestamp" : 1559827888988
}
},
"58fe50ae-db93-4a22-996f-7a28d82583ba" : {
"-Lggze2bvHZx2gJeM8OA" : {
"timestamp" : 1559828012239
}
}
},

Meteor : How to clearInterval() onDestroyed() created in onRendered()

I have a countdown function to run every sec, So i proffered setInterval(). After I moved to another template, the interval function keep on running. How to destroy it onDestroyed(). Below code will help you to understand well.
<template name="Home">
<h4>{{timeremaining}}</h4>
</template>
Template.Home.helpers({
timeremaining : function(){
return Session.get('timeremaining');
}
});
Template.Home.onRendered(function () {
// time functions begin
var end_date = new Date(1476337380000); // I am getting timestamp from the db.
var run_every_sec = setInterval(function () {
var current_date = new Date();
var remaining = end_date.getTime() - current_date.getTime();
var oneDay = 24*60*60*1000;
var diffDays = Math.round(Math.abs(remaining/oneDay));
console.log(remaining); // am getting this log in every template.
if (remaining > 0) {
//set remaining timeLeft
Session.set('timeremaining',diffDays + ' Days ' + (Math.abs(end_date.getHours()-current_date.getHours())).toString() + ' Hrs ' + (Math.abs(end_date.getMinutes()-current_date.getMinutes())).toString() + ' Min ' + (60 - end_date.getSeconds()-current_date.getSeconds()).toString() + ' Sec ')
} else {
clearInterval(run_every_sec);
}
}, 1000);
//time functions end
}.bind(this));
Template.Home.onDestroyed(function () {
clearInterval(run_every_sec); // Here I cant remove this time interval
});
We can declare run_every_sec as global function. If so How to pass end_date. I dont think its wise idea to declare end_date inside the run_every_sec because its coming from db.

If you store the interval in file scope like Repo suggested, you'll have problems if there's ever more than one instance of the template at a time: both instances will use the same run_every_sec variable. In this case, you'll need to store the interval on the template instance, which can be accessed as this inside onRendered and onDestroyed:
Template.Home.onRendered(function () {
this.run_every_sec = setInterval(/* ... */);
});
Template.Home.onDestroyed(function () {
clearInterval(this.run_every_sec);
});
That way, each instance of the template will have its own run_every_sec property.

You should declare "run_every_sec" outside "onRendered".
So instead of this:
Template.Home.onRendered(function () {
// time functions begin
var end_date = new Date(1476337380000); // I am getting timestamp from the db.
var run_every_sec = setInterval(function () {
..do this:
var run_every_sec;
Template.Home.onRendered(function () {
// time functions begin
var end_date = new Date(1476337380000); // I am getting timestamp from the db.
run_every_sec = setInterval(function () {
then it will be available in "onDestroyed"

You should use Meteor's setInterval and clearInterval to make sure they run within a fiber. You can find more info here https://docs.meteor.com/api/timers.html.
var intervalID
Template.myTemplate.onRendered(function() {
intervalID = Meteor.setInterval(function() {
//do something
}, 1000)
})
Template.myTemplate.onDestroyed(function() {
Meteor.clearInterval(intervalID)
})

Google Apps Script, fastest way to retrieve data from external spreadsheets

I'm trying to load data from multiple spreadsheets(~100) into a single spreadsheet, however when I try to do this my script times out. It appears that opening each spreadsheet takes a long time. Is there any way I can speed this up or a work around?
Here's what I use to open each spreadsheet
// We set the current spreadsheet to master and get the current date.
var master = SpreadsheetApp.getActive();
var masterSheet = master.getSheetByName('Master');
var users = master.getEditors();
var today = new Date();
// Adds the menu to the spreadsheet
function onOpen() {
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
var entries = [{
name : "Update Data",
functionName : "retrievePartnerData"
}];
spreadsheet.addMenu("Submissions Menu", entries);
};
// First we get all data from the partner sheets
function retrievePartnerData() {
masterSheet.getRange(2, 1, masterSheet.getLastRow(), masterSheet.getLastColumn()).clear(); //Clear our master sheet aka Sheet All
masterSheet.hideSheet();
//Get's Promo Outline from the internal sheet and store it's values in the promoRange array
var promoRange = master.getSheetByName("Promotional Outline").getRange("A1:Z100").getValues();
var sheetPartnerArray = [];
// Row is an array that contaings the url's to the external spreadsheets
var row = master.getSheetByName('Partner Sheet Collection').getRange("B:B").getValues();
row.map(function(e){
if(e[0] != "" && e[0] != "Url"){
var ss = SpreadsheetApp.openByUrl(e[0]);
var studioName = ss.getSheets()[0].getRange("A1").getValue();
//Updates the Promotional Outline sheet in the partner sheet
var promoSheet = ss.getSheetByName("Promotional Outline");
promoSheet.getRange("A1:Z100").setValues(promoRange);
//Hide columns K to Z
promoSheet.hideColumns(11,4);
var sheet = ss.getSheets();
sheet.map(function(f){
var sheetName = f.getSheetName(); // Retrieves the sheetname of each sheet
var lastRow = 0;
if(f.getLastRow() == 1) {
lastRow = 1;
} else {
lastRow = f.getLastRow() - 1;
}
var dataRange = f.getRange(2, 1, lastRow, f.getLastColumn());
var data = dataRange.getValues();
for (var j = 0; j < data.length; j++) {
if (data[j][0].length != 0 && (data[j][5] > today || data[j][5] == "[Please Enter]")) { // We check if the promo end date is after the current day
var sheetRow = data[j];
sheetRow[1] = studioName;
sheetRow.unshift(sheetName); //Adds the Country to the beginning of the row using the sheet name from spreadsheets
sheetPartnerArray.push(sheetRow);
}
}
})
}
})
masterSheet.getRange(2, 1, sheetPartnerArray.length , sheetPartnerArray[0].length ).setValues(sheetPartnerArray);
};
Thanks!

One common approach is to set a trigger to restart your Big Job at some time in the future (just beyond the maximum execution time). Then your Big Job does as much as it can (or stops nicely at some logical point), and either gets killed or quietly exits. Either way, it gets restarted shortly after, and resumes its work.
Patt0 has taken this idea to an elegant end, providing a library that you can add to your script. With a few adaptations, you should be able to turn your retrievePartnerData() into a batch job.
Since you have a menu already, and retrievePartnerData() involves iterating over many spreadsheets, you have the opportunity to break the barrier another way, by completing each iteration (or better, a set of iterations) in a separate server script instance.
This technique appears in What happens when I "sleep" in GAS ? (execution time limit workaround)
And there is something similar in How to poll a Google Doc from an add-on. In that answer, a UI client uses a timer to repeatedly execute a server function. Here, though, iterations would be work-based, rather than time-based. This client-side function, running in your browser, would keep calling the server until there was no more work to be done:
/**
* Call the server-side 'serverProcess' function until there's no more work.
*/
function dispatchWork(){
if (window.runningProcess) {
}
google.script.run
.withSuccessHandler( //<<<< if last call was good
// After each interval, decide what to do next
function(workis) {
if (!workis.done) { //<<<<< check if we're done
// There's more work to do, keep going.
dispatchWork();
}
else {
// All done. Stop timer
stopTimer();
$('#start-process').hide();
$("#final").html(' <h2>Processing complete!</h2>');
}
})
.withFailureHandler(
function(msg, element) { //<<<<<< do this if error
showError(msg, $('#button-bar'));
element.disabled = false;
})
.serverProcess(); //<<<<< call server function
};
In your case, you first need to refactor retrievePartnerData() so it can be called from a client to process a single spreadsheet (or set of them). No doubt you have put considerable time into making that map loop work cleanly, and taking it apart will be painful, but it will be worth it.
The following spreadsheet-bound script can be adapted to your use. It consists of a menu item, a simple UI, and scripts on the client (Javascript + jQuery) and server (Google Apps Script), which control the work in intervals.
The control data is in a "SourceSheets" tab, and results will be copied to "Master".
Code.gs
var properties = PropertiesService.getScriptProperties();
// Adds the menu to the spreadsheet
function onOpen() {
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
var entries = [{
name : "Update Data",
functionName : "updateData"
}];
spreadsheet.addMenu("Big Job", entries);
};
/**
* Presents UI to user.
*/
function updateData () {
var userInterface = HtmlService.createHtmlOutputFromFile("Conductor")
.setHeight(150)
.setWidth(250)
.setTitle("What 5 minute limit?");
var ss = SpreadsheetApp.getActiveSpreadsheet();
ss.show(userInterface)
}
/**
* Called from client, this function performs the server work in
* intervals. It will exit when processing time has exceeded MAX_INTERVAL,
* 3.5 minutes. Every time this function exits, the client is provided
* with the current status object, done=true when the work queue has
* been emptied.
*
* #returns {Object} Status { done: boolean }
*/
function serverProcess() {
var MAX_INTERVAL = (3.5 * 60); // minutes * seconds
var intervalStart = Math.round(new Date() / 1000);
// Get persisted work queue, if there is one
var queueProp = properties.getProperty('work-queue') || '[]';
var queue = JSON.parse(queueProp);
if (queue.length == 0) {
queue = prepareWork();
}
// Do the work for this interval, until we're out of time
while ((Math.round(new Date() / 1000) - intervalStart) < MAX_INTERVAL) {
if (queue.length > 0) {
var ssID = queue.shift();
processSheet(ssID);
properties.setProperty('work-queue', JSON.stringify(queue));
}
else break;
}
// Report result of this interval to client
var result = { done : (queue.length == 0) };
return( result );
}
/**
* Set up work queue & clear Master sheet, ready to import data from source sheets.
*
* #return {String[]} work queue
*/
function prepareWork() {
// No work yet, so set up work
var ss = SpreadsheetApp.getActive();
var masterSheet = ss.getSheetByName('Master');
var rowsToDelete = masterSheet.getMaxRows()-1;
if (rowsToDelete)
masterSheet.deleteRows(2, rowsToDelete); //Clear our master sheet aka Sheet All
// Build work queue
var queue = [];
var data = ss.getSheetByName('SourceSheets') // get all data
.getDataRange().getValues();
var headers = data.splice(0,1)[0]; // take headers off it
var ssIDcol = headers.indexOf('Spreadsheet ID'); // find column with work
for (var i=0; i<data.length; i++) {
queue.push(data[i][ssIDcol]); // queue up the work
}
// Persist the work queue as a scriptProperty
properties.setProperty('work-queue', JSON.stringify(queue));
return queue;
}
/**
* Do whatever work item we need. In this example, we'll import all data from
* the source sheet and append it to our Master.
*
* #param {String} ssID Source spreadsheet ID
*/
function processSheet(ssID) {
var masterSheet = SpreadsheetApp.getActive().getSheetByName('Master');
var sourceSheet = SpreadsheetApp.openById(ssID).getSheetByName('Sheet1');
Utilities.sleep(60000); // You probably don't want to do this... just wasting time.
var masterLastRow = masterSheet.getLastRow();
var sourceRows = sourceSheet.getLastRow();
masterSheet.insertRowsAfter(masterSheet.getLastRow(), sourceSheet.getLastRow());
var sourceData = sourceSheet.getDataRange().getValues().slice(1);
var destRange = masterSheet.getRange(masterLastRow+1, 1, sourceData.length, sourceData[0].length);
destRange.setValues(sourceData);
}
Conductor.html
<link rel="stylesheet" href="https://ssl.gstatic.com/docs/script/css/add-ons.css">
<!-- The CSS package above applies Google styling to buttons and other elements. -->
<div id="form-div" class="sidebar branding-below">
<span id="final"></span>
<form>
<div class="block" id="button-bar">
<button class="blue" id="start-process">Start processing</button>
</div>
</form>
</div>
<div class="bottom">
Elapsed processing time: <span id="elapsed">--:--:--</span>
</div>
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js">
</script>
<script>
/**
* On document load, assign click handlers to button(s), add
* elements that should start hidden (avoids "flashing"), and
* start polling for document selections.
*/
$(function() {
// assign click handler(s)
$('#start-process').click(startProcess);
});
/**
* Call the server-side 'serverProcess' function until there's no more work.
*/
function dispatchWork(){
if (window.runningProcess) {
}
google.script.run
.withSuccessHandler(
// After each interval, decide what to do next
function(workis) {
if (!workis.done) {
// There's more work to do, keep going.
dispatchWork();
}
else {
// All done. Stop timer
stopTimer();
$('#start-process').hide();
$("#final").html(' <h2>Processing complete!</h2>');
}
})
.withFailureHandler(
function(msg, element) {
showError(msg, $('#button-bar'));
element.disabled = false;
})
.serverProcess();
};
/**
* Runs a server-side function to retrieve the currently
* selected text.
*/
function startProcess() {
this.disabled = true; // Disable the button
$('#error').remove(); // Clear previous error messages, if any
startTimer(); // Start a work timer, for display to user
window.runningProcess = true;
dispatchWork(); // Start our work on the server
}
// Timer adapted from http://codingforums.com/javascript-programming/159873-displaying-elapsed-time.html
/**
* Kicks off the tick function.
*/
function startTimer( )
{
window.seconds = null;
window.ticker = null;
window.seconds = -1;
window.ticker = setInterval(tick, 1000);
tick( );
}
/**
* Stop ticking
*/
function stopTimer()
{
clearInterval(window.ticker);
}
/*
* Updates the timer display, between sleeps.
*/
function tick( )
{
++window.seconds;
var secs = window.seconds;
var hrs = Math.floor( secs / 3600 );
secs %= 3600;
var mns = Math.floor( secs / 60 );
secs %= 60;
var pretty = ( hrs < 10 ? "0" : "" ) + hrs
+ ":" + ( mns < 10 ? "0" : "" ) + mns
+ ":" + ( secs < 10 ? "0" : "" ) + secs;
$("#elapsed").text(pretty);
}
/**
* Inserts a div that contains an error message after a given element.
*
* #param msg The error message to display.
* #param element The element after which to display the error.
*/
function showError(msg, element) {
var div = $('<div id="error" class="error">' + msg + '</div>');
$(element).after(div);
}
</script>

firebase presence becomes more and more wrong over time

I have a simple presence user-count set up for firebase based on their example. The problem is that it relies on removing counts on disconnect. However, firebase seems to go down every 2 months and removes the ondisconnect handlers. This means that over time the counts get more and more wrong. Is there any way to fix this?
ty.Presence = function() {
this.rooms = {}
this.presence = fb.child('presence')
this.connectedRef = fb.child('.info/connected');
if (!localStorage.fb_presence_id) {
localStorage.fb_presence_id = Math.random().toString(36).slice(2)
}
this.browserID = localStorage.fb_presence_id
var first = false
}
ty.Presence.prototype.add = function(roomID, userobj) {
var self = this
var userListRef = this.presence.child(roomID)
// Generate a reference to a new location for my user with push.
var obj = {
s: "on",
id: this.browserID
}
if (userobj) {
obj.u = {
_id: userobj._id,
n: userobj.username
}
if (userobj.a) {
obj.u.a = userobj.a
}
}
var myUserRef = userListRef.push(obj)
this.rooms[roomID] = myUserRef
this.connectedRef.on("value", function(isOnline) {
if (isOnline.val()) {
// If we lose our internet connection, we want ourselves removed from the list.
myUserRef.onDisconnect().remove();
}
});
};
ty.Presence.prototype.count = function(roomID, cb) {
var self = this
var userListRef = this.presence.child(roomID)
var count = 0
function res () {
var usersArr = _.pluck(users, 'id')
usersArr = _.uniq(usersArr)
count = usersArr.length
if (cb) cb(count)
}
var users = {}
userListRef.on("child_added", function(css) {
users[css.name()] = css.val();
res()
});
userListRef.on("child_removed", function(css) {
delete users[css.name()]
res()
});
cb(count)
};
ty.Presence.prototype.get = function(ref) {
return this[ref]
};
ty.Presence.prototype.setGlobal = function(object) {
var self = this
_.each(this.rooms, function (myUserRef) {
myUserRef.set(object)
})
};
ty.Presence.prototype.remove = function(roomID) {
if (this.rooms[roomID])
this.rooms[roomID].remove();
};
ty.Presence.prototype.off = function(roomID) {
var userListRef = this.presence.child(roomID)
userListRef.off()
};
ty.presence = new ty.Presence()
ty.presence.add('all')

The onDisconnect handlers can be lost if a Firebase is restarted (e.g. when a new release is pushed live). One simple approach is to attach a timestamp as a priority to the records when they are stored. As long as the client remains online, have him update the timestamp occasionally.
setInterval(function() {
connectedRef.setPriority(Date.now());
}, 1000*60*60*4 /* every 4 hours */ );
Thus, any record which reaches, say, 24 hours old, would obviously be an orphan. A challenge could take place by clients (e.g. when a new client receives the list for the first time) or by a server process (e.g. a node.js script with a setInterval() to check for records older than X).
presenceRef.endAt(Date.now()-24*60*60*1000 /* 24 hours ago */).remove();
Less than ideal, sure, but a functional workaround I've utilized in apps.

Meteor JS: obscene amount of data loaded in loop

I have an app that loads a Jobs collection
Deps.autorun(function(){
var onet = Session.get('currentIndustryOnet');
var city_id = Session.get('currentMapArea');
jobsSubscription = Meteor.subscribe('jobs', onet, city_id);
console.log(onet);
if(jobsSubscription.ready) {
Session.set('jobCount', Jobs.find().count());
}
});
Template.selector.events({
'click div.select-block ul.dropdown-menu li': function(e) {
var selectedIndex = $(e.currentTarget).attr("rel");
var val = $('select#industryPicker option:eq(' + selectedIndex + ')').attr('value');
var oldVal = Session.get('currentIndustryOnet');
if(val != oldVal) {
Session.set('jobsLoaded', false);
Session.set('currentIndustryOnet', val);
}
}
});
The console logs 20+ values for what the var onet is. It appears that Meteor.autorun doesn't run just once. Is this normal? If not, how do I fix this to only run once?
Updated:
Jobs = new Meteor.Collection('jobs');
Cities = new Meteor.Collection('cities');
Pagination.style('bootstrap');
Session.setDefault('jobCount', null);
Session.setDefault('jobsLoaded', false);
Meteor.subscribe('cities');
Session.set('jobCount', Jobs.find().count());
Deps.autorun(function(){
var onet = Session.get('currentIndustryOnet');
var city_id = Session.get('currentMapArea');
Meteor.subscribe('jobs', onet, city_id, function onReady(){
Session.set('jobsLoaded', true);
});
Session.set('jobCount', Jobs.find().count());
});
function plotCities() {
console.log("CITIES PLOTTING");
// var jobs = Jobs.find().fetch();
// var addresses = _.chain(jobs)
// .countBy('address')
// .pairs()
// .sortBy(function(j) {return -j[1];})
// .map(function(j) {return j[0];})
// .slice(0, 50)
// .value();
// gmaps.clearMap();
// $.each(_.uniq(addresses), function(k, v){
// var addr = v.split(', ');
// Meteor.call('getCity', addr[0].toUpperCase(), addr[1], function(error, city){
// if(city) {
// var opts = {};
// opts.lng = city.loc[1];
// opts.lat = city.loc[0];
// opts.population = city.pop;
// opts._id = city._id;
// gmaps.addMarker(opts);
// }
// });
// })
}
Template.list.jobs = function() {
plotCities();
return Pagination.collection(Jobs.find({}).fetch());
}
The console.log('CITIES PLOTTING') gets called around 8 times the first time the page loads and then if I switch the Sessioned onet, and the jobs reloads the data, the call is 30+ times
Update 2:
Here is my code:
Session.set('jobsLoaded', false);
Meteor.subscribe('cities');
Session.set('jobCount', Jobs.find().count());
Deps.autorun(function(){
var onet = Session.get('currentIndustryOnet');
var city_id = Session.get('currentMapArea');
Meteor.subscribe('jobs', onet, city_id, function onReady(){
Session.set('jobsLoaded', true);
});
Session.set('jobCount', Jobs.find().count());
});
function plotCities() {
var jobs = Jobs.find().fetch();
var addresses = _.chain(jobs)
.countBy('address')
.pairs()
.sortBy(function(j) {return -j[1];})
.map(function(j) {return j[0];})
.slice(0, 50)
.value();
gmaps.clearMap();
$.each(_.uniq(addresses), function(k, v){
var addr = v.split(', ');
Meteor.call('getCity', addr[0].toUpperCase(), addr[1], function(error, city){
if(city) {
var opts = {};
opts.lng = city.loc[1];
opts.lat = city.loc[0];
opts.population = city.pop;
opts._id = city._id;
gmaps.addMarker(opts);
}
});
})
}
Template.list.jobs = function() {
if(Session.equals('jobsLoaded', true)) {
console.log("LOADED PLOT");
plotCities();
}
return Pagination.collection(Jobs.find({}).fetch());
}
When console.log("LOADED PLOT") is called... the first time it loads 8 times, the second, almost 40...

Deps.autorun rerun whenever a reactive item used inside is updated. You've got three such items in your function: two session variables and .ready() handle. Most probably the last one is causing the multiple rerun. If you're certain that the session variables were not touched during that time, that's the only option.
While I'm not certain about this, .ready() might be invalidated each time a new item is pulled up in the subscription channel. So having this check inside your autorun would result in several initial reruns as the first batch of data is pulled.
Move that check outside of autorun (it's possible as the subscription is visible from outside) and the problem should be solved.
Ah, now it's something else: you're calling plotCities from Template.list.jobs, which is also reactive and get rerun each time something in Jobs.find({}) changes – so again, each time a new initial item is loaded.
You've got a session variable in which you mark that your subscription is ready. Use it to filter the call:
Template.list.jobs = function() {
if(Session.equals('jobsLoaded', true)) plotCities();
return Pagination.collection(Jobs.find({}).fetch());
}