Meteor JS: obscene amount of data loaded in loop - javascript

I have an app that loads a Jobs collection
Deps.autorun(function(){
var onet = Session.get('currentIndustryOnet');
var city_id = Session.get('currentMapArea');
jobsSubscription = Meteor.subscribe('jobs', onet, city_id);
console.log(onet);
if(jobsSubscription.ready) {
Session.set('jobCount', Jobs.find().count());
}
});
Template.selector.events({
'click div.select-block ul.dropdown-menu li': function(e) {
var selectedIndex = $(e.currentTarget).attr("rel");
var val = $('select#industryPicker option:eq(' + selectedIndex + ')').attr('value');
var oldVal = Session.get('currentIndustryOnet');
if(val != oldVal) {
Session.set('jobsLoaded', false);
Session.set('currentIndustryOnet', val);
}
}
});
The console logs 20+ values for what the var onet is. It appears that Meteor.autorun doesn't run just once. Is this normal? If not, how do I fix this to only run once?
Updated:
Jobs = new Meteor.Collection('jobs');
Cities = new Meteor.Collection('cities');
Pagination.style('bootstrap');
Session.setDefault('jobCount', null);
Session.setDefault('jobsLoaded', false);
Meteor.subscribe('cities');
Session.set('jobCount', Jobs.find().count());
Deps.autorun(function(){
var onet = Session.get('currentIndustryOnet');
var city_id = Session.get('currentMapArea');
Meteor.subscribe('jobs', onet, city_id, function onReady(){
Session.set('jobsLoaded', true);
});
Session.set('jobCount', Jobs.find().count());
});
function plotCities() {
console.log("CITIES PLOTTING");
// var jobs = Jobs.find().fetch();
// var addresses = _.chain(jobs)
// .countBy('address')
// .pairs()
// .sortBy(function(j) {return -j[1];})
// .map(function(j) {return j[0];})
// .slice(0, 50)
// .value();
// gmaps.clearMap();
// $.each(_.uniq(addresses), function(k, v){
// var addr = v.split(', ');
// Meteor.call('getCity', addr[0].toUpperCase(), addr[1], function(error, city){
// if(city) {
// var opts = {};
// opts.lng = city.loc[1];
// opts.lat = city.loc[0];
// opts.population = city.pop;
// opts._id = city._id;
// gmaps.addMarker(opts);
// }
// });
// })
}
Template.list.jobs = function() {
plotCities();
return Pagination.collection(Jobs.find({}).fetch());
}
The console.log('CITIES PLOTTING') gets called around 8 times the first time the page loads and then if I switch the Sessioned onet, and the jobs reloads the data, the call is 30+ times
Update 2:
Here is my code:
Session.set('jobsLoaded', false);
Meteor.subscribe('cities');
Session.set('jobCount', Jobs.find().count());
Deps.autorun(function(){
var onet = Session.get('currentIndustryOnet');
var city_id = Session.get('currentMapArea');
Meteor.subscribe('jobs', onet, city_id, function onReady(){
Session.set('jobsLoaded', true);
});
Session.set('jobCount', Jobs.find().count());
});
function plotCities() {
var jobs = Jobs.find().fetch();
var addresses = _.chain(jobs)
.countBy('address')
.pairs()
.sortBy(function(j) {return -j[1];})
.map(function(j) {return j[0];})
.slice(0, 50)
.value();
gmaps.clearMap();
$.each(_.uniq(addresses), function(k, v){
var addr = v.split(', ');
Meteor.call('getCity', addr[0].toUpperCase(), addr[1], function(error, city){
if(city) {
var opts = {};
opts.lng = city.loc[1];
opts.lat = city.loc[0];
opts.population = city.pop;
opts._id = city._id;
gmaps.addMarker(opts);
}
});
})
}
Template.list.jobs = function() {
if(Session.equals('jobsLoaded', true)) {
console.log("LOADED PLOT");
plotCities();
}
return Pagination.collection(Jobs.find({}).fetch());
}
When console.log("LOADED PLOT") is called... the first time it loads 8 times, the second, almost 40...

Deps.autorun rerun whenever a reactive item used inside is updated. You've got three such items in your function: two session variables and .ready() handle. Most probably the last one is causing the multiple rerun. If you're certain that the session variables were not touched during that time, that's the only option.
While I'm not certain about this, .ready() might be invalidated each time a new item is pulled up in the subscription channel. So having this check inside your autorun would result in several initial reruns as the first batch of data is pulled.
Move that check outside of autorun (it's possible as the subscription is visible from outside) and the problem should be solved.
Ah, now it's something else: you're calling plotCities from Template.list.jobs, which is also reactive and get rerun each time something in Jobs.find({}) changes – so again, each time a new initial item is loaded.
You've got a session variable in which you mark that your subscription is ready. Use it to filter the call:
Template.list.jobs = function() {
if(Session.equals('jobsLoaded', true)) plotCities();
return Pagination.collection(Jobs.find({}).fetch());
}

Related

JavaScript - Issues recovering a map in an object after being saved in localStorage

I've been dealing with this for some time. I've a list of sections in which the user checks some checkboxes and that is sent to the server via AJAX. However, since the user can return to previous sections, I'm using some objects of mine to store some things the user has done (if he/she already finished working in that section, which checkboxes checked, etc). I'm doing this to not overload the database and only send new requests to store information if the user effectively changes a previous checkbox, not if he just starts clicking "Save" randomly. I'm using objects to see the sections of the page, and storing the previous state of the checkboxes in a Map. Here's my "supervisor":
function Supervisor(id) {
this.id = id;
this.verif = null;
this.selections = new Map();
var children = $("#ContentPlaceHolder1_checkboxes_div_" + id).children().length;
for (var i = 0; i < children; i++) {
if (i % 2 == 0) {
var checkbox = $("#ContentPlaceHolder1_checkboxes_div_" + id).children()[i];
var idCheck = checkbox.id.split("_")[2];
this.selections.set(idCheck, false);
}
}
console.log("Length " + this.selections.size);
this.change = false;
}
The console.log gives me the expected output, so I assume my Map is created and initialized correctly. Since the session of the user can expire before he finishes his work, or he can close his browser by accident, I'm storing this object using local storage, so I can change the page accordingly to what he has done should anything happen. Here are my functions:
function setObj(id, supervisor) {
localStorage.setItem(id, JSON.stringify(supervisor));
}
function getObj(key) {
var supervisor = JSON.parse(localStorage.getItem(key));
return supervisor;
}
So, I'm trying to add to the record whenever an user clicks in a checkbox. And this is where the problem happens. Here's the function:
function checkboxClicked(idCbx) {
var idSection = $("#ContentPlaceHolder1_hdnActualField").val();
var supervisor = getObj(idSection);
console.log(typeof (supervisor)); //Returns object, everythings fine
console.log(typeof (supervisor.change)); //Returns boolean
supervisor.change = true;
var idCheck = idCbx.split("_")[2]; //I just want a part of the name
console.log(typeof(supervisor.selections)); //Prints object
console.log("Length " + supervisor.selections.size); //Undefined!
supervisor.selections.set(idCheck, true); //Error! Note: The true is just for testing purposes
setObj(idSection, supervisor);
}
What am I doing wrong? Thanks!
Please look at this example, I removed the jquery id discovery for clarity. You'll need to adapt this to meet your needs but it should get you mostly there.
const mapToJSON = (map) => [...map];
const mapFromJSON = (json) => new Map(json);
function Supervisor(id) {
this.id = id;
this.verif = null;
this.selections = new Map();
this.change = false;
this.selections.set('blah', 'hello');
}
Supervisor.from = function (data) {
const id = data.id;
const supervisor = new Supervisor(id);
supervisor.verif = data.verif;
supervisor.selections = new Map(data.selections);
return supervisor;
};
Supervisor.prototype.toJSON = function() {
return {
id: this.id,
verif: this.verif,
selections: mapToJSON(this.selections)
}
}
const expected = new Supervisor(1);
console.log(expected);
const json = JSON.stringify(expected);
const actual = Supervisor.from(JSON.parse(json));
console.log(actual);
If you cant use the spread operation in 'mapToJSON' you could loop and push.
const mapToJSON = (map) => {
const result = [];
for (let entry of map.entries()) {
result.push(entry);
}
return result;
}
Really the only thing id change is have the constructor do less, just accept values, assign with minimal fiddling, and have a factory query the dom and populate the constructor with values. Maybe something like fromDOM() or something. This will make Supervisor more flexible and easier to test.
function Supervisor(options) {
this.id = options.id;
this.verif = null;
this.selections = options.selections || new Map();
this.change = false;
}
Supervisor.fromDOM = function(id) {
const selections = new Map();
const children = $("#ContentPlaceHolder1_checkboxes_div_" + id).children();
for (var i = 0; i < children.length; i++) {
if (i % 2 == 0) {
var checkbox = children[i];
var idCheck = checkbox.id.split("_")[2];
selections.set(idCheck, false);
}
}
return new Supervisor({ id: id, selections: selections });
};
console.log(Supervisor.fromDOM(2));
You can keep going and have another method that tries to parse a Supervisor from localStorageand default to the dom based factory if the localStorage one returns null.

Google Cloud SQL not updating with script

I have a long script which is designed to edit a specific row in the Cloud SQL table. The code is long so i will shorten it.
Client Side:
function build_profile(){
var cbid = sessionStorage.getItem("client_id");
var self = this;
var createSuccess = function(data){
var statuse = ["Active", "Wiating", "Discharged"];
if(data !== false){
data = data.split(",");
var dec = app.pages.Profile.descendants;
dec.fname.text = data[1];
dec.sname.text = data[3];
sessionStorage.setItem("school_id", data[9]);
app.popups.Loading.visible = false;
}
};
var init = function() {google.script.run.withSuccessHandler(createSuccess).get_user_data(cbid);};
app.popups.Loading.visible = true;
init();
}
function save_profile() {
var createSuccess = function(data){
var dec = app.pages.Profile.descendants;
console.log(data);
if(data !== -1){
var ds = app.datasources.Clients;
ds.load(function(){
ds.selectIndex(data);
console.log("editing:"+ds.item.CBID);
ds.item.fname = dec.fname_edit.value;
ds.item.sname = dec.sname_edit.value;
ds.load(function(){build_profile();});
});
}
}};
var init = function() {google.script.run.withSuccessHandler(createSuccess).update_client(sessionStorage.getItem("client_id"));};
init();
}
Server Side:
function get_user_data(cbid){
try{
var query = app.models.Clients.newQuery();
query.filters.CBID._equals = parseInt(cbid);
var results = query.run();
if(results.length > 0){
var arr = [
results[0].Id, //0
results[0].fname, //1
results[0].sname //3
];
return arr.join(",");
}else{
return false;
}
}catch(e){
console.error(e);
console.log("function get_user_data");
return false;
}
}
function update_client(cbid) {
try{
var ds = app.models.Clients;
var query = ds.newQuery();
query.filters.CBID._equals = parseInt(cbid);
var results = query.run();
if(results.length > 0){
var id = results[0]._key;
return id+1;
}else{
return -1;
}
}catch(e){
console.error(e);
return -1;
}
}
This gets the Clients table and updates the row for the selected client, then rebuilds the profile with the new information.
EDIT: I have managed to get to a point where its telling me that i cannot run the query (ds.load()) while processing its results. There does not seem to be a manual check to see if it has processed?
Note: datasource.saveChanges() does not work as it saves automatically.
You error is being produced by the client side function save_profile() and it is exactly in this block:
ds.load(function(){
ds.selectIndex(data);
console.log("editing:"+ds.item.CBID);
ds.item.fname = dec.fname_edit.value;
ds.item.sname = dec.sname_edit.value;
ds.load(function(){build_profile();});
});
So what you are doing is reloading the datasource almost immediately before it finishes loading hence you are getting that error
cannot run the query (ds.load()) while processing its results
This is just a matter of timing. A setTimeout can take of the issue. Just do the following:
ds.load(function(){
ds.selectIndex(data);
console.log("editing:"+ds.item.CBID);
ds.item.fname = dec.fname_edit.value;
ds.item.sname = dec.sname_edit.value;
setTimeout(function(){
ds.load(function(){build_profile();});
},1000);
});
I have manage to find a solution to this particular issue. It requires Manual Saving but it saves a lot of hassle as one of the inbuilt solutions can be used rather than relying on dealing with errors or timeouts.
function client_query_and_result(){
var createSuccess = function(data){ //callback function
console.log(data);
};
app.datasources.SomeTable.saveChanges(function(){//ensures all changes have been saved
app.datasources.SomeTable.load(function(){//makes sure to reload the datasource
google.script.run.withSuccessHandler(createSuccess).server_query_and_result(); //at this point All data has been saved and reloaded
});
});
}
The Server side code is the exact same methods. To enable manual saving you can select the table in App Maker -> Datasources -> check "Manual save mode".
Hope this can be useful to someone else.

One local storage JavaScript for fields on different pages

Three different web pages have three contenteditable areas each (content1, content2, and content3).
Each page links to one JavaScript which uses local storage to store the user's input and present it again on their return.
When I change the content on one page, it changes the content in the same editable area all three pages.
I want each page to be able to use the same script to save it's own data independently of the other pages.
I've tried adding page location (url) to the local storage key, to get each page to use the same script to store and retrieve it's own data, but I can't get it to work. Am new to JavaScript - would be grateful for any help. Thanks!
window.addEventListener('load', onLoad);
function onLoad() {
checkEdits();
}
// Get page location
var loc = encodeURIComponent(window.location.href);
// Add location to local storage key
function checkEdits() {
if (localStorage.userEdits1 != null) {
var userEdits1 = (loc + userEdits1);
document.getElementById('content1').innerHTML = localStorage.userEdits1;
}
if (localStorage.userEdits2 != null) {
var userEdits2 = (loc + userEdits2);
document.getElementById('content2').innerHTML = localStorage.userEdits2;
}
if (localStorage.userEdits3 != null) {
var userEdits3 = (loc + userEdits3);
document.getElementById('content3').innerHTML = localStorage.userEdits3;
}
};
document.onkeyup = function (e) {
e = e || window.event;
console.log(e.keyCode);
saveEdits();
};
function saveEdits() {
// Get editable elements
var editElem1 = document.getElementById('content1');
var editElem2 = document.getElementById('content2');
var editElem3 = document.getElementById('content3');
// Get edited elements contents
var userVersion1 = editElem1.innerHTML;
var userVersion2 = editElem2.innerHTML;
var userVersion3 = editElem3.innerHTML;
// Add page location to storage key
var userEdits1 = (loc + userEdits1);
var userEdits2 = (loc + userEdits2);
var userEdits3 = (loc + userEdits3);
// Save the content to local storage
localStorage.userEdits1 = userVersion1;
localStorage.userEdits2 = userVersion2;
localStorage.userEdits3 = userVersion3;
};
function clearLocal() {
if (confirm('Are you sure you want to clear your notes on this page?')) {
localStorage.setItem("userEdits1", "");
localStorage.setItem("userEdits2", "");
localStorage.setItem("userEdits3", "");
document.getElementById('content1').innerHTML = "";
document.getElementById('content2').innerHTML = "";
document.getElementById('content3').innerHTML = "";
alert('Notes cleared');
}
}
The actual problem of your script is this:
localStorage.userEdits1
To access the property of an object with a string (e.g. stored in a variable) you have to use the bracket notation:
locationStorage[userEdits1]
But I would propose a slightly more generic (and, imho, cleaner) solution...
Store the content of the editable elements in an object
var cache = {
<elementX id>: <content>,
<elementY id>: <content>,
<elementZ id>: <content>,
...
};
And then store this "cache" in the local storage with a page-specific key
localStorage.setItem(window.location.pathName, JSON.stringify(cache));
A possible implementation could be:
window.addEventListener('load', checkEdits);
getContentEditables().forEach(function(editable) {
// This prevents the function to execute on every keyup event
// Instead it will only be executed 100ms after the last keyup
var debouncedFunc = debounce(100, function(e) {
saveEdits();
});
editable.addEventListener("keyup", debouncedFunc);
});
function checkEdits() {
var cache = localStorage.getItem(window.location.pathName);
if (cache !== null) {
cache = JSON.parse(cache);
Object.keys(cache)
.forEach(function(key) {
var element = document.getElementById(key);
if (element !== null) {
element.innerHTML = cache[key];
}
});
}
}
function saveEdits() {
var cache = {};
getContentEditables().forEach(function(element) {
cache[element.id] = element.innerHTML;
});
localStorage.setItem(window.location.pathName, JSON.stringify(cache));
};
function clearLocal() {
if (confirm('Are you sure you want to clear your notes on this page?')) {
localStorage.removeItem(window.location.pathName);
getContentEditables().forEach(function(element) {
element.innerHTML = "";
});
alert('Notes cleared');
}
}
// helper
function getContentEditables() {
var elements = [];
document.querySelectorAll("[contenteditable]")
.forEach(function(element) {
if (element.id) {
elements.push(element);
}
});
return elements;
}
function debounce(timeout, func) {
var timeoutId;
return function() {
var that = this,
args = arguments;
clearTimeout(timeoutId);
timeoutId = setTimeout(function() {
func.apply(that, args);
}, timeout);
}
}
Use
localStorage[userEdits1]
Instead of
localStorage.userEdits1

Iterate over IDs and generate a report pdf for each ID using PhantomJS

This code generates one pdf for the first employee with his id in the url address. I would like to iterate over many ids and generate several pdfs one per each employee with unique id.
The records of employees are is a CSV file which has been read and parsed somewhere else. Also for iteraring over ids I have created an array containing ids which is called idArray. (e.g. idArray = ['123', '127', '156']). Would you please help me create a pdf per id from idArray?
var page = require('webpage').create(),
system = require('system'),
id = system.args[1];
page.open('http://127.0.0.1:3000/report.html?id=' + id, function () {
var intervalHandle;
// poll until
var ready = function () {
var ready = page.evaluate(function () {
return reportReady;
});
if (ready) {
clearInterval(intervalHandle);
page.render('report-id.pdf');
phantom.exit();
} else {
console.log("Not ready yet");
}
};
intervalHandle = setInterval(ready, 100);
});
The problem is that you can't simply iterate over the IDs. page.open() is asynchronous, so you would tell PhantomJS to load the page with the next ID before the previous one can finished loading.
The solution is to use recursion. Define a function that contains the logic to do one iteration and use that to string many callbacks together:
var page = require('webpage').create(),
system = require('system'),
idArray = system.args[1].split(",");
iterate(); // let it run
function iterate() {
var id = idArray.shift(); // changes the idArray
page.open('http://127.0.0.1:3000/report.html?id=' + id, function () {
var intervalHandle;
// poll until
var ready = function () {
var ready = page.evaluate(function () {
return reportReady;
});
if (ready) {
clearInterval(intervalHandle);
page.render('report-id.pdf');
if (idArray.length > 0) {
iterate();
} else {
phantom.exit();
}
} else {
console.log("Not ready yet");
}
};
intervalHandle = setInterval(ready, 100);
});
}
I assume that the IDs are passed in this way:
$ phantomjs script.js 4,8,15,16,23,42

How to deal with asyncronous javascript in loops?

I have a forloop like this:
for (var name in myperson.firstname){
var myphone = new phone(myperson, firstname);
myphone.get(function(phonenumbers){
if(myphone.phonearray){
myperson.save();
//Can I put a break here?;
}
});
}
What it does is that it searches for phone-numbers in a database based on various first-names. What I want to achieve is that once it finds a number associated with any of the first names, it performs myperson.save and then stops all the iterations, so that no duplicates get saved. Sometimes, none of the names return any phone-numbers.
myphone.get contains a server request and the callback is triggered on success
If I put a break inside the response, what will happen with the other iterations of the loop? Most likely the other http-requests have already been initiated. I don't want them to perform the save. One solution I have thought of is to put a variable outside of the forloop and set it to save, and then check when the other callbacks get's triggered, but I'm not sure if that's the best way to go.
You could write a helper function to restrict invocations:
function callUntilTrue(cb) {
var done = false;
return function () {
if (done) {
log("previous callback succeeded. not calling others.");
return;
}
var res = cb.apply(null, arguments);
done = !! res;
};
}
var myperson = {
firstname: {
"tom": null,
"jerry": null,
"micky": null
},
save: function () {
log("save " + JSON.stringify(this, null, 2));
}
};
var cb = function (myperson_, phonenumbers) {
if (myperson_.phonearray) {
log("person already has phone numbers. returning.");
return false;
}
if (phonenumbers.length < 1) {
log("response has no phone numbers. returning.");
return false;
}
log("person has no existing phone numbers. saving ", phonenumbers);
myperson_.phonearray = phonenumbers;
myperson_.save();
return true;
};
var restrictedCb = callUntilTrue(cb.bind(null, myperson));
for (var name in myperson.firstname) {
var myphone = new phone(myperson, name);
myphone.get(restrictedCb);
}
Sample Console:
results for tom-0 after 1675 ms
response has no phone numbers. returning.
results for jerry-1 after 1943 ms
person has no existing phone numbers. saving , [
"jerry-1-0-number"
]
save {
"firstname": {
"tom": null,
"jerry": null,
"micky": null
},
"phonearray": [
"jerry-1-0-number"
]
}
results for micky-2 after 4440 ms
previous callback succeeded. not calling others.
Full example in this jsfiddle with fake timeouts.
EDIT Added HTML output as well as console.log.
The first result callback will only ever happen after the loop, because of the single-threaded nature of javascript and because running code isn't interrupted if events arrive.
If you you still want requests to happen in parallel, you may use a flag
var saved = false;
for (var name in myperson.firstname){
var myphone = new phone(myperson, firstname /* name? */);
myphone.get(function(phonenumbers){
if (!saved && myphone.phonearray){
saved = true;
myperson.save();
}
});
}
This will not cancel any pending requests, however, just prevent the save once they return.
It would be better if your .get() would return something cancelable (the request itself, maybe).
var saved = false;
var requests = [];
for (var name in myperson.firstname){
var myphone = new phone(myperson, firstname /* name? */);
var r;
requests.push(r = myphone.get(function(phonenumbers){
// Remove current request.
requests = requests.filter(function(i) {
return r !== i;
});
if (saved || !myphone.phonearray) {
return;
}
saved = true;
// Kill other pending/unfinished requests.
requests.forEach(function(r) {
 r.abort();
});
myperson.save();
}));
}
Even better, don't start all requests at once. Instead construct an array of all possible combinations, have a counter (a semaphore) and only start X requests.
var saved = false;
var requests = [];
// Use requests.length as the implicit counter.
var waiting = []; // Wait queue.
for (var name in myperson.firstname){
var myphone = new phone(myperson, firstname /* name? */);
var r;
if (requests.length >= 4) {
// Put in wait queue instead.
waiting.push(myphone);
continue;
}
requests.push(r = myphone.get(function cb(phonenumbers){
// Remove current request.
requests = requests.filter(function(i) {
return r !== i;
});
if (saved) {
return;
}
if (!myphone.phonearray) {
// Start next request.
var w = waiting.shift();
if (w) {
requests.push(w.get(cb));
)
return;
}
saved = true;
// Kill other pending/unfinished requests.
requests.forEach(function(r) {
r.abort();
});
myperson.save();
}));
}

Categories

Resources