Develop Reference

Workaround for new Edge Indexeddb bug?

Microsoft released update kb4088776 in the past couple days, which has had a devastating effect on performance of indexedDb openCursor.
The simple fiddle here shows the problem. With the update, the "retrieval" time is 40 seconds or more. Prior to the update, it is around 1 second.
https://jsfiddle.net/L7q55ad6/23/
Relevant retrieval portion is here:
var _currentVer = 1;
function _openDatabase(fnSuccess) {
var _custDb = window.indexedDB.open("MyDatabase", _currentVer);
_custDb.onsuccess = function (event) {
var db = event.target.result;
fnSuccess(db);
}
_custDb.onerror = function (event) {
_custDb = null;
fnSuccess(null); // should use localData
}
_custDb.onupgradeneeded = function (event) {
var db = event.target.result;
var txn = event.target.transaction;
// Create an objectStore for this database
if (event.oldVersion < _currentVer) {
var customer = db.createObjectStore("customer", { keyPath: "guid" });
var index = customer.createIndex("by_id", "id", { unique: false });
}
};
}
function _retrieveCustomers(fn) {
_openDatabase(function (db) {
if (db == null)
{
alert("not supported");
return;
}
var customers = [];
var transaction = db.transaction("customer", "readonly");
var objectStore = transaction.objectStore("customer");
if (typeof objectStore.getAll === 'function') {
console.log("using getAll");
objectStore.getAll().onsuccess = function (event) {
fn(event.target.result);
};
}
else {
console.log("using openCursor");
objectStore.openCursor().onsuccess = function (event) {
var cursor = event.target.result;
if (cursor) {
customers.push(cursor.value);
cursor.continue();
}
else {
fn(customers);
}
};
}
});
}
The time to create and add the customers is basically normal, only the retrieval is bad. Edge has never supported the getAll method and it still doesn't after the update.
The only workaround I can think of would be to use localStorage instead, but unfortunately our data set is too large to fit into the 10MB limit. It is actually faster now to retrieve from our servers and convert the text to javascript objects, defeating the main purpose of indexeddb.

I don't have Edge so I can't test this, but does it happen with get too, or just openCursor? If get still performs well, you could store an index (in your example, the list of primary keys; in your real app, maybe something more complicated) in localStorage, and then use that to call get on each one.

Associative array not filling up in server-sent-event environement

I think I'm on the wrong track here:
I have an event source that gives me updates on the underlying system oprations. The page is intended to show said events in a jquery powered treetable. I receieve the events perfectly but I realized that there were a case I did not handle, the case where an event arrives but is missing it's parent. In this case I need to fetch the missing root plus all potentially missing children of that root node from the database. This works fine too.
//init fct
//...
eventSource.addEventListener("new_node", onEventSourceNewNodeEvent);
//...
function onEventSourceNewNodeEvent(event) {
let data = event.data;
if (!data)
return;
let rows = $(data).filter("tr");
rows.each(function (index, row) {
let parentEventId = row.getAttribute("data-tt-parent-id");
let parentNode = _table.treetable("node", parentEventId);
// if headless state is not fully
// resolved yet keep adding new rows to array
if (headlessRows[parentEventId]) {
headlessRows[parentEventId].push(row);
return;
} else if (parentEventId && !parentNode) { // headless state found
if (!headlessRows[parentEventId])
headlessRows[parentEventId] = [];
headlessRows[parentEventId].push(row);
fetchMissingNodes(parentEventId);
return;
}
insertNode(row, parentNode);
});
}
function fetchMissingNodes(parentEventId) {
let url = _table.data("url") + parentEventId;
$.get(url, function (data, textStatus, request) {
if (!data)
return;
let rows = $(data).filter("tr");
//insert root and children into table
_table.treetable("loadBranch", null, rows);
let parentNode = _table.treetable("node", parentEventId);
let lastLoadedRow = $(rows.last());
let headlessRowsArray = headlessRows[parentEventId];
while (headlessRowsArray && headlessRowsArray.length > 0) {
let row = headlessRowsArray.shift();
let rowId = row.getAttribute("data-tt-id");
if (rowId <= lastLoadedRow) // already loaded event from previous fetch
continue;
insertNode(row, parentNode);
let pendingUpdatesArray = pendingUpdates[rowId];
// shouldn't be more than one but who know future versions
while (pendingUpdatesArray && pendingUpdatesArray.length > 0) {
let updateEvent = headlessRowsArray.shift();
updateNode(updateEvent)
}
delete pendingUpdates[rowId]; // <- something better here?
}
delete headlessRows[parentEventId]; // <- something better here too?
});
}
The problem is around the line if (headlessRows[parentEventId]).
When I run it step by step (putting a debugger instruction just before) everything works fine, the headless array is created and filled correctly.
But as soon as I let it run full speed everything breaks.
The logs I printed seems to indicate that the array is not behaving in the way I was expecting it to. If I print the array with a console.log it shows as follow :
(2957754) [empty × 2957754]
length : 2957754
__proto__ : Array(0)
It seems to be missing any actual data. whereas it shows as follow when I execute it step by step:
(2957748) [empty × 2957747, Array(1)]
2957747:[tr.node.UNDETERMINED]
length:2957748
__proto__:Array(0)
I'm missing something but it is still eluding me.

your code is async, you do http request but you treat him as synchronized code.
try this fix
//init fct
//...
eventSource.addEventListener("new_node", onEventSourceNewNodeEvent);
//...
async function onEventSourceNewNodeEvent(event) {
let data = event.data;
if (!data)
return;
let rows = $(data).filter("tr");
rows.each(function (index, row) {
let parentEventId = row.getAttribute("data-tt-parent-id");
let parentNode = _table.treetable("node", parentEventId);
// if headless state is not fully
// resolved yet keep adding new rows to array
if (headlessRows[parentEventId]) {
headlessRows[parentEventId].push(row);
return;
} else if (parentEventId && !parentNode) { // headless state found
if (!headlessRows[parentEventId])
headlessRows[parentEventId] = [];
headlessRows[parentEventId].push(row);
await fetchMissingNodes(parentEventId);
return;
}
insertNode(row, parentNode);
});
}
function fetchMissingNodes(parentEventId) {
return new Promise((resolve,reject) =>{
let url = _table.data("url") + parentEventId;
$.get(url, function (data, textStatus, request) {
if (!data){
resolve()
return;
}
let rows = $(data).filter("tr");
//insert root and children into table
_table.treetable("loadBranch", null, rows);
let parentNode = _table.treetable("node", parentEventId);
let lastLoadedRow = $(rows.last());
let headlessRowsArray = headlessRows[parentEventId];
while (headlessRowsArray && headlessRowsArray.length > 0) {
let row = headlessRowsArray.shift();
let rowId = row.getAttribute("data-tt-id");
if (rowId <= lastLoadedRow) // already loaded event from previous fetch
continue;
insertNode(row, parentNode);
let pendingUpdatesArray = pendingUpdates[rowId];
// shouldn't be more than one but who know future versions
while (pendingUpdatesArray && pendingUpdatesArray.length > 0) {
let updateEvent = headlessRowsArray.shift();
updateNode(updateEvent)
}
delete pendingUpdates[rowId]; // <- something better here?
}
delete headlessRows[parentEventId]; // <- something better here too?
resolve()
});
})
}

Javascript class returning different values in console.log

I have the following classes (unnecessary details cut out here to make it more readable):
class CollectionManager {
constructor(){
this.collectionList = {};
}
initialize(collections){
...
}
populate(){
var collectionObjs = Object.keys(this.collectionList).map(function(key){
return collectionManager.collectionList[key];
});
return Promise.all(collectionObjs.map(function(collection){
collection.populateVideos();
}));
}
}
.
class Collection {
constructor(data){
this.collectionInfo = data;
this.videoArray = [];
}
populateVideos(){
var collectionKey = this.collectionInfo.COLLECTIONID;
var vChannels = Object.keys(this.collectionInfo.channels);
return Promise.all(vChannels.map(requestVideos))
.then(function (results) {
var videoIdArray = [];
return videoIdArray = [].concat.apply([], results);
}).then(function(arrVideoIds){
var groups = [];
for (var i = 0; i < arrVideoIds.length; i += 50) {
groups.push(arrVideoIds.slice(i, i + 50));
}
return groups;
}).then(function(chunkedArrVideoIds){
return Promise.all(chunkedArrVideoIds.map(requestVideoData)).then(function (results) {
var videoTileArray = [].concat.apply([], results);
collectionManager.collectionList[collectionKey].videoArray = videoTileArray;
return videoTileArray;
});
});
}
displayCollection(){
console.log(this.collectionInfo.COLLECTIONID);
console.log(collectionManager.collectionList);
console.log(collectionManager.collectionList[1]);
console.log(collectionManager.collectionList[1].videoArray);
And I call these classes like I would any normal promise.
collectionManager.populate().then(
function(){
collectionManager.displayCollections()
}
);
Now my problem is that when I call this code and read what is on the console, the videoArray is completely empty in the 4th console log. collectionManager.collectionList[1] contains a full object that has a videoArray with a length of 100 with all of my videos properly inside of it. But if I call collectionManager.collectionList[1].videoArray it is empty like it hasn't been filled. As far as I'm aware those should be calling the same exact place but it is giving different results.
Does anyone see where I messed up?

In the populate function, your Promise.all ... map is returning an array of undefined, which would be immediately resolved by Promise.all
You should do as follows
populate(){
var collectionObjs = Object.keys(this.collectionList).map(function(key){
return collectionManager.collectionList[key];
});
return Promise.all(collectionObjs.map(function(collection){
return collection.populateVideos();
}));
}
but, as you are using Class - you're already using more modern javascript
so
populate(){
var collectionObjs = Object.keys(this.collectionList).map(key => collectionManager.collectionList[key]);
return Promise.all(collectionObjs.map(collection => collection.populateVideos()));
}
Would be quite acceptable
as an aside, your class Collection can also be made cleaner (in my opinion) using arrow functions, and better promise chaining
class Collection {
constructor(data) {
this.collectionInfo = data;
this.videoArray = [];
}
populateVideos() {
var collectionKey = this.collectionInfo.COLLECTIONID;
var vChannels = Object.keys(this.collectionInfo.channels);
return Promise.all(vChannels.map(requestVideos))
.then(results => [].concat.apply([], results))
.then(arrVideoIds => {
var groups = [];
for (var i = 0; i < arrVideoIds.length; i += 50) {
groups.push(arrVideoIds.slice(i, i + 50));
}
return groups;
)
.then(chunkedArrVideoIds => Promise.all(chunkedArrVideoIds.map(requestVideoData)))
.then(function(results) {
var videoTileArray = [].concat.apply([], results);
collectionManager.collectionList[collectionKey].videoArray = videoTileArray;
return videoTileArray;
});
}
displayCollection() {
console.log(this.collectionInfo.COLLECTIONID);
console.log(collectionManager.collectionList);
console.log(collectionManager.collectionList[1]);
console.log(collectionManager.collectionList[1].videoArray);
}
}

I would avoid console.log() debugging, unless you really want to frustrate yourself. This could be a simple matter of console.log() not behaving as you expect. Instead, I would suggest puttting a debugger statement in displayCollection(). All you have to do is literally add the line debugger; into the code of that function and have chrome dev tools open when you run it. Execution will halt on that line and allow you to inspect application state with chrome dev tools(or the dev tools of whatever browser you're using). Based on the four print statements you have there, I think it might just be that it's not printing as you expect.

Waiting for multiple async operations in Nightwatch.js

I am attempting to test multiple sites for section headers being in the correct order. Of course everything is asynchronous in Nightwatch, including getting text from an element. The following code leads to the timeout never being called.
client.url(`www.oneofmyrealestatesites.com`);
client.waitForElementPresent("body", 5000);
var _ = require("underscore");
// This is the order I expect things to be in
var expected = ["Homes For Sale", "New Homes", "Apartments & Rentals"];
client.elements("css selector", ".listings .module-title .label", function (data) {
var listings = [];
data.value.forEach(function (element) {
client.elementIdText(element.ELEMENT, function (result) {
listings.push(result.value);
})
})
setTimeout(function () {
// Some of the sites have extra sections
var diff = _.intersection(listings, expected);
client.assert.ok(listings == diff);
}, 5000);
});
It would appear that no matter how much delay I give, listings is ALWAYS empty. If I console.log listings as it's being pushed to, it is getting populated, so that's not the issue. client.pause is always ignored as well.
Is there a way to make sure that listings is populated before asserting the diff?

I'm using async library for such cases https://github.com/caolan/async
Docs: https://github.com/caolan/async/blob/v1.5.2/README.md
var async = require("async");
/*use each, eachSeries or eachLimit - check docs for differences */
async.eachSeries(data.value, function (element, cb) {
client.elementIdText(element.ELEMENT, function (result) {
listings.push(result.value);
// this job is done
cb();
});
}, function() {
// Some of the sites have extra sections
var diff = _.intersection(listings, expected);
client.assert.ok(listings == diff);
});

setTimeout can only be called from .execute or .executeAsync because its actual javascript. The function below was only working until I used .executeAsync
Hope this works for you.
Cheers, Rody
LoopQuestionsLogSymptom: function() {
this.waitForElementVisible('.next-button', constants.timeout.medium, false);
this.api.executeAsync(function() {
let checkQuestion = function() {
// GET THE ELEMENTS
let nextButton = document.querySelectorAll('.next-button');
let answers = document.getElementsByClassName('flex-row');
let blueButton = document.querySelector('.blue-inverse-button');
let doneButton = document.querySelector('#doneButton');
let monitor = document.querySelector('.monitor');
//CHECK THE TYPES OF QUESTIONS AND CLICK THE RIGHT BUTTONS
if (!blueButton) {
answers[0].click();
nextButton[0].click()
} else if(blueButton){
blueButton.click();
}
setTimeout(() => {
if(!doneButton) {
console.log('Answering another question!');
checkQuestion();
}
if(doneButton){
doneButton.click();
}
else if(monitor) {
console.log("Exiting?")
.assert(monitor);
return this;
}
}, 2000);
};
// Initiating the check question function
return checkQuestion();
},[], function(){
console.log('Done?')
});
this.waitForElementVisible('.monitor', constants.timeout.medium);
this.assert.elementPresent('.monitor');
this.assert.urlEquals('https://member-portal.env-dev4.vivantehealth.org/progress');
return this;
},

IndexedDB via Lawnchair gets larger with every save

I am using Lawnchair.js on a mobile app I am building at work targeting iOS, Android, and Windows phones. My question is I have a relatively simple function(see below), that reads data from an object and saves it in the indexeddb database. It's about 4MB of data and on the first go round when I inspect in Internet explorer(via internet options), I can see the database is about 7MB. If I reload the page and re-run the same function with the same data, it increases to 14MB and then 20MB. Im using the same keys so my understanding is that this should just update the record but it's almost as if it's just inserting all new records every time. I have also had similar behavior using Lawnchair on mobile safari using websql adapter. Has anyone seen this before or have any suggestions as to why this might be ??.
The following code is from a function I am using to populate the database.
populateDatabase: function(database,callback) {
'use strict';
var key;
try {
for(key in MasterData){
if(MasterData.hasOwnProperty(key)){
var itemInfo = DataConfig.checkForDataUpdates[DataConfig.keyMap[key]];
database.save({key:itemInfo["name"],hash:itemInfo["version"],url:itemInfo["url"],data:MasterData[key]});
}
}
callback(true);
} catch(e){
callback(false);
}
}
MasterData is the large data file and itemInfo contains the key name, a hash that is later used to check an api for updates, and the relative url of where to update from. After I create the database I pass it into this function and then pass back true if the inserts are successful and false otherwise.
As previously mentioned, I have seen similar issues in iOS where calling database.save() was allocating a lot of memory but not releasing it and eventually causing a crash if it populated the database and then tried to update some records. Removing Lawnchair from the equation has kept it from crashing but it is still allocating a lot of memory when saving data. Not sure if this is normal for persistent storage on mobile devices, a bug in Lawnchair, or me being a noob and doing something terribly wrong but I could use some pointers on this as well as why indexeddb just keeps getting larger and larger on every save (at least during initial testing in IE10)??
EDIT: Source Code for indexed-db adapter is here:
https://github.com/brianleroux/lawnchair/blob/master/src/adapters/indexed-db.js
and here is the code for the save function I am using:
save:function(obj, callback) {
var self = this;
if(!this.store) {
this.waiting.push(function() {
this.save(obj, callback);
});
return;
}
var objs = (this.isArray(obj) ? obj : [obj]).map(function(o){if(!o.key) { o.key = self.uuid()} return o})
var win = function (e) {
if (callback) { self.lambda(callback).call(self, self.isArray(obj) ? objs : objs[0] ) }
};
var trans = this.db.transaction(this.record, READ_WRITE);
var store = trans.objectStore(this.record);
for (var i = 0; i < objs.length; i++) {
var o = objs[i];
store.put(o, o.key);
}
store.transaction.oncomplete = win;
store.transaction.onabort = fail;
return this;
},
When Creating a new instance, Lawnchair uses the init function from the indexed-db adapter which is the following.
init:function(options, callback) {
this.idb = getIDB();
this.waiting = [];
this.useAutoIncrement = useAutoIncrement();
var request = this.idb.open(this.name, STORE_VERSION);
var self = this;
var cb = self.fn(self.name, callback);
if (cb && typeof cb != 'function') throw 'callback not valid';
var win = function() {
// manually clean up event handlers on request; this helps on chrome
request.onupgradeneeded = request.onsuccess = request.error = null;
if(cb) return cb.call(self, self);
};
var upgrade = function(from, to) {
// don't try to migrate dbs, just recreate
try {
self.db.deleteObjectStore('teststore'); // old adapter
} catch (e1) { /* ignore */ }
try {
self.db.deleteObjectStore(self.record);
} catch (e2) { /* ignore */ }
// ok, create object store.
var params = {};
if (self.useAutoIncrement) { params.autoIncrement = true; }
self.db.createObjectStore(self.record, params);
self.store = true;
};
request.onupgradeneeded = function(event) {
self.db = request.result;
self.transaction = request.transaction;
upgrade(event.oldVersion, event.newVersion);
// will end up in onsuccess callback
};
request.onsuccess = function(event) {
self.db = event.target.result;
if(self.db.version != (''+STORE_VERSION)) {
// DEPRECATED API: modern implementations will fire the
// upgradeneeded event instead.
var oldVersion = self.db.version;
var setVrequest = self.db.setVersion(''+STORE_VERSION);
// onsuccess is the only place we can create Object Stores
setVrequest.onsuccess = function(event) {
var transaction = setVrequest.result;
setVrequest.onsuccess = setVrequest.onerror = null;
// can't upgrade w/o versionchange transaction.
upgrade(oldVersion, STORE_VERSION);
transaction.oncomplete = function() {
for (var i = 0; i < self.waiting.length; i++) {
self.waiting[i].call(self);
}
self.waiting = [];
win();
};
};
setVrequest.onerror = function(e) {
setVrequest.onsuccess = setVrequest.onerror = null;
console.error("Failed to create objectstore " + e);
fail(e);
};
} else {
self.store = true;
for (var i = 0; i < self.waiting.length; i++) {
self.waiting[i].call(self);
}
self.waiting = [];
win();
}
}
request.onerror = function(ev) {
if (request.errorCode === getIDBDatabaseException().VERSION_ERR) {
// xxx blow it away
self.idb.deleteDatabase(self.name);
// try it again.
return self.init(options, callback);
}
console.error('Failed to open database');
};
},

I think you keep adding data instead of updating the present data.
Can you provide some more information about the configuration of the store. Are you using an inline or external key? If it's an internal what is the keypath.