Develop Reference

Get ServiceWorkerGlobalScope's variable from window context

I have a service worker in sw.js, it uses a template engine to get the commit numbre as a version number. I set the cache name like this:
var version = {{ commit_hash }};
self.cacheName = `cache-` + version;
I have some scripts being added to the cache on the worker's install, but there are scripts that are dynamically loaded on the page. I would like to load all the scripts/css on the first load without forcing the user to wait for the app to install first.
I can get all the content on the page with the following code in the bottom of index.html:
var toCache = ['/'];
var css = document.getElementsByTagName("link");
for(el of css) {
var href = el.getAttribute("href");
if(href) {
toCache.push(href);
}
}
var js = document.getElementsByTagName("script");
for(el of js) {
var src = el.getAttribute("src");
if(src) {
toCache.push(src);
}
}
That works fine, now I would just need to open the correct cache, fetch files that aren't already present, and store them. Something like:
toCache.forEach(function(url) {
caches.match(url).then(function(result) {
if(!result) {
fetch(url).then(function(response) {
caches.open(cacheName).then(cache => {
cache.put(url, response)
});
});
}
});
});
Is there a way to get the cacheName from the service worker inside a script tag in a different file?
And yes, I know that I could simplify this greatly by doing the check in the for/of loops. I broke it apart so it would be easier to describe.

No.
JavaScript executing in the window context cannot access SW's context and vice versa. You have to implement a workaround of some sort.
Remember that you can use postMessage to communicate between the two.

Using this blog I was able to pass messages from the service worker and back. First, I added the following function at the top of sw.js:
function clientPostMessage(client, message){
return new Promise(function(resolve, reject){
var channel = new MessageChannel();
channel.port1.onmessage = function(event){
if(event.data.error){
reject(event.data.error);
}
else {
resolve(event.data);
}
};
client.postMessage(message, [channel.port2]);
});
}
This allows my service worker to post a message to the window, and then do a callback with a promise.
Then, in my index.html file I added the following to a script tag:
navigator.serviceWorker.addEventListener('message', event => {
switch(event.data) {
case "addAll":
var toCache = [];
var css = document.getElementsByTagName("link");
for(el of css) {
var href = el.getAttribute("href");
if(href) {
toCache.push(href);
}
}
var js = document.getElementsByTagName("script");
for(el of js) {
var src = el.getAttribute("src");
if(src) {
toCache.push(src);
}
}
event.ports[0].postMessage(toCache);
break;
default:
console.log(event.data);
}
});
This listens to any service workers asking for messages, and if it is a "addAll" message, it will get all the scripts and linked content on the page and return an array of the scripts.
Finally, I added the following to my activate event listener function in sw.js:
// Get all the clients, and for each post a message
clients.matchAll().then(clients => {
clients.forEach(client => {
// Post "addAll" to get a list of files to cache
clientPostMessage(client, "addAll").then(message => {
// For each file, check if it already exists in the cache
message.forEach(url => {
caches.match(url).then(result => {
// If there's nothing in the cache, fetch the file and cache it
if(!result) {
fetch(url).then(response => {
caches.open(cacheName).then(cache => {
cache.put(url, response);
});
});
}
})
});
});
})
});
For all clients the service worker sends an "addAll" message to the page and gets the result. For each item in the result, it checks if the value is already in the cache and if not, fetches and adds it.
With this method, the install listener of the service worker only needs to contain:
self.addEventListener('install', event => {
if(self.skipWaiting) {
self.skipWaiting();
}
event.waitUntil(
caches.open(cacheName).then(cache => {
return cache.addAll([
'/',
'/index.html',
])
})
);
});
It seems to be working well so far, if anyone has any suggestions or sees any errors I'd be happy to hear! You can also tell me how improper this is, but it makes my life a lot easier for adding service workers for pre-existing projects that rely on scripts that aren't bundled together.

What should I do with the redundant state of a ServiceWorker?

I gotta a companion script for a serviceworker and I'm trialling right now.
The script works like so:
((n, d) => {
if (!(n.serviceWorker && (typeof Cache !== 'undefined' && Cache.prototype.addAll))) return;
n.serviceWorker.register('/serviceworker.js', { scope: './book/' })
.then(function(reg) {
if (!n.serviceWorker.controller) return;
reg.onupdatefound = () => {
let installingWorker = reg.installing;
installingWorker.onstatechange = () => {
switch (installingWorker.state) {
case 'installed':
if (navigator.serviceWorker.controller) {
updateReady(reg.waiting);
} else {
// This is the initial serviceworker…
console.log('May be skipwaiting here?');
}
break;
case 'waiting':
updateReady(reg.waiting);
break;
case 'redundant':
// Something went wrong?
console.log('[Companion] new SW could not install…')
break;
}
};
};
}).catch((err) => {
//console.log('[Companion] Something went wrong…', err);
});
function updateReady(worker) {
d.getElementById('swNotifier').classList.remove('hidden');
λ('refreshServiceWorkerButton').on('click', function(event) {
event.preventDefault();
worker.postMessage({ 'refreshServiceWorker': true } );
});
λ('cancelRefresh').on('click', function(event) {
event.preventDefault();
d.getElementById('swNotifier').classList.add('hidden');
});
}
function λ(selector) {
let self = {};
self.selector = selector;
self.element = d.getElementById(self.selector);
self.on = function(type, callback) {
self.element['on' + type] = callback;
};
return self;
}
let refreshing;
n.serviceWorker.addEventListener('controllerchange', function() {
if (refreshing) return;
window.location.reload();
refreshing = true;
});
})(navigator, document);
I'm a bit overwhelmed right now by the enormity of the service workers api and unable to "see" what one would do with reg.installing returning a redundant state?
Apologies if this seems like a dumb question but I'm new to serviceworkers.

It's kinda difficult to work out what your intent is here so I'll try and answer the question generally.
A service worker will become redundant if it fails to install or if it's superseded by a newer service worker.
What you do when this happens is up to you. What do you want to do in these cases?

Based on the definition here https://www.w3.org/TR/service-workers/#service-worker-state-attribute I am guessing just print a log in case it comes up in debugging otherwise do nothing.

You should remove any UI prompts you created that ask the user to do something in order to activate the latest service worker. And be patient a little longer.
You have 3 service workers, as you can see on the registration:
active: the one that is running
waiting: the one that was downloaded, and is ready to become active
installing: the one that we just found, being downloaded, after which it becomes waiting
When a service worker reaches #2, you may display a prompt to the user about the new version of the app being just a click away. Let's say they don't act on it.
Then you publish a new version. Your app detects the new version, and starts to download it. At this point, you have 3 service workers. The one at #2 changes to redundant. The one at #3 is not ready yet. You should remove that prompt.
Once #3 is downloaded, it takes the place of #2, and you can show that prompt again.

Write catch function to see the error. It could be SSL issue.
/* In main.js */
if ('serviceWorker' in navigator) {
navigator.serviceWorker.register('./sw.js')
.then(function(registration) {
console.log("Service Worker Registered", registration);
})
.catch(function(err) {
console.log("Service Worker Failed to Register", err);
})
}

Update HTML object with node.js and javascript

I'm new to nodejs and jquery, and I'm trying to update one single html object using a script.
I am using a Raspberry pi 2 and a ultrasonic sensor, to measure distance. I want to measure continuous, and update the html document at the same time with the real time values.
When I try to run my code it behaves like a server and not a client. Everything that i console.log() prints in the cmd and not in the browesers' console. When I run my code now i do it with "sudo node surveyor.js", but nothing happens in the html-document. I have linked it properly to the script. I have also tried document.getElementsByTagName("h6").innerHTML = distance.toFixed(2), but the error is "document is not defiend".
Is there any easy way to fix this?
My code this far is:
var statistics = require('math-statistics');
var usonic = require('r-pi-usonic');
var fs = require("fs");
var path = require("path");
var jsdom = require("jsdom");
var htmlSource = fs.readFileSync("../index.html", "utf8");
var init = function(config) {
usonic.init(function (error) {
if (error) {
console.log('error');
} else {
var sensor = usonic.createSensor(config.echoPin, config.triggerPin, config.timeout);
//console.log(config);
var distances;
(function measure() {
if (!distances || distances.length === config.rate) {
if (distances) {
print(distances);
}
distances = [];
}
setTimeout(function() {
distances.push(sensor());
measure();
}, config.delay);
}());
}
});
};
var print = function(distances) {
var distance = statistics.median(distances);
process.stdout.clearLine();
process.stdout.cursorTo(0);
if (distance < 0) {
process.stdout.write('Error: Measurement timeout.\n');
} else {
process.stdout.write('Distance: ' + distance.toFixed(2) + ' cm');
call_jsdom(htmlSource, function (window) {
var $ = window.$;
$("h6").replaceWith(distance.toFixed(2));
console.log(documentToSource(window.document));
});
}
};
function documentToSource(doc) {
// The non-standard window.document.outerHTML also exists,
// but currently does not preserve source code structure as well
// The following two operations are non-standard
return doc.doctype.toString()+doc.innerHTML;
}
function call_jsdom(source, callback) {
jsdom.env(
source,
[ 'jquery-1.7.1.min.js' ],
function(errors, window) {
process.nextTick(
function () {
if (errors) {
throw new Error("There were errors: "+errors);
}
callback(window);
}
);
}
);
}
init({
echoPin: 15, //Echo pin
triggerPin: 14, //Trigger pin
timeout: 1000, //Measurement timeout in µs
delay: 60, //Measurement delay in ms
rate: 5 //Measurements per sample
});

Node.js is a server-side implementation of JavaScript. It's ok to do all the sensors operations and calculations on server-side, but you need some mechanism to provide the results to your clients. If they are going to use your application by using a web browser, you must run a HTTP server, like Express.js, and create a route (something like http://localhost/surveyor or just http://localhost/) that calls a method you have implemented on server-side and do something with the result. One possible way to return this resulting data to the clients is by rendering an HTML page that shows them. For that you should use a Template Engine.
Any DOM manipulation should be done on client-side (you could, for example, include a <script> tag inside your template HTML just to try and understand how it works, but it is not recommended to do this in production environments).
Try searching google for Node.js examples and tutorials and you will get it :)

Can't run Selenium PhantomJS instances in parallell

I'm using Selenium's node.js API to run PhantomJS instances against a series of web pages. The code I use to execute the actions on the pages work fine, but it seems only one instance of Selenium/PhantomJS can run at a time. This function is called multiple times from the same module and steps through pages in a webshop where the pagination is handled client side (which is why I need the Selenium/PhantomJS environment - to extract data from each page).
Once again, the code in and of itself works fine, but it can't execute in parallell. What could be causing this?
module.exports = function (crawler, page, parsePage, done) {
"use strict";
var _ = require("lodash"),
format = require("util").format,
path = require("path"),
webdriver = require("selenium-webdriver"),
By = webdriver.By,
until = webdriver.until;
var phantomPath = path.resolve(__dirname, "../node_modules/.bin/phantomjs"),
isWin = process.platform === "win32";
var driver = new webdriver.Builder()
.withCapabilities({
"phantomjs.binary.path": isWin ? phantomPath + ".cmd" : phantomPath
})
.forBrowser("phantomjs")
.build();
var windowHandle = new webdriver.WebDriver.Window(driver);
windowHandle.setSize(1100, 1000);
var getAllPagesContent = function (driver) {
var pagesContent = [],
pageNo = 1;
var getNextPage = function () {
var nextPageLink;
return driver.findElements(By.css(".pagination li")).then(function (elements) {
return elements[elements.length - 1];
}).then(function (element) {
nextPageLink = element;
return element.getAttribute("class");
}).then(function (className) {
return _.includes(className, "active");
}).then(function (isLastPage) {
return (!isLastPage) ? driver.getPageSource() : false;
}).then(function (content) {
if (content)
pagesContent.push(content);
content && console.log("Got page %d", pageNo++);
return nextPageLink.findElement(By.css("a")).then(function (element) {
return element.click();
}).then(function () {
return driver.wait(until.stalenessOf(nextPageLink), 10 * 1000);
}).then(function () {
return content ? getNextPage() : pagesContent;
});
});
};
return getNextPage();
};
var processTimeout = setTimeout(function () {
console.log("PhantomJS for page %s took too long to execute", page.url);
driver.quit().then(done);
}, 60 * 1000);
driver.get(page.url).then(function () {
var pageOverlay = driver.findElement(By.css("#overlay-the-new"));
return pageOverlay.isDisplayed().then(function (visible) {
if (visible) {
pageOverlay.click();
return driver.wait(until.elementIsNotVisible(pageOverlay), 10000);
}
}).then(function () {
return getAllPagesContent(driver);
});
}).then(function (contents) {
clearTimeout(processTimeout);
console.log("Got %d pages for %s", contents.length, page.url);
_.forEach(contents, function (pageContent) {
parsePage(page.url, pageContent);
});
return driver.quit();
}).then(function () {
done();
});
}

Although PhantomJS is now deprecated you can still run it in parallel isolated Docker containers by using Selenoid. There is a ready to use image with latest release here: https://hub.docker.com/r/selenoid/phantomjs/tags/

Parallel execution with Selenium tends to be done using Remote WebDrivers and the Selenium Grid2 Framework.
This tutorial at WeDoQA seems to be the sort of thing you want. At a brief glance it has each test in a separate class, while a central test base class points towards Grid2's hub, which then (in the tutorial) executes the tests in parallel using a Firefox driver. You could easily retool this to use phantomjs, but you might have to rework your test structure.

It seems you're only using one driver. I'd initialize a second driver, then use threading to run in parallel. I think this could get the job done.

Use Thread for running in parallel or you can use any test framework which can take care of running the tests in parallel.

Hot Code Push NodeJS

I've been trying to figure out this "Hot Code Push" on Node.js. Basically, my main file (that is run when you type node app.js) consists of some settings, configurations, and initializations. In that file I have a file watcher, using chokidar. When I file has been added, I simply require the file. If a file has been changed or updated I would delete the cache delete require.cache[path] and then re-require it. All these modules don't export anything, it just works with the single global Storm object.
Storm.watch = function() {
var chokidar, directories, self = this;
chokidar = require('chokidar');
directories = ['server/', 'app/server', 'app/server/config', 'public'];
clientPath = new RegExp(_.regexpEscape(path.join('app', 'client')));
watcher = chokidar.watch(directories, {
ignored: function(_path) {
if (_path.match(/\./)) {
!_path.match(/\.(js|coffee|iced|styl)$/);
} else {
!_path.match(/(app|config|public)/);
}
},
persistent: true
});
watcher.on('add', function(_path){
self.fileCreated(path.resolve(Storm.root, _path));
//Storm.logger.log(Storm.cliColor.green("File Added: ", _path));
//_console.info("File Updated");
console.log(Storm.css.compile(' {name}: {file}', "" +
"name" +
"{" +
"color: white;" +
"font-weight:bold;" +
"}" +
"hr {" +
"background: grey" +
"}")({name: "File Added", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
});
watcher.on('change', function(_path){
_path = path.resolve(Storm.root, _path);
if (fs.existsSync(_path)) {
if (_path.match(/\.styl$/)) {
self.clientFileUpdated(_path);
} else {
self.fileUpdated(_path);
}
} else {
self.fileDeleted(_path);
}
//Storm.logger.log(Storm.cliColor.green("File Changed: ", _path));
console.log(Storm.css.compile(' {name}: {file}', "" +
"name" +
"{" +
"color: yellow;" +
"font-weight:bold;" +
"}" +
"hr {" +
"background: grey" +
"}")({name: "File Changed", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
});
watcher.on('unlink', function(_path){
self.fileDeleted(path.resolve(Storm.root, _path));
//Storm.logger.log(Storm.cliColor.green("File Deleted: ", _path));
console.log(Storm.css.compile(' {name}: {file}', "" +
"name" +
"{" +
"color: red;" +
"font-weight:bold;" +
"}" +
"hr {" +
"background: grey" +
"}")({name: "File Deleted", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
});
watcher.on('error', function(error){
console.log(error);
});
};
Storm.watch.prototype.fileCreated = function(_path) {
if (_path.match('views')) {
return;
}
try {
require.resolve(_path);
} catch (error) {
require(_path);
}
};
Storm.watch.prototype.fileDeleted = function(_path) {
delete require.cache[require.resolve(_path)];
};
Storm.watch.prototype.fileUpdated = function(_path) {
var self = this;
pattern = function(string) {
return new RegExp(_.regexpEscape(string));
};
if (_path.match(pattern(path.join('app', 'templates')))) {
Storm.View.cache = {};
} else if (_path.match(pattern(path.join('app', 'helpers')))) {
self.reloadPath(path, function(){
self.reloadPaths(path.join(Storm.root, 'app', 'controllers'));
});
} else if (_path.match(pattern(path.join('config', 'assets.coffee')))) {
self.reloadPath(_path, function(error, config) {
//Storm.config.assets = config || {};
});
} else if (_path.match(/app\/server\/(models|controllers)\/.+\.(?:coffee|js|iced)/)) {
var isController, directory, klassName, klass;
self.reloadPath(_path, function(error, config) {
if (error) {
throw new Error(error);
}
});
Storm.serverRefresh();
isController = RegExp.$1 == 'controllers';
directory = 'app/' + RegExp.$1;
klassName = _path.split('/');
klassName = klassName[klassName.length - 1];
klassName = klassName.split('.');
klassName.pop();
klassName = klassName.join('.');
klassName = _.camelize(klassName);
if (!klass) {
require(_path);
} else {
console.log(_path);
self.reloadPath(_path)
}
} else if (_path.match(/config\/routes\.(?:coffee|js|iced)/)) {
self.reloadPath(_path);
} else {
this.reloadPath(_path);
}
};
Storm.watch.prototype.reloadPath = function(_path, cb) {
_path = require.resolve(path.resolve(Storm.root, path.relative(Storm.root, _path)));
delete require.cache[_path];
delete require.cache[path.resolve(path.join(Storm.root, "server", "application", "server.js"))];
//console.log(require.cache[path.resolve(path.join(Storm.root, "server", "application", "server.js"))]);
require("./server.js");
Storm.App.use(Storm.router);
process.nextTick(function(){
Storm.serverRefresh();
var result = require(_path);
if (cb) {
cb(null, result);
}
});
};
Storm.watch.prototype.reloadPaths = function(directory, cb) {
};
Some of the code is incomplete / not used as I'm trying a lot of different methods.
What's Working:
For code like the following:
function run() {
console.log(123);
}
Works perfectly. But any asynchronous code fails to update.
Problem = Asynchronous Code
app.get('/', function(req, res){
// code here..
});
If I then update the file when the nodejs process is running, nothing happens, though it goes through the file watcher and the cache is deleted, then re-established. Another instance where it doesn't work is:
// middleware.js
function hello(req, res, next) {
// code here...
}
// another file:
app.use(hello);
As app.use would still be using the old version of that method.
Question:
How could I fix the problem? Is there something I'm missing?
Please don't throw suggestions to use 3rd party modules like forever. I'm trying to incorporate the functionality within the single instance.
EDIT:
After studying meteors codebase (there's surprisingly little resources on "Hot Code Push" in node.js or browser.) and tinkering around with my own implementation I've successfully made a working solution. https://github.com/TheHydroImpulse/Refresh.js . This is still at an early stage of development, but it seems solid right now. I'll be implementing a browser solution too, just for sake of completion.

Deleting require's cache doesn't actually "unload" your old code, nor does it undo what that code did.
Take for example the following function:
var callbacks=[];
registerCallback = function(cb) {
callbacks.push(cb);
};
Now let's say you have a module that calls that global function.
registerCallback(function() { console.log('foo'); });
After your app starts up, callbacks will have one item. Now we'll modify the module.
registerCallback(function() { console.log('bar'); });
Your 'hot patching' code runs, deletes the require.cached version and re-loads the module.
What you must realize is that now callbacks has two items. First, it has a reference to the function that logs foo (which was added on app startup) and a reference to the function that logs bar (which was just added).
Even though you deleted the cached reference to the module's exports, you can't actually delete the module. As far as the JavaScript runtime is concerned, you simply removed one reference out of many. Any other part of your application can still be hanging on to a reference to something in the old module.
This is exactly what is happening with your HTTP app. When the app first starts up, your modules attach anonymous callbacks to routes. When you modify those modules, they attach a new callback to the same routes; the old callbacks are not deleted. I'm guessing that you're using Express, and it calls route handlers in the order they were added. Thus, the new callback never gets a chance to run.
To be honest, I wouldn't use this approach to reloading you app on modification. Most people write app initialization code under the assumption of a clean environment; you're violating that assumption by running initialization code in a dirty environment – that is, one which is already up and running.
Trying to clean up the environment to allow your initialization code to run is almost certainly more trouble than it's worth. I'd simply restart the entire app when your underlying files have changed.

Meteor solves this problem by allowing modules to "register" themselves as part of the hot code push process.
They implement this in their reload package:
https://github.com/meteor/meteor/blob/master/packages/reload/reload.js#L105-L109
I've seen that Meteor.reload API used in some plugins on GitHub, but they also use it in the session package:
https://github.com/meteor/meteor/blob/master/packages/session/session.js#L103-L115
if (Meteor._reload) {
Meteor._reload.onMigrate('session', function () {
return [true, {keys: Session.keys}];
});
(function () {
var migrationData = Meteor._reload.migrationData('session');
if (migrationData && migrationData.keys) {
Session.keys = migrationData.keys;
}
})();
}
So basically, when the page/window loads, meteor runs a "migration", and it's up to the package to define the data/methods/etc. that get recomputed when a hot code push is made.
It's also being used by their livedata package (search reload).
Between refreshes they're saving the "state" using window.sessionStorage.