Hot Code Push NodeJS - javascript

I've been trying to figure out this "Hot Code Push" on Node.js. Basically, my main file (that is run when you type node app.js) consists of some settings, configurations, and initializations. In that file I have a file watcher, using chokidar. When I file has been added, I simply require the file. If a file has been changed or updated I would delete the cache delete require.cache[path] and then re-require it. All these modules don't export anything, it just works with the single global Storm object.
Storm.watch = function() {
var chokidar, directories, self = this;
chokidar = require('chokidar');
directories = ['server/', 'app/server', 'app/server/config', 'public'];
clientPath = new RegExp(_.regexpEscape(path.join('app', 'client')));
watcher = chokidar.watch(directories, {
ignored: function(_path) {
if (_path.match(/\./)) {
!_path.match(/\.(js|coffee|iced|styl)$/);
} else {
!_path.match(/(app|config|public)/);
}
},
persistent: true
});
watcher.on('add', function(_path){
self.fileCreated(path.resolve(Storm.root, _path));
//Storm.logger.log(Storm.cliColor.green("File Added: ", _path));
//_console.info("File Updated");
console.log(Storm.css.compile(' {name}: {file}', "" +
"name" +
"{" +
"color: white;" +
"font-weight:bold;" +
"}" +
"hr {" +
"background: grey" +
"}")({name: "File Added", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
});
watcher.on('change', function(_path){
_path = path.resolve(Storm.root, _path);
if (fs.existsSync(_path)) {
if (_path.match(/\.styl$/)) {
self.clientFileUpdated(_path);
} else {
self.fileUpdated(_path);
}
} else {
self.fileDeleted(_path);
}
//Storm.logger.log(Storm.cliColor.green("File Changed: ", _path));
console.log(Storm.css.compile(' {name}: {file}', "" +
"name" +
"{" +
"color: yellow;" +
"font-weight:bold;" +
"}" +
"hr {" +
"background: grey" +
"}")({name: "File Changed", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
});
watcher.on('unlink', function(_path){
self.fileDeleted(path.resolve(Storm.root, _path));
//Storm.logger.log(Storm.cliColor.green("File Deleted: ", _path));
console.log(Storm.css.compile(' {name}: {file}', "" +
"name" +
"{" +
"color: red;" +
"font-weight:bold;" +
"}" +
"hr {" +
"background: grey" +
"}")({name: "File Deleted", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
});
watcher.on('error', function(error){
console.log(error);
});
};
Storm.watch.prototype.fileCreated = function(_path) {
if (_path.match('views')) {
return;
}
try {
require.resolve(_path);
} catch (error) {
require(_path);
}
};
Storm.watch.prototype.fileDeleted = function(_path) {
delete require.cache[require.resolve(_path)];
};
Storm.watch.prototype.fileUpdated = function(_path) {
var self = this;
pattern = function(string) {
return new RegExp(_.regexpEscape(string));
};
if (_path.match(pattern(path.join('app', 'templates')))) {
Storm.View.cache = {};
} else if (_path.match(pattern(path.join('app', 'helpers')))) {
self.reloadPath(path, function(){
self.reloadPaths(path.join(Storm.root, 'app', 'controllers'));
});
} else if (_path.match(pattern(path.join('config', 'assets.coffee')))) {
self.reloadPath(_path, function(error, config) {
//Storm.config.assets = config || {};
});
} else if (_path.match(/app\/server\/(models|controllers)\/.+\.(?:coffee|js|iced)/)) {
var isController, directory, klassName, klass;
self.reloadPath(_path, function(error, config) {
if (error) {
throw new Error(error);
}
});
Storm.serverRefresh();
isController = RegExp.$1 == 'controllers';
directory = 'app/' + RegExp.$1;
klassName = _path.split('/');
klassName = klassName[klassName.length - 1];
klassName = klassName.split('.');
klassName.pop();
klassName = klassName.join('.');
klassName = _.camelize(klassName);
if (!klass) {
require(_path);
} else {
console.log(_path);
self.reloadPath(_path)
}
} else if (_path.match(/config\/routes\.(?:coffee|js|iced)/)) {
self.reloadPath(_path);
} else {
this.reloadPath(_path);
}
};
Storm.watch.prototype.reloadPath = function(_path, cb) {
_path = require.resolve(path.resolve(Storm.root, path.relative(Storm.root, _path)));
delete require.cache[_path];
delete require.cache[path.resolve(path.join(Storm.root, "server", "application", "server.js"))];
//console.log(require.cache[path.resolve(path.join(Storm.root, "server", "application", "server.js"))]);
require("./server.js");
Storm.App.use(Storm.router);
process.nextTick(function(){
Storm.serverRefresh();
var result = require(_path);
if (cb) {
cb(null, result);
}
});
};
Storm.watch.prototype.reloadPaths = function(directory, cb) {
};
Some of the code is incomplete / not used as I'm trying a lot of different methods.
What's Working:
For code like the following:
function run() {
console.log(123);
}
Works perfectly. But any asynchronous code fails to update.
Problem = Asynchronous Code
app.get('/', function(req, res){
// code here..
});
If I then update the file when the nodejs process is running, nothing happens, though it goes through the file watcher and the cache is deleted, then re-established. Another instance where it doesn't work is:
// middleware.js
function hello(req, res, next) {
// code here...
}
// another file:
app.use(hello);
As app.use would still be using the old version of that method.
Question:
How could I fix the problem? Is there something I'm missing?
Please don't throw suggestions to use 3rd party modules like forever. I'm trying to incorporate the functionality within the single instance.
EDIT:
After studying meteors codebase (there's surprisingly little resources on "Hot Code Push" in node.js or browser.) and tinkering around with my own implementation I've successfully made a working solution. https://github.com/TheHydroImpulse/Refresh.js . This is still at an early stage of development, but it seems solid right now. I'll be implementing a browser solution too, just for sake of completion.

Deleting require's cache doesn't actually "unload" your old code, nor does it undo what that code did.
Take for example the following function:
var callbacks=[];
registerCallback = function(cb) {
callbacks.push(cb);
};
Now let's say you have a module that calls that global function.
registerCallback(function() { console.log('foo'); });
After your app starts up, callbacks will have one item. Now we'll modify the module.
registerCallback(function() { console.log('bar'); });
Your 'hot patching' code runs, deletes the require.cached version and re-loads the module.
What you must realize is that now callbacks has two items. First, it has a reference to the function that logs foo (which was added on app startup) and a reference to the function that logs bar (which was just added).
Even though you deleted the cached reference to the module's exports, you can't actually delete the module. As far as the JavaScript runtime is concerned, you simply removed one reference out of many. Any other part of your application can still be hanging on to a reference to something in the old module.
This is exactly what is happening with your HTTP app. When the app first starts up, your modules attach anonymous callbacks to routes. When you modify those modules, they attach a new callback to the same routes; the old callbacks are not deleted. I'm guessing that you're using Express, and it calls route handlers in the order they were added. Thus, the new callback never gets a chance to run.
To be honest, I wouldn't use this approach to reloading you app on modification. Most people write app initialization code under the assumption of a clean environment; you're violating that assumption by running initialization code in a dirty environment – that is, one which is already up and running.
Trying to clean up the environment to allow your initialization code to run is almost certainly more trouble than it's worth. I'd simply restart the entire app when your underlying files have changed.

Meteor solves this problem by allowing modules to "register" themselves as part of the hot code push process.
They implement this in their reload package:
https://github.com/meteor/meteor/blob/master/packages/reload/reload.js#L105-L109
I've seen that Meteor.reload API used in some plugins on GitHub, but they also use it in the session package:
https://github.com/meteor/meteor/blob/master/packages/session/session.js#L103-L115
if (Meteor._reload) {
Meteor._reload.onMigrate('session', function () {
return [true, {keys: Session.keys}];
});
(function () {
var migrationData = Meteor._reload.migrationData('session');
if (migrationData && migrationData.keys) {
Session.keys = migrationData.keys;
}
})();
}
So basically, when the page/window loads, meteor runs a "migration", and it's up to the package to define the data/methods/etc. that get recomputed when a hot code push is made.
It's also being used by their livedata package (search reload).
Between refreshes they're saving the "state" using window.sessionStorage.

Related

Nightwatch.js function not 'closing'

I'm trying to perform a function at the beginning of my test, then the rest of the test should be executed.
This is my custom-command (named internalAdviceLinksHtml):
var solr = require('solr-client')
exports.command = function() {
this
var client = solr.createClient('solr.dev.bauerhosting.com', 8080, 'cms', '/www.parkers.co.uk');
var globalSettingsQuery = client.createQuery()
.q({TypeName:'Bauer.Parkers.GlobalSettings'})
.start(0)
.rows(10);
client.search(globalSettingsQuery,function(err,obj) {
if (err) {
console.log(err);
} else {
var myresult = (obj.response.docs[0].s_InternalAdviceLinksHtml);
console.log(myresult.length);
if (myresult.length === 0) {
console.log('content block not configured');
} else {
console.log('content block configured');
}
}
});
return this;
};
Test-file (script):
module.exports = {
'set up the solr query': function (browser) {
browser
.solr_query.global_settings.internalAdviceLinksHtml();
},
'links above footer on advice landing page displayed': function (browser) {
browser
.url(browser.launch_url + browser.globals.carAdvice)
.assert.elementPresent('section.seo-internal-links')
},
'closing the browser': function (browser) {
browser
.browserEnd();
},
};
The function works correctly (i.e. if myresult length is 0 then "content block is not configured" is displayed, etc), but the following test ("links above footer on advice landing page displayed") is never invoked.
It seems like the execution stops after the custom-command. I'm sure this will be something quite obvious to someone, but I just can't seem to see what it is.
Any help would be greatly appreciated.
Regarding your internalAdviceLinksHtml custom-command, everything looks good from my point of view (I presume that lonely this was a typo).
Your hunch is correct, it seems that the Nightwatch test-runner fails to go to the next test, which is probably due to some promise not being resolved upstream (client.search function from internalAdviceLinksHtml).
I would recommend doing a return this immediately after outputting to console (content block not configured, or content block configured) and see if that fixes the problem:
client.search(globalSettingsQuery,function(err,obj) {
if (err) {
console.log(err);
} else {
var myresult = (obj.response.docs[0].s_InternalAdviceLinksHtml);
console.log(myresult.length);
if (myresult.length === 0) {
console.log('content block not configured');
} else {
console.log('content block configured');
}
}
return this
});
Also, a few extra pointers:
make use of the Nightwatch test-hooks to make your tests easier to read/maintain & create a separation of concern (setup => before/beforeEach hooks | teardown (e.g: browser.end()) => after/afterEach hooks);
you need not do an explicit browser.end() at the end of your test case. Check this answer for more information on the matter.
Your test-file would become:
module.exports = {
// > do your setup here <
before(browser) {
browser
.solr_query.global_settings.internalAdviceLinksHtml();
},
'links above footer on advice landing page displayed': function (browser) {
browser
.url(browser.launch_url + browser.globals.carAdvice)
.assert.elementPresent('section.seo-internal-links');
},
// > do your cleanup here <
after(browser) {
browser
.browserEnd();
},
};

Update HTML object with node.js and javascript

I'm new to nodejs and jquery, and I'm trying to update one single html object using a script.
I am using a Raspberry pi 2 and a ultrasonic sensor, to measure distance. I want to measure continuous, and update the html document at the same time with the real time values.
When I try to run my code it behaves like a server and not a client. Everything that i console.log() prints in the cmd and not in the browesers' console. When I run my code now i do it with "sudo node surveyor.js", but nothing happens in the html-document. I have linked it properly to the script. I have also tried document.getElementsByTagName("h6").innerHTML = distance.toFixed(2), but the error is "document is not defiend".
Is there any easy way to fix this?
My code this far is:
var statistics = require('math-statistics');
var usonic = require('r-pi-usonic');
var fs = require("fs");
var path = require("path");
var jsdom = require("jsdom");
var htmlSource = fs.readFileSync("../index.html", "utf8");
var init = function(config) {
usonic.init(function (error) {
if (error) {
console.log('error');
} else {
var sensor = usonic.createSensor(config.echoPin, config.triggerPin, config.timeout);
//console.log(config);
var distances;
(function measure() {
if (!distances || distances.length === config.rate) {
if (distances) {
print(distances);
}
distances = [];
}
setTimeout(function() {
distances.push(sensor());
measure();
}, config.delay);
}());
}
});
};
var print = function(distances) {
var distance = statistics.median(distances);
process.stdout.clearLine();
process.stdout.cursorTo(0);
if (distance < 0) {
process.stdout.write('Error: Measurement timeout.\n');
} else {
process.stdout.write('Distance: ' + distance.toFixed(2) + ' cm');
call_jsdom(htmlSource, function (window) {
var $ = window.$;
$("h6").replaceWith(distance.toFixed(2));
console.log(documentToSource(window.document));
});
}
};
function documentToSource(doc) {
// The non-standard window.document.outerHTML also exists,
// but currently does not preserve source code structure as well
// The following two operations are non-standard
return doc.doctype.toString()+doc.innerHTML;
}
function call_jsdom(source, callback) {
jsdom.env(
source,
[ 'jquery-1.7.1.min.js' ],
function(errors, window) {
process.nextTick(
function () {
if (errors) {
throw new Error("There were errors: "+errors);
}
callback(window);
}
);
}
);
}
init({
echoPin: 15, //Echo pin
triggerPin: 14, //Trigger pin
timeout: 1000, //Measurement timeout in µs
delay: 60, //Measurement delay in ms
rate: 5 //Measurements per sample
});
Node.js is a server-side implementation of JavaScript. It's ok to do all the sensors operations and calculations on server-side, but you need some mechanism to provide the results to your clients. If they are going to use your application by using a web browser, you must run a HTTP server, like Express.js, and create a route (something like http://localhost/surveyor or just http://localhost/) that calls a method you have implemented on server-side and do something with the result. One possible way to return this resulting data to the clients is by rendering an HTML page that shows them. For that you should use a Template Engine.
Any DOM manipulation should be done on client-side (you could, for example, include a <script> tag inside your template HTML just to try and understand how it works, but it is not recommended to do this in production environments).
Try searching google for Node.js examples and tutorials and you will get it :)

How to setup yeoman test for subgenerator that reads package.json

I have a subgenerator that uses the name from the package.json. Now I want to test that function and wrote a before() that is supposed to create a dummy package.json for the test.
Problem is that the subgenerator cannot read the dummy json file.
test file:
before(function (done) {
helpers.run(path.join( __dirname, '../addcomponent'))
.inDir(path.join( __dirname, './tmp'), function(dir) {
fs.copyTpl(
path.join(__dirname, '../app/templates/_package.json'),
dir + 'package.json',
{ ProjectName: 'foo' }
);
var test = fs.readJSON(dir + 'package.json');
console.log('test: ' + test); // returns the object
console.log('test.name: ' + test.name); // returns the correct name
})
.withArguments(['foo'])
.withPrompts(prompts)
.withOptions(options)
.on('end', done);
});
but in my sub-generator:
var memFs = require('mem-fs');
var editor = require('mem-fs-editor');
var store = memFs.create();
var fs = editor.create(store);
...
init: function() {
this.pkg = this.fs.readJSON('package.json');
console.log('this.pkg: ' + this.pkg); // returns undefined
}
// or
init: function() {
this.on('ready', function() {
this.pkg = this.fs.readJSON('package.json');
console.log('this.pkg: ' + this.pkg); // returns undefined
});
}
// or
anyOther: function() {
this.pkg = this.fs.readJSON('package.json');
console.log('this.pkg: ' + this.pkg); // returns undefined
}
The whole setup can be found here: https://travis-ci.org/markusfalk/generator-kickstart/builds/58892092
thanks for any help
Edit: I'll keep the old answer underneath and that's probably relevant to most people running into this issue, but not to you.
The idea behind mem-fs is to have an in memory store. It doesn't write anything to disk automatically. As so, it keep the state in the mem-fs instance. In this case, you're creating your own mem-fs instance, while yeoman use another instance. This mean the file you write is never seen by Yeoman (and never written to disk).
For you, the fix would be to use the generator instance provided as the first parameter of the ready event.
helpers.run(path.join( __dirname, '../addcomponent'))
.on('ready', function (generator) {
generator.fs.write('file.txt', 'foo');
});
Another option is to use the node.js sync fs methods. (fs.writeFileSync(), etc)
My guess is you're using this.fs.readJSON() inside your generator constructor.
The constructor is initialized before the ready event is triggered. This mean you read the file before it is actually written.
The usual fix is to never read inside the constructor. You can delay this step until the initializing phase where the inDir() (or the ready event) callback has run.
As a side note, you should use inTmpDir() rather than inDir()

How to set custom header and footer content by using phantom package in nodeJS?

Actually, I am working with generating pdf reports using phantom package in nodeJS. I've found that we can use phantom.callback method for setting it. But I have a problem that when this callback returns simple text it works fine, but when I try to use complicated function which use closures and jade engine for generating html I have error in phantom output that jade variable is not defined, I think this problem was occurred because callback which is mentioned about work in context of child phantom process, therefore all variables which are defined in my code in that callback don't work. So, how can I solve this issue? Maybe you know better phantomJS wrapper for doing this stuff?
I use this package phantom": "0.7.x"
//there I define all varaibles (jade, fs, etc., so I am sure that they are correct)
function generatePage(_page, reportConfig, phantom, html, report) {
_page.set('viewportSize', reportConfig.viewportSize);
var config = _.extend(reportConfig.paperSize, {
header: {
height: "2cm",
//contents: phantom.callback(headerCallback)
contents: phantom.callback(function (pageNum, numPages) {
var fn = jade.compile(headerTemplate); //Jade in undefined in phantom stdout
var templateData = _.extend({ currentPage: pageNum, numberPages: numPages }, report);
var generatedHtml = fn(templateData);
return "<h1>HEADER</h1><br />" /*+ generatedHtml*/;
})
}
, footer: {
height: "1cm",
contents: phantom.callback(function (pageNum, numPages) {
return "<p>Page " + pageNum + " of " + numPages + "</p>"; //WORKS fine
})
}
}
);
_page.set('paperSize', config);
_page.setContent(html);
return _page;
}
The phantom callback doesn't work this way, the function you are sending as callback will be stringified and recompiled in the phantom context where your dependencies will be unknown.
Is a late answer but i didn't find anything like this out there in the wild. So maybe will help others.
You have to generate the html out of the page context, right after you define jade and others, and compile the result into a function which you'll send as callback:
//there I define all varaibles (jade, fs, etc., so I am sure that they are correct)
var fn = jade.compile(headerTemplate); //Jade is undefined in phantom context
var templateData = report;
var generatedHtml = fn(templateData);
//You can use some patterns in your template for pageNum and numPages like #pageNum# and replace them after compilation.
//here you are compiling the result into a function which you will send
//as callback(you have to remove all spaces and all breaklines to avoid compilation errors)
var headerCallbak = 'function(pageNum, numPages) { var x = \''+ generatedHtml .trim().replace(/(\r\n|\n|\r)/gm,"") +'\'; return x.relpace("#pageNumber#", pageNum).replace("#numPages#",numPages);}';
function generatePage(_page, reportConfig, phantom, html, report) {
_page.set('viewportSize', reportConfig.viewportSize);
var config = _.extend(reportConfig.paperSize, {
header: {
height: "2cm",
contents: phantom.callback(headerCallbak)
}
, footer: {
height: "1cm",
contents: phantom.callback(function (pageNum, numPages) {
return "<p>Page " + pageNum + " of " + numPages + "</p>"; //WORKS fine
})
}
}
);
_page.set('paperSize', config);
_page.setContent(html);
return _page;
}

AngularJS - Stack trace ignoring source map

I've written an AngularJS app but it's proving a bit of a nightmare to debug. I'm using Grunt + uglify to concatenate and minify my application code. It also creates a source map alongside the minified JS file.
The source map seems to work properly when there is a JS error in the file, but outside of the AngularJS application. e.g. If I write console.log('a.b'); at the top of one of the files, the error logged in the Chrome debugger displays line + file info for the original file, not the minified one.
The problem occurs when there is a problem with code that Angular runs itself (e.g. in Controller code). I get a nice stack trace from Angular, but it only details the minified file not the original.
Is there anything I can do to get Angular to acknowledge the source map?
Example error below:
TypeError: Cannot call method 'getElement' of undefined
at Object.addMapControls (http://my-site/wp-content/plugins/my-maps/assets/js/app.min.js:1:2848)
at Object.g [as init] (http://my-site/wp-content/plugins/my-maps/assets/js/app.min.js:1:344)
at new a (http://my-site/wp-content/plugins/my-maps/assets/js/app.min.js:1:591)
at d (http://ajax.googleapis.com/ajax/libs/angularjs/1.2.0-rc.2/angular.min.js:29:495)
at Object.instantiate (http://ajax.googleapis.com/ajax/libs/angularjs/1.2.0-rc.2/angular.min.js:30:123)
Larrifax's answer is good but there is an improved version of the function documented in the same issue report:
.config(function($provide) {
// Fix sourcemaps
// #url https://github.com/angular/angular.js/issues/5217#issuecomment-50993513
$provide.decorator('$exceptionHandler', function($delegate) {
return function(exception, cause) {
$delegate(exception, cause);
setTimeout(function() {
throw exception;
});
};
});
})
This will generate two stack traces, as Andrew Magee noted: one formatted by Angular, then a second one formatted by the browser. The second trace will apply sourcemaps. It's probably not a great idea to disable the duplicates, because you may have other Angular modules that also do work with exceptions that could be called after this via the delegation.
The only solution I could find is to bite the bullet and parse the source maps yourself. Here is some code that will do this. First you need to add source-map to your page. Then add this code:
angular.module('Shared').factory('$exceptionHandler',
function($log, $window, $injector) {
var getSourceMappedStackTrace = function(exception) {
var $q = $injector.get('$q'),
$http = $injector.get('$http'),
SMConsumer = window.sourceMap.SourceMapConsumer,
cache = {};
// Retrieve a SourceMap object for a minified script URL
var getMapForScript = function(url) {
if (cache[url]) {
return cache[url];
} else {
var promise = $http.get(url).then(function(response) {
var m = response.data.match(/\/\/# sourceMappingURL=(.+\.map)/);
if (m) {
var path = url.match(/^(.+)\/[^/]+$/);
path = path && path[1];
return $http.get(path + '/' + m[1]).then(function(response) {
return new SMConsumer(response.data);
});
} else {
return $q.reject();
}
});
cache[url] = promise;
return promise;
}
};
if (exception.stack) { // not all browsers support stack traces
return $q.all(_.map(exception.stack.split(/\n/), function(stackLine) {
var match = stackLine.match(/^(.+)(http.+):(\d+):(\d+)/);
if (match) {
var prefix = match[1], url = match[2], line = match[3], col = match[4];
return getMapForScript(url).then(function(map) {
var pos = map.originalPositionFor({
line: parseInt(line, 10),
column: parseInt(col, 10)
});
var mangledName = prefix.match(/\s*(at)?\s*(.*?)\s*(\(|#)/);
mangledName = (mangledName && mangledName[2]) || '';
return ' at ' + (pos.name ? pos.name : mangledName) + ' ' +
$window.location.origin + pos.source + ':' + pos.line + ':' +
pos.column;
}, function() {
return stackLine;
});
} else {
return $q.when(stackLine);
}
})).then(function (lines) {
return lines.join('\n');
});
} else {
return $q.when('');
}
};
return function(exception) {
getSourceMappedStackTrace(exception).then($log.error);
};
});
This code will download the source, then download the sourcemaps, parse them, and finally attempt to replace the locations in the stack trace the mapped locations. This works perfectly in Chrome, and quite acceptably in Firefox. The disadvantage is that you are adding a fairly large dependency to your code base and that you move from very fast synchronous error reporting to fairly slow async error reporting.
I just had the same issue and have been hunting around for a solution - apparently it's a Chrome issue with stack traces in general and happens to apply to Angular because it uses stack traces in error reporting. See:
Will the source mapping in Google Chrome push to Error.stack
I would take a look at the following project: https://github.com/novocaine/sourcemapped-stacktrace
It does essentially the same thing as the answer from #jakub-hampl but might be useful.
According to this issue it seems that Angular's $logProvider breaks sourcemapping. A workaround like this is suggested in the issue:
var module = angular.module('source-map-exception-handler', [])
module.config(function($provide) {
$provide.decorator('$exceptionHandler', function($delegate) {
return function(exception, cause) {
$delegate(exception, cause);
throw exception;
};
});
});
As the bug has been fixed in Chrome (but the issue persists in Angular), a workaround that doesn’t print out the stack trace twice would be this:
app.factory('$exceptionHandler', function() {
return function(exception, cause) {
console.error(exception.stack);
};
});

Categories

Resources