I'm new to Node.js, and realized that one of the big differences with it and client side javascript is how asynchronous everything is.
To try and address this, I'm attempting to use fibrous to turn my code back into a more functional style of programming, but having some problems:
How can I make the following fibrous code work ?
for example, the I'd like the following code to print 1,2,3, but it prints 1,3,2
function test()
{
var fibrous = require('fibrous');
var fs = require('fs');
fibrous.run(function() {
var data = fs.sync.readFile('/etc/passwd');
console.log('2');
});
}
function runTest()
{
console.log('1');
test();
console.log('3');
}
runTest();
// This prints 1,3,2 to the console, not 1,2,3 as I'd like.
in a real use case, the above routine would be wrap a DB method that runs async, and make it so I could write things like:
var dbTable = new dbTableWrapper();
var data = dbTable.getData();
/*
... do things with the data.
The "getData" routine is the same as my "test" function above.
*/
Is the answer to run the (newly added) "runTest" routine using a fibrous.run call itself?
That's part of it, yeah. Fibrous will need to call runTest itself to be able to manage its execution.
Then, test just needs to be wrapped rather than .run():
var test = fibrous(function () {
var data = fs.sync.readFile('/etc/passwd');
console.log('2');
});
And should be called with .sync():
test.sync();
var fibrous = require('fibrous');
var fs = require('fs');
var test = fibrous(function () {
var data = fs.sync.readFile('/etc/passwd');
console.log('2');
});
function runTest() {
console.log('1');
test.sync();
console.log('3');
}
fibrous.run(runTest);
Other a Express using :
var express = require('express');
var router = express.Router();
var fibrous = require('fibrous');
router.use(fibrous.middleware);
router.get('/sync', function(req, res, next) {
var order_categories = Order_Category.sync.list(options);
console.log("Order_Category count : " , order_categories.length);
var content_tags = Content_Tag.sync.list(options);
console.log("content_tags count : " , content_tags.length);
var creatives = Creative.sync.list(options);
console.log("creatives count : " , creatives.length);
return res.send( {
order_categories: order_categories,
content_tags: content_tags,
creatives: creatives
}
);
});
Related
I'm messing around with Javascript, node js, and sails right now. I'm trying to learn how to do some stuff.
Right now, I've got an analogue for a basic situation I want to do. I want to return a json returned from a rest call to a controller, and display it on the webpage.
So far, this is what I have:
module.exports = {
/**
* `PersonController.scream()`
*/
scream: function (req, res) {
function subscream(){
var per = {voice: 'AAAAAAAAAAAAAAA'};
return per;
}
var loudness = subscream()
return loudness;
}
};
The console.log statement does print the json to the console, but the function never seems to return. I know I'm missing something obvious, but I don't know enough about Javascript or node js to know what to call this problem, so I can't Google it yet unfortunately.
To print the JSON to the client, you have to send a response and not return the JSON in the function.
scream: function (req, res) {
function subscream(){
var per = {voice: 'AAAAAAAAAAAAAAA'};
var perString = JSON.stringify(per);
console.log(perString)
return perString;
}
var loudness = subscream()
res.send(loudness);
}
module.exports = {
/**
* `PersonController.scream()`
*/
scream: function (req, res) {
function subscream(){
return new Promise(function(resolve, reject){
var per = {voice: 'AAAAAAAAAAAAAAA'};
var perString = JSON.stringify(per);
console.log(perString)
resolve(perString);
})
}
var scream = subscream()
scream.then(function(resolvedPromise){
res.send(resolvedPromise)
}).catch(function(){
res.send("error")
})
}
};
For the life of me I cannot work this one out. Have look around and tried many many different ways of trying to get this to go. Currently have the following code.
var config = require("./config.js");
var cradle = require('cradle')
var MikroNode = require('mikronode');
var WebServer = require('./bin/www');
var Routers = "Hasnt changed";
var conndb = new(cradle.Connection)(config.couchdb.host);
var db = conndb.database(config.couchdb.db);
db.exists(function(err, exists){
if (err) { console.log('error', err);}
else if (exists) { console.log('Seems the Force is with you - Database Exists');}
else { db.create(); }
});
db.temporaryView({
map: function (doc){
if (doc.type=='ConfigRouter') emit(doc.name, doc);
}
}, function (err, res){
Routers = JSON.stringify(res);
}
);
console.log(Routers);
As it stands it will respond with:
E:\Dev\MM>npm start
> MM#0.0.1 start E:\Dev\MM
> node ./Start.js
Hasnt changed
Seems the Force is with you - Database Exists
I am assuming it is an asynchronous call to the CouchDB and is not filling the result in time before it displays the result. How do I get around this issue?
You are right, the call is asynchronous so when console.log(Routers); is processed, Routers is "Hasnt changed".
One way of doing it would be to use promises thanks to the Q npm module:
var Q = require('q');
var deferred = Q.defer();
db.temporaryView({
map: function (doc) {
if (doc.type=='ConfigRouter') emit(doc.name, doc);
}
}, function (err, res) {
deferred.resolve(JSON.stringify(res));
});
deferred.promise
.then(function (data) {
Routers = data;
console.log(Routers);
// do some stuff...
})
.done();
Maybe it's possible to do something better without using Q.defer and adapting directly the callback:
https://github.com/kriskowal/q#adapting-node
I have a function which generates a checksum for a given path
function getHash(path) {
var fs = require('fs');
var crypto = require('crypto');
var fd = fs.createReadStream(path);
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
fd.on('end', function () {
hash.end();
// *** Here is my problem ***
console.log(hash.read());
});
fd.pipe(hash);
};
I want to call the calcNewHash function so that it returns the hash, the problem is, that I haven't found an asynchronous version of this, maybe some one can help.
Just add a return statement doesn't work, because the function is in a Listener
Later I want to add the checksum to an object, so I could give the reference to it as parameter, but this still does not change that this is ansynchronous ...
You basically have 2 solutions:
#1 Work with promises (i.e. q - npm install q --save)
function getHash(path) {
var Q = require('q');
var fs = require('fs');
var crypto = require('crypto');
var deferred = Q.defer();
var fd = fs.createReadStream(path);
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
fd.on('end', function () {
hash.end();
// *** Here is my problem ***
console.log(hash.read());
deferred.resolve(hash.read());
});
fd.pipe(hash);
return deferred.promise;
};
getHash('c:\\')
.then(function(result) {
// do something with the hash result
});
#2 Or use a callback function
function getHash(path, cb) {
var fs = require('fs');
var crypto = require('crypto');
var fd = fs.createReadStream(path);
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
fd.on('end', function () {
hash.end();
// *** Here is my problem ***
console.log(hash.read());
if (cb) {
cb(null, hash.read());
}
});
fd.pipe(hash);
};
getHash('C:\\', function (error, data) {
if (!error) {
// do something with data
}
});
If you don't have deep nesting in callback functions I would go for option #2.
(Note: behind the scenes promises are also using callbacks, it's just a 'cleaner' solution if you have deep nesting)
I know this is old but it is in the top results when searching for something along these lines. So for those that land here looking for a solution to this, here you go: (note that this only works well if you know the file is small. Otherwise for larger files refer to the answer provided by Dieterg)
const fs = require('fs');
const crypto = require('crypto');
function fileHashSync(filePath){
var fileData;
try{ fileData = fs.readFileSync(filePath, 'utf8'); }
catch(err){
if(err.code === 'ENOENT') return console.error('File does not exist. Error: ', err);
return console.error('Error: ', err);
}
return crypto.createHash('sha1').update(fileData, 'utf8').digest('hex');
}
Using koa.js, I want to make an API which runs a generator function that runs a long time in the background, but sends a token back to the user immediately.
The user can then use that token to retrieve status of their job later.
'use strict';
var generateToken = function(){
//...
};
var processData = function *(data, token) {
//...
var a = yield analysis(data);
console.log(a) // a is undefined
};
app.post('/process_data', validate, function *(next) {
var token = generateToken();
var that = this;
setTimeout(function() {
for (var i of processData(that.request.body, token)){
continue;
}
});
this.body = "this should return immediately " + token;
return next;
});
Running it within the setTimeout, variable 'a' is not saved. How do I construct this so that processData runs exactly like a normal yield?
You would probably want to have the long running process get handled by a job queue such as Kue
You would queue the job with a http post
then check on the job with a http get
Here is a rough outline of what I think you want to be doing:
var kue = require('kue'),
koa = require('koa'),
route = require('koa-router'),
thunkify = require('thunkify'),
parse = require('co-body'),
co = require('co'),
app = koa(),
jobs = kue.createQueue();
app.use(route(app));
// turn callbacks into thunks for generators
var createJob = thunkify(jobs.create);
var findJob = thunkify(kue.Job.get);
// Process the job here
jobs.process('longProcess', function(job, done){
// do work in here
// call done(err) when completed
// EDIT: if you want to handle job using generators/yield
// you could use a library like co
co(function *(){
var qs = yield doWork(job.data);
done();
}).error(done);
});
// Queue/Start the Job here
app.post('/jobs', function *(){
var body = yield parse(this);
var job = yield createJob('longProcess', body);
this.body = job.id;
});
// Check Status of job here
app.get('/jobs/:token', function *(){
var job = yield findJob(this.params.token);
this.body = job;
// job.status === 'complete' || ...
});
app.listen(3000);
Thanks to Bergi for the solution.
app.post('/process_data', validate, function *(next) {
var token = generateToken();
co(processData(this.request.body, token));
this.body = "this should return immediately " + token;
return next;
});
I have a NodeJS readstream listener within a function and I am trying to get my function to return a value (contents of the file) which is derived from within the listener.
e.g. :
MyObj.prototype.read = function(){
var file3 = fs.createReadStream('test.txt', {encoding: 'utf8'});
var contentRead = '';
file3.addListener('data', function(data) {
contentRead += data.toString('utf-8');
return contentRead;
});
}
I would like to do something like var contents = myDerivedObj.read() to return the contents of the file.
However, the return from inside the listener is not getting returned correctly - getting 'undefined'. And, returning from outside the listener just returns an empty string.
I cannot change the signature of read(), so I cannot add a callback as an argument.
In general, this is a poor pattern: NodeJS REALLY REALLY REALLY doesn't like it when you do things like this 'cause you block the main thread. You will discover that your performance TRULY sucks. So DON'T DO THIS. But if you MUST then you can try this:
MyObj.prototype.read = function(){
var file3 = fs.createReadStream('test.txt', {encoding: 'utf8'});
var contentRead = '';
var done = false;
file3.addListener('data', function(data) {
contentRead += data.toString('utf-8');
});
file3.addListener("end", function () {
done = true;
});
while(!done);
return contentRead;
}
EDIT: #Brandon is right, and I'm wrong. I just tested this and although I'd thought the callbacks would work the entire node process locks up. Do this:
MyObj.prototype.read = function(){
return fs.readFileSync('test.txt', 'utf-8');
}