I am trying to create my first node module to publish to NPM and have hit an issue for which I can't find an answer.
I decided to write the module using Promises and keep everything asynchronous. During initialization the module calls a few functions that spend some time accessing the file system.
My problem is the module is not being fully initialized before being called by consuming code that has required it.
The full module is on GitHub however here is the initialization code:
var uuid = require('node-uuid')
var fsBlobStoreFactory = require('fs-blob-store')
var Promise = require('bluebird')
var validator = require('validator')
var mkdirp = require('mkdirp')
var path = require('path')
var fs = require('fs')
module.exports = BlobStore
function BlobStore(opts) {
if (!(this instanceof BlobStore)) {
return new BlobStore(opts)
}
this._parseOpts(opts) // <-- Synchronous
this.currentBlobPath = ''
this.fsBlobStore = fsBlobStoreFactory(this.opts.blobStoreRoot) // <-- Synchronous
this._buildBlobPath().then(() => {
console.log('Blob Store Initialized');
console.log('Current Blob Path: ' + this.currentBlobPath);
}) // <-- This takes a while and is asynchronous
}
Here is a quick js file I am using for manual tests:
var crispyStream = require('crispy-stream');
var opts = {
blobStoreRoot: '/some/dir/blobs',
dirDepth: 3,
dirWidth: 3
}
var sbs = require('./index')(opts)
// Any code under here that uses sbs will fail because the module has not finished loading.
var input = 'pipe this';
var pipable = crispyStream.createReadStream(input);
sbs.write().then((ws) => {
pipable.pipe(ws);
}
As far as I can tell, the module loader within node is synchronous but because the function calls internally are asynchronous, the require is returning before the module has finished initializing.
The only solutions I can find are to have the consumers of the module use the 'then' method of the Promise, or use a callback.
Any pointers in the right direction would be greatly appreciated.
Related
I'm using this module https://www.npmjs.com/package/limiter to limit http requests to a site using something like this:
let running = true;
const limiter = new RateLimiter(
1,
'second'
);
function removeTokens(limiter, tokens){
const self = this;
return new Promise((resolve) => {
if (!running) throw new Error('Terminated prematurely');
limiter.removeTokens(tokens, resolve)
});
}
Array.apply(null, {length: 1000}).map(Number.call, Number).map(number=>{
return removeTokens(limiter,1).then(()=>{
request('http://test.com/'+number);
});
});
setTimeout(()=>{running=false},1000);
The problem I have is that I'm not able to clear underlying timeouts that the module uses so the requests keep getting called. I have tried to fork the repository and create my own solutions but they have all failed. I can supply my attempted solutions if needed. Is there any simple way that I can clear the timeouts (without building my own queueing system) that I'm missing?
I'm reading about requireing a JSON object from a file. I can read the JSON correctly, but after I add data to it and save it into the file, re-requiring doesn't get the updated JSON, but retrieves the old file. Can someone explain why this is so? How can I fix this?
var file = process.env[(process.platform == 'win32') ? 'USERPROFILE' : 'HOME'] + "\\AppData\\Roaming\\BetterDiscord\\plugins\\Test.json";
var json = require(file);
console.log(JSON.stringify(json.img));
json.img.push({name:"Test"});
console.log(JSON.stringify(json.img));
save(JSON.stringify(json), file);
json = require(file);
console.log(JSON.stringify(json.img));
This is the save method:
var save = function(value, file){
var fs = require('fs');
fs.writeFile(file, value, function(err) {
if (err) {
console.log(err);
}
});
};
This is the Output:
Output 1: [{"name":"A"},{"name":"B"}]
Output 2: [{"name":"A"},{"name":"B"}, {"name":"Test"}]
Output 3: [{"name":"A"},{"name":"B"}]
There's two major problems with your code. The first is that save is an asynchronous function. That means that any code you write directly after it will run before it completes. It works just like setTimeout in that regard.
console.log(1);
setTimeout(function() {
console.log(2);
}, 100);
console.log(3);
Notice how it outputs 3 before 2. That's because setTimeout, like fs.writeFile, is asynchronous. To make sure you run code after it's done, you can pass a callback or use Promises (outside of the scope of this question but very useful). So your code could look something like this.
const save = (path, cb) => {
setTimeout(() => {
cb();
}, 100);
};
let path = 'path/to/file';
save(path, () => {
console.log(`Saved ${path}`);
});
console.log(`Saving ${path}...`);
This whole thing could also be avoided by using the sync version of writeFile.
fs.writeFileSync(file, value);
The next problem stems from how require caches the results. It does this because it's intended for loading in modules and they will not be changing dynamically, most of the time. Instead, load the file from the system directly. This can also be used asynchronously or synchronously. Here's how I would rewrite your code using synchronous code (although asynchronous is generally preferable).
var fs = require('fs');
var save = function(value, file){
return fs.writeFileSync(file, value);
};
var file = process.env[(process.platform == 'win32') ? 'USERPROFILE' : 'HOME'] + "\\AppData\\Roaming\\BetterDiscord\\plugins\\Test.json";
var obj = JSON.parse(fs.readFileSync(file, 'utf8'));
obj.img.push({name:"Test"});
console.log(JSON.stringify(obj.img));
save(JSON.stringify(obj), file);
obj = JSON.parse(fs.readFileSync(file, 'utf8'));
console.log(JSON.stringify(obj.img));
require will cache the file. Instead of requiring, use the fs.readFileSync method to read the file again.
json = fs.readFileSync(file);
Also, as #MikeC points out, you're writing to the file asynchronously, so either switch that out for the synchronous version writeFileSync, or rewrite your code using with readFile and writeFile and use callbacks.
I know this is wrong, but essentially I want to
connect to a db/orm via a promise
wait on that promise to fulfill and get the models (the return from the promise)
use the results for form a middleware generator function to place the models on the request
I suspect that this isn't the best approach, so essentially I have two questions:
Should I be rewriting my db/orm connect to a generator function (I have a feeling that is more inline with koa style)
Back to the original question (as I am sure I will not get a chance to rewrite all my business logic) - how do I wait on a promise to fulfill and to then return a generator function?
This is my poor attempt - which is not working, and honestly I didn't expect it to, but I wanted to start by writing code, to have something to work with to figure this out:
var connectImpl = function() {
var qPromise = q.nbind(object.method, object);
return qPromise ;
}
var domainMiddlewareImpl = function() {
let connectPromise = connectImpl()
return connectPromise.then(function(models){
return function *(next){
this.request.models = models ;
}
})
}
var app = koa()
app.use(domainMiddlewareImpl())
According to this, you can do the following:
var domainMiddlewareImpl = function() {
return function *(){
this.request.models = yield connectImpl();
};
};
A context sensitive answer based on the info provided by Hugo (thx):
var connectImpl = function() {
var qPromise = q.nbind(object.method, object);
return qPromise ;
}
var domainMiddlewareImpl = function () {
let models = null ;
return function *(next) {
if(models == null){
//////////////////////////////////////////////////////
// we only want one "connection", if that sets up a
// pool so be it
//////////////////////////////////////////////////////
models = yield connectImpl() ;
}
this.request.models = models.collections;
this.request.connections = models.connections;
yield next
};
};
My example the connectImpl is setting up domain models in an ORM (waterline for now), connecting to a database (pooled), and returning a promise for the ORM models and DB connections. I only want that to happen once, and then for every request through my Koa middleware, add the objects to the request.
My setup is as follows:
Nodejs Server
server.js requires utils.js
utils.js loads data from mongodb into memory and exports it
server.js uses a variable that utils.js exports
The issue that I am worried about is the fact that the mongodb call is asynchronous. utils.js returns before the mongodb call is finished, meaning that server.js will use an undefined variable when it continues execution after the require.
What is the best to address this issue? The only thing I could think of is wrapping my server.js code in a giant callback and pass that to the function that makes the mongodb call. It seems a bit messy to me, is there a better way to do it?
Code:
server.js
var utils = require("./modules/utils.js");
console.log(utils);
//Do whatever
utils.js
var mods = [];
var db = require("mongojs").connect("localhost", ["modules"]);
db.modules.find({}, function(err, modules){
mods = modules;
});
module.exports = mods;
What you're referring to is called "callback hell". The easiest way to get out of that is to use a Promise library that simplifies it.
I used a node package called bluebird.
var mysql = require("mysql");
var hash = require("password-hash");
var Promise = require("bluebird");
var settings = require("../settings");
Promise.promisifyAll(require("mysql/lib/Connection").prototype);
Promise.promisifyAll(require("mysql/lib/Pool").prototype);
var db_config = {
user:settings.db.user,
password:settings.db.password,
database:settings.db.database
};
var con = mysql.createPool(db_config);
function query(sql) {
return con.getConnectionAsync().then(function(connection) {
return connection.queryAsync(sql)
.spread(function(rows,fields) {
return rows;
}).finally(function() {
connection.release();
});
});
}
This is a very basic database module I wrote that uses bluebird to promisify the database object.
And here's how it's used. It returns a promise! The benefit here is that not only does it return the clutter of callback hell, it makes sure that your code runs asynchronously and the function does not return before things have stopped processing, like in this case, a database query.
function login(user) {
//check for player existance
var query = 'SELECT p.name,p.password,p.id, pd.x, pd.y FROM player p INNER JOIN player_data pd ON p.id = pd.id WHERE p.name='+mysql.escape(user);
return db.select(query).then(function(rows) {
if (!rows.length) return;
return [
rows[0]
];
});
}
Notice how you return a promise, so that you call the then or spread method to get those database values you just queried, not having to worry about if rows will be undefined by the time you want to use it.
As you say, you need to wrap the entire server in a callback. Node.js works this way, it's asynchronous by nature. A server needs to pass through 3 stages: init, serve and deinit. In your case, that database code goes inside the init stage. You could write something like this.
//server.js
var utils = require ("./modules/utils");
var init = function (cb){
//init the utils module, the http server and whatever you need
utils.init (function (error){
if (error) return handleError (error);
cb ();
});
};
var serve = function (){
//Start listening to the http requests
};
var deinit = function (cb){
//This is typically executed when a SIGINT is received, see link1
};
init (serve);
//utils.js
//You could write a wrapper for the db instance, see link2
var mongodb = require ("mongojs");
var db;
module.exports.init = function (cb){
db = mongodb.connect ("localhost", ["modules"]);
db.modules.find ({}, function (err, modules){
if (err) return cb (err);
cb (null, modules);
});
};
I don't recommend using promises, they are slower than raw callbacks. You don't need them at all.
link1 - SIGINT
link2 - db wrapper
I want to get MX records for hostname www.example.com. Node has got function for it.
dns.resolveMx(domain, callback)
But i don't want that callback thingy. I want something like sync call. e.g.
var records = dns.resolveMx(domain);
Is this possible ?
(Note:- I found that function in Node documentation. link:- http://nodejs.org/api/dns.html)
Update 2021
There is now a promises submodule under dns module if some one is looking for sync calls.
Check more here
https://nodejs.org/api/dns.html#dns_class_dnspromises_resolver
const dnsPromises = require('dns').promises;
const demo1 = await dnsPromises.resolveMx("gmail.com");
Is there any reason you want to block your application with a network operation? The DNS resolvers are called at the C level by the c-ares library which is asynchronous by design. Therefore, you can't make it synchronous. This is the code from the DNS module with the unneeded parts removed:
var cares = process.binding('cares_wrap');
function resolver(bindingName) {
var binding = cares[bindingName];
return function query(name, callback) {
callback = makeAsync(callback);
var req = {
bindingName: bindingName,
callback: callback,
oncomplete: onresolve
};
var err = binding(req, name);
if (err) throw errnoException(err, bindingName);
callback.immediately = true;
return req;
}
}
var resolveMap = {};
exports.resolveMx = resolveMap.MX = resolver('queryMx');