Develop Reference

How should a server implement middlewares chaining?

I'm struggling a lot trying to understand the logic behind the ExpressJS module, in particular i'm focusing on the implementation of the middlewares chain.
My objective here is to understand how is it possible to implement the logic of a server listening for requests, and once arrived, passes the request packet through an array of functions each of which:
should be able to modify the packet for the next middleware
should be able to send the response back but it should not prevent other middlewares to respond as well, so there should be only one response at the end of the chain (i guess)
should access some function to call in order to run the next middleware in chain, but without the need to pass it the request and response parameters (e.g. just calling "next( )" )
Note on point 1
I imagine that with languages like javascript is possible to pass the object around using concepts like closure, but i would like to understand it in a language-independent way and use javascript features only if it provides clever ways to handle it simply, otherwise i'll write the logic as i was using any other language.
So my big questions are:
How should the request handling be implemented?
How do i allow every middleware to modify the packet?
How do i pass the updated packet to the next middleware?
How do i implement the "next" function to pass to every middleware?
Thank you

Broadly, this is how a chain of middlewares with a next function can be called.
function myMiddleware(req, res, next) {
// do something with req or res
next();
}
const mws = [
myMiddleware,
anotherMiddleware
];
/**
* Calls a chain of middlewares.
*
* mws is an array of middlewares
*/
function callMwChain(req, res, mws) {
if (mws.length === 0) {
// We're done here
return;
}
/**
* Take the first middleware
*/
const firstMw = mws[0];
/**
* Call it, and give it a next function that continues the chain
*/
firstMw(req, res, () => {
callMwChain(req, res, mws.slice(1));
});
}
I've written this as a recursive function, but it can also be rewritten as stack. This is easier though.
Express middlewares have a lot of bonus magic, so their implementation is going to be more complex, but this is broadly how it works.
Note that there's no event-loop in sight.

Here's a little working server that supports middleware and GET routes. It uses a similar calling convention on the route handlers as Express (req, res, next), but has far, far, far fewer features than Express. But, hopefully you can see how the dispatch function cycles through the routes, advancing to the next route only if the previous handler calls next().
You can actually run this code and define routes and play with it (though it is definitely barebones basic):
const http = require('http');
function dispatch(req, res, array, index, cb) {
console.log(req.method, req.url, array, index);
if (array && array.length > index) {
let item = array[index];
if (!item.path || item.path.toLowerCase() === req.url.toLowerCase()) {
item.fn(req, res, function() {
// previous route called next(), so we advance to the next item in the array
++index;
dispatch(req, res, array, index, cb);
});
return;
}
}
cb();
}
const server = http.createServer((req, res) => {
// incoming request here, initiate calling the middleware
dispatch(req, res, server.middleware, 0, function() {
if (req.method === "GET") {
dispatch(req, res, server.gets, 0, function() {
// done with gets here, apparently didn't send a response
res.statusCode = 404;
res.end();
})
} else {
// ... fill in code for other methods here
res.statusCode = 404;
res.end();
}
});
});
// server route implementation
// save arrays for each type of route handler
// upon a request, run all the middleware requests, then advance
// to the routes for the right type of request and see if any of them
// match the URL. If so, run them.
server.middleware = [];
server.gets = [];
server.posts = [];
server.use = function(fn) {
server.middleware.push({path: "", fn: fn});
}
server.get = function(path, fn) {
server.gets.push({path, fn});
}
// route definitions
server.use((req, res, next) => {
console.log(req.url);
req.myCustom = "hi";
next();
});
server.use((req, res, next) => {
console.log(req.myCustom);
next();
});
server.get("/", (req, res, next) => {
res.write("Hello World");
res.end();
});
server.listen(80);

How to create a reusable function in Node without writing boilerplate code

I am using Express.js as http server. Defined all my routes.
Most endpoints need to verify session before returning a response. E.g. below code serves users in the system and list of services respectively:
function getUsers(req, res, next) {
verifyUser(req, res, next, function () {
//serve users
});
}
function getServices(req, res, next) {
verifyUser(req, res, next, function () {
//serve services
});
}
You probably noticed there is a verifyUser function which validates the session. Which is as below.
function verifyUser(req, res, next, callback) {
var sessionKey = req.cookies.sessionKey;
var user = users.userBySession(sessionKey);
if (user) {
callback(req, res, next, user);
} else {
res.status(401).send({
message: 'Unauthorized'
});
}
}
As you can see I keep passing in req, res and next parameters along with a callback whenever I use this function.
I tried to use apply function to make it easier. Changed my getUsers function like this:
function getUsers(req, res, next) {
verifyUser
.apply(null, arguments, function () {
//serve users
});
}
The problem with this approach is callback is not passed into verifyUser function. And I don't really like passing null as scope with each call.
How can I achieve this by writing less and better code ? Any ideas?

You could use bind to create a 'partial function':
// create bound responseHelper object
var responseHelper = verifyUser.bind(null, req, res, next);
// usage
responseHelper(getUsersCallback); // same as verifyUser(req, res, next, getusersCallBack);

I think you're looking to turn verifyUser into a middleware function.
function verifyUser (req, res, next) {
var user = // yadda yadda session stuff
if (user) {
req.user = user; // [1] what you do to the req object here...
} else {
return res.status(401).send({ message: "No way Smokey Joe"});
/**
* alternatively, do something like
* var err = new Error("Not authorized");
* err.statusCode = 401;
* return next(err);
*
* this will kick off Express' error handling mechanism,
* which you should read about in the docs (see the link below)
*/
}
next();
// very important to call next after this verifyUser has done its job
// if you don't, the next middleware won't go off,
// and the request will just hang
}
function getUsers (req, res, next) {
// [2] will show up on the req object here, assuming you chain these
// two functions together as middleware
}
app.get("/users", verifyUser, getUsers);
app.get("/services", verifyUser, getServices);
// here's a route that needs no session auth, so no need to verifyUser
app.get("/latest-posts", getLatestPosts);
When you tell Express to use a function or attach a function to a route path via get('/my/route', hanlderFun) or some such, you've basically turned handlerFun into a middleware.
You can define however many middleware as handlers on a route as you like, and they'll all execute in turn as long as you keep calling next.
app.post("/checkout", verifyUser, tallyCart, checkInventory, doPayment, sendInvoice);
The job of next is to pass control from the current middelware to the next one. It's an object
You can do other stuff with next, too, which you should read up on in the docs.
http://expressjs.com/en/guide/writing-middleware.html
http://expressjs.com/en/guide/using-middleware.html
The docs on routing have good info on middleware as well:
http://expressjs.com/en/guide/routing.html
For extra credit, check out error handling middleware, too:
http://expressjs.com/en/guide/error-handling.html

Node.js API design and route handling

I am not really sure what to title this, but I'm new to Node.js. I just found a neat REST API project on GitHub to implement but I'm not sure how I can split all GET and POST etc. to separate files.
I have one singular api.js file where I have
function API_ROUTER(router, connection, md5) {
var self = this;
self.handleRoutes(router, connection, md5);
}
API_ROUTER.prototype.handleRoutes = function(router, connection, md5) {
router.get("/", function(req, res) {
res.json({"Message" : "Hello World !"});
});
};
module.exports = API_ROUTER;
Now how can I create a sibling other.js and use:
var api = require('./api.js');
// Create router.get, router.post etc. here?

but I'm not sure how I can split all GET and POST etc. to separate files.
One way you can organize your routes would be to have a separate object for each route that has the handlers (separated by HTTP methods) and other needed info such as the path:
api/home.js
module.exports = {
path: '/',
handlers: {
'get': function(req, res) {
res.json({"Message" : "Hello World !"});
},
'post': {
// ...
}
// ...
}
}
api/other.js
module.exports = {
path: '/other',
handlers: {
'get': function(req, res) {
res.json({"Message" : "Other !"});
},
// ...
Then you can load all of these inside the handleRoutes method:
API_ROUTER.prototype.handleRoutes = function(router, connection, md5) {
var routes = ['home', 'other'];
routes.forEach(function(name) {
// load the current route object (NOTE: you should use the path module for determining file path in a cross-platform manner)
var routeObject = require('./' + name + '.js');
var apiPath = routeObject.path;
var handlers = routeObject.handlers;
var methods = Object.keys(handlers);
// assign handlers for each method
methods.forEach(function(method) {
router[method](apiPath, handlers[method]);
});
});
};
This will install all your routes with the appropriate information and handlers.
Now you can call this code by instantiating your API_ROUTER with the necessary data:
// initialize the api (and handle the routes internally)
var Api = new require('./api.js')(router, connection, md5);

If you implement a RESTful API, then you should keep in mind that this is just one way how you can provide data, and you might want to change it in future, as of that the API will most of the time only be a translation layer.
Normally you will split your code based on the resources, and the code that is handling the request won't have so much logic, it will just take the request and pass it to you internal API. For that purpose you not really need an additional layer if you already use express.js or a similar library.
In express the app.use([path,] function [, function...]), already provides the functionality you would need to modularize your code. For each resource your will create an own express.Router that itself also might mount another sub module. So for this part you do not really need a library.
When might a library be useful:
if it automatically translates thrown errors to the correct response codes
if it includes a tool to automatically create a documentation to your API
if it fully abstracts the underlaying routing system so that you can hook into express, hapi, ... without the need to change the code.
Here how a setup with express.js could look like
./lib/rest/customer.js
var customerSystem = require('../customer-system');
var express = require('express');
var router = new express.Router();
router.get('/:id', function(req, res, next) {
customerSystem.find({
id: req.params.id
}, function(err, customer) {
if (err) {
res.status( /*correct status code*/ ).send( /*depending on the api return json, xml, ....*/ )
} else {
res.send( /*depending on the api return json, xml, ....*/ )
}
})
});
router.delete('/:id', function(req, res, next) {
customerSystem.delete({
id: req.params.id
}, function(err) {
//...
});
});
router.post('/', function(req, res, next) {
//...
});
//save the customer id for the pass to the sub routers
router.use('/:id', function(req, res, next) {
req.customerId = req.params.id;
next();
});
router.use('/:id/addresses', require('./customer-address') )
module.exports = router;
./lib/rest/customer-address.js
var customerSystem = require('../customer-system');
var express = require('express');
var router = new express.Router();
router.get('/:id', function(req, res, next) {
customerSystem.find({
id: req.customerId
}, function(err, customer) {
// ...
})
});
/* ..... */
//save the address id for the pass to the sub routers
router.use('/:id', function(req, res, next) {
req.addressId = req.params.id;
next();
});
router.use('/:id/addresses', require('./customer-address') )
module.exports = router;

Express middleware, next and Promises

There is very simple Express router with handler:
router.get('/users/:userId/roles/:roleId', function(req, res, next){
const roleId = req.params.roleId;
res.rest.resource = UserModel.findOne({ _id: req.params.userId}).exec().then(function(usr) {
console.log(req.params.roleId); // => undefined
console.log(roleId); // => okay here
const result = usr.roles.find( role => String(role._id) === String(roleId));
return result;
});
next();
});
As it seen accessing req.params.roleId within promise returnes undefined. This is true only for cases when next() called outside promise's then.
I'm okay with asyncing and promises and understand that next() will be called before handler in then. But what happening with req.params.roleId? Why and where it mutates? Does middleware called by next() gets same but mutated req?
Note: res.rest.resource used by middleware called later to build right REST-like response.

The code as it is is kind of indeterministic in its execution.
Something mutates the role ID in the next() handler, and since it takes a while for findOne() to eventually dispatch to the then handler, that mutation has already happened.
Without knowing further details of your app, it looks like this might be the correct implementation.
router.get('/users/:userId/roles/:roleId', function(req, res, next) {
const roleId = req.params.roleId;
UserModel.findOne({ _id: req.params.userId}).exec().then((usr) => {
const result = usr.roles.find(role => String(role._id) === String(roleId));
res.rest.resource = result;
next(); // <-- only dispatch to next after we find the resource result
});
});
Edit:
I dug a little deeper. See this little example app:
var express = require('express');
var app = express();
app.use(function (req, res, next) {
var v = 0 | +new Date();
console.log("middleware 1 setting foos to ", v);
req.params.foo = v;
req.foo = v;
next();
});
app.use(function (req, res, next) {
console.log("middleware 2 reading foos and starting timer:", req.params.foo, req.foo);
setTimeout(function() {
console.log("middleware 2: foos are now", req.params.foo, req.foo);
}, 1000);
next();
});
app.get("/", function(req, res) {
res.send("params = " + JSON.stringify(req.params) + " and foo = " + req.foo);
});
app.listen(3000);
The output for a request is
middleware 1 setting foos to -902674369
middleware 2 reading foos and starting timer: undefined -902674369
middleware 2: foos are now undefined -902674369
middleware 1 setting foos to -902673113
middleware 2 reading foos and starting timer: undefined -902673113
middleware 2: foos are now undefined -902673113
and the browser output is params = {} and foo = -902673113, so it turns out that you are not allowed to touch req.params, but you can add any other properties to the req object and they will travel along fine.
This seems to be because of the route matching layer rewriting params on each step.

Express app.all declaring req/res vs not

In express, everything else held constant, is there a difference between:
app.all('/', mongoProxy(config.mongo.dbUrl, config.mongo.apiKey));
and
app.all('/', function (req, res) {
mongoProxy(config.mongo.dbUrl, config.mongo.apiKey);
});
The former is able to return the return value from mongoProxy while the latter is not, where mongoProxy looks something like this:
module.exports = function(basePath, apiKey) {
basePath = url.parse(basePath);
// Map the request url to the mongolab url
// #Returns a parsed Url object
var mapUrl = module.exports.mapUrl = function(reqUrlString) {
//use the basePath to Parse the URL
return newUrl;
};
var mapRequest = module.exports.mapRequest = function(req) {
var newReq = mapUrl(req.url);
// Make a new request and return it..
return newReq;
};
var proxy = function(req, res, next) {
try {
var options = mapRequest(req);
// Create the request to the db
var dbReq = https.request(options, function(dbRes) {
// Save result
});
// { send result }
res.send(data);
res.end();
});
});
// send request
dbReq.end(JSON.stringify(req.body));
} catch (error) {
//..
}
};
return proxy;
};
The documentation is not clear on explaining the conceptual difference between the two; in the examples I've seen, the former function
app.all('/', mongoProxy(config.mongo.dbUrl, config.mongo.apiKey));
is able to access the req and res object without having it actually passed in as done in the latter, function (req, res).
What is the difference between the two, and is one preferable?

tl;dr
Yes, there is a difference: the first will work, whereas the second will hang (you don't invoke the anonymous function returned by mongoProxy). The first is preferable because it is more idiomatic to express (you're using middleware).
First, note how in mongoProxy, you return proxy, an anonymous function:
module.exports = function(basePath, apiKey) {
/* snip */
var proxy = function(req, res, next) { // <-- here
/* snip */
};
return proxy; // <-- and here
};
Let's break it down:
var proxy = mongoProxy(config.mongo.dbUrl, config.mongo.apiKey)
// proxy is an anonymous function which accepts: (req, res, next)
app.all('/', proxy);
// express will use proxy as the callback (middleware), which means this is the same as:
app.all('/', function (req, res, next) {
proxy(req, res, next)
})
Let's rewrite the second example—which should make clear why it doesn't work:
var proxy = mongoProxy(config.mongo.dbUrl, config.mongo.apiKey)
app.all('/', function (req, res) {
proxy // nothing happens because you don't invoke the function
});
If you want to use the second example, you could invoke proxy with proxy(req, res, next), but this is not idiomatic (in general and especially for express). Express is all about middleware, so use the first example.
Here's another example, which uses a closure (much like your mongoProxy function):
function getPermissionLevelMiddleware (level) {
// returns an anonymous function which verifies users based on `level`
return function (req, res, next) {
if (req.isAuthenticated() && req.user.permission.level > level)
return next()
return res.redirect('/no/permission')
}
}
var isAdmin = getPermissionLevelMiddleware(9000)
// `isAdmin` only allows users with more than 9000 `user.permission.level`
var isPleb = getPermissionLevelMiddleware(1)
// `isPleb` allows users with more than 1 `user.permission.level`
app.get('/admin', isAdmin, function (req, res) {
res.render('admin.jade')
})

Obviously the first one will return result because req and res objects are accessible to it, where as second case you need to send req and res in the method params of mongoProxy. If you won't send, req and res won't be accessible to it. So, for second scenario to work, the method signature need to be changed to :
module.exports = function(basePath, apiKey, req, res) {
I have done