Develop Reference

Node.js flat-cache, when to clear caches

I have a Node.js server which queries MySQL database. It serves as an api end point where it returns JSON and also backend server for my Express application where it returns the retrieved list as an object to the view.
I am looking into implementing flat-cache for increasing the response time. Below is the code snippet.
const flatCache = require('flat-cache');
var cache = flatCache.load('productsCache');
//get all products for the given customer id
router.get('/all/:customer_id', flatCacheMiddleware, function(req, res){
var customerId = req.params.customer_id;
//implemented custom handler for querying
queryHandler.queryRecordsWithParam('select * from products where idCustomers = ? order by CreatedDateTime DESC', customerId, function(err, rows){
if(err) {
res.status(500).send(err.message);
return;
}
res.status(200).send(rows);
});
});
//caching middleware
function flatCacheMiddleware(req, res, next) {
var key = '__express__' + req.originalUrl || req.url;
var cacheContent = cache.getKey(key);
if(cacheContent){
res.send(cacheContent);
} else{
res.sendResponse = res.send;
res.send = (body) => {
cache.setKey(key,body);
cache.save();
res.sendResponse(body)
}
next();
}
}
I ran the node.js server locally and the caching has indeed greatly reduced the response time.
However there are two issues I am facing that I need your help with.
Before putting that flatCacheMiddleware middleware, I received the response in JSON, now when I test, it sends me an HTML. I am not too well versed with JS strict mode (planning to learn it soon), but I am sure the answer lies in the flatCacheMiddleware function.
So what do I modify in the flatCacheMiddleware function so it would send me JSON?
I manually added a new row to the products table for that customer and when I called the end point, it still showed me the old rows. So at what point do I clear the cache?
In a web app it would ideally be when the user logs out, but if I am using this as an api endpoint (or even on webapp there is no guarantee that the user will log out the traditional way), how do I determine if new records have been added and the cache needs to be cleared.
Appreciate the help. If there are any other node.js caching related suggestions you all can give, it would be truly helpful.

I found a solution to the issue by parsing the content to JSON format.
Change line:
res.send(cacheContent);
To:
res.send(JSON.parse(cacheContent));
I created cache 'brute force' invalidation method. Calling clear method will clear both cache file and data stored in memory. You have to call it after db change. You can also try delete specified key using cache.removeKey('key');.
function clear(req, res, next) {
try {
cache.destroy()
} catch (err) {
logger.error(`cache invalidation error ${JSON.stringify(err)}`);
res.status(500).json({
'message' : 'cache invalidation error',
'error' : JSON.stringify(err)
});
} finally {
res.status(200).json({'message' : 'cache invalidated'})
}
}
Notice, that calling the cache.save() function will remove other cached API function. Change it into cache.save(true) will 'prevent the removal of non visited keys' (like mentioned in comment in the flat-cache documentation.

node.js application - how to connect to mongodb and "share" connection via an include?

Background Information
I'm attempting my first node.js API/application. As a learning exercise, I'm trying to create some test cases initially delete all records in a table, insert 3 specific records, and then query for those 3 records.
Code
Here's the code I have cobbled together:
http://pastebin.com/duQQu3fm
Problem
As you can see from the code, I'm trying to put the database connection logic in a dbSession.js file and pass it around.
I am able to start up the http server by doing the following:
dev#devbox:~/nimble_node$ sudo nodejs src/backend/index.js
Server started and listening on port: 8080
Database connection successful
However, when I try to run my jasmine tests, it fails with the following error:
F
Failures:
1) The API should respond to a GET request at /api/widgets/
Message:
TypeError: Object #<MongoClient> has no method 'collection'
Stacktrace:
TypeError: Object #<MongoClient> has no method 'collection'
at resetDatabase (/home/dev/nimble_node/spec/resetDatabase.js:6:29)
at /home/dev/nimble_node/spec/e2e/apiSpec.js:23:25
at /home/dev/nimble_node/node_modules/async/lib/async.js:683:13
at iterate (/home/dev/nimble_node/node_modules/async/lib/async.js:260:13)
at async.forEachOfSeries.async.eachOfSeries (/home/dev/nimble_node/node_modules/async/lib/async.js:279:9)
at _parallel (/home/dev/nimble_node/node_modules/async/lib/async.js:682:9)
at Object.async.series (/home/dev/nimble_node/node_modules/async/lib/async.js:704:9)
at null.<anonymous> (/home/dev/nimble_node/spec/e2e/apiSpec.js:19:9)
at null.<anonymous> (/home/dev/nimble_node/node_modules/jasmine-node/lib/jasmine-node/async-callback.js:45:37)
Finished in 0.01 seconds
1 test, 1 assertion, 1 failure, 0 skipped
Database connection successful
Line 6 of resetDatabase is:
var collection = dbSession.collection('widgets');
Given that after the error appears, I get the "Database connection successful" message, I think what's happening is that when the tests request the dbSession library, the database hasn't finished running the code to connect. And therefore, I can't get the collection object.
I'm currently reading through the mongodb online manual to see if I can find some hints as to how to do something like this.
Any suggestions or pointers would be appreciated.
EDIT 1
To prove that there is a collection method on the MongoClient object, I changed the dbSession.js code to look like this:
'use strict';
var DBWrapper = require('mongodb').MongoClient;
var dbWrapper = new DBWrapper;
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
var collection = dbWrapper.collection('widgets');
console.log('just created a collection...');
}
});
module.exports = dbWrapper;
And now, when I start up the http server (index.js), notice the messages:
dev#devbox:~/nimble_node$ sudo nodejs src/backend/index.js
Server started and listening on port: 8080
Database connection successful
just created a collection...

It could be an async issue.
Your code in dbSessionjs
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
}
});
module.exports = dbWrapper;
Starts the connection at dbWrapper asynchronously, but exports dbWrapper right away, which is then imported in resetDatabase. Thus yes, the connect function may have not yet returned from the async function when you call it in resetDatabase (and is what the log suggests,as the error appears before the success log).
You could add a callback after dbWrapper.connect() returns, in order to actually only be able to use dbWrapper when the connection finished.
(With sqlite, this may not happen as it accesses the DB faster on the commandline).
This may not be your problem but looks like a candidate.
EDIT: Here's a possible example for a callback, but please take note it depends on what you need to do so there are a lot of different solutions. The key is to call a callback function when you are done initializing.
Another solution could be to simply wait, and/or poll (e.g. chcke a variable 'initialized').
'use strict';
var DBWrapper = require('mongodb').MongoClient;
var dbWrapper = new DBWrapper;
function doConnect(callback) {
console.log("Initializing DB connection...");
dbWrapper.connect("mongodb://localhost:27017/test", function(err, db) {
if (!err) {
console.log("Database connection successful");
dbWrapper = db;
var collection = dbWrapper.collection('widgets');
console.log('just created a collection...');
console.log('calling callback...');
callback(dbWrapper);
} else {
console.log("Error connectingi: " + err);
}
});
};
doConnect(function(correctDbWrapper) {
//Now you can use the wrapper
console.log("Inside callback, now consuming the dbWrapper");
dbWrapper = correctDbWrapper;
var collection = dbWrapper.collection('widgets');
});
It's interesting though I never ran into this issue, although I have generally used similar code like yours. I guess because normally I have this DB initialization right at the top, and then have to do lots of initializations on the node app, which gives the app time enough to return from the connect call....

connecting MongoDB in Node.js not working in for-loop

I have a function which connect my local mongoDB database, but the connection suddenly doesn't work when I try to put it in a for-loop.
var connectMongo = require("./ConnectToMongoDB");
var insertDocument = require("./InsertDocument");
function spamMongoDBtest(){
process.nextTick(function(){
var max = 500;
for(var i = 0; i<max;i++){
setTimeout(function(){
connectMongo(insertDocument);
}, 50);
}
});
}
Why do I get this AssertionError:
AssertionError: null == { [MongoError: connect ECONNREFUSED] name:
'MongoError', message: 'connect ECONNREFUSED' }
connectMongo(insertDocument); DOES work in this function:
function spamMongoDB(){
process.nextTick(function(){
setInterval(function(){connectMongo(insertDocument); }, 100); });
}

Could you also share the connectMongo and insertDocument functions?
Perhaps you have too many open connections to Mongo. Try closing the connection to Mongo after each time you write the document.
You can use the async library to limit the concurrency of your test.
For example:
var concurrency = 25; //limit to 25 concurrent connections
async.eachLimit(new Array(500), concurrency, function(item, callback) {
connectMongo(function(){
insertDocument();
callback();
});
});

You are getting a connection error from MongoDb because you are trying to connect to database multiple times. You need to connect to database only once, and from that moment if the connection has been established the communication with database is opened.
You are opening the connection and once opened you can communicate with the database.
var mongoURI = "mongodb://localhost:3030/";
var MongoDB = mongoose.connect(mongoURI).connection;
MongoDB.on('error', function(err) { console.log(err.message); });
MongoDB.once('open', function() {
console.log("mongodb connection open");
insertDocument;
});
Inside the open function you can trigger the setInterval. Or if you want to stick with your code, each time you open a connection in the for loop you have to close it too. I don't know what connectMongo method contains, but after you invoke this method outside setTimeout method you have to close the connection. But this approach anyway will put on heavy duty the server, so my advice is to follow the method provided.

Get mongodb collection with node

I'm very new to express/mongo stack, so the question is pretty simple, couldn't find anything on stackoverflow which would resolve my problem, so here it is:
I have a index.js file which looks more or less like this:
var mongoose = require('mongoose');
// Connections
var developmentDb = 'mongodb://localhost/admin';
var usedDb;
// If we're in development...
if (process.env.NODE_ENV === 'development') {
// set our database to the development one
usedDb = developmentDb;
// connect to it via mongoose
mongoose.connect(usedDb);
}
// get an instance of our connection to our database
var db = mongoose.connection;
db.once('open', function callback () {
console.log('Databsae Connection Successfully Opened at ' + usedDb);
});
module.exports = db;
Then I'm requireing it in my express route like this:
var express = require('express');
var router = express.Router();
var db = require('../../database');
/* GET users listing. */
router.get('/', function(req, res) {
var users = db.collection('users');
var test = users.find();
res.send(test);
});
module.exports = router;
I'm sending request and the result I'm getting is 'undefined' so nothing is returned from the back end.
Db connection is 100% correct and works.
I'm not entirely sure, do I have to have a schema definition on express side, or is it possible to query any data, without knowing the schema?

What you are missing here is a bit of the "mongoose magic" that is actually happening "behind the scenes " as it were. It's also a base concept of most operations in node.js that the operations ( particularly where IO is concerned ) are largely asynchronous in nature, so you are generally working with callbacks that fire when the operation is complete.
Take this part of your listing:
// get an instance of our connection to our database
var db = mongoose.connection;
db.once('open', function callback () {
console.log('Databsae Connection Successfully Opened at ' + usedDb);
});
module.exports = db;
So while you may have coded that in sequence, the actual order of events is not as you might think. While you can call the db object from mongoose.connection ( and actually this is a connection object and not theDb` object as implemented by the underlying driver ) there is no guarantee that the database is actually connected at this time. In fact, in all likelihood it is not.
The sort point here is that your database connection actually happens after you export the variable from the module, and not before. It does not wait for the preceding line to complete, nor can you make it do so.
Mongoose itself rather has a concept of "models" to represent the collections in your database. So the general approach is to define these model objects and use them for accessing the data:
var Model = mongoose.model( 'Model', modelSchema, 'optionalCollectionName' );
Model.find({}, function(err,data) {
// do stuff in the callback
});
Part of the reason for this ( aside from the schema definitions that are attached ) is that there is actually something else going on here related to the connection. These objects in fact have internal logic that only processes the actions on the tied "collection" object only when the connection to the database is available. So there is an "internal callback" function that is happening here where an internal connection object is actually being used.
Here is some simple code that "overrides" the usage of the internal methods to just try and get the underlying collection object from the driver. It will fail:
var mongoose = require('mongoose'),
Schema = mongoose.Schema
mongoose.connect('mongodb://localhost/test');
var modelSchema = new Schema({},{ strict: false });
var Model = mongoose.model( 'Model', modelSchema, 'optionalCollectionName' );
Model.collection.find({}, function(err,data) {
// do stuff in the callback
});
Since this asked for the collection object as implemented in the underlying driver to be returned and the native .find() method from that object to be used, the problem occurs that the database is not actually connected yet. So in order to make this work, you would need to wrap the call inside an event handler that fired only when the database was truly connected. Or otherwise make sure that a connection had been made before this was called:
mongoose.connection.on('open',function(err,conn) {
// Now we know we are connected.
Model.collection.find({}, function(err,data) {
// do stuff in the callback
});
});
So the model objects are actually doing this for you, and are providing their own implementations of the standard methods such as "find", "update" etc.
If you don't want to do this kind of wrapping, and the definition of models seems like too much work, and even using the { strict: false } modifier here, which relaxes the constraints on the schema to effectively allow any data, then you are probably better off using the base driver rather than mongoose.
But of course you want something smarter than just wrapping all the code inside a callback to the connection. Here is one approach to define an object that you can use to "fetch" the database connection for you, and make sure that nothing is executed until that connection is made:
var async = require('async'),
mongodb = require('mongodb'),
MongoClient = mongodb.MongoClient;
var Model = (function() {
var _db,
conlock;
return {
getDb: function(callback) {
var err = null;
if ( _db == null && conlock == null ) {
conlock = 1;
MongoClient.connect('mongodb://localhost/test',function(err,db) {
_db = db;
conlock == null;
if (!err) {
console.log("Connected")
}
callback(err,_db);
});
} else if ( conlock != null ) {
var count = 0;
async.whilst(
function() { return ( _db == null ) && (count < 5) },
function(callback) {
count++
setTimeout(callback,500);
},
function(err) {
if ( count == 5 )
err = new Error("connect wait exceeded");
callback(err,_db);
}
);
} else {
callback(err,_db);
}
}
};
})();
async.parallel(
[
function(callback) {
console.log("call model");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined");
callback();
});
},
function(callback) {
console.log("call model again");
Model.getDb(function(err,db) {
if (err) throw err;
if (db != undefined)
console.log("db is defined here as well");
callback();
});
}
],
function(err) {
Model.getDb(function(err,db) {
db.close();
});
}
);
That basically wraps up an object which we called "Model" here, with a single method to .getDb(). That method simply accepts a callback, which is there to be your actual piece of code that you want to access the database with, which in turn exposes the Db object from the connection.
The Db object is stored internally in that object, so it is basically a singleton which only ever connects to the database once. But as your logic is passed in a a callback function, then it will either just pass in the existing stored object, or wait until the connection is made before passing in the code.
Output from the sample usage should be:
call model
call model again
Connected
db is defined
db is defined here as well
And that shows the order of events to how they actually happen.
So there are different ways to handle this. Mongoose models "abstract" a lot of that for you. You can of course either take a base approach with the base driver as given in the example, or take that further and implement your own connection system including overridden methods that does much of the same thing that mongoose is doing underneath. There are also other wrapper libraries that already do this without the schema concepts that is generally inherent to mongoose as well.
Basically though, every higher level library above the base driver is doing much the same as described, where there are wrappers around the methods to make sure the connection is there without needing to embed all of your code in an event listener that is checking for that.

Multiple connections with node-mongodb-native

I am working on a function to insert a document in a mongoDb database using the node-mongodb-native module. Everything is working, except if I call insert multiple documents back-to-back. I use a for loop to test how my function is reacting to multiple document insert at the same time.
var server = new Server("xxx.xxx.xxx.xxx", 27017, {auto_reconnect: true, poolSize: 100});
var db = new Db("testDb", server, {safe: false});
module.exports.insert = function(document){
var database;
function db_open(err, db_local){
if(err) throw err;
database = db_local;
database.collection("rooms", handle_insert);
}
function handle_insert(err, collection){
if(err) throw err;
collection.insert(document);
database.close();
}
db.open(db_open);
};
for(var i=0; i<100; i++){
module.exports.insert({name : "test"});
}
When I'm running this code I get the error db object already connecting, open cannot be called multiple times
To resolve the problem, I decided to create a new instance of Server and Db at each call of the function :
module.exports.insert = function(document){
var database;
var server = new Server("xxx.xxx.xxx.xxx", 27017, {auto_reconnect: true, poolSize: 100});
var db = new Db("testDb", server, {safe: false});
function db_open(err, db_local){
if(err) throw err;
database = db_local;
database.collection("rooms", handle_insert);
}
function handle_insert(err, collection){
if(err) throw err;
collection.insert(document);
database.close();
}
db.open(db_open);
};
for(var i=0; i<100; i++){
module.exports.insert({name : "test"});
}
But now I'm getting connection closed thrown by the db_open function
I really don't understand why my connection is closing between the moment when I'm creating db and when my code call db_open.
Have you an idea of what is happening?
Thank you :)
(Sorry if my English is not really good)
EDIT
I found this website explaining that the problem is caused by the too long tcp_keepalive time. The problem with this solution is my workstation (Cloud 9). I don't have the permission to access the file /proc/sys/net/ipv4/tcp_keepalive_time

I don't think your problem has anything to do with TCP keep-alive. Like the error message says, you're simply attempting to open the same connection multiple times (every time the insert method is called). Instead of calling open() every time, just call it once, and make sure the callback returns, before you call insert() for the first time. There is no hard limit to the number of simultaneous inserts you can perform on the same connection since it's all asynchronous.