Develop Reference

Nodejs mongodb fetching document size without actually fetching cursor

My question is, how can I get a cursor size (in KBs) without actually fetching it ?
I've already examined a lot of question such as here But I don't want to fetch query result to learn how much KB is it.
I just want something like:
var MongoClient = require('mongodb').MongoClient,
test = require('assert');
MongoClient.connect('mongodb://localhost:27017/test', function(err, db) {
var collection = db.collection('simple_query');
// Insert a bunch of documents for the testing
collection.insertMany([{a:1}, {a:2}, {a:3}], {w:1}, function(err, result) {
test.equal(null, err);
collection.find(/**SOME QUERY*/).size(function(err, SIZE) {
test.equal(null, err);
test.equal(32111351, SIZE); // in bytes or kilobytes whatever
db.close();
});
});
});

Something like this?
var avgSize = db.collectionName.stats().avgObjSize;
// ...
collection.count(/* some query */, function(err, count) {
var approximateSize = count*avgSize; // This could work for simple database models
}
I know its not perfect, but it is the best way i found.

How to use 64-bit long auto-increment counters in MongoDB

I'm implementing a logger database using MongoDB. The capped collection will contain log messages collected from several sources across the network. Since I want to do $lte/$gte queries on _id afterwards I need to have an _id that grows as a monotonic function.
To achieve that I've implemented the auto-incremented counter described in this article http://docs.mongodb.org/manual/tutorial/create-an-auto-incrementing-field/
My code looks like that:
var mongo = require("mongodb");
var Promise = require('es6-promise').Promise;
function connectToDB(mongo, uri) {
return new Promise(function(resolve, reject) {
mongo.MongoClient.connect(uri, function (err, db) {
if(err) reject(err);
else resolve(db);
});
});
}
function getNextSequenceNumber(db, counterName) {
return new Promise(function(resolve, reject) {
db.collection("counters", function (err, collection) {
if (err) reject(err);
else {
var criteria = { _id: counterName };
var sort = {$natural: 1};
var update = { $inc: { seq: 1 } };
var options = {remove: false, new: true, upsert: true};
collection.findAndModify(criteria, sort, update, options, function(err, res) {
if(err) reject(err);
else resolve(res.seq);
});
}
});
});
}
It works perfectly fine, but I've read that by default the number fields used in MongoDB are actually floats. The problem is that my database is a capped collection of log entries and it is going to have lots of entries. Moreover, since this is a capped collection the old entries will be overwritten but the counter will keep growing. Having counter as a float I cannot guarantee the system will keep on working after a few years.
My question is how can I force MongoDB to use 64 bit counter in this particular case.
Please provide some code examples.

MongoDB (or rather BSON) has the NumberLong type, which is a 64-bit signed integer.
From Node.js you can use it in your update statement to create the seq property of that type:
var update = { $inc : { seq : mongo.Long(1) } };
This also seems to convert the seq property of existing documents to NumberLong.

mongo / monk - not firing promise.done?

I'm using mongo/monk to try and get promises to reduce some CB ugliness but it seems to be creating more unusual problems to debug ("now you have two problems").
based off:
https://gentlenode.com/journal/node-4-monk-cheatsheet/45
I have a little routine to clear a collection and insert some data,
however the on.complete or on.success isn't firing
QuizPlugin.collection.remove({}, function(err, doc) {
Clog.log("QP", "removed"); // get this
var raw = FileReader.readDataFile(jsonfile);
var that = this;
raw.records.map(item => {
var p = QuizPlugin.collection.insert(item.fields);
console.log("p.type", p.type);
p.on("success", function(doc) {
console.log("done", doc) // never
});
p.on("complete", function(doc) {
console.log("done", doc) // not this one either
});
p.on("error", function() {
console.log("error") // or this ever show up
});
// this method also doesn't emit anything
// QuizPlugin.collection.insert(item.fields, function(err, doc) {
// console.log("inserted")
// });
// Clog.log("inserted:", item.fields.question);
})
})
Is there some other reason the inner on("success") aren't firing?
the docs seem pretty clear, if spartan
https://github.com/Automattic/monk#promises

node.js mongodb - collection.find().toArray(callback) - callback doesn't get called

I am just starting out with mongodb, but I am running into a problem when trying to use .find() on a collection.
I've created a DataAccessObject which opens a specific databate and then lets your perform operations on it. Here is the code:
The constructor:
var DataAccessObject = function(db_name, host, port){
this.db = new Db(db_name, new Server(host, port, {auto_reconnect: true}, {}));
this.db.open(function(){});
}
A getCollection function:
DataAccessObject.prototype.getCollection = function(collection_name, callback) {
this.db.collection(collection_name, function(error, collection) {
if(error) callback(error);
else callback(null, collection);
});
};
A save function:
DataAccessObject.prototype.save = function(collection_name, data, callback){
this.getCollection(collection_name, function(error, collection){
if(error) callback(error);
else{
//in case it's just one article and not an array of articles
if(typeof (data.length) === 'undefined'){
data = [data];
}
//insert to collection
collection.insert(data, function(){
callback(null, data);
});
}
});
}
And what seems to be the problematic one - a findAll function:
DataAccessObject.prototype.findAll = function(collection_name, callback) {
this.getCollection(collection_name, function(error, collection) {
if(error) callback(error)
else {
collection.find().toArray(function(error, results){
if(error) callback(error);
else callback(null, results);
});
}
});
};
Whenever I try to dao.findAll(error, callback), the callback never gets called.
I've narrowed the problem down to the following part of the code:
collection.find().toArray(function(error, result){
//... whatever is in here never gets executed
});
I've looked at how other people do it. In fact, I'm following this tutorial very closely. No one else seems to have this problem with colelction.find().toArray(), and it doesn't come up in my searches.
Thanks,
Xaan.

You are not using the open callback so if you are trying to make the findall request right after creating the dao then it won't be ready.
If your code is like this, it will not work.
var dao = new DataAccessObject("my_dbase", "localhost", 27017);
dao.findAll("my_collection",function() {console.log(arguments);});
I tested it and it doesn't find records, and it also gives no error. I think it should give an error.
But if you change it so that you give a callback to the constructor, then it should work.
var DataAccessObject = function(db_name, host, port, callback){
this.db = new Db(db_name, new Server(host, port, {auto_reconnect: true}, {}));
this.db.open(callback);
}
And make your code like this.
var dao = new DataAccessObject("my_dbase", "localhost", 27017, function() {
dao.findAll("my_collection",function() {console.log(arguments);});
});

Flattening out nested callback

I have frustrating problem with learning to work with callback style of programming in Node.js. I have a query to a MongoDB database. If I pass in a function to execute on the result it works but I'd rather flatten it out and have it return the value. Any help or direction on how to do this correctly is appreciated. Here's my code:
var getLots = function(response){
db.open(function(err, db){
db.collection('lots', function(err, collection){
collection.find(function(err, cursor){
cursor.toArray(function(err, items){
response(items);
})
})
})
})
}
I want something more like this:
lots = function(){
console.log("Getting lots")
return db.open(openCollection(err, db));
}
openCollection = function(err, db){
console.log("Connected to lots");
return (db.collection('lots',findLots(err, collection))
);
}
findLots = function(err, collection){
console.log("querying 2");
return collection.find(getLots(err, cursor));
}
getLots = function(err, cursor) {
console.log("Getting lots");
return cursor.toArray();
}
Where the final set of data would bubble back up through the function calls.
The problem is that I get an error from Node.js saying that err is not defined or that the collection is not defined. For some reason when I nest the callbacks the correct object is getting passed down. When I try going to this flattened style it complains that things are not defined. I don't know how to get it to pass the necessary objects.

What you need is one of the many control flow libraries available for node via npm and catalogued on the Node.js wiki. My specific recommendation is caolan/async, and you would use the async.waterfall function to accomplish this type of flow where each async operation must be executed in order and each requires the results from the previous operation.
Pseudocode example:
function getLots(db, callback) {
db.collection("lots", callback);
}
function findLots(collection, callback) {
collection.find(callback);
}
function toArray(cursor, callback) {
cursor.toArray(callback);
}
async.waterfall([db.open, getLots, find, toArray], function (err, items) {
//items is the array of results
//Do whatever you need here
response(items);
});

async is a good flow control library. Frame.js offers some specific advantages like better debugging, and better arrangement for synchronous function execution. (though it is not currently in npm like async is)
Here is what it would look like in Frame:
Frame(function(next){
db.open(next);
});
Frame(function(next, err, db){
db.collection('lots', next);
});
Frame(function(next, err, collection){
collection.find(next);
});
Frame(function(next, err, cursor){
cursor.toArray(next);
});
Frame(function(next, err, items){
response(items);
next();
});
Frame.init();