I'm implementing a logger database using MongoDB. The capped collection will contain log messages collected from several sources across the network. Since I want to do $lte/$gte queries on _id afterwards I need to have an _id that grows as a monotonic function.
To achieve that I've implemented the auto-incremented counter described in this article http://docs.mongodb.org/manual/tutorial/create-an-auto-incrementing-field/
My code looks like that:
var mongo = require("mongodb");
var Promise = require('es6-promise').Promise;
function connectToDB(mongo, uri) {
return new Promise(function(resolve, reject) {
mongo.MongoClient.connect(uri, function (err, db) {
if(err) reject(err);
else resolve(db);
});
});
}
function getNextSequenceNumber(db, counterName) {
return new Promise(function(resolve, reject) {
db.collection("counters", function (err, collection) {
if (err) reject(err);
else {
var criteria = { _id: counterName };
var sort = {$natural: 1};
var update = { $inc: { seq: 1 } };
var options = {remove: false, new: true, upsert: true};
collection.findAndModify(criteria, sort, update, options, function(err, res) {
if(err) reject(err);
else resolve(res.seq);
});
}
});
});
}
It works perfectly fine, but I've read that by default the number fields used in MongoDB are actually floats. The problem is that my database is a capped collection of log entries and it is going to have lots of entries. Moreover, since this is a capped collection the old entries will be overwritten but the counter will keep growing. Having counter as a float I cannot guarantee the system will keep on working after a few years.
My question is how can I force MongoDB to use 64 bit counter in this particular case.
Please provide some code examples.
MongoDB (or rather BSON) has the NumberLong type, which is a 64-bit signed integer.
From Node.js you can use it in your update statement to create the seq property of that type:
var update = { $inc : { seq : mongo.Long(1) } };
This also seems to convert the seq property of existing documents to NumberLong.
Related
I'm writing an application that lets you add visits (visit date, visit type, notes) to a case, via an Ajax POST call from a form. The visit creation functionality lets you add the same visit type and notes on several dates. So I end up with a visit object that has an array of dates in it, but the same notes and visit type. Because SQL isn't where I should be doing any looping, I want to do it in Node as I'll be able to handle any failures in the array or results returned from the individual SQL calls.
I tried setting up the procedure call so that it took an array of arrays in an array as a parameter as per here, but I couldn't get it to work, so am falling back to looping through.
The issue I'm having is with callbacks completing before I've got any results. Obviously it's because I don't understand callbacks enough and no amount of reading is making it any clearer, so I've ended up here to ask for help.
Below is the code that is executed. The visit object that is the parameter of the insertVisit function is the class as mentioned above with the array of dates.
this.insertVisit = function (req, res, visit)
{
var insertVisit = new Visit();
insertVisit = visit;
var success = 0;
var visitId = 0;
//Split the visits into an array of individual dates
var allVisits = insertVisit.visitDates.split(',');
//Attemp to call insertVisits using a callback
insertVisits(0, function(err){
if( err ) {
console.log('yeah, that insert didnt work: '+ err)
}
});
console.log('finished');
function insertVisits(v)
{
//Loop through all of the visits
if (v < allVisits.length )
{
//Attempt to call the next function
singleDate(allVisits[v], function(err)
{
if(err)
{
console.log(err);
}
else
{
//if everything is successful, insert the next individual date
allVisits[v + 1];
}
})
}
}
function singleDate(singleVisitDate)
{
var query = 'CALL aau.sp_InsertVisit (?,?,?,?,?,?,?,#visitId,#success); SELECT #visitId, #success;';
var parts = singleVisitDate.split('-');
var formattedDate = new Date(parts[2], parts[1] - 1, parts[0]);
connection.init();
//Everything runs fine up to here, but as soon as we go to the next line, the program
//continues back at the end of the loop in the insertVisits function an exits the function.
//At this point the below code executes asynchronously and inserts one of the dates before returning
//and doesn't call any further dates.
connection.acquire(function (err, con)
{
con.query(query,
[
insertVisit.caseId,
formattedDate,
parseInt(insertVisit.visitTypeId),
parseInt(insertVisit.visitStatusId),
insertVisit.adminNotes,
insertVisit.operatorNotes,
insertVisit.isDeleted,
visitId,
success
]
, function (err, result)
{
if(err)
{
console.log(err);
}
else
{
con.release();
res.write(JSON.stringify(result));
}
})
})
}
So I'm trying to loop through each of the dates and call the stored procedure for each date and add the results to response using res.write.
This is a brand new project, so happy to rewrite it with promises or asynch/await. But any examples would be greatly appreciated of looping through multiple procedure calls
Ok,
So I looked at using async.eachSeries and managed to get it to work when I put the callback at the bottom of the 'tree'.
Hopefully this can be helpful to anyone else trying to run the same proc multiple times.
this.insertVisit = function (req, res, visit)
{
var insertVisit = new Visit();
insertVisit = visit;
var success = 0;
var visitId = 0;
var allVisits = insertVisit.visitDates.split(',');
async.eachSeries(allVisits, function(singleVisitDate, callback)
{
var query = 'CALL aau.sp_InsertVisit (?,?,?,?,?,?,?,#visitId,#success); SELECT #visitId, #success;';
var parts = singleVisitDate.split('-');
var formattedDate = new Date(parts[2], parts[1] - 1, parts[0]);
connection.init();
connection.acquire(function (err, con)
{
con.query(query,
[
insertVisit.caseId,
formattedDate,
parseInt(insertVisit.visitTypeId),
parseInt(insertVisit.visitStatusId),
insertVisit.adminNotes,
insertVisit.operatorNotes,
insertVisit.isDeleted,
visitId,
success
]
, function (err, result)
{
if(err)
{
console.log(err);
}
else
{
con.release();
res.write(JSON.stringify(result));
callback();
}
})
})
},
function(err)
{
if(err)
{
console.log(err);
}
else
{
res.end();
}
});
Struggling with this mongoose function. I'm trying to add an embedded object inside another embedded object, but it is adding the new bid object to the wrong place - the new listings variable I created for my iterator.
My for loops are trying to find the exact Listing to update so I think the action of assigning them is messing them up. For instance if the Listing was users[2].listings[10].bids , how do I get to that object so I can update it?
function create(req, res) {
db.Listing.findById(req.params.listingId, function(err, foundListing) {
// console.log(foundListing._id );
if (err) {
console.log('Error', err);
}
// res.json(foundListing);
// get array of all users
db.User.find({}, function(err, users) {
// for loop iterates through all users' listings
for (let i = 0; i < users.length; i++) {
let listings = users[i].listings
// for loop iterates through all listings ids
for (let j = 0; j < listings.length; j++) {
// finds match
// comparing _id with _id returning false. Not sure why, will return later
if (listings[j].topic === foundListing.topic && listings[j].created === foundListing.created) {
console.log("Found match: " + foundListing.topic);
// get current user id to add to bid object
db.User.findById(req.user, function(err, user) {
if (err) {
console.log(err);
}
var newBid = new db.Bid(req.body); // add data validation later
newBid.uid = user._id
// pushes new bid object into embedded listing
listings[j].bids.push(newBid);
listings[j].save(function(err, savedBid) {
console.log('newBid created: ', newBid);
console.log(listings[j]);
res.json(newBid);
});
});
}
}
}
})
if (err) {
console.log(err);
}
});
};
EDIT - Got this far, but now it doesn't seem like my array is saving.
function create(req, res) {
db.Listing.findById(req.params.listingId, function(err, foundListing) {
if (err) {
console.log('Error:', err);
}
db.User.findOne({ 'listings._id': req.params.listingId }, function(err, foundUser) {
// console.log(foundUser.listings);
if (err) {
console.log('Error: ', err);
}
for (let i = 0; i < foundUser.listings.length; i++) {
// console.log(foundUser.listings[i]._id);
if (foundUser.listings[i]._id == req.params.listingId) {
console.log( 'found it! - ' + foundUser.listings[i].topic);
var newBid = new db.Bid(req.body);
// console.log(newBid)
foundUser.listings[i].bids.push(newBid);
console.log(foundUser.listings[i].bids)
foundUser.listings[i].save(function(err, savedListing) {
// console.log(foundUser.listings[i])
if (err) {
console.log('Error: ', err);
return;
}
console.log('newBid created: ', newBid);
console.log('savedListing', savedListing);
res.json(newBid);
})
}
}
})
});
};
The code you have is pretty hard to grasp. I think you might be using the wrong tools to solve the problem. MongoDb and Mongoose seem to have pretty advanced querying, so that you can specify what exact user, listing etc. you're interested in.
I think it would be worth your time to look a bit at the documentation for Mongoose queries and maybe MongoDb queries as well. The . used for listings.id in db.User.findOne({ 'listings.id': req.params.listingId } is described here.
I can't really test that this code compiles or that it would work, because I don't know enough about your database models and so on, but something along these lines should be possible:
function create(req, res) {
db.Listing.findById(req.params.listingId, function(err, foundListing) {
// use the power of mongo db and mongoose.
// you can specify more exactly what you're looking for
// in this case we're interested in ONE user that
// has a listing with a specific id
// (if you still problems with comparing identical IDs, then that's a serious issue..
// maybe try logging req.params.listingId and check if it really exists
// in your database.)
db.User.findOne({ 'listings.id': req.params.listingId }, function(err, foundUser) {
var newBid = new db.Bid(req.body); // add data validation later
newBid.uid = foundUser._id
// Old code: foundListing.bids.push(newBid)
// Change according to first question/answer at http://mongoosejs.com/docs/faq.html
// which was linked in the thread you linked to in comment.
// The essence is that Mongoose doesn't get
// notified about changes to arrays by default
// To get around this you can update using "set"
// but then you need both a value and an index:
var oldBidsLength = foundListing.bids.length;
foundListing.bids.set(oldBidsLength, newBid);
foundListing.save(function(err, savedBid) {
// savedBid = saved LISTING?
console.log('newBid created: ', newBid);
console.log('savedBid', savedBid);
res.json(savedBid);
}
}
})
});
};
Things that may be off with this code example is things like if db.User.findOne({ 'listings.id': req.params.listingId } should be used with listings._id. I don't know enough about your models.
I was wondering what is the correct way to do bulk inserts into Mongodb (although could be any other database) with Node.js
I have written the following code as an example, although I believe it is floored as db.close() may be run before all the asynchronous collection.insert calls have completed.
MongoClient.connect('mongodb://127.0.0.1:27017/test', function (err, db) {
var i, collection;
if (err) {
throw err;
}
collection = db.collection('entries');
for (i = 0; i < entries.length; i++) {
collection.insert(entries[i].entry);
}
db.close();
});
If your MongoDB server is 2.6 or newer, it would be better to take advantage of using a write commands Bulk API that allow for the execution of bulk insert operations which are simply abstractions on top of the server to make it easy to build bulk operations and thus get perfomance gains with your update over large collections.
Sending the bulk insert operations in batches results in less traffic to the server and thus performs efficient wire transactions by not sending everything all in individual statements, but rather breaking up into manageable chunks for server commitment. There is also less time waiting for the response in the callback with this approach.
These bulk operations come mainly in two flavours:
Ordered bulk operations. These operations execute all the operation in order and error out on the first write error.
Unordered bulk operations. These operations execute all the operations in parallel and aggregates up all the errors. Unordered bulk operations do not guarantee order of execution.
Note, for older servers than 2.6 the API will downconvert the operations. However it's not possible to downconvert 100% so there might be some edge cases where it cannot correctly report the right numbers.
In your case, you could implement the Bulk API insert operation in batches of 1000 like this:
For MongoDB 3.2+ using bulkWrite
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
var entries = [ ... ] // a huge array containing the entry objects
var createNewEntries = function(db, entries, callback) {
// Get the collection and bulk api artefacts
var collection = db.collection('entries'),
bulkUpdateOps = [];
entries.forEach(function(doc) {
bulkUpdateOps.push({ "insertOne": { "document": doc } });
if (bulkUpdateOps.length === 1000) {
collection.bulkWrite(bulkUpdateOps).then(function(r) {
// do something with result
});
bulkUpdateOps = [];
}
})
if (bulkUpdateOps.length > 0) {
collection.bulkWrite(bulkUpdateOps).then(function(r) {
// do something with result
});
}
};
For MongoDB <3.2
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
var entries = [ ... ] // a huge array containing the entry objects
var createNewEntries = function(db, entries, callback) {
// Get the collection and bulk api artefacts
var collection = db.collection('entries'),
bulk = collection.initializeOrderedBulkOp(), // Initialize the Ordered Batch
counter = 0;
// Execute the forEach method, triggers for each entry in the array
entries.forEach(function(obj) {
bulk.insert(obj);
counter++;
if (counter % 1000 == 0 ) {
// Execute the operation
bulk.execute(function(err, result) {
// re-initialise batch operation
bulk = collection.initializeOrderedBulkOp();
callback();
});
}
});
if (counter % 1000 != 0 ){
bulk.execute(function(err, result) {
// do something with result
callback();
});
}
};
Call the createNewEntries() function.
MongoClient.connect(url, function(err, db) {
createNewEntries(db, entries, function() {
db.close();
});
});
You can use insertMany. It accepts an array of objects. Check the API.
New in version 3.2.
The db.collection.bulkWrite() method provides the ability to perform bulk insert, update, and remove operations. MongoDB also supports bulk insert through the db.collection.insertMany().
In bulkWrite it is supporting only insertOne, updateOne, updateMany, replaceOne, deleteOne, deleteMany
In your case to insert data using single line of code, it can use insertMany option.
MongoClient.connect('mongodb://127.0.0.1:27017/test', function (err, db) {
var i, collection;
if (err) {
throw err;
}
collection = db.collection('entries');
collection.insertMany(entries)
db.close();
});
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
var data1={
name:'Data1',
work:'student',
No:4355453,
Date_of_birth:new Date(1996,10,17)
};
var data2={
name:'Data2',
work:'student',
No:4355453,
Date_of_birth:new Date(1996,10,17)
};
MongoClient.connect(url, function(err, db) {
if(err!=null){
return console.log(err.message)
}
//insertOne
db.collection("App").insertOne(data1,function (err,data) {
if(err!=null){
return console.log(err);
}
console.log(data.ops[0]);
});
//insertMany
var Data=[data1,data2];
db.collection("App").insertMany(Data,forceServerObjectId=true,function (err,data) {
if(err!=null){
return console.log(err);
}
console.log(data.ops);
});
db.close();
});
My question is, how can I get a cursor size (in KBs) without actually fetching it ?
I've already examined a lot of question such as here But I don't want to fetch query result to learn how much KB is it.
I just want something like:
var MongoClient = require('mongodb').MongoClient,
test = require('assert');
MongoClient.connect('mongodb://localhost:27017/test', function(err, db) {
var collection = db.collection('simple_query');
// Insert a bunch of documents for the testing
collection.insertMany([{a:1}, {a:2}, {a:3}], {w:1}, function(err, result) {
test.equal(null, err);
collection.find(/**SOME QUERY*/).size(function(err, SIZE) {
test.equal(null, err);
test.equal(32111351, SIZE); // in bytes or kilobytes whatever
db.close();
});
});
});
Something like this?
var avgSize = db.collectionName.stats().avgObjSize;
// ...
collection.count(/* some query */, function(err, count) {
var approximateSize = count*avgSize; // This could work for simple database models
}
I know its not perfect, but it is the best way i found.
Is there anyway to duplicate an collection through the nodejs mongodb driver?
i.e. collection.copyTo("duplicate_collection");
You can eval copyTo() server-side though it will block the entire mongod process and won't create indexes on the new collection.
var copyTo = "function() { db['source'].copyTo('target') };"
db.eval(copyTo, [], function(err, result) {
console.log(err);
});
Also note the field type warning.
"When using db.collection.copyTo() check field types to ensure that the operation does not remove type information from documents during the translation from BSON to JSON. Consider using cloneCollection() to maintain type fidelity."
Try to avoid .eval() if this is something you want to do regularly on a production system. It's fast, but there are problems.
A better approach would be to use The "Bulk" operations API, and with a little help from the "async" library:
db.collection("target",function(err,target) {
var batch = target.initializeOrderedBulkOp();
counter = 0;
var cursor = db.collection("source").find();
var current = null;
async.whilst(
function() {
cursor.nextObject(function(err,doc) {
if (err) throw err;
// .nextObject() returns null when the cursor is depleted
if ( doc != null ) {
current = doc;
return true;
} else {
return false;
}
})
},
function(callback) {
batch.insert(current);
counter++;
if ( counter % 1000 == 0 ) {
batch.execute(function(err,result) {
if (err) throw err;
var batch = target.initializeOrderedBulkOp();
callback();
});
}
},
function(err) {
if (err) throw err;
if ( counter % 1000 != 0 )
batch.execute(function(err,result) {
if (err) throw err;
// job done
});
}
);
});
It's fast, not as fast as .eval() but does not block either the application or server.
Batch operations will generally take as many operations as you throw at them, but using a modulo as a limiter allows a little more control and essentially avoids loading an unreasonable amount of documents in memory at a time. Keep in mind that whatever the the case the batch size that is sent cannot exceed more that 16MB between executions.
Another option to duplicate a collection would be to use aggregate method on a collection and the $out parameter. Here is an example inside of an async function:
const client = await MongoClient.connect("mongodb://alt_dev:aaaaa:27018/meteor");
const db = client.db('meteor');
const planPrice = await db.collection('plan_price');
const planPriceCopy = await planPrice.aggregate([{$match: {}}, {$out: planPriceUpdateCollection}]);
await planPriceCopy.toArray();
This will create a copy of the original collection with all of its content.