Develop Reference

Override Mongoose save method to retry on `duplicate key error`

My Mongoose schema uses a custom _id value and the code I inherited does something like this
const sampleSchema = new mongoose.Schema({
_id: String,
key: String,
});
sampleSchema.statics.generateId = async function() {
let id;
do {
id = randomStringGenerator.generate({length: 8, charset: 'hex', capitalization: 'uppercase'});
} while (await this.exists({_id: id}));
return id;
};
let SampleModel = mongoose.model('Sample', sampleSchema);
A simple usage looks like this:
let mySample = new SampleModel({_id: await SampleModel.generateId(), key: 'a' });
await mySample.save();
There are at least three problems with this:
Every save will require at least two trips to the database, one to test for a unique id and one to save the document.
For this to work, it is necessary to manually call generateId() before each save. An ideal solution would handle that for me, like Mongoose does with ids of type ObjectId.
Most significantly, there is a potential race condition that will result in duplicate key error. Consider two clients running this code. Both coincidentally generate the same id at the same time, both look in the database and find the id absent, both try to write the record to the database. The second will fail.
An ideal solution would, on save, generate an id, save it to the database and on duplicate key error, generate a new id and retry. Do this in a loop until the document is stored successfully.
The trouble is, I don't know how to get Mongoose to let me do this.
Here's what I tried: Based on this SO Question, I found a rather old sample (using a very old mongoose version) of overriding the save function to accomplish something similar and based this attempt off it.
// First, change generateId() to force a collision
let ids = ['a', 'a', 'a', 'b'];
let index = 0;
let generateId = function() {
return ids[index++];
};
// Configure middleware to generate the id before a save
sampleSchema.pre('validate', function(next) {
if (this.isNew)
this._id = generateId();
next();
});
// Now override the save function
SampleModel.prototype.save_original = SampleModel.prototype.save;
SampleModel.prototype.save = function(options, callback) {
let self = this;
let retryOnDuplicate = function(err, savedDoc) {
if (err) {
if (err.code === 11000 && err.name === 'MongoError') {
self.save(options, retryOnDuplicate);
return;
}
}
if (callback) {
callback(err, savedDoc);
}
};
return self.save_original(options, retryOnDuplicate);
}
This gets me close but I'm leaking a promise and I'm not sure where.
let sampleA = new SampleModel({key: 'a'});
let sampleADoc = await sampleA.save();
console.log('sampleADoc', sampleADoc); // prints undefined, but should print the document
let sampleB = new SampleModel({key: 'b'});
let sampleBDoc = await sampleB.save();
console.log('sampleBDoc', sampleBDoc); // prints undefined, but should print the document
let all = await SampleModel.find();
console.log('all', all); // prints `[]`, but should be an array of two documents
Output
sampleADoc undefined
sampleBDoc undefined
all []
The documents eventually get written to the database, but not before the console.log calls are made.
Where am I leaking a promise? Is there an easier way to do this that addresses the three problems I outlined?
Edit 1:
Mongoose version: 5.11.15

I fixed the problem by changing the save override. The full solution looks like this:
const sampleSchema = new mongoose.Schema({
_id: String,
color: String,
});
let generateId = function() {
return randomStringGenerator.generate({length: 8, charset: 'hex', capitalization: 'uppercase'});
};
sampleSchema.pre('validate', function() {
if (this.isNew)
this._id = generateId();
});
let SampleModel = mongoose.model('Sample', sampleSchema);
SampleModel.prototype.save_original = SampleModel.prototype.save;
SampleModel.prototype.save = function(options, callback) {
let self = this;
let isDupKeyError = (error, field) => {
// Determine whether the error is a duplicate key error on the given field
return error?.code === 11000 && error?.name === 'MongoError' && error?.keyValue[field];
}
let saveWithRetries = (options, callback) => {
// save() returns undefined if used with callback or a Promise otherwise.
// https://mongoosejs.com/docs/api/document.html#document_Document-save
let promise = self.save_original(options, callback);
if (promise) {
return promise.catch((error) => {
if (isDupKeyError(error, '_id')) {
return saveWithRetries(options, callback);
}
throw error;
});
}
};
let retryCallback;
if (callback) {
retryCallback = (error, saved, rows) => {
if (isDupKeyError(error, '_id')) {
saveWithRetries(options, retryCallback);
} else {
callback(error, saved, rows);
}
}
}
return saveWithRetries(options, retryCallback);
}
This will generate an _id repeatedly until a successful save is called and addresses the three problems outlined in the original question:
The minimum trips to the database has been reduced from two to one. Of course, if there are collisions, more trips will occur but that's the exceptional case.
This implementation takes care of generating the id itself with no manual step to take before saving. This reduces complexity and removes the required knowledge of prerequisites for saving that are present in the original method.
The race condition has been addressed. It won't matter if two clients attempt to use the same key. One will succeed and the other will generate a new key and save again.
To improve this:
There ought to be a maximum number of save attempts for a single document followed by failure. In this case, you've perhaps used up all the available keys in whatever domain you're using.
The unique field may not be named _id or you might have multiple fields that require a unique generated value. The embedded helper function isDupKeyError() could be updated to look for multiple keys. Then on error you could add logic to regenerate just the failed key.

Make old links backwards compatible in express

You have an application created in express and angular that allows the user to perform a search. The URL is built based upon the search that was just performed. So if you perform a search on “Will” the url looks like http://localhost.com:9000/search/query?q=Will Everything works fine but you forgot that the app previously performed searches without the /query?= and now all of your old links like http://localhost.com:9000/search/will or http://localhost.com:9000/search/roberto no longer work.
What would be the correct approach to get the old links working again?
Should you use JavaScript on the frontend to look for /query?= missing in the URL and add after the search path but before the queried text?

It'd be easier to do a redirect on the Express back-end.
Say your code for the /search/query path is initially like this :
app.get("/search/query", function (req, res) {
// Do your query validation and fetch your search result.
// Here, I just check if a query value was given or not for the q param.
// I recommend you use better ways to check for empty queries.
// (ex: lodash's `isEmpty()` function)
if (req.query.q) {
// Serve the page !
res.send("What you want to render if the search result finds something.");
}
else {
// Return an error !
res.status(404).send("Nothing was found with the criterias entered.");
}
});
This is probably similar to what you have. Now, here is the answer to your question, based on the initial implementation above :
app.get("/search/query", function (req, res, next) {
// Check if a query value was given AND if the value isn't equal to "query".
// The later condition is to prevent infinite loops.
if (req.query.q && req.query.q !== "query") {
// Redirect using the value assigned to the q query param.
res.redirect("/search/" + req.query.q);
}
else {
// Since there is no query parameter named `q` in the request,
// we can be sure that `query` reffers to a search term.
next();
}
});
app.param("srchterm", function (req, res, next, value) {
// Check, for example, if the value isn't empty.
if (value) {
// Do your query validation and fetch your search result HERE.
// Add those results in an array in the res.locals object.
// Those results can be used later.
res.locals.results = ["all", "your", "search", "results"];
}
next();
});
app.get("/search/:srchterm", function (req, res) {
console.log("another blah");
// We don't need to fetch the data here anymore, since it's handled by the param parser above!
// However, it's still necessary to check if the search gave back some results.
if (res.locals.results) {
// Serve the results !
res.send("A total of " + res.locals.results.length + " results were found for " + req.params['srchterm']);
}
else {
// Return an error !
res.status(404).send("Nothing was found with the criterias entered.");
}
});
So from now on, every query using /search/query?q=123 will redirect towards /search/123. It even lets you use query as the search term!

Just use a regex and redirect
app.use(function(req, res, next) {
var searchRegEx = /\/search/g;
var searchedTerm = req.originalUrl.replace(searchRegEx, '');
var queryPath = req.originalUrl.match(/\/query[?]q=/);
if(!queryPath) {
var regexSlash = /\//g;
res.redirect('query?q=' + searchedTerm.replace(regexSlash, ''));
}
else {
next();
}
});

How use next() properly in routes (node.js)

var s_bookingController = require('s/controllers);
app.get('/dashboard/:page/:param', s_bookingController.index)
app.get('/dashboard/show/:id', s_bookingController.show);
Controllers:
exports.index = function(req, res, next) {
var page = parseInt(req.param("id"));
data = {};
data.page = page;
data.nextPage = page + 1;
data.prevPage = page - 1;
MyModel.find().sort('brand').skip((page-1)*11).limit(11).exec(function(err, result) {
res.render('index', {
data: data,
booking: result,
});
});
};
And
exports.show = function(req, res, next) {
var id = req.param("id");
res.send(id);
};
I'm using this controllers, but there is something wrong with the code of the exports.index, because it's stuck in the code.
If I change the routes to:
app.get('/dashboard/:page', s_bookingController.index)
(Note that I'm take off the second parameter that I was passing)
the show will work, but if I use the second parameter, the show will not run, it will be stuck in the index page.
Why is this? I was wondering if I need use the next();.

Expanding my comment:
You should have the following order of the routes:
app.get('/dashboard/show/:id', s_bookingController.show);
app.get('/dashboard/:page/:param', s_bookingController.index);
Express routing requires that a more specific route should be placed above the more general one.
The /dashboard/show/:id is more specific in this case as /dashboard/:page/:param covers it, so that /dashboard/show is handled by it. When the route is handled next routes are not executed.

In Node.js, a loop calling a function defined in another JS file is making mistakes

I have a server (made with Express in Node.js) that gets notifications of RSS feeds, gets data from their entries (title, date, link) and then "does something" with the data by calling a function defined in another JS file ("article_filter_toDB.js"). The code on the server-side is:
// parts omitted
var article_filter_toDB = require('./article_filter_toDB.js');
// parts omitted
client.on('notification', function (notification) {
// gets notifications of RSS feeds
entries = notification.entries;
for (index = 0; index < entries.length; ++index) {
title = entries[index].title;
date = entries[index].published;
link = entries[index].link.href;
// gets data from the entry of the feed
miniwords = 1000;
// a variable that I set
article_filter_toDB(link, title, miniwords);
// "does something" by calling a function defined in another JS file ("article_filter_toDB.js")
}
});
// parts omitted
What the function "article_filter_toDB" does is to get the content of the article given by the link from the RSS feed (using Request), parsing HTML code to count the words of the article, and, if this length is above "miniwords" (here 1000), save the data relative to the article (title, link, date...) to a database (MongoDB, via Mongoose).
Sometimes it works well. But sometimes it computes a length equal to 1 (that is, it was unable to really count the words) although, if I run the function "article_filter_toDB" separately (that is, the separate JS file, applied to same "link", "title", "miniwords" that I copy to it), it is able to correctly count the words.
Do you know what I'm doing wrong? Thanks!
To be more complete, here is the code of the "article_filter_toDB.js" file:
// parts omitted
article_filter_toDB = function (link, title, miniwords) {
Article.findOne({
title: title
}, 'title', function (err, articles) {
if (err) return console.error(err);
if (articles == null) {
// ...if an article with this title is not already present in my database...
// parts omitted here, that set the variable "balise" depending on the link
request(link, function (err, resp, body) {
$ = cheerio.load(body);
texte = $(balise).text();
content = texte.split(" ");
length = content.length;
// ...let's count its words with Request and Cheerio...
if ((length > miniwords)) {
var newArticle = new Article({
site: url.parse(link).hostname.replace(/^www\./, ''),
date: date,
link: link,
title: title,
length: length,
});
newArticle.save(function (err, newArticle) {
if (err) return console.error(err)
});
// if the article's length is more than the number given by "miniwords", let's save its data in my database
}
});
}
});
}
module.exports = article_filter_toDB;
// exportation of the function to use it elsewhere

This how you call functions from another file properly in node.js
// otherfile.js
// ========
module.exports = {
article_filter_toDB: function (link, title, miniwords) {
// do stuff here
},
};
Then on your code:
var otherfile = require('./otherfile');
...
otherfile.article_filter_toDB(link, title, miniwords);

Partial update while supporting seo

Using NodeJs I'm trying to do something quite similar to Meteor: I want to send only the parts of a page that actually changed. My dilemma is that I know how to create such a framework to respond to link clicks and send partial updates but such a framework won't cater to direct browsing to a page other than the index (which is what is required for search engines and people without javascript to use your site).
I can also figure out how to make a framework to support entire page reloads, handlebars and a simple node server instance would take care of that. Hoeever, I can't figure out how to create a way that would allow me to write one method to tell the framework the partial updates for a page and let the framework figure out what else needs to be loaded.
A way I can think of would be to create the index page every time (for entire page loads) and apply partial updates to that but that can quickly become expensive if a subpage differs a lot from a very crowded index.
An example method would look something like this:
function images(id) {
if (id == null) {
// load all images from database
template.images = db.result();
template.content = template.loadblock('gallery');
}
else {
// retrieve single image
template.content = template.loadblock('single_image');
template.image = db.result();
}
}
On a partisl updste calling this method for domain.com/images would work just fine because it's clear what had changed.
For an entire page load this function would miss things like a header, footer ,navigation, etc.
In an answer I would look for an example where this has been done or some tips that Can point me in the right direction. I'm sorry for any typoes I wrote this post on an ipad. If you have any questions about my question just ask and I'll update as needed.
Update:
A possible example of a solution might be the following code. It's to give an idea, it probably won't actually run
// As a convention, don't pass around raw values if they aren't static but pass around functions such as
data.images = function () {
// run db query
// return an object with the images
}
// This constraint might be limited to the index() method
var routes = {
// This now allows us to do things like this:
index: function() {
var data;
// Initialise everything needed for the index
data.title = 'Index';
data.nav = { Index: '/', Images: '/images' };
data.content = 'Hello World';
},
categories: function() {
var data;
data.content = render('gallery', function () { /* load and return images as object */ }); // Not sure about this dynamic subtemplating but oh well
}
// This now allows us to do the following:
function request(page, type) {
if (type == 'update') {
if (routes[page] != undefined && typeof routes[page] == 'function') {
respond(routes[page]());
}
}
else {
if (routes[page] != undefined && typeof routes[page] == 'function') {
var data = mergeArrays(routes['index'](), routes[page]());
// index.html which is just a Handlebars template
respond(data);
}
}
}

Here is a pattern I often use (in Express apps):
function respond(req, res, name, resource) {
if(req.accepts('json')) {
// Send JSON to clients who want it
res.send(resource);
} else {
// Render with layout only for non-XHR requests
resource.layout = !req.xhr;
res.render('resources/' + name, resource);
}
}
Example usage:
app.get('/images', function(req, res, next) {
getImages(function(err, images) {
if(err) return next(err);
respond(req, res, 'images', images);
});
});
app.get('/images/:id', function(req, res, next) {
getImage(req.params.id, function(err, image) {
if(err) return next(err);
respond(req, res, 'image', image);
});
});
image.jade:
img(src=uri, alt=title)
images.jade:
#gallery
for image in images
include image
Clients who ask for JSON get that, otherwise they get the full page only if it's a non-XHR request. XHR requests get just the HTML snippet for the requested resource. This works well for quite simple apps, where resources mostly correspond to pages.