I'm using Mongoose in a Node.js backend and I need to update a subset of elements of an array within a document based on a condition. I used to perform the operations using save(), like this:
const channel = await Channel.findById(id);
channel.messages.forEach((i) =>
i._id.toString() === messageId && i.views < channel.counter
? i.views++
: null
);
await channel.save();
I'd like to change this code by using findByIdAndUpdate since it is only an increment and for my use case, there isn't the need of retrieving the document. Any suggestion on how I can perform the operation?
Of course, channel.messages is the array under discussion. views and counter are both of type Number.
EDIT - Example document:
{
"_id": {
"$oid": "61546b9c86a9fc19ac643924"
},
"counter": 0,
"name": "#TEST",
"messages": [{
"views": 0,
"_id": {
"$oid": "61546bc386a9fc19ac64392e"
},
"body": "test",
"sentDate": {
"$date": "2021-09-29T13:36:03.092Z"
}
}, {
"views": 0,
"_id": {
"$oid": "61546dc086a9fc19ac643934"
},
"body": "test",
"sentDate": {
"$date": "2021-09-29T13:44:32.382Z"
}
}],
"date": {
"$date": "2021-09-29T13:35:33.011Z"
},
"__v": 2
}
You can try updateOne method if you don't want to retrieve document in result,
match both fields id and messageId conditions
check expression condition, $filter to iterate loop of messages array and check if messageId and views is less than counter then it will return result and $ne condition will check the result should not empty
$inc to increment the views by 1 if query matches using $ positional operator
messageId = mongoose.Types.ObjectId(messageId);
await Channel.updateOne(
{
_id: id,
"messages._id": messageId,
$expr: {
$ne: [
{
$filter: {
input: "$messages",
cond: {
$and: [
{ $eq: ["$$this._id", messageId] },
{ $lt: ["$$this.views", "$counter"] }
]
}
}
},
[]
]
}
},
{ $inc: { "messages.$.views": 1 } }
)
Playground
Related
I have page like this. And the data is from my MongoDB server.
At first, I create User collection and put those user data. But I add the Counter collection because, I want to show their View Count.
Each user has multiple pages, and the View Count data is the sum of the Counter data.
User collection's are look like this and I only need login, id(unique) data.
Users Collection
{
"_id": {
"$oid": "5f7e92b88dc8f64cb4e6c2c0"
},
"login": "mojombo",
"id": 1,
"avatar_url": "https://avatars0.githubusercontent.com/u/1?v=4",
"url": "https://api.github.com/users/mojombo",
"html_url": "https://github.com/mojombo",
"type": "User",
"site_admin": "false",
"name": "Tom Preston-Werner",
"company": null,
"blog": "http://tom.preston-werner.com",
"location": "San Francisco",
"email": "tom#mojombo.com",
"hireable": null,
"bio": null,
"created_at": {
"$date": "2007-10-20T05:24:19.000Z"
},
"updated_at": {
"$date": "2020-09-22T15:50:44.000Z"
},
"registerDate": {
"$date": "2020-10-08T04:16:56.459Z"
},
"__v": 0
}
Counters collection
{
"_id": {
"$oid": "5f8e5bde9054ba2477dc2c57"
},
"repoName": "ale",
"repoNumber": 171780764,
"userName": "technicalpickles",
"userNumber": 159,
"viewDate": "2020-10-20",
"count": 1
}
In Counters collection I only need userName, count field to calculate counter data.
So I try for loop and aggregate method but the problem is User data is too much so I have to send too many queries to server. (If I have 10000 users data I have to send 10000 requests) So, I must not to use for loop method.
router.get(`/sitemap/0`, async (req, res, next) => {
let name;
let dataArray = [];
let pageNumber = (Number(req.params.page) * 10000); // Current Page Number
let nextPage = (Number(req.params.page) + 1) * 10000; // Next Page Number
let pageResult; // Page Result
try {
let users = await User.find({}, 'login id').limit(1000)
for (let i = 0; i < 1000; i++) {
name = users[i].login
let counters = await Counter.aggregate([{
$match: {
id: users.id,
userName: users[i].login
}
},
{
$group: {
_id: `${users[i].login}`,
count: {
$sum: "$count"
}
}
}
])
dataArray.push(counters)
console.log(counters)
}
console.log(dataArray)
} catch (e) {
// throw an error
throw e;
}
res.render("sitemap", {
dataArray
})
});
Codes result
So I want to send single query and I think it should be use 'aggregate' method. But without 'for loop'.
I want to join Users collection and Counters collection but I heard that in MongoDB & Mongoose there are no join function.
I just want to make the result like this. Is there anyway to make it like this?
{
"_id": {
"$oid": "5f7e92b88dc8f64cb4e6c2c0"
},
"login": "mojombo", --------------Match login with counter collection 'userName' value
"id": 1, ------------------------Match id with counter collection 'id' value
"count": [Sum of counters count data and it should be Number] ------- Only this field is added
"avatar_url": "https://avatars0.githubusercontent.com/u/1?v=4",
"url": "https://api.github.com/users/mojombo",
"html_url": "https://github.com/mojombo",
"type": "User",
"site_admin": "false",
"name": "Tom Preston-Werner",
"company": null,
"blog": "http://tom.preston-werner.com",
"location": "San Francisco",
"email": "tom#mojombo.com",
"hireable": null,
"bio": null,
"created_at": {
"$date": "2007-10-20T05:24:19.000Z"
},
"updated_at": {
"$date": "2020-09-22T15:50:44.000Z"
},
"registerDate": {
"$date": "2020-10-08T04:16:56.459Z"
},
"__v": 0
}
$lookup with counters collection using pipeline stage, let to pass 2 fields id, login to match with counters collection
$match both fields conditions
$addFields to count sum of count field from counters array, using $reduce to iterate loop and $add to sum values of count field
$skip for pagination
$limit pass your limit of documents
let page = 1; // start pagination from first
let limit = 1000;
let skip = (page - 1) * limit;
let users = await User.aggregate([
{
$lookup: {
from: "counters",
let: {
id: "$id",
login: "$login"
},
pipeline: [
{
$match: {
$expr: {
$eq: ["$userName", "$$login"],
$eq: ["$userNumber", "$$id"]
}
}
}
],
as: "count"
}
},
{
$addFields: {
count: {
$reduce: {
input: "$count",
initialValue: 0,
in: { $add: ["$$value", "$$this.count"] }
}
}
}
},
{ $skip: skip },
{ $limit: limit }
])
Playground
I have this document in my database:
{
"_id": "ObjectId(...)",
"chapters": [
{
"_id": "ObjectId(...)",
"link": "128371.html",
"content": ""
}
]
}
The chapters array can have up to 3k items, and I have to populate each content attribute with some info. I want to be able to save the info I want inside the right object. Until now I was able to change the content attribute generally (in all items), but I am having trouble filtering it. This is what I managed to code using what I found in other questions:
let content = "Testing";
await models.ListNovel.updateOne(
{ link: novel_link },
{ $set: { "chapters.$[].content": content } }
);
I saw that { arrayFilters: [{ link: { $eq: chapter_link } }], multi: false } may work in some cases, but I don't use the link identifier in the update.
Thank you!
UPDATE
Similar to Suleyman's solution, I ended up with the following working code, I hope it may be useful for you.
await models.ListNovel.updateOne(
{ link: novel.link },
{ $set: { "chapters.$[elem].content": content } },
{
multi: true,
arrayFilters: [{ "elem.link": { $eq: chapter.link } }]
}
);
The condition in updateOne must match parent object, but you are using { link: novel_link } which belongs to the inner array object field, so it cannot find the document, and update doesn't happen.
To illustrate this, let's say your schema is like this:
const mongoose = require("mongoose");
const schema = new mongoose.Schema({
name: String,
chapters: [
new mongoose.Schema({
link: String,
content: String
})
]
});
module.exports = mongoose.model("ListNovel", schema);
Let's have this existing document in this collection:
{
"_id": "5e498a1fe21eea0e10690e39",
"name": "Novel1",
"chapters": [
{
"_id": "5e498a1fe21eea0e10690e3b",
"link": "128371.html",
"content": ""
},
{
"_id": "5e498a1fe21eea0e10690e3a",
"link": "222222.html",
"content": ""
}
],
"__v": 0
}
If we want to update this document's chapter with "link": "128371.html", first we need to find it with name or _id field, and update it using the filtered positional operator $.
router.put("/novels/:name", async (req, res) => {
const novel_link = "128371.html";
const content = "Testing";
const result = await ListNovel.findOneAndUpdate(
{ name: req.params.name },
{
$set: { "chapters.$[chapter].content": content }
},
{
arrayFilters: [{ "chapter.link": novel_link }],
new: true
}
);
res.send(result);
});
Here I used findOneAndUpdate to immediately retrieve the updated document, but you can also use the updateOne instead of findOneAndUpdate.
The result will be like this:
{
"_id": "5e498a1fe21eea0e10690e39",
"name": "Novel1",
"chapters": [
{
"_id": "5e498a1fe21eea0e10690e3b",
"link": "128371.html",
"content": "Testing" // => UPDATED
},
{
"_id": "5e498a1fe21eea0e10690e3a",
"link": "222222.html",
"content": ""
}
],
"__v": 0
}
Is it possible to have facet to return as an object instead of an array? It seems a bit counter intuitive to need to access result[0].total instead of just result.total
code (using mongoose):
Model
.aggregate()
.match({
"name": { "$regex": name },
"user_id": ObjectId(req.session.user.id),
"_id": { "$nin": except }
})
.facet({
"results": [
{ "$skip": start },
{ "$limit": finish },
{
"$project": {
"map_levels": 0,
"template": 0
}
}
],
"total": [
{ "$count": "total" },
]
})
.exec()
Each field you get using $facet represents separate aggregation pipeline and that's why you always get an array. You can use $addFields to overwrite existing total with single element. To get that first item you can use $arrayElemAt
Model
.aggregate()
.match({
"name": { "$regex": name },
"user_id": ObjectId(req.session.user.id),
"_id": { "$nin": except }
})
.facet({
"results": [
{ "$skip": start },
{ "$limit": finish },
{
"$project": {
"map_levels": 0,
"template": 0
}
}
],
"total": [
{ "$count": "total" },
]
})
.addFields({
"total": {
$arrayElemAt: [ "$total", 0 ]
}
})
.exec()
You can try this as well
Model
.aggregate()
.match({
"name": { "$regex": name },
"user_id": ObjectId(req.session.user.id),
"_id": { "$nin": except }
})
.facet({
"results": [
{ "$skip": start },
{ "$limit": finish },
{
"$project": {
"map_levels": 0,
"template": 0
}
}
],
"total": [
{ "$count": "total" },
]
})
.addFields({
"total": {
"$ifNull": [{ "$arrayElemAt": [ "$total.total", 0 ] }, 0]
}
})
.exec()
imagine that you want to pass the result of $facet to the next stage, let's say $match. well $match accepts an array of documents as input and return an array of documents that matched an expression, if the output of $facet was just an element we can't pass its output to $match because the type of output of $facet is not the same as the type of input of $match ($match is just an example). In my opinion it's better to keep the output of $facet as array to avoid handling those types of situations.
PS : nothing official in what i said
I have the following schema:
{ "_id": {
"$oid": "58c0204d9f10810115f13e5d"
},"OrgName": "A",
"modules": [
{
"name": "test",
"fullName": "john smith",
"_id": {
"$oid": "58c0204d9f10810115f13e5e"
},
"TimeSavedPlanning": 520,
"TimeSavedWorking": 1000,
"costSaved": 0
},
{
"name": "test1",
"fullName": "john smith",
"_id": {
"$oid": "58c020f85437c22215be92cc"
},
"TimeSavedPlanning": 0,
"TimeSavedWorking": 1000,
"costSaved": 500
}
]
}
I want to aggregate the data within the "modules" array for all documents where OrgName = A and outputs the following totals.
TimeSavedPlanning = 520 (because 520 + 0 = 520)
TimeSavedWorking = 2000 (because 1000 + 1000 = 2000)
costSaved = 500 (because 0 + 500)
Just supply each field for the $group accumulators. And use the "double barreled" $sum to "sum" both from arrays, and from documents:
Model.aggregate([
{ "$match": { "OrgName": "A" } },
{ "$group": {
"_id": null,
"TimeSavedPlanning": { "$sum": { "$sum":"$modules.TimeSavedPlanning" } },
"TimeSavedWorking": { "$sum": { "$sum": "$modules.TimeSavedWorking" } },
"costSaved": { "$sum": { "$sum": { "$modules.costSaved" } }
}}
])
You have been allowed to use $sum like that since MongoDB 3.2. Since that release it has "two" functions:
Takes an "array" of values and "sums" them together.
Acts and an "accumulator" within $group to "sum" values provided from documents.
So here you use "both" functions by "reducing" the arrays down to numeric values per document, and then "accumulating" via the $group.
Of course the $match does the "selection" right at the beginning of the operation chain. Since that determines the selection of data, and you put that there for that purpose, as well as the fact you can use an "index" from that "first" stage.
My data looks like this:
{
"foo_list": [
{
"id": "98aa4987-d812-4aba-ac20-92d1079f87b2",
"name": "Foo 1",
"slug": "foo-1"
},
{
"id": "98aa4987-d812-4aba-ac20-92d1079f87b2",
"name": "Foo 1",
"slug": "foo-1"
},
{
"id": "157569ec-abab-4bfb-b732-55e9c8f4a57d",
"name": "Foo 3",
"slug": "foo-3"
}
]
}
Where foo_list is a field in a model called Bar. Notice that the first and second objects in the array are complete duplicates.
Aside from the obvious solution of switching to PostgresSQL, what MongoDB query can I run to remove duplicate entries from foo_list?
Similar answers that do not quite cut it:
https://stackoverflow.com/a/16907596/432
https://stackoverflow.com/a/18804460/432
These questions answer the question if the array had bare strings in it. However in my situation the array is filled with objects.
I hope it is clear that I am not interested querying the database; I want the duplicates to be gone from the database forever.
Purely from an aggregation framework point of view there are a few approaches to this.
You can either just apply $setUnion in modern releases:
db.collection.aggregate([
{ "$project": {
"foo_list": { "$setUnion": [ "$foo_list", "$foo_list" ] }
}}
])
Or more traditionally with $unwind and $addToSet:
db.collection.aggregate([
{ "$unwind": "$foo_list" },
{ "$group": {
"_id": "$_id",
"foo_list": { "$addToSet": "$foo_list" }
}}
])
Or if you were just interested in the duplicates only then by general grouping:
db.collection.aggregate([
{ "$unwind": "$foo_list" },
{ "$group": {
"_id": {
"_id": "$_id",
"foo_list": "$foo_list"
},
"count": { "$sum": 1 }
}},
{ "$match": { "count": { "$ne": 1 } } },
{ "$group": {
"_id": "$_id._id",
"foo_list": { "$push": "$_id.foo_list" }
}}
])
The last form could be useful to you if you actually want to "remove" the duplicates from your data with another update statement as it identifies the elements which are duplicates.
So in that last form the returned result from your sample data identifies the duplicate:
{
"_id" : ObjectId("53f5f7314ffa9b02cf01c076"),
"foo_list" : [
{
"id" : "98aa4987-d812-4aba-ac20-92d1079f87b2",
"name" : "Foo 1",
"slug" : "foo-1"
}
]
}
Where results are returned from your collection per document that contains duplicate entries in the array and which entries are duplicated. This is the information you need to update, and you loop the results as you need to specify the update information from the results in order to remove duplicates.
This is actually done with two update statements per document, as a simple $pull operation would remove "both" items, which is not what you want:
var cursor = db.collection.aggregate([
{ "$unwind": "$foo_list" },
{ "$group": {
"_id": {
"_id": "$_id",
"foo_list": "$foo_list"
},
"count": { "$sum": 1 }
}},
{ "$match": { "count": { "$ne": 1 } } },
{ "$group": {
"_id": "$_id._id",
"foo_list": { "$push": "$_id.foo_list" }
}}
])
var batch = db.collection.initializeOrderedBulkOp();
var count = 0;
cursor.forEach(function(doc) {
doc.foo_list.forEach(function(dup) {
batch.find({ "_id": doc._id, "foo_list": { "$elemMatch": dup } }).updateOne({
"$unset": { "foo_list.$": "" }
});
batch.find({ "_id": doc._id }).updateOne({
"$pull": { "foo_list": null }
});
});
count++;
if ( count % 500 == 0 ) {
batch.execute();
batch = db.collection.initializeOrderedBulkOp();
}
});
if ( count % 500 != 0 ) {
batch.execute();
}
That's the modern MongoDB 2.6 and above way to do it, with a cursor result from aggregation and Bulk operations for updates. But the principles remain the same:
Identify the duplicates in documents
Loop the results to issue the updates to the affected documents
Use $unset with the positional $ operator to set the "first" matched array element to null
Use $pull to remove the null entry from the array
So after processing the above operations your sample now looks like this:
{
"_id" : ObjectId("53f5f7314ffa9b02cf01c076"),
"foo_list" : [
{
"id" : "98aa4987-d812-4aba-ac20-92d1079f87b2",
"name" : "Foo 1",
"slug" : "foo-1"
},
{
"id" : "157569ec-abab-4bfb-b732-55e9c8f4a57d",
"name" : "Foo 3",
"slug" : "foo-3"
}
]
}
The duplicate is removed with the "duplicated" item still intact. That is how you process to identify and remove the duplicate data from your collection.