I have two data structures "database" and "components"
const DatabaseSchema = mongoose.Schema({
components: [{ type: Schema.Types.ObjectId, ref: 'Components', required: false }],
});
const ComponentsSchema = mongoose.Schema({
name: { type: String, required: true, trim: true, unique: true, lowercase: true },
updatedAt: Date,
});
I want to filter all items in the database by component names
search rule I'm using
Database.find({
components: { $elemMatch: { name: /antr/i } }
}).populate({
path: 'components',
select: 'name -_id'
}).select(['descript','components']).exec( (err,data) => {
console.log(err);
res.json(data);
});
however always return an empty element
Please try this :
As I've already suggested you can use this :
Database.find({})
.populate({ path: 'components', match: { name: /antr/i }, select: 'name -_id' })
.exec((err, data) => { console.log(err); res.json(data); });
Since you're seeing empty array's is because of the filter query in match which doesn't find appropriate documents in components collection w.r.t. ObjectIds in components array of database document, this is normal. May be you can filter those out in code, as you aren't looking in that way, You can use mongoDB's $lookup from aggregation framework which is equivalent to .populate() from mongoose.
Database.aggregate(
[{
$lookup: {
from: "components",
"let": { "ids": "$components" },
pipeline: [
{ $match: { $expr: { $in: ['$_id', '$$ids'] } } }],
as: "dbComponentsArray"
}
}, { $unwind: '$dbComponentsArray' }, { $match: { 'dbComponentsArray.name': /antr/i } },
{ $group: { _id: '$_id', dbComponentsArray: { $push: '$dbComponentsArray' }, data: { $first: '$$ROOT' } } }, { $addFields: { 'data.dbComponentsArray': '$dbComponentsArray' } },
{ $replaceRoot: { 'newRoot': '$data' } }])
Sample Data in collections :
components :
/* 1 */
{
"_id" : ObjectId("5d481cd098ba991c0857959f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
/* 2 */
{
"_id" : ObjectId("5d481cd098ba991c0857958f"),
"name" : "anacito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
/* 3 */
{
"_id" : ObjectId("5d481cd098ba991c0857951f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
/* 4 */
{
"_id" : ObjectId("5d481cd098ba991c0857952f"),
"name" : "anacito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
database :
/* 1 */
{
"_id" : ObjectId("5d4979d52a17d10a6c8de81b"),
"components" : [
ObjectId("5d481cd098ba991c0857951f"),
ObjectId("5d481cd098ba991c0857952f"),
ObjectId("5d481cd098ba991c0857953f"),
ObjectId("5d481cd098ba991c0857959f")
]
}
Output :
/* 1 */
{
"_id" : ObjectId("5d4979d52a17d10a6c8de81b"),
"components" : [
ObjectId("5d481cd098ba991c0857951f"),
ObjectId("5d481cd098ba991c0857952f"),
ObjectId("5d481cd098ba991c0857953f"),
ObjectId("5d481cd098ba991c0857959f")
],
"dbComponentsArray" : [
{
"_id" : ObjectId("5d481cd098ba991c0857959f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
},
{
"_id" : ObjectId("5d481cd098ba991c0857951f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
]
}
Related
I am trying to implement Pagination and Filtering at the backend.
The input to this controller is Page number and Filtering conditions.
Controller:-
const getPosts = asyncHandler(async (req, res) => {
const {
page,
statusFilter,
typeFilter,
sourceFilter,
} = JSON.parse(req.query.filterData);
var query = [
{
$addFields: {
paramType: typeFilter,
paramSource: sourceFilter,
paramStatus: statusFilter,
},
},
{
$match: {
$expr: {
$and: [
{ user: req.user.id },
{
$or: [
{
$eq: ["$paramType", "All"],
},
{
$eq: ["$paramType", "$type"],
},
],
},
{
$or: [
{
$eq: ["$paramSource", "All"],
},
{
$eq: ["$paramSource", "$source"],
},
],
},
{
$or: [
{
$eq: ["$paramStatus", "All"],
},
{
$eq: ["$paramStatus", "$status"],
},
],
},
],
},
},
},
{
$project: {
paramSource: false,
paramType: false,
paramStatus: false,
},
},
];
//pagination
const PAGE_SIZE = 5;
const PAGE = parseInt(page) || 0;
// aggregate query
const aggregateQuery = await Post.aggregate([query]);
const total = aggregateQuery.length;
const Allposts = await Post.aggregate([query])
.limit(PAGE_SIZE)
.skip(PAGE_SIZE * PAGE)
.sort({ createdAt: -1 });
const totalPages = Math.ceil(total / PAGE_SIZE);
res.status(200).json({ totalPages, Allposts });
});
Problem:-
The pagination and filtering part works as intended but only for the first page, when I go to second page the Allposts object is empty.
Why is the Allposts object empty after first page?
Edit:-
Sample Data:-
{
"_id" : 1,
"type" : "Type A",
"source" : "Source A",
"status" : "Status A",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
},
{
"_id" : 2,
"type" : "Type B",
"source" : "Source C",
"status" : "Status B",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
},
{
"_id" : 3,
"type" : "Type A",
"source" : "Source A",
"status" : "Status A",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
},
{
"_id" : 4,
"type" : "Type A",
"source" : "Source C",
"status" : "Status B",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
}
Updated aggreation query:-
var query = [
{
$addFields: {
paramType: typeFilter,
paramSource: sourceFilter,
paramStatus: statusFilter,
},
},
{
$match: {
$expr: {
$and: [
{ user: req.user.id },
{
$or: [
{
$eq: ["$paramType", "All"],
},
{
$eq: ["$paramType", "$type"],
},
],
},
{
$or: [
{
$eq: ["$paramSource", "All"],
},
{
$eq: ["$paramSource", "$source"],
},
],
},
{
$or: [
{
$eq: ["$paramStatus", "All"],
},
{
$eq: ["$paramStatus", "$status"],
},
],
},
],
},
},
},{ $sort : { createdAt : -1 } },
{
$project: {
paramSource: false,
paramType: false,
paramStatus: false,
createdAt : 1,
},
}
];
If you want pagination + total count with one query, you can do something like this:
db.collection.aggregate([
{
$addFields: {
paramType: "typeFilter",
paramSource: "sourceFilter",
paramStatus: "statusFilter"
}
},
{
"$match": {
// complete here
}
},
{
$setWindowFields: {
output: {totalCount: {$count: {}}}}
},
{$sort: {createdAt: -1}},
{$skip: PAGE_SIZE * PAGE},
{$limit: PAGE_SIZE},
{
$facet: {
results: [
{
$project: {
// here put whatever you want to send to FE
type: 1,
source: 1,
}
}
],
totalCount: [
{$limit: 1},
{$project: {totalCount: 1, _id: 0}}
]
}
}
])
As you can see on the playground
The $setWindowFields allows you to add the total count to all documents. $sort, $skip and $limit allow the pagination. The $facet allows you to get different outputs from the same documents.
I have the following documents and here I want to sort them by the fields 'ranks.rank' within a restricted range.
How to do this kind of sorting ? 'ranks.date': { '$gte': 20200516 }
I have tried something like
{ $match: selector },
{
$project: {
views: 1,
'ranks': {
$cond: {
if: { $gte: ["$ranks.date", 20200516] },
then: "$ranks",
else: "$$REMOVE"
}
},
}
},
{ $addFields: { totalRank: { $sum: '$ranks.rank' } } },
{ $sort: { 'totalRank': 1 } }
Documents
{
"_id" : "Qvpbjpjqexko4XFGH",
"views" : NumberInt(15),
"ranks" : [
{
"date" : NumberInt(20200415),
"rank" : NumberInt(1)
},
{
"date" : NumberInt(20200418),
"rank" : NumberInt(13)
},
{
"date" : NumberInt(20200503),
"rank" : NumberInt(1)
}
]{
"_id" : "bLQKR39qmJcwuzm8r",
"views" : NumberInt(16),
"ranks" : [
{
"date" : NumberInt(20200415),
"rank" : NumberInt(1)
},
{
"date" : NumberInt(20200418),
"rank" : NumberInt(12)
},
{
"date" : NumberInt(20200501),
"rank" : NumberInt(2)
},
{
"date" : NumberInt(20200521),
"rank" : NumberInt(1)
}
]
It looks like your post is mostly code; please add some more details.
I think i figured out
{ $match: selector },
{
$project: {
views: 1,
ranks: {
$filter: {
input: "$ranks",
as: "rank",
cond: { $gte: ["$$rank.date", monAgo] }
}
},
},
},
{ $addFields: { totalRank: { $sum: '$ranks.rank' } } },
{ $sort: { 'totalRank': -1 } }
My mongodb data is like this,i want to filter the memoryLine.
{
"_id" : ObjectId("5e36950f65fae21293937594"),
"userId" : "5e33ee0b4a3895a6d246f3ee",
"notes" : [
{
"noteId" : ObjectId("5e36953665fae212939375a0"),
"time" : ISODate("2020-02-02T17:24:06.460Z"),
"memoryLine" : [
{
"_id" : ObjectId("5e36953665fae212939375ab"),
"memoryTime" : ISODate("2020-02-03T17:54:06.460Z")
},
{
"_id" : ObjectId("5e36953665fae212939375aa"),
"memoryTime" : ISODate("2020-02-03T05:24:06.460Z")
}
]
}
]}
i want to get the item which memoryTime is great than now as expected like this,
"userId" : "5e33ee0b4a3895a6d246f3ee",
"notes" : [
{
"noteId" : ObjectId("5e36953665fae212939375a0"),
"time" : ISODate("2020-02-02T17:24:06.460Z"),
"memoryLine" : [
{
"_id" : ObjectId("5e36953665fae212939375ab"),
"memoryTime" : ISODate("2020-02-03T17:54:06.460Z")
},
{
"_id" : ObjectId("5e36953665fae212939375aa"),
"memoryTime" : ISODate("2020-02-03T05:24:06.460Z")
}
]
}]
so is use code as below.i use a $filter in memoryLine to filter to get the right item.
aggregate([{
$match: {
"$and": [
{ userId: "5e33ee0b4a3895a6d246f3ee"},
]
}
}, {
$project: {
userId: 1,
notes: {
noteId: 1,
time: 1,
memoryLine: {
$filter: {
input: "$memoryLine",
as: "mLine",
cond: { $gt: ["$$mLine.memoryTime", new Date(new Date().getTime() + 8 * 1000 * 3600)] }
}
}
}
}
}]).then(doc => {
res.json({
code: 200,
message: 'success',
result: doc
})
});
but i got this,memoryLine is null,why?I try to change $gt to $lt, but also got null.
"userId" : "5e33ee0b4a3895a6d246f3ee",
"notes" : [
{
"noteId" : ObjectId("5e36953665fae212939375a0"),
"time" : ISODate("2020-02-02T17:24:06.460Z"),
"memoryLine" : null <<<------------- here is not right
}]
You can use $addFields to replace existing field, $map for outer collection and $filter for inner:
db.collection.aggregate([
{
$addFields: {
notes: {
$map: {
input: "$notes",
in: {
$mergeObjects: [
"$$this",
{
memoryLine: {
$filter: {
input: "$$this.memoryLine",
as: "ml",
cond: {
$gt: [ "$$ml.memoryTime", new Date() ]
}
}
}
}
]
}
}
}
}
}
])
$mergeObjects is used to avoid repeating fields from source memoryLine object.
Mongo Playground
I have the following Product Schema: (partial, using mongoose)
attributes: [
{
set: {
ref: 'AttributeSet',
type: Schema.Types.ObjectId
},
items: [
{
attribute: {
ref: 'Attributes',
type: Schema.Types.ObjectId
},
values: [
{
ref: 'AttributeValues',
type: Schema.Types.ObjectId
}
]
}
],
_id: 0
}
],
example document 1: (partial)
"attributes" : [
{
"set" : ObjectId("5ccc079c846de44116182890"),
"items" : [
{
"values" : [
ObjectId("5ccc0900846de441161828a1")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c4"),
"attribute" : ObjectId("5ccc0900846de441161828a0")
},
{
"values" : [
ObjectId("5ccc0a51846de441161828cc")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c3"),
"attribute" : ObjectId("5ccc0a51846de441161828cb")
},
{
"values" : [
ObjectId("5ccc0c7d846de44116182906")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c2"),
"attribute" : ObjectId("5ccc0c7d846de44116182904")
},
{
"values" : [
ObjectId("5ccc0d64846de44116182911")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c1"),
"attribute" : ObjectId("5ccc0d64846de4411618290f")
},
{
"values" : [
ObjectId("5ccc079f846de44116182892")
],
"_id" : ObjectId("5def6acf66910405e07e1e9f"),
"attribute" : ObjectId("5ccc079f846de44116182891")
}
]
}
]
example document 2: (partial)
"attributes" : [
{
"set" : ObjectId("5ccc079c846de44116182890"),
"items" : [
{
"values" : [
ObjectId("5ccc079f846de44116182892")
],
"_id" : ObjectId("5dee635c72fa520f53d0d1c0"),
"attribute" : ObjectId("5ccc079f846de44116182891")
},
{
"values" : [
ObjectId("5ccc0900846de441161828a2")
],
"_id" : ObjectId("5dee635c72fa520f53d0d1bf"),
"attribute" : ObjectId("5ccc0900846de441161828a0")
},
{
"values" : [
ObjectId("5ccc0ea4846de44116182941")
],
"_id" : ObjectId("5dee635c72fa520f53d0d1be"),
"attribute" : ObjectId("5ccc0ea4846de44116182940")
},
{
"values" : [
ObjectId("5ccc08ba846de4411618289c")
],
"_id" : ObjectId("5def56c537e877042d5abeb5"),
"attribute" : ObjectId("5ccc08ba846de4411618289a")
},
{
"values" : [
ObjectId("5ccc09ca846de441161828aa"),
ObjectId("5ccc09ca846de441161828a9")
],
"_id" : ObjectId("5def56c537e877042d5abeb4"),
"attribute" : ObjectId("5ccc09ca846de441161828a7")
}
]
}
],
I want to aggregate and find all products that have attributes and then group the attributes in the output.
Pipeline:
db.getCollection("products").aggregate(
[
{ $unwind: "$attributes" },
{
$group: {
_id: "$attributes",
attributes: { $first: "$attributes.items" }
}
},
{ $unwind: "$attributes" },
{
$lookup: {
from: "attributes",
let: { attribute: "$attributes.attribute" },
pipeline: [
{
$match: {
$expr: {
$eq: ["$_id", "$$attribute"]
}
}
},
{ $project: { display_name: 1, _id: 1 } }
],
as: "attrs"
}
},
{
$lookup: {
from: "attributevalues",
let: { attribute: "$attributes.values" },
pipeline: [
{
$match: {
$expr: {
$in: ["$_id", "$$attribute"]
}
}
}
],
as: "values"
}
},
{ $project: { attrs: 1, values: 1, _id: 0 } },
{
$group: { _id: "$attrs", items: { $push: "$values" }, total: { $sum: 1 } }
}
],
{
allowDiskUse: true
}
);
Pipeline output:
[{
"_id" : [
{
"_id" : ObjectId("5ccc079f846de44116182891"),
"display_name" : "Caliber (cal.)"
}
],
"items" : [
[
{
"_id" : ObjectId("5ccc079f846de44116182892"),
"sort_order" : NumberInt(0),
"label" : "12",
"attribute_id" : ObjectId("5ccc079f846de44116182891")
}
],
[
{
"_id" : ObjectId("5ccc079f846de44116182892"),
"sort_order" : NumberInt(0),
"label" : "12",
"attribute_id" : ObjectId("5ccc079f846de44116182891")
}
]
],
"total" : 2.0
},
{
"_id" : [
{
"_id" : ObjectId("5ccc0900846de441161828a0"),
"display_name" : "Mechanism"
}
],
"items" : [
[
{
"_id" : ObjectId("5ccc0900846de441161828a2"),
"sort_order" : NumberInt(1),
"label" : "Inaction",
"attribute_id" : ObjectId("5ccc0900846de441161828a0")
}
],
[
{
"_id" : ObjectId("5ccc0900846de441161828a1"),
"sort_order" : NumberInt(0),
"label" : "Gas",
"attribute_id" : ObjectId("5ccc0900846de441161828a0")
}
]
],
"total" : 2.0
}]
Problem is that e.x. in 1st element in the array, I have a duplicate inside the items array.
This is the desired output:
[{
"_id" : [
{
"_id" : ObjectId("5ccc079f846de44116182891"),
"display_name" : "Caliber (cal.)"
}
],
"items" : [
[
{
"_id" : ObjectId("5ccc079f846de44116182892"),
"sort_order" : NumberInt(0),
"label" : "12",
"attribute_id" : ObjectId("5ccc079f846de44116182891"),
"total": 'current _id total, in this case it should be 2'
}
],
...other items goes below, grouped as above
]
}]
Add the solution to group the values by attribute and values and count the occurrences followed by look up and pushing all the values for the attribute with their count.
db.products.aggregate(
[
{"$unwind":"$attributes"},
{"$unwind":"$attributes.items"},
{"$replaceRoot":{"newRoot":"$attributes.items"}},
{"$unwind":"$values"},
{"$group":{
"_id":{"attribute":"$attribute","values":"$values"},
"total":{"$sum":1}
}},
{"$lookup":{
"from":"attributes",
"let":{"attribute":"$_id.attribute"},
"pipeline":[
{"$match":{"$expr":{"$eq":["$_id","$$attribute"]}}},
{"$project":{"display_name":1,"_id":1}}],
"as":"attrs"
}},
{"$lookup":{
"from":"attributevalues",
"localField":"_id.values",
"foreignField":"_id",
"as":"values"
}},
{"$unwind":"$values"},
{"$addFields":{"values.total":"$total"}},
{"$group":{
"_id":{"$arrayElemAt":["$attrs", 0]},
"values":{"$push":"$values"}
}}
])
Use the below aggregation query. Use $addToSet to keep unique values.
db.products.aggregate(
[
{"$unwind":"$attributes"},
{"$unwind":"$attributes.items"},
{"$replaceRoot":{"newRoot":"$attributes.items"}},
{"$unwind":"$values"},
{"$group":{
"_id":"$attribute",
"values":{"$addToSet":"$values"},
"total":{"$sum":1}
}},
{"$lookup":{
"from":"attributes",
"let":{"attribute":"$_id"},
"pipeline":[
{"$match":{"$expr":{"$eq":["$_id","$$attribute"]}}},
{"$project":{"display_name":1,"_id":1}}],
"as":"attrs"
}},
{"$addFields":{"attrs":{"$arrayElemAt":["$attrs", 0]}},
{"$lookup":{
"from":"attributevalues",
"localField":"values",
"foreignField":"_id",
"as":"values"
}}
])
Old answer
You could use below aggregation query. I tried to clean up your current query and change to group only by values field.
Something like
db.products.aggregate(
[
{"$unwind":"$attributes"},
{"$unwind":"$attributes.items"},
{"$replaceRoot":{"newRoot":"$attributes.items"}},
{"$unwind":"$values"},
{"$group":{
"_id":"$values",
"items":{"$first":"$$ROOT"},
"total":{"$sum":1}
}},
{"$lookup":{
"from":"attributes",
"let":{"attribute":"$items.attribute"},
"pipeline":[
{"$match":{"$expr":{"$eq":["$_id","$$attribute"]}}},
{"$project":{"display_name":1,"_id":1}}],
"as":"attrs"
}},
{"$lookup":{
"from":"attributevalues",
"localField":"items.values",
"foreignField":"_id",
"as":"values"
}},
{"$unwind":"$values"},
{"$group":{
"_id":{"$arrayElemAt":["$attrs", 0]},
"values":{"$push":"$values"},
"total":{"$first":"$total"}
}},
{"$addFields":{"_id":0, "attribute":"$_id"}}
])
This is my model schema
{
"_id" : ObjectId("59999454bad567268f78dc09"),
"chapters" : [
{
"_id" : ObjectId("59999454bad567268f78dc0a"),
"individualChapters" : [
{
"_id" : ObjectId("59999454bad567268f78dc0b"),
"photos" : [
{
"_id" : ObjectId("59999454bad567268f78dc0c"),
image: ObjectId("599993e980c3ba2681639163")
},
{
"_id" : ObjectId("59999595bad567268f78dc12"),
image: ObjectId("599993e980c3ba2681639163")
}
]
},
{
"_id" : ObjectId("5999957dbad567268f78dc10"),
"photos" : [
{
"_id" : ObjectId("5999957dbad567268f78dc11"),
image: ObjectId("599993e980c3ba2681639163")
}
]
}
]
}
]
}
i am trying to push another element in "photos" of second "individualChapters" using the below query, but its always pushing in "photos" of first "individualChapters".
Model.findAndModify(
{
'chapters.individualChapters._id' : ObjectId("5999957dbad567268f78dc10"),
'chapters._id' : ObjectId("59999454bad567268f78dc0a"),
_id: ObjectId("59999454bad567268f78dc09")
},
{
'$push': {
'chapters.0.individualChapters.$.photos': {
image: ObjectId("599993e980c3ba2681639163")
}
}
}, { new: false, upsert: false })
Its always updating first individualChapters. Please help!!
You are referring to the wrong element. $push should have this
chapters.0.individualChapters.1.photos.0._id like this
Here's what works in your specific case.
Model.findAndModify(
{
'chapters.individualChapters._id' : ObjectId("5999957dbad567268f78dc10"),
'chapters._id' : ObjectId("59999454bad567268f78dc0a"),
'_id': ObjectId("59999454bad567268f78dc09")
},
{
'$push': {
'chapters.0.individualChapters.1.photos': {
'image': ObjectId("599993e980c3ba2681639163")
}
}
}, { new: false, upsert: false })