I have the following Product Schema: (partial, using mongoose)
attributes: [
{
set: {
ref: 'AttributeSet',
type: Schema.Types.ObjectId
},
items: [
{
attribute: {
ref: 'Attributes',
type: Schema.Types.ObjectId
},
values: [
{
ref: 'AttributeValues',
type: Schema.Types.ObjectId
}
]
}
],
_id: 0
}
],
example document 1: (partial)
"attributes" : [
{
"set" : ObjectId("5ccc079c846de44116182890"),
"items" : [
{
"values" : [
ObjectId("5ccc0900846de441161828a1")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c4"),
"attribute" : ObjectId("5ccc0900846de441161828a0")
},
{
"values" : [
ObjectId("5ccc0a51846de441161828cc")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c3"),
"attribute" : ObjectId("5ccc0a51846de441161828cb")
},
{
"values" : [
ObjectId("5ccc0c7d846de44116182906")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c2"),
"attribute" : ObjectId("5ccc0c7d846de44116182904")
},
{
"values" : [
ObjectId("5ccc0d64846de44116182911")
],
"_id" : ObjectId("5dee638d72fa520f53d0d1c1"),
"attribute" : ObjectId("5ccc0d64846de4411618290f")
},
{
"values" : [
ObjectId("5ccc079f846de44116182892")
],
"_id" : ObjectId("5def6acf66910405e07e1e9f"),
"attribute" : ObjectId("5ccc079f846de44116182891")
}
]
}
]
example document 2: (partial)
"attributes" : [
{
"set" : ObjectId("5ccc079c846de44116182890"),
"items" : [
{
"values" : [
ObjectId("5ccc079f846de44116182892")
],
"_id" : ObjectId("5dee635c72fa520f53d0d1c0"),
"attribute" : ObjectId("5ccc079f846de44116182891")
},
{
"values" : [
ObjectId("5ccc0900846de441161828a2")
],
"_id" : ObjectId("5dee635c72fa520f53d0d1bf"),
"attribute" : ObjectId("5ccc0900846de441161828a0")
},
{
"values" : [
ObjectId("5ccc0ea4846de44116182941")
],
"_id" : ObjectId("5dee635c72fa520f53d0d1be"),
"attribute" : ObjectId("5ccc0ea4846de44116182940")
},
{
"values" : [
ObjectId("5ccc08ba846de4411618289c")
],
"_id" : ObjectId("5def56c537e877042d5abeb5"),
"attribute" : ObjectId("5ccc08ba846de4411618289a")
},
{
"values" : [
ObjectId("5ccc09ca846de441161828aa"),
ObjectId("5ccc09ca846de441161828a9")
],
"_id" : ObjectId("5def56c537e877042d5abeb4"),
"attribute" : ObjectId("5ccc09ca846de441161828a7")
}
]
}
],
I want to aggregate and find all products that have attributes and then group the attributes in the output.
Pipeline:
db.getCollection("products").aggregate(
[
{ $unwind: "$attributes" },
{
$group: {
_id: "$attributes",
attributes: { $first: "$attributes.items" }
}
},
{ $unwind: "$attributes" },
{
$lookup: {
from: "attributes",
let: { attribute: "$attributes.attribute" },
pipeline: [
{
$match: {
$expr: {
$eq: ["$_id", "$$attribute"]
}
}
},
{ $project: { display_name: 1, _id: 1 } }
],
as: "attrs"
}
},
{
$lookup: {
from: "attributevalues",
let: { attribute: "$attributes.values" },
pipeline: [
{
$match: {
$expr: {
$in: ["$_id", "$$attribute"]
}
}
}
],
as: "values"
}
},
{ $project: { attrs: 1, values: 1, _id: 0 } },
{
$group: { _id: "$attrs", items: { $push: "$values" }, total: { $sum: 1 } }
}
],
{
allowDiskUse: true
}
);
Pipeline output:
[{
"_id" : [
{
"_id" : ObjectId("5ccc079f846de44116182891"),
"display_name" : "Caliber (cal.)"
}
],
"items" : [
[
{
"_id" : ObjectId("5ccc079f846de44116182892"),
"sort_order" : NumberInt(0),
"label" : "12",
"attribute_id" : ObjectId("5ccc079f846de44116182891")
}
],
[
{
"_id" : ObjectId("5ccc079f846de44116182892"),
"sort_order" : NumberInt(0),
"label" : "12",
"attribute_id" : ObjectId("5ccc079f846de44116182891")
}
]
],
"total" : 2.0
},
{
"_id" : [
{
"_id" : ObjectId("5ccc0900846de441161828a0"),
"display_name" : "Mechanism"
}
],
"items" : [
[
{
"_id" : ObjectId("5ccc0900846de441161828a2"),
"sort_order" : NumberInt(1),
"label" : "Inaction",
"attribute_id" : ObjectId("5ccc0900846de441161828a0")
}
],
[
{
"_id" : ObjectId("5ccc0900846de441161828a1"),
"sort_order" : NumberInt(0),
"label" : "Gas",
"attribute_id" : ObjectId("5ccc0900846de441161828a0")
}
]
],
"total" : 2.0
}]
Problem is that e.x. in 1st element in the array, I have a duplicate inside the items array.
This is the desired output:
[{
"_id" : [
{
"_id" : ObjectId("5ccc079f846de44116182891"),
"display_name" : "Caliber (cal.)"
}
],
"items" : [
[
{
"_id" : ObjectId("5ccc079f846de44116182892"),
"sort_order" : NumberInt(0),
"label" : "12",
"attribute_id" : ObjectId("5ccc079f846de44116182891"),
"total": 'current _id total, in this case it should be 2'
}
],
...other items goes below, grouped as above
]
}]
Add the solution to group the values by attribute and values and count the occurrences followed by look up and pushing all the values for the attribute with their count.
db.products.aggregate(
[
{"$unwind":"$attributes"},
{"$unwind":"$attributes.items"},
{"$replaceRoot":{"newRoot":"$attributes.items"}},
{"$unwind":"$values"},
{"$group":{
"_id":{"attribute":"$attribute","values":"$values"},
"total":{"$sum":1}
}},
{"$lookup":{
"from":"attributes",
"let":{"attribute":"$_id.attribute"},
"pipeline":[
{"$match":{"$expr":{"$eq":["$_id","$$attribute"]}}},
{"$project":{"display_name":1,"_id":1}}],
"as":"attrs"
}},
{"$lookup":{
"from":"attributevalues",
"localField":"_id.values",
"foreignField":"_id",
"as":"values"
}},
{"$unwind":"$values"},
{"$addFields":{"values.total":"$total"}},
{"$group":{
"_id":{"$arrayElemAt":["$attrs", 0]},
"values":{"$push":"$values"}
}}
])
Use the below aggregation query. Use $addToSet to keep unique values.
db.products.aggregate(
[
{"$unwind":"$attributes"},
{"$unwind":"$attributes.items"},
{"$replaceRoot":{"newRoot":"$attributes.items"}},
{"$unwind":"$values"},
{"$group":{
"_id":"$attribute",
"values":{"$addToSet":"$values"},
"total":{"$sum":1}
}},
{"$lookup":{
"from":"attributes",
"let":{"attribute":"$_id"},
"pipeline":[
{"$match":{"$expr":{"$eq":["$_id","$$attribute"]}}},
{"$project":{"display_name":1,"_id":1}}],
"as":"attrs"
}},
{"$addFields":{"attrs":{"$arrayElemAt":["$attrs", 0]}},
{"$lookup":{
"from":"attributevalues",
"localField":"values",
"foreignField":"_id",
"as":"values"
}}
])
Old answer
You could use below aggregation query. I tried to clean up your current query and change to group only by values field.
Something like
db.products.aggregate(
[
{"$unwind":"$attributes"},
{"$unwind":"$attributes.items"},
{"$replaceRoot":{"newRoot":"$attributes.items"}},
{"$unwind":"$values"},
{"$group":{
"_id":"$values",
"items":{"$first":"$$ROOT"},
"total":{"$sum":1}
}},
{"$lookup":{
"from":"attributes",
"let":{"attribute":"$items.attribute"},
"pipeline":[
{"$match":{"$expr":{"$eq":["$_id","$$attribute"]}}},
{"$project":{"display_name":1,"_id":1}}],
"as":"attrs"
}},
{"$lookup":{
"from":"attributevalues",
"localField":"items.values",
"foreignField":"_id",
"as":"values"
}},
{"$unwind":"$values"},
{"$group":{
"_id":{"$arrayElemAt":["$attrs", 0]},
"values":{"$push":"$values"},
"total":{"$first":"$total"}
}},
{"$addFields":{"_id":0, "attribute":"$_id"}}
])
Related
So i want to check if an element is in few arrays. the number of arrays can change between 2 to 4. the structure of data is as follows:
{"question" : {
"answers" : [
{
"answerText" : "7000",
"_answerVotes" : []
},
{
"answerText" : "15000",
"_answerVotes" : []
},
{
"answerText" : "45000",
"_answerVotes" : []
},
{
"answerText" : "80000",
"_answerVotes" : []
}
],
"expireTime" : ISODate("2022-12-14T15:58:11.453Z")
}}
aggregation stage:
(addFields.isUserVotedAnswerd = {
$cond: {
if: {
$in: [ObjectId(user), "question.answers.$._answerVotes"],
},
// $or: [
// { $in: [ObjectId(user), "$question.answers.0._answerVotes"] },
// { $in: [ObjectId(user), "$question.answers.1._answerVotes"] },
// { $in: [ObjectId(user), "$question.answers.2._answerVotes"] },
// { $in: [ObjectId(user), "$question.answers.3._answerVotes"] },
// ],
// },
then: true,
else: false,
},
}),
Tried both approaches. And I keep getting this error:
"$in requires an array as a second argument, found: string "
Anybody know how to address this?
Many thanks
Given this input:
var r = [
{question: {
"answers" : [
{"answerText" : "7000", "_answerVotes" : ["U1"] },
{"answerText" : "15000", "_answerVotes" : ["U2","U7"] }
]
}},
{question: {
"answers" : [
{"answerText" : "7000", "_answerVotes" : ["U2"] },
{"answerText" : "15000", "_answerVotes" : ["U9"] }
]
}},
{question: {
"answers" : [
{"answerText" : "7000", "_answerVotes" : ["U7"] }
]
}}
];
then the following will select only those docs where at least one user ID in any of the _answerVotes in the answers array matches the target:
var target_user = 'U2'; // user is string here but could be ObjectId
db.foo.aggregate([
{$project: {'X': {$filter: {input: '$question.answers',
as: 'z',
cond: {$in:[target_user,'$$z._answerVotes']}
}}
}}
,{$match: {$expr: {$ne:[0,{$size:'$X'}] } }}
]);
I am trying to implement Pagination and Filtering at the backend.
The input to this controller is Page number and Filtering conditions.
Controller:-
const getPosts = asyncHandler(async (req, res) => {
const {
page,
statusFilter,
typeFilter,
sourceFilter,
} = JSON.parse(req.query.filterData);
var query = [
{
$addFields: {
paramType: typeFilter,
paramSource: sourceFilter,
paramStatus: statusFilter,
},
},
{
$match: {
$expr: {
$and: [
{ user: req.user.id },
{
$or: [
{
$eq: ["$paramType", "All"],
},
{
$eq: ["$paramType", "$type"],
},
],
},
{
$or: [
{
$eq: ["$paramSource", "All"],
},
{
$eq: ["$paramSource", "$source"],
},
],
},
{
$or: [
{
$eq: ["$paramStatus", "All"],
},
{
$eq: ["$paramStatus", "$status"],
},
],
},
],
},
},
},
{
$project: {
paramSource: false,
paramType: false,
paramStatus: false,
},
},
];
//pagination
const PAGE_SIZE = 5;
const PAGE = parseInt(page) || 0;
// aggregate query
const aggregateQuery = await Post.aggregate([query]);
const total = aggregateQuery.length;
const Allposts = await Post.aggregate([query])
.limit(PAGE_SIZE)
.skip(PAGE_SIZE * PAGE)
.sort({ createdAt: -1 });
const totalPages = Math.ceil(total / PAGE_SIZE);
res.status(200).json({ totalPages, Allposts });
});
Problem:-
The pagination and filtering part works as intended but only for the first page, when I go to second page the Allposts object is empty.
Why is the Allposts object empty after first page?
Edit:-
Sample Data:-
{
"_id" : 1,
"type" : "Type A",
"source" : "Source A",
"status" : "Status A",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
},
{
"_id" : 2,
"type" : "Type B",
"source" : "Source C",
"status" : "Status B",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
},
{
"_id" : 3,
"type" : "Type A",
"source" : "Source A",
"status" : "Status A",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
},
{
"_id" : 4,
"type" : "Type A",
"source" : "Source C",
"status" : "Status B",
"createdAt" : ISODate("2022-04-13T17:12:28.096Z"),
"updatedAt" : ISODate("2022-04-13T17:12:28.096Z"),
"__v" : 0
}
Updated aggreation query:-
var query = [
{
$addFields: {
paramType: typeFilter,
paramSource: sourceFilter,
paramStatus: statusFilter,
},
},
{
$match: {
$expr: {
$and: [
{ user: req.user.id },
{
$or: [
{
$eq: ["$paramType", "All"],
},
{
$eq: ["$paramType", "$type"],
},
],
},
{
$or: [
{
$eq: ["$paramSource", "All"],
},
{
$eq: ["$paramSource", "$source"],
},
],
},
{
$or: [
{
$eq: ["$paramStatus", "All"],
},
{
$eq: ["$paramStatus", "$status"],
},
],
},
],
},
},
},{ $sort : { createdAt : -1 } },
{
$project: {
paramSource: false,
paramType: false,
paramStatus: false,
createdAt : 1,
},
}
];
If you want pagination + total count with one query, you can do something like this:
db.collection.aggregate([
{
$addFields: {
paramType: "typeFilter",
paramSource: "sourceFilter",
paramStatus: "statusFilter"
}
},
{
"$match": {
// complete here
}
},
{
$setWindowFields: {
output: {totalCount: {$count: {}}}}
},
{$sort: {createdAt: -1}},
{$skip: PAGE_SIZE * PAGE},
{$limit: PAGE_SIZE},
{
$facet: {
results: [
{
$project: {
// here put whatever you want to send to FE
type: 1,
source: 1,
}
}
],
totalCount: [
{$limit: 1},
{$project: {totalCount: 1, _id: 0}}
]
}
}
])
As you can see on the playground
The $setWindowFields allows you to add the total count to all documents. $sort, $skip and $limit allow the pagination. The $facet allows you to get different outputs from the same documents.
My mongodb data is like this,i want to filter the memoryLine.
{
"_id" : ObjectId("5e36950f65fae21293937594"),
"userId" : "5e33ee0b4a3895a6d246f3ee",
"notes" : [
{
"noteId" : ObjectId("5e36953665fae212939375a0"),
"time" : ISODate("2020-02-02T17:24:06.460Z"),
"memoryLine" : [
{
"_id" : ObjectId("5e36953665fae212939375ab"),
"memoryTime" : ISODate("2020-02-03T17:54:06.460Z")
},
{
"_id" : ObjectId("5e36953665fae212939375aa"),
"memoryTime" : ISODate("2020-02-03T05:24:06.460Z")
}
]
}
]}
i want to get the item which memoryTime is great than now as expected like this,
"userId" : "5e33ee0b4a3895a6d246f3ee",
"notes" : [
{
"noteId" : ObjectId("5e36953665fae212939375a0"),
"time" : ISODate("2020-02-02T17:24:06.460Z"),
"memoryLine" : [
{
"_id" : ObjectId("5e36953665fae212939375ab"),
"memoryTime" : ISODate("2020-02-03T17:54:06.460Z")
},
{
"_id" : ObjectId("5e36953665fae212939375aa"),
"memoryTime" : ISODate("2020-02-03T05:24:06.460Z")
}
]
}]
so is use code as below.i use a $filter in memoryLine to filter to get the right item.
aggregate([{
$match: {
"$and": [
{ userId: "5e33ee0b4a3895a6d246f3ee"},
]
}
}, {
$project: {
userId: 1,
notes: {
noteId: 1,
time: 1,
memoryLine: {
$filter: {
input: "$memoryLine",
as: "mLine",
cond: { $gt: ["$$mLine.memoryTime", new Date(new Date().getTime() + 8 * 1000 * 3600)] }
}
}
}
}
}]).then(doc => {
res.json({
code: 200,
message: 'success',
result: doc
})
});
but i got this,memoryLine is null,why?I try to change $gt to $lt, but also got null.
"userId" : "5e33ee0b4a3895a6d246f3ee",
"notes" : [
{
"noteId" : ObjectId("5e36953665fae212939375a0"),
"time" : ISODate("2020-02-02T17:24:06.460Z"),
"memoryLine" : null <<<------------- here is not right
}]
You can use $addFields to replace existing field, $map for outer collection and $filter for inner:
db.collection.aggregate([
{
$addFields: {
notes: {
$map: {
input: "$notes",
in: {
$mergeObjects: [
"$$this",
{
memoryLine: {
$filter: {
input: "$$this.memoryLine",
as: "ml",
cond: {
$gt: [ "$$ml.memoryTime", new Date() ]
}
}
}
}
]
}
}
}
}
}
])
$mergeObjects is used to avoid repeating fields from source memoryLine object.
Mongo Playground
I have two data structures "database" and "components"
const DatabaseSchema = mongoose.Schema({
components: [{ type: Schema.Types.ObjectId, ref: 'Components', required: false }],
});
const ComponentsSchema = mongoose.Schema({
name: { type: String, required: true, trim: true, unique: true, lowercase: true },
updatedAt: Date,
});
I want to filter all items in the database by component names
search rule I'm using
Database.find({
components: { $elemMatch: { name: /antr/i } }
}).populate({
path: 'components',
select: 'name -_id'
}).select(['descript','components']).exec( (err,data) => {
console.log(err);
res.json(data);
});
however always return an empty element
Please try this :
As I've already suggested you can use this :
Database.find({})
.populate({ path: 'components', match: { name: /antr/i }, select: 'name -_id' })
.exec((err, data) => { console.log(err); res.json(data); });
Since you're seeing empty array's is because of the filter query in match which doesn't find appropriate documents in components collection w.r.t. ObjectIds in components array of database document, this is normal. May be you can filter those out in code, as you aren't looking in that way, You can use mongoDB's $lookup from aggregation framework which is equivalent to .populate() from mongoose.
Database.aggregate(
[{
$lookup: {
from: "components",
"let": { "ids": "$components" },
pipeline: [
{ $match: { $expr: { $in: ['$_id', '$$ids'] } } }],
as: "dbComponentsArray"
}
}, { $unwind: '$dbComponentsArray' }, { $match: { 'dbComponentsArray.name': /antr/i } },
{ $group: { _id: '$_id', dbComponentsArray: { $push: '$dbComponentsArray' }, data: { $first: '$$ROOT' } } }, { $addFields: { 'data.dbComponentsArray': '$dbComponentsArray' } },
{ $replaceRoot: { 'newRoot': '$data' } }])
Sample Data in collections :
components :
/* 1 */
{
"_id" : ObjectId("5d481cd098ba991c0857959f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
/* 2 */
{
"_id" : ObjectId("5d481cd098ba991c0857958f"),
"name" : "anacito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
/* 3 */
{
"_id" : ObjectId("5d481cd098ba991c0857951f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
/* 4 */
{
"_id" : ObjectId("5d481cd098ba991c0857952f"),
"name" : "anacito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
database :
/* 1 */
{
"_id" : ObjectId("5d4979d52a17d10a6c8de81b"),
"components" : [
ObjectId("5d481cd098ba991c0857951f"),
ObjectId("5d481cd098ba991c0857952f"),
ObjectId("5d481cd098ba991c0857953f"),
ObjectId("5d481cd098ba991c0857959f")
]
}
Output :
/* 1 */
{
"_id" : ObjectId("5d4979d52a17d10a6c8de81b"),
"components" : [
ObjectId("5d481cd098ba991c0857951f"),
ObjectId("5d481cd098ba991c0857952f"),
ObjectId("5d481cd098ba991c0857953f"),
ObjectId("5d481cd098ba991c0857959f")
],
"dbComponentsArray" : [
{
"_id" : ObjectId("5d481cd098ba991c0857959f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
},
{
"_id" : ObjectId("5d481cd098ba991c0857951f"),
"name" : "antracito",
"updatedAt" : ISODate("2019-08-05T12:10:56.777Z"),
"__v" : 0
}
]
}
I have the data structure like this:
{
"_id" : ObjectId("5c4404906736bd2608e30b5e"),
"assets": [
{
"name" : "xa",
"id" : 1
},
{
"name" : "xs",
"id" : 2
}
]
},
{
"_id" : ObjectId("5c4404906736bd2608e30b5f"),
"assets": [
{
"name" : "xa",
"id" : 3
}
]
},
{
"_id" : ObjectId("5c4404906736bd2608e30b5g"),
"assets": [
{
"name" : "xa",
"id" : 4
},
{
"name" : "xd",
"id" : 5
},
{
"name" : "xs",
"id" : 6
}
]
}
Now I want to implement the MongoDB aggregation by which I got the Answer like this:
[
{
"assets": "xa",
"count": 3
},
{
"assets": "xs",
"count": 2
},
{
"assets": "xd",
"count": 1
},
]
I have to get this done by javascript but need to implement this on aggregation. My code for acheiveing with js is like this for set of array of object i.e
var arr = [
{ asset: "xa" },
{ asset: "xs" },
{ asset: "xa" },
{ asset: "xs" },
{ asset: "xa" },
{ asset: "xd" }
];
var userDict = arr.reduce((acc, el) => {
if (!acc.hasOwnProperty(el.asset)) {
acc[el.asset] = { count: 0 };
}
acc[el.asset].count++;
return acc;
}, {});
var result = Object.entries(userDict).map(([k, v]) => ({
asset: k,
count: v.count
}));
console.log(result);
Any help is really appreciated
You can $unwind assets before applying $group with count:
db.col.aggregate([
{
$unwind: "$assets"
},
{
$group: {
_id: "$assets.name",
count: { $sum: 1 }
}
},
{
$project: {
_id: 0,
asset: "$_id",
count: 1
}
}
])