Develop Reference

Add new field to report - Script - NodeJS and MongoDB

I have to run a report from a script.
The script works fine
Need to add a new field that indicates if the user is admin or not.
I am not sure how to add that with a true or false condition.
Right now it is returning undefined or false.
The field in mongo database is this one:
mongo-database
var orderTotalsByUserId = {};
db.orders.aggregate([{ $match: { completed: { $exists: true }, completed: { $gt : new Date('2020-08-01') } } }, { $group: { _id: { userId : "$userId" }, total : { $sum: "$total"} } }]).forEach( d => orderTotalsByUserId[d._id.userId] = d.total )
var userIdsByOrders = db.orders.aggregate([{ $match: { completed: { $exists: true }, completed: { $gt : new Date('2020-08-01') } } }, { $group: { _id: { userId : "$userId" }, total : { $sum: "$total"} } }]).map( d => ObjectId(d._id.userId) )
var customers = {};
db.organizations.find().forEach(c => { customers[c._id.valueOf()] = c });
db.users.find({ _id: { $in: userIdsByOrders } }).forEach(d => print(`${d.firstName}, ${d.lastName}, ${d.profile.email}, ${customers[d.customerId].name},${d.sensitive.active.globalAdmin}, ${(orderTotalsByUserId[d._id.valueOf()] * .01).toFixed(2)}`) )

After some time, I've been able to solve it.
Hope it can be helpful for someone else.
First filtering the featurePermissions field in the pipeline.
Then, created an if statement, indicating that if the user has a Customer, it should print the following values.
The solution was:
var orderTotalsByUserId = {};
db.orders.aggregate([{ $match: { completed: { $exists: true }, completed: { $gt : new Date('2020-08-01') } } }, { $group: { _id: { userId : "$userId" }, total : { $sum: "$total"} } }]).forEach( d => orderTotalsByUserId[d._id.userId] = d.total );
var customers = {};
db.organizations.find().forEach(c => { customers[c._id.valueOf()] = c });
db.users.find({ featurePermissions: { $exists: true } }, { _id: 1, firstName: 1, lastName: 1, profile: 1, customerId: 1, featurePermissions: 1 }).forEach(d => { if (customers[d.customerId]) { print(`${d.firstName}, ${d.lastName}, ${d.profile.email}, ${customers[d.customerId].name}, ${orderTotalsByUserId[d._id.valueOf()] ? (orderTotalsByUserId[d._id.valueOf()] * .01).toFixed(2) : 0 }, ${d.featurePermissions.user_management ? 'true' : 'false' }`)}});

Mongo find by sum of subdoc array

I'm trying to find stocks in the Stock collection where the sum of all owners' shares is less than 100. Here is my schema.
const stockSchema = new mongoose.Schema({
owners: [
{
owner: {
type: Schema.Types.ObjectId,
ref: "Owner"
},
shares: {
type: Number,
min: 0,
max: 100
}
}
]
}
const Stock = mongoose.model("Stock", stockSchema);
I've tried to use aggregate but it returns a single object computed over all stocks in the collection, as opposed to multiple objects with the sum of each stock's shares.
stockSchema.statics.getUnderfundedStocks = async () => {
const result = await Stock.aggregate([
{ $unwind: "$owners" },
{ $group: { _id: null, shares: { $sum: "$owners.shares" } } },
{ $match: { shares: { $lt: 100 } } }
]);
return result;
};
So, rather than getting:
[ { _id: null, shares: 150 } ] from getUnderfundedStocks, I'm looking to get:
[ { _id: null, shares: 90 }, { _id: null, shares: 60 } ].
I've come across $expr, which looks useful, but documentation is scarce and not sure if that's the appropriate path to take.
Edit: Some document examples:
/* 1 */
{
"_id" : ObjectId("5ea699fb201db57b8e4e2e8a"),
"owners" : [
{
"owner" : ObjectId("5ea62a94ccb1b974d40a2c72"),
"shares" : 85
}
]
}
/* 2 */
{
"_id" : ObjectId("5ea699fb201db57b8e4e2e1e"),
"owners" : [
{
"owner" : ObjectId("5ea62a94ccb1b974d40a2c72"),
"shares" : 20
},
{
"owner" : ObjectId("5ea62a94ccb1b974d40a2c73"),
"shares" : 50
},
{
"owner" : ObjectId("5ea62a94ccb1b974d40a2c74"),
"shares" : 30
}
]
}
I'd like to return an array that just includes document #1.

You do not need to use $group here. Simply use $project with $sum operator.
db.collection.aggregate([
{ "$project": {
"shares": { "$sum": "$owners.shares" }
}},
{ "$match": { "shares": { "$lt": 100 } } }
])
Or even you do not need to use aggregation here
db.collection.find({
"$expr": { "$lt": [{ "$sum": "$owners.shares" }, 100] }
})
MongoPlayground

Implement feed with retweets in MongoDB

I want to implement retweet feature in my app. I use Mongoose and have User and Message models, and I store retweets as array of objects of type {userId, createdAt} where createdAt is time when retweet occurred. Message model has it's own createdAt field.
I need to create feed of original and retweeted messages merged together based on createdAt fields. I am stuck with merging, whether to do it in a single query or separate and do the merge in JavaScript. Can I do it all in Mongoose with a single query? If not how to find merge insertion points and index of the last message?
So far I just have fetching of original messages.
My Message model:
const messageSchema = new mongoose.Schema(
{
fileId: {
type: mongoose.Schema.Types.ObjectId,
ref: 'File',
required: true,
},
userId: {
type: mongoose.Schema.Types.ObjectId,
ref: 'User',
required: true,
},
likesIds: [{ type: mongoose.Schema.Types.ObjectId, ref: 'User' }],
reposts: [
{
reposterId: {
type: mongoose.Schema.Types.ObjectId,
ref: 'User',
},
createdAt: { type: Date, default: Date.now },
},
],
},
{
timestamps: true,
},
);
Edit: Now I have this but pagination is broken. I am trying to use newCreatedAt field for cursor, that doesn't seem to work. It returns empty array in second call when newCreatedAt is passed from the frontend.
messages: async (
parent,
{ cursor, limit = 100, username },
{ models },
) => {
const user = username
? await models.User.findOne({
username,
})
: null;
const options = {
...(cursor && {
newCreatedAt: {
$lt: new Date(fromCursorHash(cursor)),
},
}),
...(username && {
userId: mongoose.Types.ObjectId(user.id),
}),
};
console.log(options);
const aMessages = await models.Message.aggregate([
{
$addFields: {
newReposts: {
$concatArrays: [
[{ createdAt: '$createdAt', original: true }],
'$reposts',
],
},
},
},
{
$unwind: '$newReposts',
},
{
$addFields: {
newCreatedAt: '$newReposts.createdAt',
original: '$newReposts.original',
},
},
{ $match: options },
{
$sort: {
newCreatedAt: -1,
},
},
{
$limit: limit + 1,
},
]);
const messages = aMessages.map(m => {
m.id = m._id.toString();
return m;
});
//console.log(messages);
const hasNextPage = messages.length > limit;
const edges = hasNextPage ? messages.slice(0, -1) : messages;
return {
edges,
pageInfo: {
hasNextPage,
endCursor: toCursorHash(
edges[edges.length - 1].newCreatedAt.toString(),
),
},
};
},
Here are the queries. The working one:
Mongoose: messages.aggregate([{
'$match': {
createdAt: {
'$lt': 2020 - 02 - 02 T19: 48: 54.000 Z
}
}
}, {
'$sort': {
createdAt: -1
}
}, {
'$limit': 3
}], {})
And the non working one:
Mongoose: messages.aggregate([{
'$match': {
newCreatedAt: {
'$lt': 2020 - 02 - 02 T19: 51: 39.000 Z
}
}
}, {
'$addFields': {
newReposts: {
'$concatArrays': [
[{
createdAt: '$createdAt',
original: true
}], '$reposts'
]
}
}
}, {
'$unwind': '$newReposts'
}, {
'$addFields': {
newCreatedAt: '$newReposts.createdAt',
original: '$newReposts.original'
}
}, {
'$sort': {
newCreatedAt: -1
}
}, {
'$limit': 3
}], {})

This can be done in one query, although its a little hack-ish:
db.collection.aggregate([
{
$addFields: {
reposts: {
$concatArrays: [[{createdAt: "$createdAt", original: true}],"$reports"]
}
}
},
{
$unwind: "$reposts"
},
{
$addFields: {
createdAt: "$reposts.createdAt",
original: "$reposts.original"
}
},
{
$sort: {
createdAt: -1
}
}
]);
You can add any other logic you want to the query using the original field, documents with original: true are the original posts while the others are retweets.

Mongoose $slice and get orginal size array

I'm currently trying to get the total amount of items in my News object, and return a slice of the items as objects.
I found out how to use the $slice operator in my query, but I don't know how to get the original size of the array of items.
The code I'm currently using in NodeJS:
if (req.query.limit) {
limit = 5;
}
News.findOne({ connected: club._id }, {items: {$slice: limit}}).exec(function (err, news) {
if (err) {
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
} else if (!news || news.items.length === 0) {
res.jsonp([]);
} else {
const returnObj = { items: [], totalNumber: 0 };
const items = news.items.sort(function (a, b) {
return b.date - a.date
});
res.jsonp({
items: items,
totalNumber: news.items.length
});
}
});
The Mongo model:
var mongoose = require('mongoose'),
validator = require('validator'),
Schema = mongoose.Schema;
var NewsSchema = new Schema({
connected: {
type: Schema.Types.ObjectId,
required: 'Gelieve een club toe te wijzen.',
ref: 'Club'
},
items: [{
userFirstName: String,
action: String,
date: Date,
targetName: String
}],
created: {
type: Date,
default: Date.now
}
});
module.exports = mongoose.model('News', NewsSchema);
How would I do this efficiently?
Thanks!
EDIT: final code which works:
News.aggregate([{
$match: {
connected: club._id
}
}, {
$project: {
totalNumber: {
$size: '$items'
},
items: {
$slice: ['$items', limit]
}
}
}
]).exec(function (err, news) {
console.log(news);
if (!news || news[0].items.length === 0) {
res.jsonp([]);
} else {
res.jsonp(news[0]);
}
});

You cannot have both information at once using find and $slice.
The soluce you have :
Use aggregate to return the count and only the sliced values.
Like :
[{
$project: {
count: {
$size: "$params",
},
params: {
$slice: ["$params", 5],
},
},
}]
To help you out making aggregate, you can use the awesome mongodb-compass software and its aggregate utility tool.
Use a find without $slice, get the number of item there, and then slice in javascript the array before returning it.
EDIT :
[{
$sort: {
'items.date': -1,
},
}, {
$project: {
count: {
$size: "$items",
},
params: {
$slice: ["$items", 5],
},
},
}]

Mongoose, sort query by populated field

As far as I know, it's possible to sort populated docs with Mongoose (source).
I'm searching for a way to sort a query by one or more populated fields.
Consider this two Mongoose schemas :
var Wizard = new Schema({
name : { type: String }
, spells : { [{ type: Schema.ObjectId, ref: 'Spell' }] }
});
var Spell = new Schema({
name : { type: String }
, damages : { type: Number }
});
Sample JSON:
[{
name: 'Gandalf',
spells: [{
name: 'Fireball',
damages: 20
}]
}, {
name: 'Saruman',
spells: [{
name: 'Frozenball',
damages: 10
}]
}, {
name: 'Radagast',
spells: [{
name: 'Lightball',
damages: 15
}]
}]
I would like to sort those wizards by their spell damages, using something like :
WizardModel
.find({})
.populate('spells', myfields, myconditions, { sort: [['damages', 'asc']] })
// Should return in the right order: Saruman, Radagast, Gandalf
I'm actually doing those sorts by hands after querying and would like to optimize that.

You can explicitly specify only required parameters of populate method:
WizardModel
.find({})
.populate({path: 'spells', options: { sort: [['damages', 'asc']] }})
Have a look at http://mongoosejs.com/docs/api.html#document_Document-populate
Here is an example from a link above.
doc
.populate('company')
.populate({
path: 'notes',
match: /airline/,
select: 'text',
model: 'modelName'
options: opts
}, function (err, user) {
assert(doc._id == user._id) // the document itself is passed
})

Even though this is rather an old post, I'd like to share a solution through the MongoDB aggregation lookup pipeline
The important part is this:
{
$lookup: {
from: 'spells',
localField: 'spells',
foreignField:'_id',
as: 'spells'
}
},
{
$project: {
_id: 1,
name: 1,
// project the values from damages in the spells array in a new array called damages
damages: '$spells.damages',
spells: {
name: 1,
damages: 1
}
}
},
// take the maximum damage from the damages array
{
$project: {
_id: 1,
spells: 1,
name: 1,
maxDamage: {$max: '$damages'}
}
},
// do the sorting
{
$sort: {'maxDamage' : -1}
}
Find below a complete example
'use strict';
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/lotr');
const db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', () => {
let SpellSchema = new Schema({
name : { type: String },
damages : { type: Number }
});
let Spell = mongoose.model('Spell', SpellSchema);
let WizardSchema = new Schema({
name: { type: String },
spells: [{ type: Schema.Types.ObjectId, ref: 'Spell' }]
});
let Wizard = mongoose.model('Wizard', WizardSchema);
let fireball = new Spell({
name: 'Fireball',
damages: 20
});
let frozenball = new Spell({
name: 'Frozenball',
damages: 10
});
let lightball = new Spell({
name: 'Lightball',
damages: 15
});
let spells = [fireball, frozenball, lightball];
let wizards = [{
name: 'Gandalf',
spells:[fireball]
}, {
name: 'Saruman',
spells:[frozenball]
}, {
name: 'Radagast',
spells:[lightball]
}];
let aggregation = [
{
$match: {}
},
// find all spells in the spells collection related to wizards and fill populate into wizards.spells
{
$lookup: {
from: 'spells',
localField: 'spells',
foreignField:'_id',
as: 'spells'
}
},
{
$project: {
_id: 1,
name: 1,
// project the values from damages in the spells array in a new array called damages
damages: '$spells.damages',
spells: {
name: 1,
damages: 1
}
}
},
// take the maximum damage from the damages array
{
$project: {
_id: 1,
spells: 1,
name: 1,
maxDamage: {$max: '$damages'}
}
},
// do the sorting
{
$sort: {'maxDamage' : -1}
}
];
Spell.create(spells, (err, spells) => {
if (err) throw(err);
else {
Wizard.create(wizards, (err, wizards) =>{
if (err) throw(err);
else {
Wizard.aggregate(aggregation)
.exec((err, models) => {
if (err) throw(err);
else {
console.log(models[0]); // eslint-disable-line
console.log(models[1]); // eslint-disable-line
console.log(models[2]); // eslint-disable-line
Wizard.remove().exec(() => {
Spell.remove().exec(() => {
process.exit(0);
});
});
}
});
}
});
}
});
});

here's the sample of mongoose doc.
var PersonSchema = new Schema({
name: String,
band: String
});
var BandSchema = new Schema({
name: String
});
BandSchema.virtual('members', {
ref: 'Person', // The model to use
localField: 'name', // Find people where `localField`
foreignField: 'band', // is equal to `foreignField`
// If `justOne` is true, 'members' will be a single doc as opposed to
// an array. `justOne` is false by default.
justOne: false,
options: { sort: { name: -1 }, limit: 5 }
});