Develop Reference

How to filter with data taken from $lookup

I am currently trying to aggregate list of documents by filtering them with data taken with $lookup
Product.aggregate([
{
$lookup: {
from: "categories",
localField: "category",
foreignField: "_id",
as: "category",
},
},
{ $unwind: "$category" }])
I was hoping adding { $match: { "category.left": {$gte: 3}} }, would be able to get all of the products with categories that's left property is greater than specified, but so far I get nothing. what would be the solution for this?
category documents
{ "_id" : ObjectId("570557d4094a4514fc1291d6"), "left" : 1, "right" : "2" }
{ "_id" : ObjectId("570557d4094a4514fc1291d7"), "left" : 3, "right" : "8"}
{ "_id" : ObjectId("570557d4094a4514fc1291d8"), "left" : 4, "right" : "5"}
{ "_id" : ObjectId("570557d4094a4514fc1291d9"), "left" : 6, "right" : "7" }
product documents
{ "_id" : ObjectId("570557d4094a4514fc129120"), "category": ObjectId("570557d4094a4514fc1291d6") }
{ "_id" : ObjectId("570557d4094a4514fc129121"), "category": ObjectId("570557d4094a4514fc1291d7")}
{ "_id" : ObjectId("570557d4094a4514fc129122"), "category": ObjectId("570557d4094a4514fc1291d8")}
{ "_id" : ObjectId("570557d4094a4514fc129123"), "category": ObjectId("570557d4094a4514fc1291d9") }
I was expecting to get
{ "_id" : ObjectId("570557d4094a4514fc129121"), "category": ObjectId("570557d4094a4514fc1291d7")}
{ "_id" : ObjectId("570557d4094a4514fc129122"), "category": ObjectId("570557d4094a4514fc1291d8")}
{ "_id" : ObjectId("570557d4094a4514fc129123"), "category": ObjectId("570557d4094a4514fc1291d9") }
for my response

How to Join 2 Collection into 1 in Mongoose without Populate

so I have the task to combine 2 collections into 1 to be able to search by additional parameter queries
like ac_key or ac_value,
but the reality is not as expected, this only works for one collection, when I do a search with the other collection nothing happens
account collection
{
"_id" : ObjectId("5ce2409399c7952d4c6f0f5d"),
"status" : "not verified",
"name" : "Monkey D Garp",
"phone" : "1234",
"email" : "ccc#ccc.com",
"password" : "$2a$10$186wQau8GBtqOORovWP7..r8bwSAW1kK9Cb0lT8ckeNFSkEDYjOuu"
},
{
"_id" : ObjectId("5ce2408b99c7952d4c6f0f5b"),
"status" : "not verified",
"name" : "Monkey D Garp",
"phone" : "1234",
"email" : "aaa#aaa.com",
"password" : "$2a$10$WskmjNldC2TQ13Rl6ZLqROJwIux1KwM2tkCqfbiMSxWKRUAgsQWn."
},
{
"_id" : ObjectId("5ce2407c99c7952d4c6f0f59"),
"status" : "not verified",
"name" : "Monkey D Garp",
"phone" : "1234",
"email" : "bbb#bbb.com",
"password" : "$2a$10$g1WRwu4Tp85hIIyw4ONd9e3CGOd7u8UN1jfF.zsVpAOE9Usdy01Bm"
}
account_meta collection
{
"_id" : ObjectId("5ce37884551b0b07f4b60598"),
"value" : "sleeping",
"key" : "speciality",
"account_id" : ObjectId("5ce2407c99c7952d4c6f0f59")
},
{
"_id" : ObjectId("5ce240fc99c7952d4c6f0f61"),
"value" : "cooking",
"key" : "hobby",
"account_id" : ObjectId("5ce2407c99c7952d4c6f0f59")
},
{
"_id" : ObjectId("5ce240f399c7952d4c6f0f60"),
"value" : "12",
"key" : "age",
"account_id" : ObjectId("5ce2407c99c7952d4c6f0f59")
},
{
"_id" : ObjectId("5ce240e799c7952d4c6f0f5f"),
"value" : "singapore",
"key" : "address",
"account_id" : ObjectId("5ce2407c99c7952d4c6f0f59")
},
{
"_id" : ObjectId("5ce2409399c7952d4c6f0f5e"),
"value" : "staff",
"account_id" : ObjectId("5ce2409399c7952d4c6f0f5d"),
"key" : "role"
},
{
"_id" : ObjectId("5ce2408b99c7952d4c6f0f5c"),
"value" : "user",
"account_id" : ObjectId("5ce2408b99c7952d4c6f0f5b"),
"key" : "role"
},
{
"_id" : ObjectId("5ce2407c99c7952d4c6f0f5a"),
"value" : "admin",
"account_id" : ObjectId("5ce2407c99c7952d4c6f0f59"),
"key" : "role"
}
expected output
[{
"status": "not verified",
"_id": "5ce2407c99c7952d4c6f0f59",
"name": "Monkey D Garp",
"phone": "1234",
"email": "bbb#bbb.com",
"password": "$2a$10$g1WRwu4Tp85hIIyw4ONd9e3CGOd7u8UN1jfF.zsVpAOE9Usdy01Bm",
"role": "admin",
"address": "singapore",
"age": "12",
"hobby": "cooking",
"speciality": "sleeping"
}]
the condition: I want to find the key and value with the parameters ac_key and ac_value, when I look for non-specific and related data it will not appear, and get all the meta related to the same account_id
assuming data to merge from account to meta account exists _id (account) = account_id (account_meta)
in reality when i hit ac_key: age, and ac_value: '12', i get:
[
{
"status": "not verified",
"_id": "5ce2407c99c7952d4c6f0f59",
"name": "Monkey D Garp",
"phone": "1234",
"email": "bbb#bbb.com",
"password": "$2a$10$g1WRwu4Tp85hIIyw4ONd9e3CGOd7u8UN1jfF.zsVpAOE9Usdy01Bm",
"role": "admin",
"address": "singapore",
"age": "12",
"hobby": "cooking",
"speciality": "sleeping"
},
{
"status": "not verified",
"_id": "5ce2408b99c7952d4c6f0f5b",
"name": "Monkey D Garp",
"phone": "1234",
"email": "aaa#aaa.com",
"password": "$2a$10$WskmjNldC2TQ13Rl6ZLqROJwIux1KwM2tkCqfbiMSxWKRUAgsQWn.",
"role": "user"
},
{
"status": "not verified",
"_id": "5ce2409399c7952d4c6f0f5d",
"name": "Monkey D Garp",
"phone": "1234",
"email": "ccc#ccc.com",
"password": "$2a$10$186wQau8GBtqOORovWP7..r8bwSAW1kK9Cb0lT8ckeNFSkEDYjOuu",
"role": "staff"
}
]
this is my controller
exports.get_Account = async (req, res) => {
const { _id, name, email, phone, status, ac_key, ac_value } = req.query
const accounts = await Account.find({
// query database query
...(_id && {_id : { $in : _id.split(",") }}),
...(name && {$text : { $search: name }}),
...(email && {email : { $in : email.split(",") }}),
...(phone && {phone : { $in : phone.split(",") }}),
...(status && {status : { $in : status.split(",") }}),
});
const newAcc = await accounts.map(async account => {
const accMeta = await AccountMeta.find({
...({account_id : account._id}),
...(ac_key && {key : ac_key}),
...(ac_value && {value : ac_value})
});
console.log('accMeta', accMeta)
const new_account = {};
await accMeta.map(editMeta => {
new_account[editMeta.key] = editMeta.value;
});
let dynamicAccount = Object.assign({}, account._doc, new_account); //cuma
return {...dynamicAccount}
});
await Promise.all(newAcc).then(result => res.status(200).json(result))
};
thanks in advance

Using Javascript aggregation in druid queries

I am trying to write javascript aggregator for my druid queries. i need to to calculate average of a metric "Base_SalesRank".
So far i have been able to this by writing:
{
"queryType": "groupBy",
"dataSource": "marketdata",
"granularity": "all",
"dimensions" : ["Item"],
"filter": { "type": "and", "fields" : [{"type": "selector", "dimension": "Item", "value": "MN10CESWW"}]},
"intervals": ["2018-06-28T00:00Z/2018-07-04T00:00Z"],
"aggregations" : [
{ "type" : "count", "name" : "rows" },
{ "type" : "doubleSum", "name" : "Base_SalesRank", "fieldName" : "Base_SalesRank" }
],
"postAggregations" : [{
"type": "javascript",
"name": "Target DOS Average",
"fieldNames": ["Base_SalesRank", "rows"],
"function": "function(Base_SalesRank, rows) {return Base_SalesRank/ rows;}"
}]
}
But I noticed that many values in Base_SalesRank is 0.
[ {
"timestamp" : "2018-06-28T05:06:03.000Z",
"result" : {
"pagingIdentifiers" : {
"marketdata_2018-06-28T00:00:00.000Z_2018-06-29T00:00:00.000Z_2018-07-06T08:11:02.499Z" : 3
},
"dimensions" : [ "Item" ],
"metrics" : [ "Base_SalesRank" ],
"events" : [ {
"segmentId" : "marketdata_2018-06-28T00:00:00.000Z_2018-06-29T00:00:00.000Z_2018-07-06T08:11:02.499Z",
"offset" : 0,
"event" : {
"timestamp" : "2018-06-28T07:10:02.000Z",
"Item" : "MN10CESWW",
"Base_SalesRank" : 0
}
},
{
"segmentId" : "marketdata_2018-06-28T00:00:00.000Z_2018-06-29T00:00:00.000Z_2018-07-06T08:11:02.499Z",
"offset" : 3,
"event" : {
"timestamp" : "2018-06-28T07:20:21.000Z",
"Item" : "MN10CESWW",
"Base_SalesRank" : 5558
}
} ]
}
} ]
So i am not getting true average. Now i need to weed out these 0 values and then calucate average. We can do this by using filters
{"type": "not", "field": {"type": "selector", "dimension": "Base_SalesRank", "value": "0"}}
But I have constraint that I have to perform this filter operation inside the javascript function only.

You can achieve the same with just adding a having query -
"having": {
"type": "greaterThan",
"aggregation": "Base_SalesRank",
"value": 0
}
If you want to do the same in javascript function than it can be done as below -
You should add a dimension (key/value) say "isValid" as "0" or "1" during pre-ingestion json data based on if Base_SalesRank is 0 than "isValid" will be 0 else 1.
Apply filter on this field in your query.
Use the rows in your post Aggregration.

Project Specific Fields from a Foreign Collection $lookup result

I'm writing aggregation to get foreign collection data with local collection.
db.getCollection('orders').aggregate([
{
$match: {
status: "UNASSIGNED",
serviceLocationId: "83177"
}
}, {
$lookup: {
from: "servicelocations",
localField: "serviceLocationId",
foreignField: "serviceLocationId",
as: "locations"
}
}, {
$unwind: "$locations"
}])
I'm getting:
{
"_id" : ObjectId("59d32b5c360198e441b67545"),
"accountId" : 1.0,
"orderId" : "AQ137O1701240",
"serviceLocationId" : "83177",
"orderDate" : "2017-09-18T18:29:00.000Z",
"description" : "AQ137O1701240",
"serviceType" : "Delivery",
"orderSource" : "Import",
"takenBy" : "KARIM",
"plannedDeliveryDate" : ISODate("2017-10-09T00:00:00.000Z"),
"plannedDeliveryTime" : "",
"actualDeliveryDate" : "",
"actualDeliveryTime" : "",
"deliveredBy" : "",
"size1" : 25.0,
"size2" : 464.0,
"size3" : 46.0,
"jobPriority" : 1.0,
"cancelReason" : "",
"cancelDate" : "",
"cancelBy" : "",
"reasonCode" : "",
"reasonText" : "",
"status" : "UNASSIGNED",
"lineItems" : [
{
"ItemId" : "MMGW001",
"size1" : 25.0,
"size2" : 464.38,
"size3" : 46.875
}
],
"locations" : {
"_id" : ObjectId("59ce18e172dbf6926093e189"),
"accountId" : 1.0,
"serviceLocationId" : "83177",
"regionId" : "1",
"zoneId" : "DXBZONE1",
"description" : "EXPRESS BLUE MART SUPERMARKET",
"locationPriority" : 1.0,
"accountTypeId" : 1.0,
"locationType" : "SERVICELOCATION",
"location" : {
"makani" : "",
"lng" : 55.179042,
"lat" : 25.098741
},
"deliveryDays" : "MTWRFSU",
"serviceTimeTypeId" : "1",
"timeWindow" : {
"timeWindowTypeId" : "1"
},
"address1" : "",
"address2" : "",
"phone" : "",
"city" : "",
"county" : "",
"state" : "",
"country" : "",
"zipcode" : "",
"imageUrl" : "",
"contact" : {
"name" : "",
"email" : ""
},
"status" : "ACTIVE",
"createdBy" : "",
"updatedBy" : "",
"updateDate" : ""
}
}
but i need:
{
"_id" : ObjectId("59d32b5c360198e441b67545"),
"accountId" : 1.0,
"orderId" : "AQ137O1701240",
"serviceLocationId" : "83177",
"orderDate" : "2017-09-18T18:29:00.000Z",
"description" : "AQ137O1701240",
"serviceType" : "Delivery",
"orderSource" : "Import",
"takenBy" : "KARIM",
"plannedDeliveryDate" : ISODate("2017-10-09T00:00:00.000Z"),
"plannedDeliveryTime" : "",
"actualDeliveryDate" : "",
"actualDeliveryTime" : "",
"deliveredBy" : "",
"size1" : 25.0,
"size2" : 464.0,
"size3" : 46.0,
"jobPriority" : 1.0,
"cancelReason" : "",
"cancelDate" : "",
"cancelBy" : "",
"reasonCode" : "",
"reasonText" : "",
"status" : "UNASSIGNED",
"lineItems" : [
{
"ItemId" : "MMGW001",
"size1" : 25.0,
"size2" : 464.38,
"size3" : 46.875
}
],
"locations" : {
"lng" : 55.179042,
"lat" : 25.098741
}
}

MongoDB less than 3.4.4
Basically, use $project as a final stage and select ALL the specific fields you want. Unfortunately $addFields is out because it will actually "merge" the sub-keys with the existing ones. So the seemingly simple:
{ "$addFields": {
"locations": {
"lng": "$locations.location.lng",
"lat": "$locations.location.lat"
}
}}
Just gives you all the existing content under "locations" as well as the those newly defined keys. Unless of course you don't $unwind directly after the $lookup, which you can do if this would not cause the BSON limit to be exceeded. ( this is called $lookup + $unwind coalescence )
Then we can use $addFields with $map, because we can simply "re-map" the array:
{ "$addFields": {
"locations": {
"$map": {
"input": "$locations",
"as": "l",
"in": {
"lng": "$$l.location.lng",
"lat": "$$l.location.lat"
}
}
}
}},
{ "$unwind": "$locations" }
And then $unwind if you still need to after that re-mapping.
So with $project it is:
{ "$project": {
"accountId" : 1,
"orderId" : 1,
"serviceLocationId" : 1,
"orderDate" : 1,
"description" : 1,
"serviceType" : 1,
"orderSource" : 1,
"takenBy" : 1,
"plannedDeliveryDate" : 1,
"plannedDeliveryTime" : 1,
"actualDeliveryDate" : 1,
"actualDeliveryTime" : 1,
"deliveredBy" : 1,
"size1" : 1,
"size2" : 1,
"size3" : 1,
"jobPriority" : 1,
"cancelReason" : 1,
"cancelDate" : 1,
"cancelBy" : 1,
"reasonCode" : 1,
"reasonText" : 1,
"status" : 1,
"lineItems" : 1,
"locations" : {
"lng": "$locations.location.lng",
"lat": "$locations.location.lat"
}
}}
Simple but long winded.
MongoDB 3.4.4 Or greater
If you have MongoDB 3.4.4 or greater with $objectToArray and $arrayToObject, then you can be a bit more fancy about it:
{ "$replaceRoot": {
"newRoot": {
"$arrayToObject": {
"$concatArrays": [
{ "$filter": {
"input": { "$objectToArray": "$$ROOT" },
"cond": { "$ne": [ "$$this.k", "locations" ] }
}},
{ "$objectToArray": {
"locations": {
"lng": "$locations.location.lng",
"lat": "$locations.location.lat"
}
}}
]
}
}
}}
Which basically takes all the fields presently in the whole document from $$ROOT, turns it into an array format. We then $filter the "location" field by the "key name" and $concatArrays it with the new "location" key and sub-keys again transformed into an array.
Finally of course $arrayToObject takes that and transforms back into an object which is supplied to newRoot of $replaceRoot as the final output.
So using either of those except $addFields after $unwind of course gives you the correct result:
/* 1 */
{
"_id" : ObjectId("59d32b5c360198e441b67545"),
"accountId" : 1.0,
"orderId" : "AQ137O1701240",
"serviceLocationId" : "83177",
"orderDate" : "2017-09-18T18:29:00.000Z",
"description" : "AQ137O1701240",
"serviceType" : "Delivery",
"orderSource" : "Import",
"takenBy" : "KARIM",
"plannedDeliveryDate" : ISODate("2017-10-09T00:00:00.000Z"),
"plannedDeliveryTime" : "",
"actualDeliveryDate" : "",
"actualDeliveryTime" : "",
"deliveredBy" : "",
"size1" : 25.0,
"size2" : 464.0,
"size3" : 46.0,
"jobPriority" : 1.0,
"cancelReason" : "",
"cancelDate" : "",
"cancelBy" : "",
"reasonCode" : "",
"reasonText" : "",
"status" : "UNASSIGNED",
"lineItems" : [
{
"ItemId" : "MMGW001",
"size1" : 25.0,
"size2" : 464.38,
"size3" : 46.875
}
],
"locations" : {
"lng" : 55.179042,
"lat" : 25.098741
}
}
MongoDB 3.6 and greater
As a bit of a preview, $lookup gets a more expressive overhaul with MongoDB 3.6. So you can actually specifically state the fields to return that way:
{ "$lookup": {
"from": "servicelocations",
"let": { "serviceLocationId": "$serviceLocationId" },
"pipeline": [
{ "$match": { "$expr": { "$eq": [ "serviceLocationId", "$$serviceLocationId" ] } }},
{ "$project": {
"_id": 0,
"lng": "$location.lng",
"lat": "$location.lat"
}}
],
"as": "locations"
}}
Little bit more handy when that is actually released. This actually uses $expr instead of the localField and foreignField to define the "join" condition in a $match stage of the sub-pipeline. Then you can simply $project the fields to return, which then go into the array targeted by $lookup.
Going forward, that's the general approach you would want to take as it limits what actually gets returned.

D3 tree/hierachical related data between nodes

I am working on a d3 project at the moment, and I am trying to map out a hierachical tree to show people and who they are responsible for. Basically I can user A and user B and they can each be responsible for the same person.
Currently to highlight this in my JSON data that builds the visualisation I am repeating data, is there away to not repeat data and use the same data point when 2 or more people are responsible for the same person?
Here is my JSfiddle example
My Hierachical Visualisation
You will see here that, Raymond Reddington & Donald Ressler have cross over between some of their responsibilites, I am repeating the data which seems inefficient, is there a better way, here is my JSON.
[
{
"name" : "Company Name",
"parent" : null,
"children": [
{
"name" : "Raymond Reddington",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Debe Zuma",
"parent" : "Raymond Reddington",
},
{
"name" : "Tom Keen",
"parent" : "Raymond Reddington",
},
{
"name" : "Aram Mojtabai",
"parent" : "Raymond Reddington",
}
]
},
{
"name" : "Elizabeth Keen",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Samar Navabi",
"parent" : "Elizabeth Keen",
},
{
"name" : "Meera Malik",
"parent" : "Elizabeth Keen",
},
{
"name" : "Mr. Kaplan",
"parent" : "Elizabeth Keen",
},
{
"name" : "Reven Wright",
"parent" : "Elizabeth Keen",
}
]
},
{
"name" : "Donald Ressler",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Matius Solomon",
"parent" : "Donald Ressler",
"size" : 3938
},
{
"name" : "Peter Kotsiopulos",
"parent" : "Donal Ressler",
"size" : 3938
},
{
"name" : "Tom Keen",
"parent" : "Raymond Reddington",
"size" : 3938
},
{
"name" : "Aram Mojtabai",
"parent" : "Raymond Reddington",
"size" : 3938
}
]
},
{
"name" : "Harold Cooper",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Samar Navabi",
"parent" : "Elizabeth Keen",
"size" : 3938
},
{
"name" : "Meera Malik",
"parent" : "Elizabeth Keen",
"size" : 3938
}
]
}
]
}
]

This website details a method of converting flat data to the hierarchical data required by d3 http://www.d3noob.org/2014/01/tree-diagrams-in-d3js_11.html
They explain it well too. As the author notes it is originally based on https://stackoverflow.com/a/17849353/1544886
I have copied and pasted their website's example below:
var data = [
{ "name" : "Level 2: A", "parent":"Top Level" },
{ "name" : "Top Level", "parent":"null" },
{ "name" : "Son of A", "parent":"Level 2: A" },
{ "name" : "Daughter of A", "parent":"Level 2: A" },
{ "name" : "Level 2: B", "parent":"Top Level" }
];
will map to:
var treeData = [
{
"name": "Top Level",
"parent": "null",
"children": [
{
"name": "Level 2: A",
"parent": "Top Level",
"children": [
{
"name": "Son of A",
"parent": "Level 2: A"
},
{
"name": "Daughter of A",
"parent": "Level 2: A"
}
]
},
{
"name": "Level 2: B",
"parent": "Top Level"
}
]
}
];
via:
var dataMap = data.reduce(function(map, node) {
map[node.name] = node;
return map;
}, {});
var treeData = [];
data.forEach(function(node) {
// add to parent
var parent = dataMap[node.parent];
if (parent) {
// create child array if it doesn't exist
(parent.children || (parent.children = []))
// add node to child array
.push(node);
} else {
// parent is null or missing
treeData.push(node);
}
});
You could extend that further replacing with Ids and using a second normalised array for the lookup:
[{
"id": 0,
"name": "Cherry Tree Lodge"
},{
"id": 1,
"name": "Tom Keen"
},{
"id": 2,
"name": "Debe Zuma"
}]
Also please note that your json data is not strictly valid, you have extra commas.