I am currently working in a project that has insisted in explicitly defining over 1,700 questions into a JSON data schema for an API and its getting out of control. I have been suggesting a more generic structure to the schema and let the data do the talking to tell you what the context is.
Whilst there are debates happening around which schema to use, we have decided on our internal systems, to go ahead and use a more generic model even if the externally facing model is the explicit one. This means we need an adapter to convert from one to the other, until such time as we can just use the one we wanted in the first place.
The business is a Java shop, I don't know whether to advise to build the adapter in Java or whether we can incorporate some lightweight JavaScript to do the work, maybe in the form of a configuration.
My question is: How would you approach converting the first JSON example into the second JSON example? It maps from explicitly defined objects to generic objects in arrays. Thanks for considering my question.
Example One
{
"nested_object" : {
"department_one" : {
"floor" : "4",
"product_one" : {
"quantity" : 10,
"size" : "L"
},
"product_two" : {
"quantity" : 5,
"size" : "S"
}
},
"department_two" : {
"floor" : "2",
"product_thirteen" : {
"quantity" : 1,
"size" : "M"
},
"product_eleven" : {
"quantity" : 8,
"size" : "L"
}
}
}
}
Example Two
{
"departments" : [
{
"department_name" : "department_one",
"floor" : "4",
"products" : [
{
"product_name" : "product_one",
"quantity" : 10,
"size" : "L"
},
{
"product_name" : "product_two",
"quantity" : 5,
"size" : "S"
}
]
},
{
"department_name" : "department_two",
"floor" : "2",
"products" : [
{
"product_name" : "product_thirteen",
"quantity" : 1,
"size" : "M"
},
{
"product_name" : "product_eleven",
"quantity" : 8,
"size" : "L"
}
]
}
]
}
You could use a combination of Object.keys (to grab product and department names). Below is a quick implementation.
const obj1 = {
"nested_object" : {
"department_one" : {
"floor" : "4",
"product_one" : {
"quantity" : 10,
"size" : "L"
},
"product_two" : {
"quantity" : 5,
"size" : "S"
}
},
"department_two" : {
"floor" : "2",
"product_thirteen" : {
"quantity" : 1,
"size" : "M"
},
"product_eleven" : {
"quantity" : 8,
"size" : "L"
}
}
}
}
const transformedObj = {
departments: [ ],
};
//holds all department names
const departmentKeys = Object.keys(obj1.nested_object)
const departmentsArr = departmentKeys.map((key) => {
const floor = obj1.nested_object[key].floor
//remove floor reference, since we already stored the value above
delete obj1.nested_object[key].floor
//holds all product names
const productsKeysArr = Object.keys(obj1.nested_object[key])
//holds all product objects for respective department
const productsArr = productsKeysArr.map((product) => {
const quantity = obj1.nested_object[key][product].quantity
const size = obj1.nested_object[key][product].size
return {
product_name: product,
quantity: quantity,
size: size
}
})
return {
department_name: key,
floor: floor,
products: productsArr
}
})
//assign departments array to transformed object
transformedObj.departments = departmentsArr
console.log(transformedObj)
This would be my take on this. I like conciseness and expressiveness in implementations:
const data = { "nested_object": { ... }}
Object.entries(data.nested_object).map(([department_name, {floor, ...ps}]) => ({
department_name,
floor,
products: Object.entries(ps).map(([product_name, p]) => ({product_name, ...p}))
}))
Related
Here is MongoDB scheme.
{
"_id" : ObjectId("222222"),
"active" : false,
"amount" : "15%",
"description" : "15% discount",
"name" : "20200628-test",
"policies" : {
"apply" : [
{
"name" : "expiryDate",
"params" : {
"date" : ISODate("2020-07-06T14:59:59.999Z")
}
},
{
"name" : "isApplyCategoryExist"
}
],
"discount" : [],
"conflict" : [
{
"name" : "exclusive"
}
],
"grant" : []
},
"target" : {
"sku" : "",
"products_ids" : [],
"category_ids" : [
ObjectId("11111111")
]
},
"title" : "15% coupon"
}
I want to access date.
For example, "policies.apply.params.date"...
I don't know how to access 'date' to Javascript.
Please let me know...
apply is an array, so you have to give it index which you want to get.
var num = 0; // pick up an array number you want
var date = policies.apply[num].params.date;
Your policies.apply is an array so if you want to access "2020-07-06T14:59:59.999Z", you should do this:
policies.apply[0].params.date
But the "policies.apply[1]" doesn't have params (params.date also) so you can write a function to get date like this:
function get_apply_date(index) {
if(policies.apply[index].params && policies.apply[index].params.date)
return policies.apply[index].params.date;
return undefined; // or null
}
I've got a lot of doc filters on my UI (date ranges, checkboxes, input fields), so the query is generated dynamically - that's why I decided to create a boolean query, and push everything to must array. This is the example of my request:
const {
body: {
hits
}
} = await esclient.search({
from: filterQuery.page || 0,
size: filterQuery.limit || 1000,
index,
body: query
});
Checkboxes (I used additional bool.should inside must array) and date range work perfectly, but term/match filtering is not working at all:
{
"query": {
"bool": {
"must": [
{"match": { "issueNumber": "TEST-10" }}
]
}
}
}
The query above gives me all the documents from the index that contains "TEST" (with their scores), if I change match to term - it returns an empty array.
As my field is of a type 'text', I've also tried filter query - ES still gives all the documents with 'TEST' word:
{
"query": {
"bool": {
"must": [
{
"bool": {
"filter": {
"match": {"issueNumber": "TEST-10"}
}
}
}
]
}
}
}
This is how my hit looks like:
{
"_index" : "test_elastic",
"_type" : "_doc",
"_id" : "bj213hj2gghg213",
"_score" : 0.0,
"_source" : {
"date" : "2019-11-26T13:27:01.586Z",
"country" : "US",
"issueNumber" : "TEST-10",
}
Can someone give me input on how to filter the docs properly in complex query?
This is the structure of my index:
{
"test_elasticsearch" : {
"aliases" : { },
"mappings" : {
"properties" : {
"country" : {
"type" : "text"
},
"date" : {
"type" : "date"
},
"issueNumber" : {
"type" : "text"
}
}
},
"settings" : {
"index" : {
"creation_date" : "1574759226800",
"number_of_shards" : "1",
"number_of_replicas" : "1",
"uuid" : "PTDsdadasd-ERERER",
"version" : {
"created" : "7040299"
},
"provided_name" : "logs"
}
}
}
}
Ok, the problem is that your issueNumber field has not the right type, it should be keyword instead of text if your goal is to make exact searches on it. Same for country. Modify your mapping like this:
"properties" : {
"country" : {
"type" : "keyword"
},
"date" : {
"type" : "date"
},
"issueNumber" : {
"type" : "keyword"
}
}
Then reindex your data and your queries will work.
I want to sum counts values for each beacon. I'm using aggregate query but it returns with counts 0. Please let me know if there is mistake in query.
Sample Data
[ {
"_id" : ObjectId("5a8166392aa41ec66efc66bf"),
"userID" : "5a7c3410bdff0f0014181874",
"date" : ISODate("2018-02-08T11:04:54.000Z"),
"beacons" : [
{
"counts" : "2",
"UUID" : "red"
},
{
"counts" : "1",
"UUID" : "blue"
}
]
},
{
"_id" : ObjectId("5a8166392aa41ec66efc66c0"),
"userID" : "5a7c3410bdff0f0014181874",
"date" : ISODate("2018-02-08T11:04:54.000Z"),
"beacons" : [
{
"counts" : "2",
"UUID" : "red"
},
{
"counts" : "1",
"UUID" : "blue"
}
]
}
]
Query
/* Query */
db.getCollection('CountsDetail')
.aggregate([
{
"$unwind":"$beacons"
},
{
"$group": {
"_id":"$beacons.UUID", "counts":{ "$sum":"$counts"}
}
}]);
Response
/* Response */
{
"_id" : "red",
"counts" : 0
}
{
"_id" : "blue",
"counts" : 0
}
Response is returning 0 in sum, which is weird. Please correct what I am doing wrong. Thanks
Sorry miss read your OP. Your counts field is nested so use a dot notation:
"counts":{ "$sum":"$beacons.counts"}
I'm trying to match some document in mongoDB:
My Document model :
profile: {
languages: [
{"name": "French", "level": 1},
{"name": "English", "level": 2},
{"name": "Spanish", "level": 4}
]
}
What I (can) have to search my result set:
lang = ["French", "English"];
objLang = [
{name: "French"},
{name: "English"}
];
What I need is to db.find() all documents that match at least one of the languages, for example :
profile.languages.name = "French"
or
profile.languages.name = "English"
WHat I mean is that if I have French or English in my option set, I need to get all the users who have a element of their languages array where name match one of my options, no matter the level of the languages.
So, unless I'm wrong, I can't do
db.find({"profile.languages": {$in: [{name: "French"}, {name: "English"}]});
How would you proceed ?
Thanks a lot.
David
Actually, you were almost right:
db.collection.find({"profile.languages.name":{$in: ["French","English"]} })
Since profile.languages is an array of subdocuments, you can call in for one of the subdocument's keys and the subsequent query gets mapped to all subdocuments containing that key.
However, without proper indexing, an added .explain() shows something pretty ugly:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.languages",
"indexFilterSet" : false,
"parsedQuery" : {
"profile.languages.name" : {
"$in" : [
"English",
"French"
]
}
},
"winningPlan" : {
"stage" : "COLLSCAN",
"filter" : {
"profile.languages.name" : {
"$in" : [
"English",
"French"
]
}
},
"direction" : "forward"
},
"rejectedPlans" : [ ]
}
}
(serverInfowas ommited).
So in order to make this query efficient, you need to create an index over the field you want to query:
db.languages.ensureIndex({"profile.languages.name":1})
An added explain now tells us that the matching documents are identified via the index:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "test.languages",
"indexFilterSet" : false,
"parsedQuery" : {
"profile.languages.name" : {
"$in" : [
"English",
"French"
]
}
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"profile.languages.name" : 1
},
"indexName" : "profile.languages.name_1",
"isMultiKey" : true,
"direction" : "forward",
"indexBounds" : {
"profile.languages.name" : [
"[\"English\", \"English\"]",
"[\"French\", \"French\"]"
]
}
}
},
"rejectedPlans" : [ ]
},
"ok" : 1
}
You could try using the dot notation to access both the elements of an array and to access the fields of an embedded document:
db.find({
"profile.languages.name": {
"$in": ["French", "English"]
}
});
Please try the below query :
Solution 1 :
db.collection.aggregate([
{
$unwind:"$profile.languages"
},
{
$match:{"profile.languages.name":{"$in":["French", "English"]}}
},
{
$group:{_id: "$_id", profile:{"$push":"$profile.languages"}}
}
])
Solution 2 :
db.collection.find(
{
"profile.languages.name":
{
"$in": [ "English", "French"]
}
}
);
I have this data in Mongo:
{'_id':1,
'name':'Root',
'taskId':1,
'parentId':"",
'path':[1],
'tasks':[ {"taskId":3,parentId:1,name:'A',type:'task'},
{"taskId":4,parentId:1,name:'D',type:'task'},
{"taskId":5,parentId:4,name:'B',type:'task'},
{'type':'project' , 'proRef':2},
{"taskId":6,parentId:3,name:'E',type:'task'},
{"taskId":7,parentId:6,name:'C',type:'task'}]
}
Now I want to update taskId 6 with new Json data .
var jsonData = {"taskId":6,"name":'Sumeet','newField1':'Val1','newField2':'Val2'}
query should update if field is available else add new key to existing .Output Like
{"taskId":6,parentId:3,name:'Sumeet',type:'task','newField1':'Val1','newField2':'Val2'}]
I have tried few query but it is completely replacing json .
db.projectPlan.update({_id:1,'tasks.taskId':6},{$set :{'tasks.$':jsonData }});
Thanks in advance for your helps!
Sumeet
You need to transform the jsonData variable into something that can be passed to update. Here's an example that does exactly what you want with your sample document:
var updateData = {};
for (f in jsonData) {
if (f != "taskId") updateData["tasks.$."+f]=jsonData[f];
};
db.projectPlan.update({_id:1, 'tasks.taskId':6}, {$set:updateData})
Result:
{ "_id" : 1,
"name" : "Root",
"taskId" : 1,
"parentId" : "",
"path" : [ 1 ],
"tasks" : [
{ "taskId" : 3, "parentId" : 1, "name" : "A", "type" : "task" },
{ "taskId" : 4, "parentId" : 1, "name" : "D", "type" : "task" },
{ "taskId" : 5, "parentId" : 4, "name" : "B", "type" : "task" },
{ "type" : "project", "proRef" : 2 },
{ "taskId" : 6, "parentId" : 3, "name" : "Sumeet", "type" : "task", "newField1" : "Val1", "newField2" : "Val2" },
{ "taskId" : 7, "parentId" : 6, "name" : "C", "type" : "task" }
] }
You will need to merge the document manually:
var jsonData = {"taskId":5,"name":'Sumeet','newField1':'Val1','newField2':'Val2'};
db.projectPlan.find({ _id: 1 }).forEach(
function(entry) {
for (var taskKey in entry.tasks) {
if (entry.tasks[taskKey].taskId === jsonData.taskId) {
printjson(entry.tasks[taskKey]);
for (var taskSubKey in jsonData) {
entry.tasks[taskKey][taskSubKey] = jsonData[taskSubKey];
}
printjson(entry.tasks[taskKey]);
}
}
db.projectPlan.save(entry);
}
);
Obviously you can leave away the printjson statements. This is simply to see that the merging of the original tasks with the new tasks works. Note that this query will only update a single document as long as the _id field is unique.