Develop Reference

Elasticsearch sorting by custom item weight

I have stored the documents which include status property. I would like to sort the documents by status priority (not status alphabetically). I have followed previous answers and composed the following function which still doesnt work as expected; the documents are sorted by status names (alphabetically):
function getESSortingByStatusQuery(query, order) {
let statusOrder = ['BLUE', 'RED', 'BLACK', 'YELLOW', 'GREEN'];
if(order == 'desc'){
statusOrder.reverse();
}
const functions = statusOrder.map((item) => {
const idx = statusOrder.indexOf(item);
return {filter: {match: {statusColor: item}},
weight: (idx + 1) * 50}
});
const queryModified = {
"function_score": {
"query": {"match_all": {}}, // this is for testing purposes and should be replaced with original query
"boost": "5",
"functions": functions,
"score_mode": "multiply",
"boost_mode": "replace"
}
}
return queryModified;
}
I would be thankful if anyone suggested the way to sort items according to predefined priority of the property (in this case status).

Below is a sample custom sort script which I think is what you are looking for. I've added sample mapping, documents, query and the response as how it appears.
Mapping:
PUT color_index
{
"mappings": {
"properties": {
"color":{
"type": "keyword"
},
"product":{
"type": "text"
}
}
}
}
Sample Documents:
POST color_index/_doc/1
{
"color": "BLUE",
"product": "adidas and nike"
}
POST color_index/_doc/2
{
"color": "GREEN",
"product": "adidas and nike and puma"
}
POST color_index/_doc/3
{
"color": "GREEN",
"product": "adidas and nike"
}
POST color_index/_doc/4
{
"color": "RED",
"product": "nike"
}
POST color_index/_doc/5
{
"color": "RED",
"product": "adidas and nike"
}
Query:
POST color_index/_search
{
"query": {
"bool": {
"must": [
{
"query_string": {
"default_field": "*",
"query": "adidas OR nike"
}
}
]
}
},
"sort": [
{ "_score": { "order": "desc"} }, <---- First sort by score
{ "_script": { <---- Second sort by Colors
"type": "number",
"script": {
"lang": "painless",
"source": "if(params.scores.containsKey(doc['color'].value)) { return params.scores[doc['color'].value];} return 100000;",
"params": {
"scores": {
"BLUE": 0,
"RED": 1,
"BLACK": 2,
"YELLOW": 3,
"GREEN": 4
}
}
},
"order": "asc"
}
}
]
}
Firstly it would return documents sorted by its score, and then it would apply the second sorting logic to that result.
For the second sorting, i.e. using script sort, notice how I have added the numeric values to the colors in the scores section. You would need to construct your query accordingly.
The logic as how it works is in the source section which I believe is self-explainable, where I used doc['color'].value as that was my field on which I'm applying custom sort logic.
Response:
{
"took" : 5,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 5,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "color_index",
"_type" : "_doc",
"_id" : "1",
"_score" : 0.5159407,
"_source" : {
"color" : "BLUE",
"product" : "adidas and nike"
},
"sort" : [
0.5159407, <--- This value is score(desc by nature)
0.0 <--- This value comes from script sort as its BLUE and I've used value 0 in the script which is in 'asc' order
]
},
{
"_index" : "color_index",
"_type" : "_doc",
"_id" : "5",
"_score" : 0.5159407,
"_source" : {
"color" : "RED",
"product" : "adidas and nike"
},
"sort" : [
0.5159407,
1.0
]
},
{
"_index" : "color_index",
"_type" : "_doc",
"_id" : "3",
"_score" : 0.5159407,
"_source" : {
"color" : "GREEN",
"product" : "adidas and nike"
},
"sort" : [
0.5159407,
4.0
]
},
{
"_index" : "color_index",
"_type" : "_doc",
"_id" : "2",
"_score" : 0.40538198,
"_source" : {
"color" : "GREEN",
"product" : "adidas and nike and puma"
},
"sort" : [
0.40538198,
4.0
]
},
{
"_index" : "color_index",
"_type" : "_doc",
"_id" : "4",
"_score" : 0.10189847,
"_source" : {
"color" : "RED",
"product" : "nike"
},
"sort" : [
0.10189847,
1.0
]
}
]
}
}
Notice the first three documents, it has exact value of product but different color and you can see that they are grouped together as we first sorted by _score then we sort that by color
Let me know if this helps!

Here's the code sample of sorting result. I think this will helps you. If you don't want to get entire documents as result you can filter results using includes.
GET testindex/_search
{
"_source": {
"includes": [
"filed1"
]
},
"aggs": {
"emp_figures": {
"terms": {
"field": "status"
}
}
}
}
This is the sample result you should retrieve
{
"took": 11,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"failed": 0
},
"hits": {
"total": 84968,
"max_score": 1,
"hits": [
{
"_index": "test",
"_type": "type",
"_id": "0001",
"_score": 1,
"_source": {
"filed1": "color1,
}
},
{
"_index": "test",
"_type": "type",
"_id": "0002",
"_score": 1,
"_source": {
"filed1": "color2,
}
}
}
}
}

Using Javascript aggregation in druid queries

I am trying to write javascript aggregator for my druid queries. i need to to calculate average of a metric "Base_SalesRank".
So far i have been able to this by writing:
{
"queryType": "groupBy",
"dataSource": "marketdata",
"granularity": "all",
"dimensions" : ["Item"],
"filter": { "type": "and", "fields" : [{"type": "selector", "dimension": "Item", "value": "MN10CESWW"}]},
"intervals": ["2018-06-28T00:00Z/2018-07-04T00:00Z"],
"aggregations" : [
{ "type" : "count", "name" : "rows" },
{ "type" : "doubleSum", "name" : "Base_SalesRank", "fieldName" : "Base_SalesRank" }
],
"postAggregations" : [{
"type": "javascript",
"name": "Target DOS Average",
"fieldNames": ["Base_SalesRank", "rows"],
"function": "function(Base_SalesRank, rows) {return Base_SalesRank/ rows;}"
}]
}
But I noticed that many values in Base_SalesRank is 0.
[ {
"timestamp" : "2018-06-28T05:06:03.000Z",
"result" : {
"pagingIdentifiers" : {
"marketdata_2018-06-28T00:00:00.000Z_2018-06-29T00:00:00.000Z_2018-07-06T08:11:02.499Z" : 3
},
"dimensions" : [ "Item" ],
"metrics" : [ "Base_SalesRank" ],
"events" : [ {
"segmentId" : "marketdata_2018-06-28T00:00:00.000Z_2018-06-29T00:00:00.000Z_2018-07-06T08:11:02.499Z",
"offset" : 0,
"event" : {
"timestamp" : "2018-06-28T07:10:02.000Z",
"Item" : "MN10CESWW",
"Base_SalesRank" : 0
}
},
{
"segmentId" : "marketdata_2018-06-28T00:00:00.000Z_2018-06-29T00:00:00.000Z_2018-07-06T08:11:02.499Z",
"offset" : 3,
"event" : {
"timestamp" : "2018-06-28T07:20:21.000Z",
"Item" : "MN10CESWW",
"Base_SalesRank" : 5558
}
} ]
}
} ]
So i am not getting true average. Now i need to weed out these 0 values and then calucate average. We can do this by using filters
{"type": "not", "field": {"type": "selector", "dimension": "Base_SalesRank", "value": "0"}}
But I have constraint that I have to perform this filter operation inside the javascript function only.

You can achieve the same with just adding a having query -
"having": {
"type": "greaterThan",
"aggregation": "Base_SalesRank",
"value": 0
}
If you want to do the same in javascript function than it can be done as below -
You should add a dimension (key/value) say "isValid" as "0" or "1" during pre-ingestion json data based on if Base_SalesRank is 0 than "isValid" will be 0 else 1.
Apply filter on this field in your query.
Use the rows in your post Aggregration.

D3 tree/hierachical related data between nodes

I am working on a d3 project at the moment, and I am trying to map out a hierachical tree to show people and who they are responsible for. Basically I can user A and user B and they can each be responsible for the same person.
Currently to highlight this in my JSON data that builds the visualisation I am repeating data, is there away to not repeat data and use the same data point when 2 or more people are responsible for the same person?
Here is my JSfiddle example
My Hierachical Visualisation
You will see here that, Raymond Reddington & Donald Ressler have cross over between some of their responsibilites, I am repeating the data which seems inefficient, is there a better way, here is my JSON.
[
{
"name" : "Company Name",
"parent" : null,
"children": [
{
"name" : "Raymond Reddington",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Debe Zuma",
"parent" : "Raymond Reddington",
},
{
"name" : "Tom Keen",
"parent" : "Raymond Reddington",
},
{
"name" : "Aram Mojtabai",
"parent" : "Raymond Reddington",
}
]
},
{
"name" : "Elizabeth Keen",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Samar Navabi",
"parent" : "Elizabeth Keen",
},
{
"name" : "Meera Malik",
"parent" : "Elizabeth Keen",
},
{
"name" : "Mr. Kaplan",
"parent" : "Elizabeth Keen",
},
{
"name" : "Reven Wright",
"parent" : "Elizabeth Keen",
}
]
},
{
"name" : "Donald Ressler",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Matius Solomon",
"parent" : "Donald Ressler",
"size" : 3938
},
{
"name" : "Peter Kotsiopulos",
"parent" : "Donal Ressler",
"size" : 3938
},
{
"name" : "Tom Keen",
"parent" : "Raymond Reddington",
"size" : 3938
},
{
"name" : "Aram Mojtabai",
"parent" : "Raymond Reddington",
"size" : 3938
}
]
},
{
"name" : "Harold Cooper",
"parent" : "Cherry Tree Lodge",
"children" : [
{
"name" : "Samar Navabi",
"parent" : "Elizabeth Keen",
"size" : 3938
},
{
"name" : "Meera Malik",
"parent" : "Elizabeth Keen",
"size" : 3938
}
]
}
]
}
]

This website details a method of converting flat data to the hierarchical data required by d3 http://www.d3noob.org/2014/01/tree-diagrams-in-d3js_11.html
They explain it well too. As the author notes it is originally based on https://stackoverflow.com/a/17849353/1544886
I have copied and pasted their website's example below:
var data = [
{ "name" : "Level 2: A", "parent":"Top Level" },
{ "name" : "Top Level", "parent":"null" },
{ "name" : "Son of A", "parent":"Level 2: A" },
{ "name" : "Daughter of A", "parent":"Level 2: A" },
{ "name" : "Level 2: B", "parent":"Top Level" }
];
will map to:
var treeData = [
{
"name": "Top Level",
"parent": "null",
"children": [
{
"name": "Level 2: A",
"parent": "Top Level",
"children": [
{
"name": "Son of A",
"parent": "Level 2: A"
},
{
"name": "Daughter of A",
"parent": "Level 2: A"
}
]
},
{
"name": "Level 2: B",
"parent": "Top Level"
}
]
}
];
via:
var dataMap = data.reduce(function(map, node) {
map[node.name] = node;
return map;
}, {});
var treeData = [];
data.forEach(function(node) {
// add to parent
var parent = dataMap[node.parent];
if (parent) {
// create child array if it doesn't exist
(parent.children || (parent.children = []))
// add node to child array
.push(node);
} else {
// parent is null or missing
treeData.push(node);
}
});
You could extend that further replacing with Ids and using a second normalised array for the lookup:
[{
"id": 0,
"name": "Cherry Tree Lodge"
},{
"id": 1,
"name": "Tom Keen"
},{
"id": 2,
"name": "Debe Zuma"
}]
Also please note that your json data is not strictly valid, you have extra commas.

Cannot access a field in JSON

I am trying to access an image in a JSON response however the field I need to access is an id value that is unique or rather is random. We are fetching this data from a server so we cannot hard code the id's.
The JSON is as follows:
{ "error" : { "occured" : "false" },
"errors" : [ ],
"executiontime" : 2500,
"metadata" : { },
"value" : [ { "activity_duration" : "1 hour, ½ day & full day packages",
"adult_rate_high_period_high_price" : 275,
"adult_rate_high_period_low_price" : 49,
"adult_rate_low_period_high_price" : "",
"adult_rate_low_period_low_price" : "",
"amenities" : [ ],
"assets" : { "logo" : { "436209" : { "asset_type" : "image",
"caption" : "",
"credit" : "",
"description" : "",
"exists" : "true",
"height" : 82,
"label" : "Copy of Monarch logo",
"latitude" : 0,
"longitude" : 0,
"market" : "$",
"o_id" : 3221685,
"type_o_id" : 2543991,
"unique_id" : 436209,
"url" : "http://c0481729.cdn2.cloudfiles.rackspacecloud.com/p-DD951E3E-C7AF-F22C-77E98D299833B38F-2544001.jpg",
"width" : 220
} },
We are trying to display the business logo for each amenity. To do this I need to access the url field in the above JSON. How do I access the url field under assest.

The Problem is to get the id of the Logo 436209.
var theid;
var l = obj.value[0].assets.logo
for (var p in l) {
if (l[p].hasOwnProperty('unique_id')) {
theid = l[p].unique_id;
break;
}
}
This is untestet. The idee is to use the in-operator to iterate over the properties of the logo-object and get the propterty that has the unique_id.

Correction:
obj.value[0].assets.logo["436209"].url = 'foo';
// or
var foo = obj.value[0].assets.logo["436209"].url;
This assumes that your object is well formed and continues with more parts of obj.value[0].
Specifically, if your object were completed, perhaps, like this:
var obj = {
"error": { "occured": "false" },
"errors": [],
"executiontime": 2500,
"metadata": {},
"value": [{
"activity_duration": "1 hour, ½ day & full day packages",
"adult_rate_high_period_high_price": 275,
"adult_rate_high_period_low_price": 49,
"adult_rate_low_period_high_price": "",
"adult_rate_low_period_low_price": "",
"amenities": [],
"assets": {
"logo": {
"436209": {
"asset_type": "image",
"caption": "",
"credit": "",
"description": "",
"exists": "true",
"height": 82,
"label": "Copy of Monarch logo",
"latitude": 0,
"longitude": 0,
"market": "$",
"o_id": 3221685,
"type_o_id": 2543991,
"unique_id": 436209,
"url": "http://c0481729.cdn2.cloudfiles.rackspacecloud.com/p-DD951E3E-C7AF-F22C-77E98D299833B38F-2544001.jpg",
"width": 220
}
}
}
}]
};

Multi options in select2

I am trying to use select2 to get remote JSON data and display it with mutli levels.
http://ivaynberg.github.io/select2/index.html
This is my response
{
"Company": [
{
"name": "athenahealth Inc"
},
{
"name": "Localiza Rent a Car"
},
{
"name": "M and B Switchgears"
}
],
"Functional": [
{
"name": "arranger"
},
{
"name": "ambassadors"
}
],
"Persons": [
{
"name": "Moustapha al"
},
{
"name": "Saleh al"
}
]
}
I want to show the result in Multi-Value format - http://ivaynberg.github.io/select2/index.html#multi
So far i am able to fetch data from server side , but then i have no idea how to enable the multi select option.

JSON in following format will work fine
Related issue - https://github.com/ivaynberg/select2/issues/58
{ "Data" : [ {
"id" :1 ,
"text" : "Subsection" ,
"children" : [{
"id" : 2,
"text" : "Paru"
},
{
"id" : 3,
"text" : "Vinu"
}]
},
{ "id" : 4 ,
"text" : "Family" ,
"children" : [{
"id" : 5,
"text" : "ChildVM"
},
{
"id" : 6,
"text" : "ChildPM"
}]
}
]
}