Related
I have a need to get data out of a NiFi flow file with somewhat complex JSON content. I'm planning to use a NiFi ExecuteScript processor because I don't think it can be done with EvaluateJSONPath. The content looks like this (snippet)
...
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
],
...
I want to extract the REF02 property value from the segments array element that has REF01 === '0F'. The array element does not necessarily have a REF02 property. So in the above case, I should get 1041558xxxxx.
Here's my current script:
var flowFile = session.get()
if (flowFile != null) {
var InputStreamCallback = Java.type('org.apache.nifi.processor.io.InputStreamCallback')
var IOUtils = Java.type('org.apache.commons.io.IOUtils')
var StandardCharsets = Java.type('java.nio.charset.StandardCharsets')
try {
var subscriber = null
session.read(flowFile,
new InputStreamCallback(function (inputStream) {
var data = JSON.parse(IOUtils.toString(inputStream, StandardCharsets.UTF_8))
var segment = data.segments.find(function (s) { return s.hasOwnProperty('REF01') && s.REF01 === '0F' })
subscriber = segment ? segment.REF02 : null
}));
session.putAttribute(flowFile, 'subscriber', subscriber ? subscriber : '')
session.transfer(flowFile, REL_SUCCESS)
} catch (e) {
log.error('script failed', e)
session.transfer(flowFile, REL_FAILURE)
}
}
When I execute the above, I get a java.lang.NoSuchMethodException. Also, are anonymous 'arrow' functions allow?
I've tried using an old-school for loop to no avail.
Thanks for your help.
You can add a JoltTransformJSON processor with specification
[
{
"operation": "shift",
"spec": {
"segments": {
"*": {
"REF01": {
"0F": {// conditional to match "REF01" with "0F"
"#2,REF02": ""// go two levels up the three to reach the level of the attributes REF01 or REF02
}
}
}
}
}
}
]
in order to return the result
"1041558xxxxx"
You can use below JSONPath with EvaluateJSONPath processor:
$.segments[?(#.REF01<="0F")]#.REF02
Note: Returned result is in the array, So you can use SplitJSON after that to get your string.
Groovy script:
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import groovy.json.JsonSlurper
flowFile = session.get()
if(!flowFile) return
def jsonSlurper = new JsonSlurper()
def subscriber = ""
flowFile = session.write(flowFile, {inputStream, outputStream ->
input = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
json = jsonSlurper.parseText(input)
segment = json.segments.find{ segment ->
if (segment.keySet().contains('REF01')) {
if (segment.REF01 == '0F') {
return true
} else {
return false
}
} else {
return false
}
}
if (segment) {
subscriber = segment.REF02
}
outputStream.write(input.getBytes(StandardCharsets.UTF_8))
} as StreamCallback)
session.putAttribute(flowFile, 'subscriber', subscriber)
session.transfer(flowFile, REL_SUCCESS)
input:
{
"test": "best",
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
]
}
output (with attribute subscriber: 1041558xxxxx):
{
"test": "best",
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
]
}
I have the following hierarchical JSON data retrieved from a PostgreSQL DB using PHP:
[{"treelevel":"1","app":"Front","lrflag":null,"ic":null,"price":null,"parentlevel":"0","seq":"27", "indexlistid":439755},
{"treelevel":"2","app":"V-Series","lrflag":null,"ic":null,"price":null,"parentlevel":"1","seq":"28", "indexlistid":439755},
{"treelevel":"3","app":"opt J56","lrflag":null,"ic":null,"price":null,"parentlevel":"2","seq":"29", "indexlistid":439755},
{"treelevel":"4","app":"R.","lrflag":"R","ic":"536-01132AR","price":"693.00","parentlevel":"3","seq":"30", "indexlistid":439755},
{"treelevel":"4","app":"L.","lrflag":"L","ic":"536-01133AL","price":"693.00","parentlevel":"3","seq":"31", "indexlistid":439755},
{"treelevel":"3","app":"opt J63","lrflag":null,"ic":null,"price":null,"parentlevel":"2","seq":"32", "indexlistid":439755},
{"treelevel":"4","app":"R.","lrflag":"R","ic":"536-01130R","price":null,"parentlevel":"3","seq":"33", "indexlistid":439755},
{"treelevel":"4","app":"L.","lrflag":"L","ic":"536-01131L","price":null,"parentlevel":"3","seq":"34", "indexlistid":439755}]
I need some way to reformat the data to this:
[{"app": "Front-V-Series-opt J56-R. R", "price": "$693", "ic": "536-01132AR"},
{"app": "Front-V-Series-opt J56-L. L", "price": "$693", "ic": "536-01132AL"},
{"app": "Front-V-Series-opt J63-R. R", "price": null, "ic": "536-01130R"},
{"app": "Front-V-Series-opt J63-L. L", "price": null, "ic": "536-01131L"}]
The parentlevel and treelevel are the 2 keys that make up the tree relationship.
The app value in the new format is a concatenation of the app and lrflag values from all the
nodes in one tree level + the price and ic from the deepest leaf node in the level. This is the
data visualized as a tree:
[
{
"app": "Front",
"children": [
{
"app": "V-Series",
"children": [
{
"app": "opt J56",
"children": [
{
"app": "R. ,
"lrflag": "R",
"ic": "536-01132AR",
"price": "$693"
},
{
"app": "L. ,
"lrflag": "L",
"ic": "536-01132AL",
"price": "$693"
}
]
},
{
"app": "opt J63",
"children": [
{
"app": "R. ,
"lrflag": "R",
"ic": "536-01130R"
},
{
"app": "L. ,
"lrflag": "L",
"ic": "536-01131L"
}
]
}
]
}
]
}
]
I've tried several different ways to do this but am just stuck. Here are some of the functions
that I've attempted to modify and use to no luck. I can't even get the tree correctly built from
these.
function buildTree(list) {
var map = {}, node, roots = [], i;
for (i = 0; i < list.length; i += 1) {
map[list[i].treelevel] = i; // initialize the map
list[i].children = []; // initialize the children
}
for (i = 0; i < list.length; i += 1) {
node = list[i];
if (node.parentlevel !== "0") {
// if you have dangling branches check that map[node.parentId] exists
list[map[node.parentlevel]].children.push(node);
} else {
roots.push(node);
}
}
return roots;
}
function listToTree(data, options) {
options = options || {};
var ID_KEY = options.idKey || 'treelevel';
var PARENT_KEY = options.parentKey || 'parentlevel';
var CHILDREN_KEY = options.childrenKey || 'children';
var tree = [],
childrenOf = {};
var item, id, parentId;
for (var i = 0, length = data.length; i < length; i++) {
item = data[i];
id = item[ID_KEY];
parentId = item[PARENT_KEY] || 0;
// every item may have children
childrenOf[id] = childrenOf[id] || [];
// init its children
item[CHILDREN_KEY] = childrenOf[id];
if (parentId != 0) {
// init its parent's children object
childrenOf[parentId] = childrenOf[parentId] || [];
// push it into its parent's children object
childrenOf[parentId].push(item);
} else {
tree.push(item);
}
};
return tree;
}
unflattenToObject = function(array, parent) {
var tree = {};
parent = typeof parent !== 'undefined' ? parent : {id: 0};
var childrenArray = array.filter(function(child) {
return child.treelevel == parent.parentlevel;
});
if (childrenArray.length > 0) {
var childrenObject = {};
// Transform children into a hash/object keyed on token
childrenArray.forEach(function(child) {
childrenObject[child.treelevel] = child;
});
if (parent.treelevel == 0) {
tree = childrenObject;
} else {
parent['children'] = childrenObject;
}
childrenArray.forEach(function(child) {
unflattenToObject(array, child);
})
}
return tree;
};
the idea is to get the first element's index in the max depth treelevel : 4 then start from there and loop backwards, making sure you don't go through the parents twice, and concatenate what you need along the way, once you reach the top level, you delete that element
wrap that in a function and call it recursively until there's no treelevel : 4 left
var data = [{
"treelevel": "1",
"app": "Front",
"lrflag": null,
"ic": null,
"price": null,
"parentlevel": "0",
"seq": "27",
"indexlistid": 439755
},
{
"treelevel": "2",
"app": "V-Series",
"lrflag": null,
"ic": null,
"price": null,
"parentlevel": "1",
"seq": "28",
"indexlistid": 439755
},
{
"treelevel": "3",
"app": "opt J56",
"lrflag": null,
"ic": null,
"price": null,
"parentlevel": "2",
"seq": "29",
"indexlistid": 439755
},
{
"treelevel": "4",
"app": "R.",
"lrflag": "R",
"ic": "536-01132AR",
"price": "693.00",
"parentlevel": "3",
"seq": "30",
"indexlistid": 439755
},
{
"treelevel": "4",
"app": "L.",
"lrflag": "L",
"ic": "536-01133AL",
"price": "693.00",
"parentlevel": "3",
"seq": "31",
"indexlistid": 439755
},
{
"treelevel": "3",
"app": "opt J63",
"lrflag": null,
"ic": null,
"price": null,
"parentlevel": "2",
"seq": "32",
"indexlistid": 439755
},
{
"treelevel": "4",
"app": "R.",
"lrflag": "R",
"ic": "536-01130R",
"price": null,
"parentlevel": "3",
"seq": "33",
"indexlistid": 439755
},
{
"treelevel": "4",
"app": "L.",
"lrflag": "L",
"ic": "536-01131L",
"price": null,
"parentlevel": "3",
"seq": "34",
"indexlistid": 439755
}
];
// first go look for the max depth in the tree, if it's always four, skip this part and put var max = 4
var max = 0;
data.forEach(function(elem) {
if (elem.treelevel >= max)
max = elem.treelevel;
});
// implement a function to get the index of the first matched element with treelevel = max depth
function getIndex() {
return data.indexOf(data.find((elem) => {
return elem.treelevel == max;
}));
}
var myTree = [];
function formObject(ndx) {
var myObj = {};
myObj.app = [];
var nextTreeLevel = 3;
// start looping from the index of the element in max depth backwards
for (var i = ndx; i >= 0; i--) {
// if the next element ( backwards ) is a parent , this is to avoid going through parent's siblings
if (nextTreeLevel == data[i].parentlevel) {
if (data[i].ic != null) myObj.ic = data[i].ic;
if (data[i].price != null) myObj.price = data[i].price;
if (data[i].lrflag != null) myObj.app.push(data[i].lrflag);
myObj.app.push(data[i].app);
nextTreeLevel = data[i].parentlevel - 1; // prent's level
}
}
data.splice(ndx, 1); // remove the lement once you're done with it
// glue the "app" together, would be better to use array.join but it's not the same join everywhere
myObj.app = myObj.app[4] + '-' + myObj.app[3] + '-' + myObj.app[2] + '-' + myObj.app[1] + ' ' + myObj.app[0];
// just to fill the price with null if there is none
if (myObj.price == undefined) myObj.price = null;
// push the object to the result's array
myTree.push(myObj);
// get the index of the next element in max depth ( 4 )
var nextIndex = getIndex();
// if there's still another element in the max depth, recall the same function with it's index
if (nextIndex > -1)
formObject(nextIndex)
}
formObject(getIndex())
console.log(myTree);
This is my JSON Object in this i want to remove "item" key from json and want to keep its inner object include with "children" and its tree like JOSN. How can I do this?
[
{
"item": {
"id": 11865,
"parentid": null,
"levelid": 63,
"name": "Total"
},
"children": [
{
"item": {
"id": 10143,
"parentid": 11865,
"levelid": 19,
"name": "Productive"
}
}
]
}
]
If I'm understanding what you want your object to look like after correctly, then this should do the trick:
var arrayOfObjects = [
{
"item": {
"id": 11865,
"parentid": null,
"levelid": 63,
"name": "Total"
},
"children": [
{
"item": {
"id": 10143,
"parentid": 11865,
"levelid": 19,
"name": "Productive"
}
}
]
}
]
arrayOfObjects.forEach(function(obj) {
obj.id = obj.item.id;
obj.parentid = obj.item.parentid;
obj.levelid = obj.item.levelid;
obj.name = obj.item.name;
delete obj.item;
});
All this is doing is manually moving the data from obj.item to obj and then deleting obj.item entirely.
I would do this:
//your original array of objects
var array = [{
"item": {
"id": 11865,
"parentid": null,
"levelid": 63,
"name": "Total"
},
"children": [
{
"item": {
"id": 10143,
"parentid": 11865,
"levelid": 19,
"name": "Productive"
}
}
]
}, ...];
array.forEach(function(parent) {
flattenKey(parent, 'item');
});
function flattenKey(parent, keyName) {
var section = parent[keyName];
var child = section ? section : {};
var keys = Object.keys(child);
keys.forEach(function(key) {
parent[key] = child[key];
})
delete parent[keyName];
}
basically, the function flattenKey would flatten any key for a given object (given its key).
logic is similar to other solutions here: iterate through child keys and assign their values to the parent object (flattening).
then it deletes the child key after step 1.
try
objArray = objArray.map( function(value){
var item = value.item;
for( var key in item )
{
value[key] = item[key];
}
delete value.item;
return value;
});
DEMO
Explanation
1) Use map to iterate on each item (value) of this given array objArray.
2) Get the item property of value, assign them to value directly
3) Finally delete the item property of the value.
Faster option
objArray = objArray.map( function(value){
var item = value.item;
var itemKeys = Object.keys(item);
for( var counter = 0; counter < itemKeys.length; counter++ )
{
value[itemKeys[counter]] = item[itemKeys[counter]];
}
delete value.item;
return value;
});
You can use a recursion, which keeps the content of item and adds the children property as well.
function delItem(a, i, aa) {
var children = a.children;
if (a.item) {
aa[i] = a.item;
aa[i].children = children;
delete a.item;
}
Array.isArray(children) && children.forEach(delItem);
}
var array = [{ "item": { "id": 11865, "parentid": null, "levelid": 63, "name": "Total" }, "children": [{ "item": { "id": 10143, "parentid": 11865, "levelid": 19, "name": "Productive" } }] }];
array.forEach(delItem);
document.write('<pre>' + JSON.stringify(array, 0, 4) + '</pre>');
This is my saved localstorage,
[{"industry_Id":1,"merchant_id":2}]
I want to filter below result, to get HP.
{
"industries": [
{
"id": 1,
"name": "oil and gas",
"merchant": [
{
"id": 1,
"name": "ABC",
},
{
"id": 2,
"name": "DEF",
},
{
"id": 3,
"name": "GHJ",
}
]
},
{
"id": 2,
"name": "IT",
"merchant": [
{
"id": 1,
"name": "Apple",
},
{
"id": 2,
"name": "HP",
},
{
"id": 3,
"name": "Google",
}
]
}
]
}
I thought of using multiple $.each but it have to iterate few times and it's quite redundant.
I would prefer using Javascript for loop, that way you can skip iterating over every object once required element is found.
Without jQuery (using for)
var i, j, merchant = null;
for(i = 0; i < data['industries'].length; i++){
if(data['industries'][i]['id'] == arg[0]['industry_Id']){
for(j = 0; j < data['industries'][i]['merchant'].length; j++){
if(data['industries'][i]['merchant'][j]['id'] == arg[0]['merchant_id']){
merchant = data['industries'][i]['merchant'][j];
break;
}
}
if(merchant !== null){ break; }
}
}
With jQuery (using $.each)
var merchant_found = null;
$.each(data['industries'], function(i, industry){
if(industry['id'] == arg[0]['industry_Id']){
$.each(industry['merchant'], function(i, merchant){
if(merchant['id'] == arg[0]['merchant_id']){
merchant_found = merchant;
}
return (!merchant_found);
});
}
return (!merchant_found);
});
var arg = [{"industry_Id":1,"merchant_id":2}];
var data = {
"industries": [
{
"id": 1,
"name": "oil and gas",
"merchant": [
{
"id": 1,
"name": "ABC",
},
{
"id": 2,
"name": "DEF",
},
{
"id": 3,
"name": "GHJ",
}
]
},
{
"id": 2,
"name": "IT",
"merchant": [
{
"id": 1,
"name": "Apple",
},
{
"id": 2,
"name": "HP",
},
{
"id": 3,
"name": "Google",
}
]
}
]
};
var i, j, merchant = null;
for(i = 0; i < data['industries'].length; i++){
if(data['industries'][i]['id'] == arg[0]['industry_Id']){
for(j = 0; j < data['industries'][i]['merchant'].length; j++){
if(data['industries'][i]['merchant'][j]['id'] == arg[0]['merchant_id']){
merchant = data['industries'][i]['merchant'][j];
break;
}
}
if(merchant !== null){ break; }
}
}
console.log(merchant);
document.writeln("<b>Without jQuery:</b><br>");
document.writeln((merchant !== null) ? "Found " + merchant['name'] : "Not found");
var merchant_found = null;
$.each(data['industries'], function(i, industry){
if(industry['id'] == arg[0]['industry_Id']){
$.each(industry['merchant'], function(i, merchant){
if(merchant['id'] == arg[0]['merchant_id']){
merchant_found = merchant;
}
return (!merchant_found);
});
}
return (!merchant_found);
});
console.log(merchant_found);
document.writeln("<br><br><b>With jQuery:</b><br>");
document.writeln((merchant_found) ? "Found " + merchant_found['name'] : "Not found");
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
selectors.map(function(selector) {
return data.industries.filter(function(industry) {
return industry.id == selector.industry_Id;
})[0].merchant.filter(function(merchant) {
return merchant.id == selector.merchant_id;
})[0].name;
});
// => DEF
If you want "HP", you want industry 2, not industry 1.
.filter(...)[0] is not really optimal. You could use .find(...), but that is not yet universally supported. Or you could use plain old JavaScript and write for loops instead to make it fast. Or you could use objects with ID keys instead of arrays to make lookups faster.
When it comes into a position where collection of data is what you're processing, I suggest you to take a look at underscore.js. It's not optimal choice for the best performance but it does make you code more readable and makes more sense especially when compared with loop.
Say data is a variable which stores your JSON data.
Try this:
// Given this selector criteria
var select = [{"industry_Id":1,"merchant_id":2}];
function filterByCriteria(criteria, data){
var match = [];
_.each(criteria, function(crit){
function matchIndustry(rec){ return rec.id===crit.industry_Id }
function matchMerchant(rec){ return rec.id===crit.merchant_id }
// Filter by industry id
var industry = _.first(_.where(data.industry, matchIndustry));
// Filter by merchant id
var merchant = _.where(industry.merchant, matchMerchant);
_.each(merchant, function addToMatchResult(m){
match.push(m.name);
});
});
return match;
}
var filteredData = filterByCriteria(select, data);
From snippet above, any merchants which match the search criteria will be taken to the match list. Is it more readable to you?
Do you even need numerical id's? Gets super easy when you don't.
/*
{
"industry": {
"oil and gas":{
"merchant": {
"ABC": {
"name": "ABC oil"
},
"DEF": {
"name": "DEF gas"
},
"GHJ" :{
"name": "GHJ oil and gas"
}
}
},
"IT": {
"merchant": {
"Apple" : {
"name": "Apple computers"
},
"HP": {
"name": "Hewlett Packard"
},
"Google": {
"name": "Google. Maw haw haw"
}
}
}
}
}
*/
var data = '{"industry": {"oil and gas":{"merchant": {"ABC": {"name": "ABC oil"},"DEF": {"name": "DEF gas"},"GHJ" :{"name": "GHJ oil and gas"}}},"IT": {"merchant": {"Apple" : {"name": "Apple computers"},"HP": {"name": "Hewlett Packard"},"Google": {"name": "Google. Maw haw haw"}}}}}';
data = JSON.parse(data);
var merchant = data.industry['IT'].merchant['HP'];
alert(merchant.name);
//console.log(merchant.name);
I am working on a solution where I need to search for an element in a deeply nested JSON by its id. I have been advised to use underscore.js which I am pretty new to.
After reading the documentation http://underscorejs.org/#find , I tried to implement the solution using find, filter and findWhere.
Here is what I tried using find :
var test = {
"menuInputRequestId": 1,
"catalog":[
{
"uid": 1,
"name": "Pizza",
"desc": "Italian cuisine",
"products": [
{
"uid": 3,
"name": "Devilled chicken",
"desc": "chicken pizza",
"prices":[
{
"uid": 7,
"name": "regular",
"price": "$10"
},
{
"uid": 8,
"name": "large",
"price": "$12"
}
]
}
]
},
{
"uid": 2,
"name": "Pasta",
"desc": "Italian cuisine pasta",
"products": [
{
"uid": 4,
"name": "Lasagne",
"desc": "chicken lasage",
"prices":[
{
"uid": 9,
"name": "small",
"price": "$10"
},
{
"uid": 10,
"name": "large",
"price": "$15"
}
]
},
{
"uid": 5,
"name": "Pasta",
"desc": "chicken pasta",
"prices":[
{
"uid": 11,
"name": "small",
"price": "$8"
},
{
"uid": 12,
"name": "large",
"price": "$12"
}
]
}
]
}
]
};
var x = _.find(test, function (item) {
return item.catalog && item.catalog.uid == 1;
});
And a Fiddle http://jsfiddle.net/8hmz0760/
The issue I faced is that these functions check the top level of the structure and not the nested properties thus returning undefined. I tried to use item.catalog && item.catalog.uid == 1; logic as suggested in a similar question Underscore.js - filtering in a nested Json but failed.
How can I find an item by value by searching the whole deeply nested structure?
EDIT:
The following code is the latest i tried. The issue in that is that it directly traverses to prices nested object and tries to find the value. But my requirement is to search for the value in all the layers of the JSON.
var x = _.filter(test, function(evt) {
return _.any(evt.items, function(itm){
return _.any(itm.outcomes, function(prc) {
return prc.uid === 1 ;
});
});
});
Here's a solution which creates an object where the keys are the uids:
var catalogues = test.catalog;
var products = _.flatten(_.pluck(catalogues, 'products'));
var prices = _.flatten(_.pluck(products, 'prices'));
var ids = _.reduce(catalogues.concat(products,prices), function(memo, value){
memo[value.uid] = value;
return memo;
}, {});
var itemWithUid2 = ids[2]
var itemWithUid12 = ids[12]
I dont use underscore.js but you can use this instead
function isArray(what) {
return Object.prototype.toString.call(what) === '[object Array]';
}
function find(json,key,value){
var result = [];
for (var property in json)
{
//console.log(property);
if (json.hasOwnProperty(property)) {
if( property == key && json[property] == value)
{
result.push(json);
}
if( isArray(json[property]))
{
for(var child in json[property])
{
//console.log(json[property][child]);
var res = find(json[property][child],key,value);
if(res.length >= 1 ){
result.push(res);}
}
}
}
}
return result;
}
console.log(find(test,"uid",4));