Develop Reference

How do I find an element of an array in NiFi ExecuteScript (ECMAScript)?

I have a need to get data out of a NiFi flow file with somewhat complex JSON content. I'm planning to use a NiFi ExecuteScript processor because I don't think it can be done with EvaluateJSONPath. The content looks like this (snippet)
...
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
],
...
I want to extract the REF02 property value from the segments array element that has REF01 === '0F'. The array element does not necessarily have a REF02 property. So in the above case, I should get 1041558xxxxx.
Here's my current script:
var flowFile = session.get()
if (flowFile != null) {
var InputStreamCallback = Java.type('org.apache.nifi.processor.io.InputStreamCallback')
var IOUtils = Java.type('org.apache.commons.io.IOUtils')
var StandardCharsets = Java.type('java.nio.charset.StandardCharsets')
try {
var subscriber = null
session.read(flowFile,
new InputStreamCallback(function (inputStream) {
var data = JSON.parse(IOUtils.toString(inputStream, StandardCharsets.UTF_8))
var segment = data.segments.find(function (s) { return s.hasOwnProperty('REF01') && s.REF01 === '0F' })
subscriber = segment ? segment.REF02 : null
}));
session.putAttribute(flowFile, 'subscriber', subscriber ? subscriber : '')
session.transfer(flowFile, REL_SUCCESS)
} catch (e) {
log.error('script failed', e)
session.transfer(flowFile, REL_FAILURE)
}
}
When I execute the above, I get a java.lang.NoSuchMethodException. Also, are anonymous 'arrow' functions allow?
I've tried using an old-school for loop to no avail.
Thanks for your help.

You can add a JoltTransformJSON processor with specification
[
{
"operation": "shift",
"spec": {
"segments": {
"*": {
"REF01": {
"0F": {// conditional to match "REF01" with "0F"
"#2,REF02": ""// go two levels up the three to reach the level of the attributes REF01 or REF02
}
}
}
}
}
}
]
in order to return the result
"1041558xxxxx"

You can use below JSONPath with EvaluateJSONPath processor:
$.segments[?(#.REF01<="0F")]#.REF02
Note: Returned result is in the array, So you can use SplitJSON after that to get your string.

Groovy script:
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import groovy.json.JsonSlurper
flowFile = session.get()
if(!flowFile) return
def jsonSlurper = new JsonSlurper()
def subscriber = ""
flowFile = session.write(flowFile, {inputStream, outputStream ->
input = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
json = jsonSlurper.parseText(input)
segment = json.segments.find{ segment ->
if (segment.keySet().contains('REF01')) {
if (segment.REF01 == '0F') {
return true
} else {
return false
}
} else {
return false
}
}
if (segment) {
subscriber = segment.REF02
}
outputStream.write(input.getBytes(StandardCharsets.UTF_8))
} as StreamCallback)
session.putAttribute(flowFile, 'subscriber', subscriber)
session.transfer(flowFile, REL_SUCCESS)
input:
{
"test": "best",
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
]
}
output (with attribute subscriber: 1041558xxxxx):
{
"test": "best",
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
]
}

How can I update this nested object's value if one of its properties match in JS?

I saw many answers, but I haven't been able to modify any to my need.
Object
{
"id": "476ky1",
"custom_id": null,
"name": "Reunião com o novo Gerente de Vendas - Airton",
"text_content": null,
"description": null,
"status": {
"id": "p3203621_11svBhbO"
},
"archived": false,
"creator": {
"id": 3184709
},
"time_spent": 0,
"custom_fields": [{
"id": "36c0de9a-9243-4259-ba57-bd590ba07fe0",
"name": "Comments",
"value": "dsdsdsds"
}],
"attachments": []
}
Within custom_fields, if the property name's value is Comments, update the value property.
I've tried it like this, using this approach, for example, but it doesn't produce the expected result.
const updatedComment = [{ name: "Comments", value: "The comment is updated"}];
updateNestedObj(taskData, updatedComment)
function updateNestedObj(obj, updates) {
const updateToApply = updates.find(upd => upd.id === obj.id);
if (updateToApply) {
obj.title = updateToApply.content;
obj.note = updateToApply.note;
}
// Apply updates to any child objects
for(let k in obj) {
if (typeof(obj[k]) === 'object') {
updateNestedObj(obj[k], updates);
}
}
}

You're using the wrong property names when you search updates for updateToApply, and then when assigning the value.
When you recurse on children, you need to distinguish between arrays and ordinary objects, so you can loop over the nested arrays. You also have to skip null properties, because typeof null == 'object'.
const updatedComment = [{
name: "Comments",
value: "The comment is updated"
}];
function updateNestedObj(obj, updates) {
let updateToApply = updates.find(upd => upd.name == obj.name);
if (updateToApply) {
obj.value = updateToApply.value;
}
// Apply updates to any child objects
Object.values(obj).forEach(child => {
if (Array.isArray(child)) {
child.forEach(el => updateNestedObj(el, updates));
} else if (typeof(child) === 'object' && child != null) {
updateNestedObj(child, updates);
}
});
}
const taskData = {
"id": "476ky1",
"custom_id": null,
"name": "Reunião com o novo Gerente de Vendas - Airton",
"text_content": null,
"description": null,
"status": {
"id": "p3203621_11svBhbO"
},
"archived": false,
"creator": {
"id": 3184709
},
"time_spent": 0,
"custom_fields": [{
"id": "36c0de9a-9243-4259-ba57-bd590ba07fe0",
"name": "Comments",
"value": "dsdsdsds"
}],
"attachments": []
};
updateNestedObj(taskData, updatedComment)
console.log(taskData);

Try this:
const updatedComment = [{ name: "Comments", value: "A new comment value" }]
// you can add as many updates as you want
function update(obj, updates) {
for (const update in updates) {
for (const field in obj.custom_fields) {
if (obj.obj.custom_fields.name == update.name) {
obj.obj.custom_fields.value = update.value
}
}
}
}
update(obj, updatedComment)

Filter an nested Object with filter and map with JavaScript

I know that this is close to a duplicate but I can't get the code to work. I have an object that I need to filter and I'm currently trying to emulate the accepted as an answer the code at Javascript filtering nested arrays
My data object is:
[{
"project_num": "5R01DA012513-23",
"principal_investigators": [{
"profile_id": 2076451,
"full_name": "PK",
"title": ""
}]
},
{
"project_num": "5R01DK118529-03",
"principal_investigators": [{
"profile_id": 8590844,
"full_name": "HW",
"title": "PROFESSOR, SCIENTIFIC DIRECTOR"
}]
},
{
"project_num": "3R01AA025365-05S1",
"principal_investigators": [{
"profile_id": 8730036,
"full_name": "JJ",
"title": "ASSOCIATE PROFESSOR OF PSYCHIATRY"
}]
},
{
"project_num": "1R01HL163963-01",
"principal_investigators": [{
"profile_id": 2084037,
"full_name": "KH",
"title": "ASSOCIATE PROFESSOR"
},
{
"profile_id": 11309656,
"full_name": "AM",
"title": "RESEARCH ASSISTANT PROFESSOR"
}
]
},
{
"project_num": "5R25HL092611-15",
"principal_investigators": [{
"profile_id": 1886512,
"full_name": "CW",
"title": "P"
}]
}
]
and my JavaScript code is:
let payLoad = 1886512
const result = this.reporterData.map(t => {
const principal_investigators = t.principal_investigators.filter(d =>
d.profile_id === payLoad);
return { ...t,
principal_investigators
};
})
I need to pass in a profile_id as a payload and return the objects that will fill a data table.
The data can be 1000's of items and the principla_investigators can be multiple entries. When I use the code that I have it return all of the objects. Can someone point out my error? Thanks

You can try doing like this:
const result = this.reporterData.filter((t) => {
const principal_investigators = t.principal_investigators.filter((d) => d.profile_id === payLoad)
return (principal_investigators.length > 0)
})

I understand that you want an array with all the investigators matching that ID, right?
Try this:
const result = this.reporterData.reduce((previous, current) => {
if (current.principal_investigators) {
current.principal_investigators.forEach(pi => {
if (pi.profile_id === payLoad) {
previous.push(current)
}
});
}
return previous
}, [])
You can also do for loops with the same result:
const result = [];
for (project of this.reporterData) {
if (project.principal_investigators) {
for (pi of project.principal_investigators) {
if (pi.profile_id == payLoad) {
result.push(pi);
}
}
}
}

Forming JSON with Key and Value if Key Exists

I have a JSON Structure something like:
[
{
"name":"angelinas"
},
{
"name":"besuto"
},
{
"name":"catch",
"cuisine":"Japanese"
},
{
"name":"center cut"
},
{
"name":"fedora"
},
{
"name":"Habanero",
"cuisine":"Mexican"
},
{
"name":"Indies"
},
{
"name":"new"
},
{
"name":"RazINN"
},
{
"name":"restaurantTestVenue779"
},
{
"name":"restaurantTestVenue9703"
},
{
"name":"Salsa ",
"cuisine":"Mexican"
},
{
"name":"Sushi Place",
"cuisine":"Japanese"
},
{
"name":"The Ashoka"
},
{
"name":"The Poboys"
},
{
"name":"the shogun"
},
{
"name":"vinyard view"
}
]
Using the JSON above i want to identify whether a cuisine is assosiated to restaurant. If yes, I want to build a JSON Structure something like:
[
{
"Mexican":{
"venueNames":[
"Habanero",
"Salsa"
]
}
},
{
"Japanese":{
"venueNames":[
"Sushi Place",
"catch"
]
}
}
]
Have tried to build the JSON using a for loop and .hasProperty but not much of a success.

Here is what you can do!
First iterate through the data and use the method "hasOwnProperty" to check if the cuisine exists and if it does then check if your cuisines object has that cuisine and if does then add it to it.
const data = [{
"name": "angelinas"
},
{
"name": "besuto"
},
{
"name": "catch",
"cuisine": "Japanese"
},
{
"name": "center cut"
},
{
"name": "fedora"
},
{
"name": "Habanero",
"cuisine": "Mexican"
},
{
"name": "Indies"
},
{
"name": "new"
},
{
"name": "RazINN"
},
{
"name": "restaurantTestVenue779"
},
{
"name": "restaurantTestVenue9703"
},
{
"name": "Salsa ",
"cuisine": "Mexican"
},
{
"name": "Sushi Place",
"cuisine": "Japanese"
},
{
"name": "The Ashoka"
},
{
"name": "The Poboys"
},
{
"name": "the shogun"
},
{
"name": "vinyard view"
}
]
let cuisines = {};
for (const resturant of data) {
if (resturant.hasOwnProperty('cuisine')) {
if (cuisines.hasOwnProperty(resturant.cuisine)) {
cuisines[resturant.cuisine].venueNames.push(resturant.name);
} else {
cuisines[resturant.cuisine] = {
venueNames: [resturant.name]
};
}
}
}

You can use in one loop below.
data.forEach(function(item) {
// if item has cuisine and cuisine not exist in new array
if(item["cuisine"] != null && typeof newArr.find(v => v[item.cuisine] != null) == 'undefined') {
// create new object with structure
let obj = {};
obj[item.cuisine] = {
"venueNames":[item.name]
};
newArr.push(obj);
}
else {
// else find existing cuisine and add new venue
let obj = newArr.find(v => v.hasOwnProperty(item.cuisine));
if(typeof obj != 'undefined') {
obj[item.cuisine].venueNames.push(item.name);
}
}
});
JSFIDDLE

It's a simple reduction of the array. If the restaurant has a defined cuisine, check if the result already has this cuisine defined. If not, create an object for it where you can push the restaurant name to.
const restaurants = [
{
"name":"angelinas"
},
{
"name":"besuto"
},
{
"name":"catch",
"cuisine":"Japanese"
},
{
"name":"center cut"
},
{
"name":"fedora"
},
{
"name":"Habanero",
"cuisine":"Mexican"
},
{
"name":"Indies"
},
{
"name":"new"
},
{
"name":"RazINN"
},
{
"name":"restaurantTestVenue779"
},
{
"name":"restaurantTestVenue9703"
},
{
"name":"Salsa ",
"cuisine":"Mexican"
},
{
"name":"Sushi Place",
"cuisine":"Japanese"
},
{
"name":"The Ashoka"
},
{
"name":"The Poboys"
},
{
"name":"the shogun"
},
{
"name":"vinyard view"
}
];
const cuisines = restaurants.reduce((result, restaurant ) => {
if ( restaurant.hasOwnProperty( 'cuisine' )) {
const { cuisine } = restaurant;
if ( !result.hasOwnProperty( cuisine )) {
result[ cuisine ] = {
venueNames: []
};
}
result[ cuisine ].venueNames.push( restaurant.name );
}
return result;
}, {});
console.log( cuisines );
In my personal opinion, I would use a slightly different structure though. If we represent collections with objects that are always the same, we can simplify most transformations. This is less efficient that doing everything in one loop, but the code used to create the transformation is almost readable english:
const restaurants = [
{ "name": "angelinas", "cuisine": null },
{ "name": "besuto", "cuisine": null },
{ "name": "catch", "cuisine": "japanese" },
{ "name": "center cut", "cuisine": null },
{ "name": "fedora", "cuisine": null },
{ "name": "habanero", "cuisine": "mexican" },
{ "name": "Indies", "cuisine": null },
{ "name": "new", "cuisine": null },
{ "name": "RazINN", "cuisine": null },
{ "name": "restaurantTestVenue779", "cuisine": null },
{ "name": "restaurantTestVenue9703", "cuisine": null },
{ "name": "Salsa ", "cuisine": "mexican" },
{ "name": "Sushi Place", "cuisine": "japanese" },
{ "name": "The Ashoka", "cuisine": null },
{ "name": "The Poboys", "cuisine": null },
{ "name": "the shogun", "cuisine": null },
{ "name": "vinyard view", "cuisine": null }
];
const create_cuisine = name => ({ name, "venues": [] });
const unique = () => {
const seen = {};
return item => {
const json = JSON.stringify( item );
return seen.hasOwnProperty( json )
? false
: ( seen[ json ] = true );
};
};
// Filter away all the restaurants without a cuisine value.
const restaurants_with_cuisine = restaurants.filter( restaurant => restaurant.cuisine );
const cuisines = restaurants_with_cuisine
// Extract the cuisine anmes from the restaurants.
.map( restaurant => restaurant.cuisine )
// Filter aways all the duplicates.
.filter( unique() )
// Create a new cuisine object.
.map( cuisine_name => create_cuisine( cuisine_name ));
// Finally add all the restaurant names to the right cuisine.
restaurants_with_cuisine.forEach( restaurant => cuisines.find( cuisine => cuisine.name === restaurant.cuisine ).venues.push( restaurant.name ));
console.log( cuisines );

Using a few es6 features, we can generate this list with Set, map and filter.
We will first map a list of cuisines, and remove invalid ones such as undefined. With that we will use a Set to create a unique list of cuisines.
Next we will take that list and map it again to return the final object, by filtering the original object where the cuisine matches the current iteration. Finally we map the filtered results to return just the name to the venueNames object.
Our result will look like this:
function getItems(places) {
// Get a unique list of cuisines
return [...new Set(places.map(p => p.cuisine).filter(c => c))]
// Build the result
.map(c => {
return {
[c]: {
// Get a list of cuisines that match the current cuisine
venueNames: places.filter(p => p.cuisine == c).map(c => c.name)
}
}
})
}
const places = [
{"name": "angelinas"},
{"name": "besuto"},
{"name": "catch","cuisine": "Japanese"},
{"name": "center cut"},
{"name": "fedora"},
{"name": "Habanero","cuisine": "Mexican"},
{"name": "Indies"},
{"name": "new"},
{"name": "RazINN"},
{"name": "restaurantTestVenue779"},
{"name": "restaurantTestVenue9703"},
{"name": "Salsa ","cuisine": "Mexican"},
{"name": "Sushi Place","cuisine": "Japanese"},
{"name": "The Ashoka"},
{"name": "The Poboys"},
{"name": "the shogun"},
{"name": "vinyard view"}
]
console.log(getItems(places))

recursion with Promises

I have a collection in MongoDB like this
[
{
"classId": "1",
"name": "Input",
"definition": [
{
"property": [
{
"classId": "12",
"name": "One"
},
{
"classId": "8",
"name": "Comment"
}
]
}
]
},
{
"classId": "8",
"name": "CommentDetail",
"definition": [
{
"property": [
{
"classId": "10",
"name": "user"
},
{
"classId": "10",
"name": "message"
}
]
}
]
},
{
"classId": "10",
"name": "String",
"definition": []
},
{
"classId": "12",
"name": "Int",
"definition": []
}
]
Based on db above, I have a model to display
data = {
name:'',
template: ''
}
With classId=1, the expectation result is
{
"Name": "Input",
"temlate": "{['One': 'Int','Comment': ['user': 'String','message':'String']]}"
}
I try to using recursive promise to implement it. When property[] is empty, the result will be return.
Here is my function:
const getObjectTypeByClassId = (snapshotId, artifactId, objectType) => {
return artifactDetailModel.find({
'snapshotId': snapshotId,
'artifactId': artifactId
})
.then(data => {
let artifact = data[0];
let definition;
let definitionData = {};
return Promise.resolve()
.then(() => {
definition = artifact.data.teamworks.twClass[0].definition[0];
if (!lodash.isUndefined(definition.property)) {
const listOfProperty = definition.property;
for (let property of listOfProperty) {
classId = commonUtil.getArtifactId(property.classRef[0]);
if (!lodash.isUndefined(classId)) {
return getObjectTypeByClassId(snapshotId, classId, objectType);
}
}
} else {
definitionData.nameType = artifact.data.teamworks.twClass[0].elementAttribute.name;
definitionData.classId = artifact.data.teamworks.twClass[0].elementAttribute.id;
definitionData.template = bpmMapping.objectType[artifact.data.teamworks.twClass[0].elementAttribute.name];
return objectTypeModel.create(definitionData)
.then(obj => {
const response = {
name: objectType.name,
isArrayOf: objectType.isArrayOf,
nameType: obj.nameType,
template: obj.template,
}
return response;
})
}
})
})
}
Run with my function, the response is
data: {
Name: Input
temlate: user: String,
}
Please advice me.

I tried it to some extent, but wasn't able to get it right. Plus your expected output is not the valid JSON "temlate": {[]} doesn't make sense.
It has nothing to do with Promise. You have to DFS you db array and created expected output. Here is what I have donup tillll now, you can think along those lines. But this is far from the solution.
let mainArray = [
{
"classId": "1",
"name": "Input",
"definition": [
{
"property": [
{
"classId": "12",
"name": "One"
},
{
"classId": "8",
"name": "Comment"
}
]
}
]
},
{
"classId": "8",
"name": "CommentDetail",
"definition": [
{
"property": [
{
"classId": "10",
"name": "user"
},
{
"classId": "10",
"name": "message"
}
]
}
]
},
{
"classId": "10",
"name": "String",
"definition": []
},
{
"classId": "12",
"name": "Int",
"definition": []
}
]
function dfs(root, createdRoot, fn, level) {
fn(root,createdRoot, level);
if(root.definition)/*if definition exists => keep traversing*/
root.definition[0].property.forEach(function (child) {
createdRoot.template = createdRoot.template || [];
let tempObj = {};
let lookupObj = lookupByClassId(child.classId);
tempObj[child.name] = lookupObj.name;
createdRoot.template.push(tempObj);
dfs(child,tempObj, fn, level + 1);
});
else /*if definition doesn't exist, look into the array*/
{
createdRoot.template = lookupByClassId(root.classId);
}
}
function lookupByClassId(classId){
for(let i=0;i<mainArray.length;++i){
let element =mainArray[i]
if(element.classId == classId)
return element;
}
}
let root = lookupByClassId(1);
createdRoot ={};
function func1(root, createdRoot, level) {
createdRoot.name = root.name;
console.log(root.classId);
}
dfs(root, createdRoot, func1, 0);

here is the solution from a guy on the internet. it works well. Thanks, #Cuong Quach
var MongoClient = require('mongodb').MongoClient;
var database = 'testdequy';
var collection = 'classes';
MongoClient.connect("mongodb://localhost:27017/" + database, function (err, db) {
findDetails("1").then((r) => {
console.log("r>>>", r);
})
function findDetails(classId) {
var results = { Name: "", Template: "" };
var template = {};
return new Promise((main_resolve, reject) => {
process(template, "" , classId)
var works = 0;
function process(parent, childKey, objectId) {
return new Promise((resolve, reject) => {
db.collection(collection).find({ classId: objectId })
.toArray((err, docs) => {
if (results.Name == "") {
results.Name = docs[0].name;
}
let objectItem;
if (childKey == "")
objectItem = parent;
else
objectItem = parent[childKey];
console.log("\ndocs", docs[0], objectId, objectItem)
if (docs[0].definition.length == 0 || docs[0].definition[0].property == undefined) {
let name = docs[0].name;
parent[childKey] = name;
console.log("\nNo child", docs[0],parent, objectItem, docs[0].name)
resolve(0);
} else {
docs[0].definition[0].property.forEach((item) => {
works++;
//console.log("item", item)
let id = item.classId;
let name = item.name;
objectItem[name] = {};
process(objectItem, name, id).then((len)=>{
works--;
if(len == 0 && works == 0) main_resolve(template);
})
})
resolve(docs[0].definition[0].property.length)
}
})
})
}
})
}
});