How do I find an element of an array in NiFi ExecuteScript (ECMAScript)? - javascript

I have a need to get data out of a NiFi flow file with somewhat complex JSON content. I'm planning to use a NiFi ExecuteScript processor because I don't think it can be done with EvaluateJSONPath. The content looks like this (snippet)
...
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
],
...
I want to extract the REF02 property value from the segments array element that has REF01 === '0F'. The array element does not necessarily have a REF02 property. So in the above case, I should get 1041558xxxxx.
Here's my current script:
var flowFile = session.get()
if (flowFile != null) {
var InputStreamCallback = Java.type('org.apache.nifi.processor.io.InputStreamCallback')
var IOUtils = Java.type('org.apache.commons.io.IOUtils')
var StandardCharsets = Java.type('java.nio.charset.StandardCharsets')
try {
var subscriber = null
session.read(flowFile,
new InputStreamCallback(function (inputStream) {
var data = JSON.parse(IOUtils.toString(inputStream, StandardCharsets.UTF_8))
var segment = data.segments.find(function (s) { return s.hasOwnProperty('REF01') && s.REF01 === '0F' })
subscriber = segment ? segment.REF02 : null
}));
session.putAttribute(flowFile, 'subscriber', subscriber ? subscriber : '')
session.transfer(flowFile, REL_SUCCESS)
} catch (e) {
log.error('script failed', e)
session.transfer(flowFile, REL_FAILURE)
}
}
When I execute the above, I get a java.lang.NoSuchMethodException. Also, are anonymous 'arrow' functions allow?
I've tried using an old-school for loop to no avail.
Thanks for your help.

You can add a JoltTransformJSON processor with specification
[
{
"operation": "shift",
"spec": {
"segments": {
"*": {
"REF01": {
"0F": {// conditional to match "REF01" with "0F"
"#2,REF02": ""// go two levels up the three to reach the level of the attributes REF01 or REF02
}
}
}
}
}
}
]
in order to return the result
"1041558xxxxx"

You can use below JSONPath with EvaluateJSONPath processor:
$.segments[?(#.REF01<="0F")]#.REF02
Note: Returned result is in the array, So you can use SplitJSON after that to get your string.

Groovy script:
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import groovy.json.JsonSlurper
flowFile = session.get()
if(!flowFile) return
def jsonSlurper = new JsonSlurper()
def subscriber = ""
flowFile = session.write(flowFile, {inputStream, outputStream ->
input = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
json = jsonSlurper.parseText(input)
segment = json.segments.find{ segment ->
if (segment.keySet().contains('REF01')) {
if (segment.REF01 == '0F') {
return true
} else {
return false
}
} else {
return false
}
}
if (segment) {
subscriber = segment.REF02
}
outputStream.write(input.getBytes(StandardCharsets.UTF_8))
} as StreamCallback)
session.putAttribute(flowFile, 'subscriber', subscriber)
session.transfer(flowFile, REL_SUCCESS)
input:
{
"test": "best",
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
]
}
output (with attribute subscriber: 1041558xxxxx):
{
"test": "best",
"segments": [
{
"INS01": "Y",
"INS03": "001",
"INS02": "18",
"INS05": "A",
"id": "INS",
"INS04": "AI",
"INS08": "FT"
},
{
"REF02": "1041558xxxxx",
"REF01": "0F",
"id": "REF"
},
{
"REF02": "ABD",
"REF01": "1L",
"id": "REF"
},
{
"REF02": "106835xxxxx",
"REF01": "23",
"id": "REF"
}
]
}

Related

How in JS to merge in one object two json objects where the ID of on object correspond on the same ID of the second object

My question relates to the fact I'm querying 2 different objects from DB and the result is in JSON. I need to merge them into one.
The 2 objects have in common this two key/value IRBId = ... and id = ... and they look as an example
OBJ 1
{
"data":{
"IRBs":{
"nodes":[
{
"id":"8",
"name":"Admin ",
},
{
"id":"9",
"name":"Again",
}
],
}
}
}
OBJ 2
{
"data":{
"informedConsentForms":{
"count":3,
"nodes":[
{
"id":"93",
...
"IRBId":"9",
},
{
"id":"92",
...
"IRBId":"8",
},
{
"id":"91",
...
"IRBId":"8",
}
],
}
},
As you will see above OBJ 2 and OBJ 1 corresponding with the same at IRBid and id.
What I need is to merge the two OBJ where IRBId OBJ 2 === id OBJ 1
The result I would expect after the merge is
OBJ merged
{
[{
"id":"93",
...
"IRBId":"9",
"irb": {
"name":"Again ",
...
}
},
{
"id":"92",
...
"IRBId":"8",
"irb": {
"name":"Admin ",
...
}
},
{
"id":"91",
...
"IRBId":"8",
"irb": {
"name":"Admin ",
...
}
],
},
I don't know how to make it looks like this.
Try using Array.reduce
Logic
Loop through second object data nodes
Find the matching nodes from object 1 data nodes.
Push to accumulator with required details. (I have added only the nodes that was mentioned in in Expected resut, you can add asmuch as you need.)
const obj1 = {
"data": {
"IRBs": {
"nodes": [
{
"id": "8",
"name": "Admin ",
},
{
"id": "9",
"name": "Again",
}
],
}
}
}
const obj2 = {
"data": {
"informedConsentForms": {
"count": 3,
"nodes": [
{
"id": "93",
"IRBId": "9",
},
{
"id": "92",
"IRBId": "8",
},
{
"id": "91",
"IRBId": "8",
}
],
}
},
};
const obj1List = obj1.data.IRBs.nodes;
const output = obj2.data.informedConsentForms.nodes.reduce((acc, curr) => {
const matchingNode = obj1List.find((item) => item.id === curr.IRBId);
if (matchingNode) {
acc.push({
id: curr.id,
IRBId: curr.IRBId,
irb: {
name: matchingNode.name
}
})
}
return acc;
}, []);
console.log(output);
You need to use the map function on the nodes in the first object to construct a new object that contains the second and first object's attributes.
const obj1 = {
"data": {
"IRBs": {
"nodes": [{
"id": "8",
"obj1": "one",
"name": "Admin ",
},
{
"id": "9",
"obj1": "two",
"name": "Again",
}
]
}
}
};
const obj2 = {
"data": {
"informedConsentForms": {
"count": 3,
"nodes": [{
"id": "93",
"obj2": "1",
"IRBId": "9",
},
{
"id": "92",
"obj2": "2",
"IRBId": "8",
},
{
"id": "91",
"obj2": "3",
"IRBId": "8",
}
],
}
}
};
const obj1Data = obj1.data.IRBs.nodes;
const obj2Data = obj2.data.informedConsentForms.nodes;
const res = obj2Data.map(item => {
const obj1Item = obj1Data.find(obj1Item => item.IRBId === obj1Item.id);
return obj1Item ? { ...item, "irb": { ...obj1Item}} : { ...item};
});
console.log(res);
i am using nested loop, try this one
const obj2 = {
"data":{
"informedConsentForms":{
"count":3,
"nodes":[
{
"id":"93",
"IRBId":"9",
},
{
"id":"92",
"IRBId":"8",
},
{
"id":"91",
"IRBId":"8",
}
],
}
},
}
const obj1 = {
"data":{
"IRBs":{
"nodes":[
{
"id":"8",
"name":"Admin ",
},
{
"id":"9",
"name":"Again",
}
],
}
}
}
const result = [];
const obj2Nodes = obj2.data.informedConsentForms.nodes;
for(let i = 0; i < obj2Nodes.length; i++) {
const obj1Nodes = obj1.data.IRBs.nodes
for(let j = 0; j < obj1Nodes.length; j++) {
if(obj2Nodes[i].IRBId === obj1Nodes[j].id) {
const {id, ...reObj1Nodes} = obj1Nodes[j];
result.push({
...obj2Nodes[i],
'irb': {
...reObj1Nodes
}
})
}
}
}
console.log(result)

How to parse FractalTransformer with normalizr

I'm trying to use paularmstrong/normalizr on JSON that comes from FractalTransformer and whose nested childs have "data" attribute. Example of JSON:
{
"data": {
"object": "Offer",
"id": "5g6aqocew4qjzl40",
"real_id": 26,
"name": "Random Name",
"created_at": {
"date": "2019-06-18 11:13:08.000000",
"timezone_type": 3,
"timezone": "UTC"
},
"readable_created_at": "1 year ago",
"site": {
"data": {
"object": "Site",
"id": "65zody8vj29vlegd",
"name": "Test Site",
"real_id": 1
}
},
"countries": {
"data": [
{
"object": "Country",
"code": "US",
"name": "United States"
},
{
"object": "Country",
"code": "DE",
"name": "Germany"
}
]
}
},
"meta": {
"include": [
"site",
"countries"
],
"custom": []
}
}
Schemas I use:
export const offerSchema = new schema.Entity('offers')
export const siteSchema = new schema.Entity('sites', {}, {
processStrategy: (value) => {
return { ...value.data }
},
idAttribute: (value) => {
return value.data.id
},
})
export const countrySchema = new schema.Entity('countries')
offerSchema.define({
site: siteSchema,
countries: [countrySchema],
})
Now the issue is that I remove 'data' from the site since it's just one object successfully, but I can't do it in the country case. Whatever I tried with custom processStrategy fails, as country is object that has data which is array (I assume this is where the issue is, going from Entity to Array). And in idAttribute function I always get complete array so can't determine the ID of single entry. So the end result is that the ID of countries is undefined. Any ides?
I actually managed with another approach. I added processStrategy on the parent, 'Offer' in this case, so all 'data' parts get stripped before they reach other child schemas.
const normalizrStripDataOptions = {
processStrategy: (value) => {
const ret = { ...value }
Object.keys(ret).forEach((key) => {
if (ret[key] !== null) {
if (ret[key].data && Array.isArray(ret[key].data)) {
ret[key] = [...ret[key].data]
}
if (ret[key].data && typeof ret[key].data === 'object') {
ret[key] = { ...ret[key].data }
}
}
})
return ret
},
}
export const offerSchema = new schema.Entity('offers', {}, normalizrStripDataOptions)
export const siteSchema = new schema.Entity('sites')
export const countrySchema = new schema.Entity('countries')
offerSchema.define({
site: siteSchema,
countries: [countrySchema],
})

Javascript Object array get last array

I have a json object array I have two functions. One to get the last message and the other to get. I need to keep the outer format the same but only return the one message.
I am getting the Json from the Telegram api and I have a Node Express script to return the reformatted Json
Here is the full Json:
{
"ok": true,
"result": [
{
"update_id": 650787950,
"channel_post": {
"message_id": 258,
"chat": {
"id": -1001497153100,
"title": "TestBot",
"type": "channel"
},
"date": 1592256395,
"text": "test messge"
}
},
{
"update_id": 650787951,
"channel_post": {
"message_id": 259,
"chat": {
"id": -1001497153100,
"title": "TestBot",
"type": "channel"
},
"date": 1592256604,
"text": "next"
}
}
]
}
I have a function to store the object after an api call to Telegram:
storeUpdates(data){
this.messageData = data;
}
For the function to get the last message:
getlastMessage() {
return
}
I am trying to return the Json:
{
"ok": true,
"result": [
{
"update_id": 650787951,
"channel_post": {
"message_id": 259,
"chat": {
"id": -1001497153100,
"title": "TestBot",
"type": "channel"
},
"date": 1592256604,
"text": "next"
}
}
]
}
And for the function to get a specific update_id
getNextMessage(update_id) {
return
}
Again I am trying to get this format of a single message matching the passed in update_id
{
"ok": true,
"result": [
{
"update_id": 650787951,
"channel_post": {
"message_id": 259,
"chat": {
"id": -1001497153100,
"title": "TestBot",
"type": "channel"
},
"date": 1592256604,
"text": "next"
}
}
]
}
I am a little confused with the layers of object and arrays mixed.
Does this work?
const messages = {
ok: true,
result: [{
update_id: 650787950,
channel_post: {
message_id: 258,
chat: {
id: -1001497153100,
title: 'TestBot',
type: 'channel',
},
date: 1592256395,
text: 'test messge',
},
},
{
update_id: 650787951,
channel_post: {
message_id: 259,
chat: {
id: -1001497153100,
title: 'TestBot',
type: 'channel',
},
date: 1592256604,
text: 'next',
},
},
],
};
const getLastMessage = (messages) => {
final = {
ok: true,
result: [],
};
final.result.push(messages.result[messages.result.length - 1]);
return final;
};
const getNextMessage = (update_id, messages) => {
final = {
ok: true
};
final.result = messages.result.filter((msg) => {
return msg.update_id === update_id;
});
return final;
};
console.log(getLastMessage(messages));
console.log(getNextMessage(650787950, messages));
You get the last message by returning the last element in the array, by getting the length of the array and -1
I used Array.prototype.filter() to find the correct object.
To get the last result you would need to go to results and return the last index:
function getlastMessage(resultObject) {
return {
ok: resultObject.ok
result: [resultObject.result[resultObject.result.length - 1]]
}
}
To get the message by update_id:
getNextMessage(update_id) {
return {
ok: resultObject.ok
result: [resultObject.result.find(message => message.update_id === update_id)]
}
}
Something along these lines
Using destructuring you can make your code a little bit more compact:
const someObject = JSON.parse(`{
"ok": true,
"result": [
{
"update_id": 650787950,
"channel_post": {
"message_id": 258,
"chat": {
"id": -1001497153100,
"title": "TestBot",
"type": "channel"
},
"date": 1592256395,
"text": "test messge"
}
},
{
"update_id": 650787951,
"channel_post": {
"message_id": 259,
"chat": {
"id": -1001497153100,
"title": "TestBot",
"type": "channel"
},
"date": 1592256604,
"text": "next"
}
}
]
}`)
const getNextMessage = (update_id) => {
return {
...someObject,
result: someObject.result.find(message => message.update_id === update_id)
}
}
const getLastMessage = () => {
const arrayLength = someObject.result.length;
return {
...someObject,
result: someObject.result[arrayLength - 1]
}
}
console.log(getNextMessage(650787950))
console.log(getLastMessage())
If you want to keep the result type as an array you can use filter instead of find and surround the last element of result array with square brackets, like this:
const getNextMessage = (update_id) => {
return {
...someObject,
result: someObject.result.filter(message => message.update_id === update_id)
}
}
const getLastMessage = () => {
const arrayLength = someObject.result.length;
return {
...someObject,
result: [someObject.result[arrayLength - 1]]
}
}

How to denormalize array in JS

I have a data set of the following form
let data = [
{
"id": {
"primary": "A1"
},
"msg": 1
}, {
"id": {
"primary": "A1"
},
"msg": 2
}, {
"id": {
"primary": "B2"
},
"msg": 3
}
]
I would like to transform it to
newData = [
{
"id": {
"primary": "A1"
},
"items": [
{ "msg": 1 },
{ "msg": 2 }
]
},
{
"id": {
"primary": "B2"
},
"items": [
{ "msg": 3 }
]
}
]
I think the method is something like the following, but am not sure how to check against undefined values in this case.
let newData = [];
for (let i = 0; i < data.length; i++) {
if (newData[i]['id']['primary'] === data[i]['id']) newData.push(data[i]['id'])
else newData[i]['items'].push(data[i]['msg'])
}
How can I transform the original data set to merge entries with a matching primary id?
One option would be to use .reduce() to create a new array from the existing.
I've added comments to clarify.
let data = [ { "id": { "primary": "A1" }, "msg": 1 }, { "id": { "primary": "A1" }, "msg": 2 }, { "id": { "primary": "B2" }, "msg": 3 } ];
let result = data.reduce((out,item) => {
let {id, ...items} = item; //Separate the "id" and "everything else"
let existing = out.find(({id}) => id.primary == item.id.primary);
existing //have we seen this ID already?
? existing.items.push(items) //yes - add the items to it
: out.push({ id: {...id}, items: [items]}); //no - create it
return out;
}, []);
console.log(result);
A couple notes:
You may notice that I've set the ID using id: {...id}, despite the id already being an object. This is because using the existing id object would create a reference, whereas {...id} creates a shallow copy.
I haven't specified the msg property anywhere. Instead, any properties that aren't id will be added to the items list (example below).
let data = [ { "id": { "primary": "A1" }, "msg": 1, "otherStuff": "Hello World!" }, { "id": { "primary": "A1" }, "msg": 2, "AnotherThing": true }, { "id": { "primary": "B2" }, "msg": 3, "someOtherProperty": false } ];
let result = data.reduce((out,item) => {
let {id, ...items} = item;
let existing = out.find(({id}) => id.primary == item.id.primary);
existing
? existing.items.push(items)
: out.push({ id: {...id}, items: [items]});
return out;
}, []);
console.log(result);
That said, if you start to nest objects (other than ID), they will likely be included as references; ...items is only a shallow copy.
If such a case, consider something like JSON.parse(JSON.stringify(...)) for a deep copy. Be sure to read the link though; there are caveats.
You could also solve this in a concise way via the Array.reduce and ES6 destructuring:
let data = [ { "id": { "primary": "A1" }, "msg": 1 }, { "id": { "primary": "A1" }, "msg": 2 }, { "id": { "primary": "B2" }, "msg": 3 } ]
let result = data.reduce((r, {id, msg}) =>
((r[id.primary] = r[id.primary] || { id, items: [] }).items.push({msg}), r), {})
console.log(Object.values(result))
In more readable format it is:
let data = [ { "id": { "primary": "A1" }, "msg": 1 }, { "id": { "primary": "A1" }, "msg": 2 }, { "id": { "primary": "B2" }, "msg": 3 } ]
let result = data.reduce((r, {id, msg}) => {
r[id.primary] = (r[id.primary] || { id, items: [] })
r[id.primary].items.push({msg})
return r
}, {})
console.log(Object.values(result))
The idea is to group by the id.primary and then once the grouping is done simply get the values via Object.values
Notice that this is one pass solution where you do not have to per each iteration do an Array.find against the current accumulator.

Remove duplicate array from response comparing attribute value

I want to remove a duplicate array from the response on the basis of the attribute value. If the attribute_value data match with other array attribute value then other should be removed.
The logic is very simple. check duplicate attribute_value in each array and remove duplicate array and return
In response. now you can see the attribute value = 1 is thrice
and attribute value = 2 is twice
How do i compare and remove whole array if I see attribute value duplicate?
I tried with filter method which seems not working. Please help.
for(var j=0; j<social_post_link.length; j++){
newFilterarray = social_post_link[j].activity_attributes[0].attribute_value.filter(function(item, index) {
if (social_post_link[j].activity_attributes[0].attribute_value.indexOf(item) == index){
return social_post_link;
}
});
}
Response
[
{
"id": "484822",
"activity_attributes": [
{
"id": "868117",
"activity_id": "484822",
"attribute_name": "position",
"attribute_value": "1",
}
]
},
{
"id": "484884",
"activity_attributes": [
{
"id": "868175",
"activity_id": "484884",
"attribute_name": "position",
"attribute_value": "1",
}
]
},
{
"id": "484888",
"activity_attributes": [
{
"id": "868182",
"activity_id": "484888",
"attribute_name": "position",
"attribute_value": "1",
}
]
},
{
"id": "484823",
"activity_attributes": [
{
"id": "868120",
"activity_id": "484823",
"attribute_name": "position",
"attribute_value": "2",
}
]
},
{
"id": "484975",
"activity_attributes": [
{
"id": "868344",
"attribute_name": "position",
"attribute_value": "2",
}
]
},
{
"id": "484891",
"activity_attributes": [
{
"id": "868189",
"attribute_name": "position",
"attribute_value": "3",
}
]
},
{
"id": "484903",
"activity_attributes": [
{
"id": "868200",
"attribute_name": "position",
"attribute_value": "4",
},
]
}
]
Desired output
[
{
"id": "484822",
"activity_attributes": [
{
"id": "868117",
"activity_id": "484822",
"attribute_name": "position",
"attribute_value": "1",
}
]
},
{
"id": "484823",
"activity_attributes": [
{
"id": "868120",
"activity_id": "484823",
"attribute_name": "position",
"attribute_value": "2",
}
]
},
{
"id": "484891",
"activity_attributes": [
{
"id": "868189",
"attribute_name": "position",
"attribute_value": "3",
}
]
},
{
"id": "484903",
"activity_attributes": [
{
"id": "868200",
"attribute_name": "position",
"attribute_value": "4",
},
]
}
]
You can probably use the lodash utility uniqBy,
where iteratee is a function that returns the value you want to compare against.
In your case, it would probably look like the following:
const uniqueLinks = _.uniqBy(social_post_link, item =>
item.activity_attributes[0].attribute_value
)
Edit:
Here is a vanilla JS function that will accomplish the same.
const filterByIteratee = (array, iteratee) => {
// Empty object to store attributes as we encounter them
const previousAttributeNames = {
}
return array.filter(item => {
// Get the right value
const itemValue = iteratee(item)
// Check if we have already stored this item
if (previousAttributeNames.hasOwnProperty(itemValue)) return false
else {
// Store the item so next time we encounter it we filter it out
previousAttributeNames[itemValue] = true
return true
}
})
}
It will loop through an array, store its identifier by some function, and return only the first instance of each item.
Use it the same way:
const uniqueLinks = filterByIteratee(social_post_link, item =>
item.activity_attributes[0].attribute_value
)
This is probably not the best performing solution. but it works for your requirements.
var resultArray = [];
for (var i = 0; i < social_post_link.length; i++) {
var currentSocialLink = social_post_link[i];
for (var j = 0; j < currentSocialLink.activity_attributes.length; j++) {
if (!resultArray.some(val =>
val.activity_attributes.some(activity =>
activity.attribute_value === currentSocialLink.activity_attributes[j].attribute_value))) {
resultArray.push(currentSocialLink);
}
}
}
function removeDuplicates(myArr, prop) { // removes duplicate objects from array
return myArr.filter((obj, pos, arr) => {
return arr.map(mapObj => mapObj[prop]).indexOf(obj[prop]) === pos;
});
};
I found this function not too long ago which removes duplicate objects from an array. Pass it the array and the property you wish to not be duplicated.

Categories

Resources