How to take data from nested JSON data in bigquery - javascript

I have table with column field as follow :
{
"Quantity":{
"id_1":1,
"id_2":4,
},
"Discount" : {
"id_1":2,
"id_2":1,
},
"sales":{
"id_1":{
"price":50,
"quantity":1
},
"id_2":{
"quantity":1,
"price":620
}}
,
"tax":{
"id_1" : 2,
"id_2" : 3
}
}
My Expected Result is a table as follow :
id
tax
sales_quantity
sales_price
Discount
Quantity
id_1
2
1
50
2
1
id_2
3
1
620
1
4
While I'm trying to create the expected result, I think I found UDF function that work in javascript as follow :
CREATE TEMP FUNCTION json2array(json STRING)
RETURNS ARRAY <STRUCT<id STRING, tax STRING, sales_quantity STRING, sales_price STRING, Discount STRING, Quantity STRING>>
LANGUAGE js AS """
var result = {};
var keys = Object.keys(json);
keys.forEach(k => {
keys2 = Object.keys(json[k])
keys2.forEach(k2 => {
if(result[k2]== null)
result[k2] ={}
if(typeof json[k][k2] === 'object' )
{
Object.keys(json[k][k2]).forEach(k3 => {
result[k2][k +"_"+k3] = json[k][k2][k3]
})
} else {
result[k2][k] =json[k][k2]
}
})
})
var final_result = []
for (const [key, value] of Object.entries(result)) {
value["id"] = key; final_result.push(value);
}
final_result.map(obj => Object.keys(obj).map(k => obj[k] = obj[k] === null ? "" : obj[k]))
return final_result
""";
the function is working in javascript compiler, but it seems the result is not expected when using BigQuery UDF.I think the main problem is in return, but I'm not sure what I'm missing here

Consider below javascript UDF and PIVOT query.
CREATE TEMP FUNCTION json2array(json STRING)
RETURNS ARRAY<STRUCT<id STRING, key STRING, value STRING>> LANGUAGE js AS """
result = [];
for (const [key, obj] of Object.entries(JSON.parse(json))) { // top-most
Object.entries(obj).forEach(o => {
const [k0, v0] = o;
if (typeof(v0) === 'object')
for (const [k, v] of Object.entries(v0))
result.push({id:k0, key:key + '_' + k, value:v});
else
result.push({id:k0, key:key, value:v0});
});
}
return result;
""";
WITH sample_table AS (
SELECT '''{
"Quantity":{ "id_1":1, "id_2":4 },
"Discount" : { "id_1":2, "id_2":1 },
"sales":{
"id_1":{ "price":50, "quantity":1 },
"id_2":{ "quantity":1, "price":620 }
},
"tax":{ "id_1" : 2, "id_2" : 3 }
}''' json
)
SELECT * FROM (
SELECT e.* FROM sample_table, UNNEST(json2array(json)) e
) PIVOT (ANY_VALUE(value) FOR key IN ('tax', 'sales_quantity', 'sales_price', 'Discount', 'Quantity'));
Query results

Related

How to split an object to an array

I want to split the object to be an array when the child value is an array. The new array length is the longest object child value length.(the longest length is three in the example)
Example:
// source
const data = {
key1: 'key1',
key2: ['key2-1', 'key2-2'],
key3: ['key3-1', 'key3-2', 'key3-3'],
};
// expect to be:
const result = [
{ key1: 'key1', key2: 'key2-1', key3: 'key3-1' },
{ key2: 'key2-2', key3: 'key3-2' },
{ key3: 'key3-3' },
];
// my low code
const transform = (data) => {
const maxLen = Object.values(data).reduce((max, value) => {
if (Array.isArray(value)) {
if (value.length > max) {
return value.length;
}
}
return max;
}, 0);
const newArray = [];
for (let index = 0; index < maxLen; index++) {
Object.entries(data).forEach(([key, value]) => {
if (!newArray[index]) {
newArray[index] = {};
}
if (Array.isArray(value) && value[index]) {
newArray[index][key] = value[index];
} else if (index === 0) {
newArray[index][key] = value;
}
});
}
return newArray;
};
but my code is very low.have you a nice answer?
One way to achieve the result:
function toArray(data) {
return Object.values(data).reduce((acc, currValue) => {
Array.isArray(currValue)
? // e.g., ['key2-1', 'key2-3']
currValue.forEach(v => addValues(acc, v))
: // e.g., 'key1'
addValues(acc, currValue)
return acc
}, [])
function addValues(acc, value) {
/*
Split 'key2-1' into 'key2' and '1'
Then assign 'key' = 'key2', 'index' = '1'
*/
let [key, index] = value.split('-')
/*
The 'index' value is used to determine which object
the 'key' and 'value' will be stored in.
Since array element indexes start from 0 and the numbering of keys
in the provided data object start from 1 the 'index' needs to be
converted from string to number to substract 1 from it.
The ' || 1' part is becuase for the first value 'key1'.split('-')
will result in only ['key1'] so in the assignment:
let [key, index] = ['key1']
index will be undefined and it will be set to 0 :
index = (Number(undefined) || 1) - 1
index = (NaN || 1) - 1
index = (1) - 1
index = 0
*/
index = (Number(index) || 1) - 1
if (!acc[index]) {
acc[index] = {}
}
acc[index][key] = value
}
}
Example:
const data = {
key1: 'key1',
key2: ['key2-1', 'key2-2'],
key3: ['key3-1', 'key3-2', 'key3-3'],
}
function toArray(data) {
return Object.values(data).reduce((acc, currValue) => {
Array.isArray(currValue)
? currValue.forEach(v => addValues(acc, v))
: addValues(acc, currValue)
return acc
}, [])
function addValues(acc, value) {
let [key, index] = value.split('-')
index = (Number(index) || 1) - 1
if (!acc[index]) acc[index] = {}
acc[index][key] = value
}
}
const result = toArray(data)
console.log(result)
.as-console-wrapper {min-height: 100%}
References:
Object.values(data).reduce((acc, currValue) => { ... }, []):
Object.values()
.reduce()
Array.isArray(currValue) ? currValue.forEach(v => ... ) : ...:
Array.isArray()
Conditional (ternary ?:) operator
.forEach()
let [key, index] = value.split('-'):
Destructuring assignment
.split()
index = (Number(index) || 1) - 1:
Number()
Logical OR (||)
You can format it like so.
const data = { key1: 'key1', key2: ['key2-1', 'key2-2'] };
// [{"key1":"key1"},{"key2":"key2-1"},{"key2":"key2-2"}]
const res = Object.keys( data ).reduce( ( p, key ) => {
if (typeof data[key] === 'string') {
return p.concat( { [key]: key } )
} else {
return p.concat( Object.values( data[key] ).map( value => ({ [key]: value }) ) )
}
}, [] )

iterate over a map and change the position of element

I have an array of object
const test = [{'type':'Material'}, {'type':''}, {'type':'ABC'}]
Here I am using map over here to itertae
export const mapToNA = values => map(test, value => type || 'NA')
mapToNA(test)
This returns the [{'type':'Material'}, {'type':'NA'}, {'type':'ABC'}]
Now I want value which is NA then it should be at the end
so Output would be like:
[{'type':'Material'},{'type':'ABC'},{'type':'NA'}, ]
How can I get this ?
Since you're already using lodash, lets use _.sortBy with a custom function:
// Data
let test = [{'type':'Material'}, {'type':''}, {'type':'ABC'}];
// Map '' to 'NA'
const mapToNA = values => _.map(test, value => { return { type: value.type || 'NA' }; } )
test = mapToNA(test)
// Sort
test = _.sortBy(test, element => (element.type === 'NA'));
// Log
console.log(test)
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.21/lodash.min.js"></script>
[
{
"type": "Material"
},
{
"type": "ABC"
},
{
"type": "NA"
}
]
You can partition the array to items with type, and items out without, and then map the relevant items' type to NA, and use spread to combine the arrays:
const data = [{'type':'Material'}, {'type':''}, {'type':'ABC'}];
// Pertition items to have type and NA
const [itemsWithType, itemsWithEmptyType] = _.partition(data, o => o.type)
// map the itemsWithEmptyType to NA and combine
const result = [...itemsWithType, ...itemsWithEmptyType.map(o => ({ ...o, type: 'NA' }))]
console.log(result)
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.21/lodash.min.js"></script>
You can easily achieve this result using map and a custom sorting algorithm.
const test = [
{ type: "ABC" },
{ type: "Material" },
{ type: "" },
{ type: "ABC" },
{ type: "" },
];
const result = test
.map((s) => (s.type === "" ? { type: "NA" } : s))
.sort((a, b) => {
if (a.type === "NA" && b.type === "NA") return 0;
if (a.type === "NA") return 1;
if (b.type === "NA") return -1;
else return 0;
});
console.log(result);
You could use 2 separate arrays to keep track of objects with and without type. Merge them after the loop. This is readable and faster.
const withType = [],
withoutType = []
for (const o of test) {
if (o.type)
withType.push(o)
else
withoutType.push({ type: 'NA' })
}
console.log( withType.concat(withoutType) )
You could also reduce with 2 separate arrays and flat them:
const group = test.reduce((acc, o) => {
if (o.type)
acc[0].push(o)
else
acc[1].push({ 'type': 'NA' })
return acc
}, [[], []])
console.log( group.flat() )
You can use default sort function from javascript to solve this.
const arr =[{'type':'Material'}, {'type':'NA'}, {'type':'ABC'}];
arr.sort((a, b) => a["type"] === 'NA' ? 1 : -1);
console.log(arr)
Do you want to switch position between last element of array with your target element using Array.prototype.map ?
If so,
const test = [{"type": "Material"}, {"type": "NA"}, {"type":"ABC"}];
const someFunction = values => {
// NAIndex remember which index of values have "NA"
let NAIndex = 0;
// copy original array values.
const copiedValues = values.slice();
return values.map((value,index) => {
if(value["type"] === "NA"){
NAIndex = index;
return copiedValues[values.length-1];
}
else if(index === values.length-1){
return copiedValues[NAIndex];
}
return value;
})
}
const array = someFunction(test);
console.log(array)
/*
(3) [Object, Object, Object]
0: Object
type: "Material"
1: Object
type: "ABC"
2: Object
type: "NA"
*/
only use map method is a bit give you constraints yourself. try use splice or something. using only map has low efficiency
You can map the array and then sort your array based on 'NA'
const test = [{'type':'Material'}, {'type':''}, {'type':'ABC'}]
let result = test
.map(val => ({type: val.type || 'NA'}))
.sort((a,b) => a.type === 'NA' ? 1 : b.type === 'NA' ? -1 : 0);
console.log(result)
We can use the sort function.
test.sort((elem1, elem2) => {
if (elem2.type !== 'NA')
if (elem1.type === 'NA')
return 1
else
return 0
return -1
})
Or you can use the shorter one
test.sort((elem1, elem2) =>
elem2.type !== 'NA' ? elem1.type === 'NA' ? 1 : 0 : -1)
Learn more about the sort method here.

Node js , replacing occurrences in a nested data structure

Basically I want to replace # occurrences in a string from an object. As you can see it replace occurences in templateName , description , comments and Name but I can't replace sections header and sections questions , how will I improve my loop to apply replaceOccurrences in sections.header and sections questions array of objects? . sections headers are array of objects I also want to include that. Any idea? thank you.
Code
const replaceOccurrences = (originalString) => (typeof originalString === 'string' ? originalString.replace(/#/g, '&num;') : originalString);
const generateTemplate = async (data) => {
for (const [k, v] of Object.entries(data)) { data[k] = replaceOccurrences(v); }
return template(data);
};
Data
data : {
Name: 'Rajesh',
sections: [
{
questions: [Array]
}
],
templateName: 'TEMPLAT#E',
description: 'Tes#t',
comments: "adasdada'dfgdfgdfg 'gfddf#gdfgdf #num;## ##fsdfds gdfgdfgfd##"
}
James's answer will work if your obj is serializable, otherwise you'll need a recursive function
const removePoundSign = cur => {
if (typeof cur === 'string') {
return cur.replace(/#/g, '&num;');
}
if (cur == null || typeof cur !== 'object') return cur;
if (Array.isArray(cur)) {
return cur.map(el => removePoundSign(el));
}
const newObj = {};
for (const key in cur) {
newObj[key] = removePoundSign(cur[key]);
}
return newObj;
};
also, theres no need for the function to be async in your code!
You can stringify the object, make your replacements, then return it to object form:
let a = JSON.stringify(data).replace(/#/g, '&num;');
let b = JSON.parse(a);
Name: "Rajesh"
comments: "adasdada'dfgdfgdfg 'gfddf&num;gdfgdf &num;num;&num;&num; &num;&num;fsdfds gdfgdfgfd&num;&num;"
description: "Tes&num;t"
sections: Array(1)
0: {header: "Testing sec&num;tion", questions: Array(1)}
length: 1
__proto__: Array(0)
templateName: "TECHNICAL TEMPLAT&num;E"

Transform an object to add new keys based on existing key-values

I need to transform this object :
myObject = {
"pageName": "home",
"dataExtract": "data1|data2=value2|data3=value3|data4=value4a,value4b,value4c"}
To this one:
myObject_mod = {
'pageName' : 'home',
'dataExtract' : {
'data1' : '', //no value for 'data1'
'data2' : 'value2',
'data3' : 'value3',
'data4' : {
'data4key1' : 'value4a',
'data4key2' : 'value4b',
'data4key3' : 'value4c'
}
}
I've started by taking the 'dataExtract' key, and split it by "|", so I get its values splited:
myObject.dataExtract.split("|");
(4) ["data1", "data2=value2", "data3=value3", "data4=value4a,value4b,value4c"]
How can I continue?
Use reduce to build up your result object for every entry in your data extract. The general form would be:
myArray.reduce((resultObject, entryString) => {
// some logic here
resultObject[key] = resultValue;
return resultObject
}, {});
For example:
array = {
"pageName": "home",
"dataExtract": "data1|data2=value2|data3=value3|data4=value4a,value4b,value4c"
};
array_mod = {
pageName: array.pageName,
dataExtract: array.dataExtract.split("|").reduce((obj, entry) => {
// handle cases like data1
if (entry.indexOf('=') === -1) {
obj[entry] = '';
return obj;
}
// handle cases like data2 and data3
const [key, value] = entry.split('=', 2);
if (value.indexOf(',') === -1) {
obj[key] = value;
return obj;
}
// handle cases like data4
const values = value.split(',');
obj[key] = values.reduce((o, v, i) => (o[`${key}key${i+1}`] = v, o), {});
return obj;
}, {})
};
console.log(array_mod);
Use a combination of Array.split(), Array.reduce(), and destructuring, to break the string into keys and values, and rebuild as an object:
const object = {
"pageName": "home",
"dataExtract": "data1|data2=value2|data3=value3|data4=value4a,value4b,value4c"
}
const result = {
...object,
dataExtract: object.dataExtract.split('|')
.reduce((r, kv) => {
const [key, value = ''] = kv.split('=')
r[key] = !value.includes(',') ?
value
:
value.split(',').reduce((ra, val, i) => {
ra[`${key}key${i + 1}`] = val;
return ra;
}, {})
return r
}, {})
}
console.log(result)
An alternative to what was already posted, also using String.split and Array.reduce:
const array = {
"pageName": "home",
"dataExtract": "data1|data2=value2|data3=value3|data4=value4a,value4b,value4c"
};
const array_mod = {
pageName: array.pageName,
dataExtract: array.dataExtract.split('|').reduce((r, cur) => {
const [k, v = ''] = cur.split('=', 2);
const vs = v.split(',');
r[k] = vs.length > 1 ? vs.reduce((sr, scur, i) => (sr[`${k}key${i + 1}`] = scur, sr), {}) : v;
return r;
}, {})
};
console.log(array_mod);
With comments and more explicit variable names:
const array = {
"pageName": "home",
"dataExtract": "data1|data2=value2|data3=value3|data4=value4a,value4b,value4c"
};
const array_mod = {
pageName: array.pageName,
// Split by | and build a key => value object from it
dataExtract: array.dataExtract.split('|').reduce((data, entry) => {
// Grab key (before `=`) and value (after `=`)
const [key, value = ''] = entry.split('=', 2);
// Grab subvalues (separated by `,`) if any
const subValues = value.split(',');
// If there are at least 2 subvalues, build a key => value object from them
data[key] = subValues.length > 1
? subValues.reduce((sub, subVal, i) => (sub[`${key}key${i + 1}`] = subVal, sub), {})
// Otherwise return the value as string
: value;
return data;
}, {})
};
console.log(array_mod);

lodash orderby with null and real values not ordering correctly

I have an Angular 2 typescript application that is using lodash for various things.
I have an array of objects that I am ordering using a property in the object...
_.orderBy(this.myArray, ['propertyName'], ['desc']);
This works well however my problem is that sometimes 'propertyName' can have a null value.
These are ordered as the first item in a descending list, the highest real values then follow.
I want to make these null values appear last in the descending ordering.
I understand why the nulls come first.
Does anyone know how to approach this?
The _.orderBy() function's iteratees can use a method instead of a string. Check the value, and if it's null return an empty string.
const myArray = [{ propertyName: 'cats' }, { propertyName: null }, { propertyName: 'dogs' }, { propertyName: 'rats' }, { propertyName: null }];
const result = _.orderBy(myArray, ({ propertyName }) => propertyName || '', ['desc']);
console.log(result);
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.2/lodash.min.js"></script>
The check can be simple (like the one I've used), which converts all falsy values to an empty string:
propertyName || ''
If you need a stricter check, you can use the ternary operator, and handle just null values:
propertyName === null ? '' : propertyName
Edit: Example with multiple ordering:
const result = _.orderBy(myArray, (item) => [get(item, 'propertyName', 0), get(item, 'propertyName2')], ['desc', 'asc']);
This will order by propertyName then propertyName2.
If propertyName is undefined/null then its default order will be set to 0. (and therefore will be displayed at last because of desc ordering on the propertyName field). In such case, propertyName2 will therefore determine the ordering.
The code I needed looks like this...
_.orderBy(this.myArray, [( o ) => { return o.myProperty || ''}], ['desc']);
Just for future reference to others you can do this to sort ascending with falsey values at the end.
items =>
orderBy(
items,
[
i => !!i.attributeToCheck,
i => {
return i.attributeToCheck ? i.attributeToCheck.toLowerCase() : ''
}
],
['desc', 'asc']
)
mine looks like this. PropName and sort are both variables in my solution
return _.orderBy( myarray, [
( data ) => {
if ( data[propName] === null ) {
data[propName] = "";
}
return data[propName].toLowerCase();
}
], [sort] );
I wanted tolowercase because otherwise the sorting is not correct if different casings
This will put bad values at the bottom, and it differentiates between numbers and strings.
const items = [] // some list
const goodValues = isAscending => ({ value }) => {
if (typeof value !== 'string' && isNaN(value)) {
return isAscending ? Infinity : -Infinity
}
return value || ''
}
const sortedItems = orderBy(
items,
[goodValues(isAscending), 'value'],
[isAscending ? 'asc' : 'desc']
)
This worked for me
orders = [{id : "1", name : "test"}, {id : "1"}];
sortBy = ["id", "name"];
orderby(
orders,
sortBy.map(s => {
return (r: any) => {
return r[s] ? r[s] : "";
};
})),
);
I created a function for this (ts code):
const orderByFix = (array: any[], orderKeys: string[], orderDirs: ('asc' | 'desc')[]) => {
const ordered = orderBy(array, orderKeys, orderDirs);
const withProp = ordered.filter((o) => orderKeys.every(k => o[k]));
const withoutProp = ordered.filter((o) => !orderKeys.every(k => o[k]));
return [...withProp, ...withoutProp];
};
I've extended gwendall's answer to also handle case when "order keys" are functions (_.orderBy allows that)
const orderByFix = (
array: any[],
orderKeys: (string | ((o: any) => any))[],
orderDirs: ('asc' | 'desc')[]
) => {
const ordered = orderBy(array, orderKeys, orderDirs)
const withProp = ordered.filter((o) =>
orderKeys.every((k) => {
if (typeof k === 'string') {
return o[k]
} else if (typeof k === 'function') {
return k(o)
} else {
throw Error(`Order key must be string or function not ${typeof k}`)
}
})
)
const withoutProp = ordered.filter(
(o) =>
!orderKeys.every((k) => {
if (typeof k === 'string') {
return o[k]
} else if (typeof k === 'function') {
return k(o)
} else {
throw Error(`Order key must be string or function not ${typeof k}`)
}
})
)
return [...withProp, ...withoutProp]
}

Categories

Resources