Develop Reference

Iterate deeply nested object with unknown level and add remove key/value based on user provided conditions

Could anyone please guide me how to achieve the below challenge which I am facing?
I have thousands of mock API request response JSON files. They are deeply nested, and they all are structured differently. I need to add/update/delete entry at the specfic location where the condition match which will be provided by user. I am not sure how to approach this problem? I have tried doing something like below. I am asking user for path for where to start looking. But this will increase time as user has to look for path in all file and pass that info to api. below code work upto 2 level only. need to search full tree where all user provides conditions matches, and at that place, I need to add/update/delete data. I took condition as an array of objects.
Draft Code
const _ = require("lodash");
const file = "./sample.json";
const actions = ["add", "delete", "update"];
const consumer = (file, key, where, data, action) => {
try {
const act = action.toLowerCase();
if(!actions.includes(act) throw new Error("invalid action provided");
if(_.isArray(where) && _.every(where, _.isObject())) throw new Error("no where clause condition provided");
let content = require(file);
let typeKeyContent = null;
let keyContent = _.get(content, key);
if(!keyContent) throw new Error("invalid key");
if(_.isArray(keyContent)) {
typeKeyContent = "array"
} else if (_.isObject(keyContent)) {
typeKeyContent = "object"
}
switch (act) {
case "add":
if (typeKeyContent === "array") {
// array logic
for (let i = 0; i < keyContent.length; i++) {
const result = where.every(element => {
for (let key in element) {
return keyContent[key] && element[key] === keyContent[key];
}
});
if (!result) {
console.log("attributes matching -> ", result);
return;
}
keyContent[i] = {...keyContent[i], ...data }
}
let newcontent = _.set(content, key, keyContent);
console.log("newcontent -> \n",JSON.stringify(newcontent, null, 2));
return;
}
const result = where.every(element => {
for (let key in element) {
return keyContent[key] && element[key] === keyContent[key];
}
});
if (!result) {
console.log("attributes matching -> ", result);
return;
}
keyContent = { ...keyContent, ...data };
let newcontent = _.set(content, key, keyContent);
console.log("newcontent -> \n",JSON.stringify(newcontent, null, 2));
// TODO :: store back in json file
break;
default:
console.log("reached default case");
return;
}
} catch(err) {
console.log("ERROR :: CONSUMER ::", error);
}
}
// AND based condition only
const conditions = [
{ name: "Essential Large" },
{ selected: true }
];
const newdata = { description: "our best service" } // wants to add new prop
consumer(file, "selected_items.essential", conditions, newdata, "add");
sample json
{
"status": 200,
"request": {},
"response": {
"ffs": false,
"customer": {
"customer_id": 1544248,
"z_cx_id": 123456
},
"selected_items": {
"essential": [
{
"id": 4122652,
"name": "Essential Large",
"selected": true,
"description": "our best service" // will be added
},
{
"id": 4122653,
"name": "Essential Large",
"selected": true,
"description": "our best service" // will be added
}
]
},
"service_partner": {
"id": 3486,
"name": "Some String",
"street": "1234 King St."
},
"subject": "Project",
"description": "Issue: (copy/paste service request details here Required"
}
}

So you want to go through every key of a nested object right?
function forEvery(object,fn){
//obj is the object, fn is the function
//this function should go through each item in an object loaded from JSON string
//fn takes in 3 arguments: current element, that element's parent, level of depth(starts at 1)
var arr=[]
function recurse(obj,map,depth){
Object.keys(obj).forEach((a,i)=>{
fn(obj[a],obj,a,depth) //because fn can affect the object so the if statement should after not before ;-;
if(typeof obj[a]=="object"&&obj[a]!=null){ //if nested value is another object
map.push(a); arr.push(map)
recurse(obj[a],[...map],depth+1)
}
})
}
recurse(object,[],1)
}
//usage would be like:
//let customerCondition=/*some logic here*/
//let testObj=JSON.parse( (require('fs')).readFileSync('dirToSomeFile.json') )
forEvery(testObj,customerCondition)
Here's a live example
let testObj={"status":200,"request":{},"response":{"ffs":false,"customer":{"customer_id":1544248,"z_cx_id":123456},"selected_items":{"essential":[{"id":4122652,"name":"Essential Large","selected":true},{"id":4122653,"name":"Essential Medium","selected":false}]},"service_partner":{"id":3486,"name":"Some String","street":"1234 King St."},"subject":"Project","description":"Issue: (copy/paste service request details here Required"}}
function forEvery(object,fn){
//obj is the object, fn is the function
//this function should go through each item in an object loaded from JSON string
//fn takes in 3 arguments: current element, that element's parent, level of depth(starts at 1)
var arr=[]
function recurse(obj,map,depth){
Object.keys(obj).forEach((a,i)=>{
fn(obj[a],obj,a,depth) //because fn can affect the object so the if statement should after not before ;-;
if(typeof obj[a]=="object"&&obj[a]!=null){ //if nested value is another object
map.push(a); arr.push(map)
recurse(obj[a],[...map],depth+1)
}
})
}
recurse(object,[],1)
}
//example usage
let userQuery=[{ name: "Essential Large" },{ selected: true }]; //the user query in the format you gave
let userCondition={} //assuming each key across userQuery is unique, I set a model object for comparisons later on
userQuery.forEach(obj=>{ //I fill the model object :D
Object.keys(obj).forEach(key=>{
userCondition[key]=obj[key]
})
})
let testFn=(elem,parent,key,depth)=>{
//I use comparisons with the model object
let condition=typeof elem!="object"?false:
Object.keys(userCondition)
.every(item=>userCondition[item]==elem[item])
//true if matches user condition(meaning elem must be an object), false otherwise
if(condition){
console.log(parent[key],"will now be deleted")
delete(parent[key]) //deletion example(if user conditions match)
}
}
forEvery(testObj,testFn)
console.log("and the changed object looks like",testObj)

How to make a nested queries on mysql with Node.js?

I'm trying to make a nested queries with mysql on Node.js with Express, the result should be a json with three nested, the problem is that the return doesn't show me the third nested, my code is:
app.get('/devices/:id', ensureToken, function(req, res){
jwt.verify(req.token, jwt_secret, function(err, data) {
if (err) {
res.sendStatus(403);
} else {
var segment_id = req.param('id');
//Select devices for segment
var ls_devices = 'SELECT d.device_id AS id, d.device_name AS name, d.device_serial AS serial, d.device_description AS description, d.device_key AS keyatom, d.device_type AS type_id, dt.device_type_name AS type, d.device_user_id AS user_id, u.user_name AS username, d.device_segment_id AS segment_id, sg.segment_name, d.device_public AS isPublic, d.device_status AS status FROM ((g_devices d LEFT JOIN g_user u ON u.user_id = d.device_user_id) LEFT JOIN g_device_type dt ON dt.device_type_id = d.device_type) LEFT JOIN g_segments sg ON sg.segment_id = d.device_segment_id WHERE d.device_status = 1 AND sg.segment_id = '+segment_id;
connection.query(ls_devices, function (error, results, fields) {
if (error) throw error;
if(results.length != 0) {
var j = JSON.parse(JSON.stringify(results));
var i = 0;
var d = [];
j.forEach(function(r,index_r){
//
var ls_controllers = 'SELECT c.device_controller_id AS id, c.device_controller_name AS name, c.device_controller_description AS description, c.device_controller_devcon_type AS type_id, ct.devcon_name AS type, ct.devcon_description AS description, d.device_name AS device, d.device_id AS device_id, d.device_serial AS serial, c.device_controller_public AS public, c.device_controller_date_register AS registered, c.device_controller_date AS date, c.device_controller_status AS status FROM (g_device_controller c LEFT JOIN g_device_controller_type ct ON ct.devcon_id = c.device_controller_devcon_type) LEFT JOIN g_devices d ON d.device_id = c.device_controller_device_id WHERE c.device_controller_status = 1 AND d.device_id = '+r.id;
connection.query(ls_controllers, function (error_c, results_c, fields_c) {
var k = JSON.parse(JSON.stringify(results_c));
d.push({device:r.name,controller:[]})
k.forEach(function(r2,index_r2){
d[index_r].controller.push({name:r2.name,action:[]})
var ls_actions = 'SELECT a.action_id AS id, a.action_name AS name, a.action_description AS description, a.action_type AS type_id, aty.action_type_name, aty.action_type_description AS type_description, a.action_controller_device_id AS device_id, a.action_control AS control, a.action_value AS value, a.action_time AS time, a.action_times AS times, a.action_date AS date, a.action_status AS status FROM g_actions a LEFT JOIN g_action_type aty ON aty.action_type_id = a.action_type WHERE a.action_controller_id = '+r2.id;
connection.query(ls_actions, function (error_a, results_a, fields_a) {
if(results_a.length > 0) {
var l = JSON.parse(JSON.stringify(results_a));
l.forEach(function(r3, index_r3){
d[index_r].controller[index_r2].action.push({id_a:r3.id,name_a:r3.name});
console.log(JSON.stringify(d))
});
}
});
});
i ++;
if(i == j.length)
{
return res.json(d);
}
});
})
}
else {
return res.json({devices:false});
}
});
}
});
});
My web response is:
[
{
"device": "device one",
"controller": [
{
"name": "device controller One",
"action": []
}
]
},
{
"device": "device two",
"controller": []
}
]
And the print of the last array push is:
[
{
"device": "device one",
"controller": [
{
"name": "device controller One",
"action": [
{
"id_a": 1,
"name_a": "device action One"
}
]
}
]
},
{
"device": "device two",
"controller": [
]
}
]

k.forEach() does not wait for connection.query() to finish, so effectively the queries are skipped. The remedy is to do something similar to what you did with j.forEach(), but i++ should not happen until all of those finish too.
(Other notes: you could use promises, or async/await and make the flow appear much more neatly, or if you want to stick with learning callbacks you could use async library to simplify some of this)

Why is batchGet not showing any records in the response?

Overview: I am using aws-sdk for javascript to get 2 records from my table "testTable". I have provided the values for the respective keys as well.
Online Help: I have followed the documentation at http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/DocumentClient.html#batchGet-property
var batchGetItem = function() {
var params = {
RequestItems: {
"testTable": {
Keys:[
{
"CaseID": "A1245",
"AssignedSupervisorName": "John"
},
{
"CaseID": "A1247",
"AssignedSupervisorName": "Bob"
}
]
}
}
};
docClient.batchGet(params, function(err, data) {
if (err) {
document.getElementById('textarea').innerHTML = "Unable to read item: " + "\n" + JSON.stringify(err, undefined, 2);
} else {
document.getElementById('textarea').innerHTML = "GetItem succeeded: " + "\n" + JSON.stringify(data, undefined, 2);
$scope.readData = data;
}
});
}
The value of data is coming as:
{
"Responses":
{ "testTable": []
},
"UnprocessedKeys" : {}
}
I was expecting the 2 records to be returned in "testTable" but it is just returning an empty array.
Where am I going wrong?

I'm suspecting your table is not created with the keys you are trying to use inside your JS code.
The keys inside your params should contain only your primary key and sort key of your dynamodb table. You should check whether AssignedSupervisorName is set as your table's sort key.

pg-promise task and map with multiple same level nested queries

I am using node and pg-promise to create a basic rest API and am having some issue querying all data for a particular user. Below is what the data returned should look like. Address, Phone Number, and Skills all live in separate tables. I am having no issue retrieving addresses or phone numbers its just skills I can't seem to get. Not quite sure how to have multiple queries after the main query that gets the user to get all these other fields, please see the attached code for reference and I will be happy to answer any questions.
{
"user_id": 1,
"first_name": "Eugene",
"last_name": "Hanson",
"display_name": "Eugene Hanson",
"email": "ehanson0#typepad.com",
"hash": "88a6aa27235d2e39dd9cb854cc246487147050f265578a3e1aee35be5db218ef",
"privilege_id": 14,
"seniority": 1,
"birthday": "19-11-1940 00:00:00.0",
"shift_count_total": 587,
"shift_count_year": 62,
"address_id": 1,
"street": "92 Schmedeman Lane",
"city": "Fort Smith",
"state": "AR",
"zip": 72905,
"phone_numbers": [
{
"phone_number": "62-(705)636-2916",
"name": "PRIMARY"
}
],
"skills": [
"Head Audio",
"Head Video",
"Head Electrician",
"Carpenter",
"rigger"
]
}
function getAllUsers() {
// console.time("answer time")
var deferred = Q.defer();
db.task(t => {
return t.map('SELECT * \
FROM users \
JOIN addresses \
ON users.address_id = addresses.address_id',[], user => {
var user_id = user.user_id;
// console.log(user_id)
console.time("answer time")
return t.manyOrNone('SELECT phone_numbers.phone_number, phone_types.name \
FROM users \
JOIN users_phone_numbers \
ON users.user_id = users_phone_numbers.user_id \
JOIN phone_numbers \
ON users_phone_numbers.phone_id = phone_numbers.phone_id \
JOIN phone_types \
ON phone_numbers.phone_type_id = phone_types.phone_type_id \
WHERE users.user_id = $1', user.user_id)
.then(phone_numbers=> {
// logger.log('info', phone_numbers)
user.phone_numbers = phone_numbers;
return user;
})
}).then(t.batch);
})
.then(data => {
// console.log(data)
console.timeEnd("answer time");
var response = {code: "200",
message: "",
payload: data};
deferred.resolve(response);
})
.catch(error => {
var response = {code: error.code,
message: error.message,
payload: ""};
logger.log('error', error)
deferred.reject(response)
});

I'm the author of pg-promise.
Simplified version of your function would be:
function getAllUsers() {
return db.task(t => {
return t.map('SELECT * FROM users', [], user => {
return t.batch([
t.any('SELECT * FROM phones'), // plus formatting params
t.any('SELECT * FROM skills'), // plus formatting params
])
.then(data => {
user.phones = data[0];
user.skills = data[1];
return user;
});
}).then(t.batch);
});
}
getAllUsers()
.then(data => {
// data tree
})
.catch(error => {
// error
});
And if you are using bluebird as the promise library, then you can replace this code:
.then(data => {
user.phones = data[0];
user.skills = data[1];
return user;
});
with this one:
.spread((phones, skills) => {
user.phones = phones;
user.skills = skills;
return user;
});
And do not use things like var deferred = Q.defer();, it is not needed there. The library is already promise-based.
For a high-performance alternative see: get JOIN table as array of results with PostgreSQL/NodeJS.

IBM Worklight - How to parse adapter response?

This is my stored procedure:
CREATE PROCEDURE PROC()
BEGIN
SELECT * FROM TABLENAME;
END//
This is my unction to call stored procedure using SQL adapter:
function callStored() {
return WL.Server.invokeSQLStoredProcedure({
procedure : "proc",
parameters : []
});
}
This is the invocationResult:
{
"isSuccessful": true,
"resultSet": [
{
"name": "a",
"pass": "123",
"time_stamp": "2014-04-07T10:13:17.000Z"
},
{
"name": "chetan",
"pass": "123456",
"time_stamp": "2014-04-07T10:13:34.000Z"
},
{
"name": "dileep",
"pass": "456321",
"time_stamp": "2014-04-07T10:13:54.000Z"
},
{
"name": "bnc",
"pass": "654321",
"time_stamp": "2014-04-07T10:19:37.000Z"
}
]
}
I need to parse this and display or alert the values of name, pass and time_stamp.
How do I accomplish this?

In your application JavaScript (common\js\main.js), you can have something like the following. In the below code an alert will be displayed with the values of the name key from the resultSet.
You can also take a look here: use resultset returned by WL.Server.invokeSQLStatement within sql adapter procedure
function wlCommonInit() {
invokeAdapter();
}
function invokeAdapter() {
var invocationData = {
adapter: "your-adapter-name",
procedure: "callStored",
parameters: []
};
WL.Client.invokeProcedure (invocationData, {
onSuccess: invocationSuccess,
onFailure: invocationFailure}
);
}
function invocationSuccess(response) {
var i,
resultSet = response.invocationResult.resultSet;
namesArray = [];
for (i = 0; i < resultSet.length; i++) {
namesArray[i] = resultSet[i].name;
}
alert (JSON.stringify(namesArray));
}
function invocationFailure(response) {
alert (response);
}
You have already asked this here: ibm worklight stored procedure
Why did you not follow through the documentation and learned how to write the above?
Please read the documentation!

Please read the "Invoking adapter procedures from client applications" and its Exercise and code sample in the Getting started with IBM Worklight page.
function wlCommonInit(){
getUsersInfo();
}
function getUsersInfo(){
var invocationData = {
adapter : 'YOUR_ADAPTER',
procedure : 'YOUR_PROCEDURE',
parameters : []
};
WL.Client.invokeProcedure(invocationData,{
onSuccess : getUsersInfoSuccess,
onFailure : getUsersInfoFailure
});
}
function getUsersInfoSuccess(result){
if (result.invocationResult.Items.length > 0) {
displayUsersInfo(result.invocationResult.Items);
} else {
getUsersInfoFailure();
}
}
function getUsersInfoFailure(result){
alert("Cannot retrieve users info");
}
function displayUsersInfo(items){
var i = 0, usersInfo = '';
for (i = 0; i < items.length; i++) {
usersInfo += ' name: ' + items[i].name;
usersInfo += ' pass: ' + items[i].pass;
usersInfo += ' time_stamp: ' + items[i].time_stamp;
}
alert(usersInfo);
}