Develop Reference

Iterate deeply nested object with unknown level and add remove key/value based on user provided conditions

Could anyone please guide me how to achieve the below challenge which I am facing?
I have thousands of mock API request response JSON files. They are deeply nested, and they all are structured differently. I need to add/update/delete entry at the specfic location where the condition match which will be provided by user. I am not sure how to approach this problem? I have tried doing something like below. I am asking user for path for where to start looking. But this will increase time as user has to look for path in all file and pass that info to api. below code work upto 2 level only. need to search full tree where all user provides conditions matches, and at that place, I need to add/update/delete data. I took condition as an array of objects.
Draft Code
const _ = require("lodash");
const file = "./sample.json";
const actions = ["add", "delete", "update"];
const consumer = (file, key, where, data, action) => {
try {
const act = action.toLowerCase();
if(!actions.includes(act) throw new Error("invalid action provided");
if(_.isArray(where) && _.every(where, _.isObject())) throw new Error("no where clause condition provided");
let content = require(file);
let typeKeyContent = null;
let keyContent = _.get(content, key);
if(!keyContent) throw new Error("invalid key");
if(_.isArray(keyContent)) {
typeKeyContent = "array"
} else if (_.isObject(keyContent)) {
typeKeyContent = "object"
}
switch (act) {
case "add":
if (typeKeyContent === "array") {
// array logic
for (let i = 0; i < keyContent.length; i++) {
const result = where.every(element => {
for (let key in element) {
return keyContent[key] && element[key] === keyContent[key];
}
});
if (!result) {
console.log("attributes matching -> ", result);
return;
}
keyContent[i] = {...keyContent[i], ...data }
}
let newcontent = _.set(content, key, keyContent);
console.log("newcontent -> \n",JSON.stringify(newcontent, null, 2));
return;
}
const result = where.every(element => {
for (let key in element) {
return keyContent[key] && element[key] === keyContent[key];
}
});
if (!result) {
console.log("attributes matching -> ", result);
return;
}
keyContent = { ...keyContent, ...data };
let newcontent = _.set(content, key, keyContent);
console.log("newcontent -> \n",JSON.stringify(newcontent, null, 2));
// TODO :: store back in json file
break;
default:
console.log("reached default case");
return;
}
} catch(err) {
console.log("ERROR :: CONSUMER ::", error);
}
}
// AND based condition only
const conditions = [
{ name: "Essential Large" },
{ selected: true }
];
const newdata = { description: "our best service" } // wants to add new prop
consumer(file, "selected_items.essential", conditions, newdata, "add");
sample json
{
"status": 200,
"request": {},
"response": {
"ffs": false,
"customer": {
"customer_id": 1544248,
"z_cx_id": 123456
},
"selected_items": {
"essential": [
{
"id": 4122652,
"name": "Essential Large",
"selected": true,
"description": "our best service" // will be added
},
{
"id": 4122653,
"name": "Essential Large",
"selected": true,
"description": "our best service" // will be added
}
]
},
"service_partner": {
"id": 3486,
"name": "Some String",
"street": "1234 King St."
},
"subject": "Project",
"description": "Issue: (copy/paste service request details here Required"
}
}

So you want to go through every key of a nested object right?
function forEvery(object,fn){
//obj is the object, fn is the function
//this function should go through each item in an object loaded from JSON string
//fn takes in 3 arguments: current element, that element's parent, level of depth(starts at 1)
var arr=[]
function recurse(obj,map,depth){
Object.keys(obj).forEach((a,i)=>{
fn(obj[a],obj,a,depth) //because fn can affect the object so the if statement should after not before ;-;
if(typeof obj[a]=="object"&&obj[a]!=null){ //if nested value is another object
map.push(a); arr.push(map)
recurse(obj[a],[...map],depth+1)
}
})
}
recurse(object,[],1)
}
//usage would be like:
//let customerCondition=/*some logic here*/
//let testObj=JSON.parse( (require('fs')).readFileSync('dirToSomeFile.json') )
forEvery(testObj,customerCondition)
Here's a live example
let testObj={"status":200,"request":{},"response":{"ffs":false,"customer":{"customer_id":1544248,"z_cx_id":123456},"selected_items":{"essential":[{"id":4122652,"name":"Essential Large","selected":true},{"id":4122653,"name":"Essential Medium","selected":false}]},"service_partner":{"id":3486,"name":"Some String","street":"1234 King St."},"subject":"Project","description":"Issue: (copy/paste service request details here Required"}}
function forEvery(object,fn){
//obj is the object, fn is the function
//this function should go through each item in an object loaded from JSON string
//fn takes in 3 arguments: current element, that element's parent, level of depth(starts at 1)
var arr=[]
function recurse(obj,map,depth){
Object.keys(obj).forEach((a,i)=>{
fn(obj[a],obj,a,depth) //because fn can affect the object so the if statement should after not before ;-;
if(typeof obj[a]=="object"&&obj[a]!=null){ //if nested value is another object
map.push(a); arr.push(map)
recurse(obj[a],[...map],depth+1)
}
})
}
recurse(object,[],1)
}
//example usage
let userQuery=[{ name: "Essential Large" },{ selected: true }]; //the user query in the format you gave
let userCondition={} //assuming each key across userQuery is unique, I set a model object for comparisons later on
userQuery.forEach(obj=>{ //I fill the model object :D
Object.keys(obj).forEach(key=>{
userCondition[key]=obj[key]
})
})
let testFn=(elem,parent,key,depth)=>{
//I use comparisons with the model object
let condition=typeof elem!="object"?false:
Object.keys(userCondition)
.every(item=>userCondition[item]==elem[item])
//true if matches user condition(meaning elem must be an object), false otherwise
if(condition){
console.log(parent[key],"will now be deleted")
delete(parent[key]) //deletion example(if user conditions match)
}
}
forEvery(testObj,testFn)
console.log("and the changed object looks like",testObj)

How can I parse a large JSON file with repeating values in JavaScript?

I am parsing a large JSON file using JSON stream. This works but it returns the file line by line. So when I try and restructure the data I can only get the data that is not repeating.
For example, this is the structure:
{
"Test": {
"id": 3454534344334554345434,
"details": {
"text": "78679786787"
},
"content": {
"text": 567566767656776
},
"content": {
"text": 567566767656776
},
"content": {
"text": 567566767656776
}
}
}
I'm able to get Test.id or Test.details.id but I can only get the First Test.content per line.
I've tried to set in an array but this still gets only the first line of Test.content.
Is there another way to transform a large file other than using JSONStream?
Parsing code:
var getStream = function () {
let stream = fs.createReadStream(path.join(__dirname, '../test.json'), {
encoding: 'utf8'
}),
parser = JSONStream.parse('*.Test')
return stream.pipe(parser);
};
getStream()
.pipe(es.mapSync(function (data) {
let dataObj = []
dataObj.push(data)
processData(dataObj)
}))
function processData(d) {
let js = JSON.parse(JSON.stringify(d))
console.log(js)
// js.forEach(function (value, index) {
// Object.keys(value).forEach(function (v, i) {});
// })
}

Search for a related json data

How can i find data that is related to the already known data?
( I'm a newb. )
For example here is my json :
[
{ "id": "1", "log": "1","pass": "1111" },
{ "id": 2, "log": "2","pass": "2222" },
{ "id": 3, "log": "3","pass": "3333" }
]
Now i know that "log" is 1 and i want to find out the data "pass" that is related to it.
i've tried to do it so :
The POST request comes with log and pass data , i search the .json file for the same log value and if there is the same data then i search for related pass
fs.readFile("file.json", "utf8", function (err, data) {
var jsonFileArr = [];
jsonFileArr = JSON.parse(data); // Parse .json objekts
var log = loginData.log; // The 'log' data that comes with POST request
/* Search through .json file for the same data*/
var gibtLog = jsonFileArr.some(function (obj) {
return obj.log == log;
});
if (gotLog) { // If there is the same 'log'
var pass = loginData.pass; // The 'pass' data that comes with POST request
var gotPass = jsonFileArr.some(function (obj) {
// How to change this part ?
return obj.pass == pass;
});
}
else
console.log("error");
});
The problem is that when i use
var gotPass = jsonFileArr.some(function (obj) {
return obj.pass == pass;
});
it searches through the whole .json file and not through only one objekt.

Your main problem is that .some() returns a boolean, whether any of the elements match your predicate or not, but not the element itself.
You want .find() (which will find and return the first element matching the predicate):
const myItem = myArray.find(item => item.log === "1"); // the first matching item
console.log(myItem.pass); // "1111"
Note that it is possible for .find() to not find anything, in which case it returns undefined.

The .some() method returns a boolean that just tells you whether there is at least one item in the array that matches the criteria, it doesn't return the matching item(s). Try .filter() instead:
var jsonFileArr = JSON.parse(data);
var log = loginData.log;
var matchingItems = jsonFileArr.filter(function (obj) {
return obj.log == log;
});
if (matchingItems.length > 0) { // Was at least 1 found?
var pass = matchingItems[0].pass; // The 'pass' data that comes with the first match
} else
console.log("error"); // no matches

Using ES6 Array#find is probably the easiest, but you could also do (among other things)
const x = [{
"id": "1",
"log": "1",
"pass": "1111"
}, {
"id": 2,
"log": "2",
"pass": "2222"
}, {
"id": 3,
"log": "3",
"pass": "3333"
}];
let myItem;
for (let item of x) {
if (item.log === '1') {
myItem = item;
break;
}
}
console.log(myItem);

How to replace old json object with a new one with NodeJS

How to replace old json object using NodeJS with a new updated object ?
Right now when i update the json file it saves the new data with the old one.
JSON :
[ {
"id": 1,
"name": "Sven",
"phone": "123123"
},
{
"id": 2,
"name": "Martin",
"phone": "2342342"
} ]
Here is my code :
var operation = POST.operation; // POST request comes with operation = update/insert/delete
if (operation == 'update') {
fs.readFile("file.json", "utf8", function (err, data) {
var jsonFileArr = [];
jsonFileArr = JSON.parse(data); //Parse the data from JSON file
var haveId = jsonFileArr.some(function (obj){ // Checks if the POST request have the same id as JSON file
return obj.id == POST.id;
})
if (haveId) { // if true
var updateData = []; // Array with POST data
updateData.push({
id: POST.id,
name: POST.name,
phone: POST.phone,
})
jsonFileArr.push(updateData);
var newUsers = JSON.stringify(jsonFileArr);
fs.writeFile("file.json", newUsers, "utf8");
console.log(err);
}
})
}
I should probably use delete object but how can i specify what object should be removed ?
So when i update data with id 1 it would delete the old id / Name / phone and write the new data.

My assumption base on your question is that you have multiple objects in one file. So the easy way to work around this would be to
if (operation == 'update') {
fs.readFile("file.json", "utf8", function (err, data) {
var jsonFileArr = [];
jsonFileArr = JSON.parse(data); //Parse the data from JSON file
var haveId = jsonFileArr.some(function (obj){ // Checks if the POST request have the same id as JSON file
return obj.id == POST.id;
})
if (haveId) { // if true
var updateData = []; // Array with POST data
updateData.push({
id: POST.id,
name: POST.name,
phone: POST.phone,
})
for(let Arr of jsonFileArr){
if (Arr.id == POST.id){
let currentIndex = jsonFileArr.indexOf(Arr);
jsonFileArr.splice(currentIndex,1,updateData) //removing the old object and adding the new one
}
}
var newUsers = JSON.stringify(jsonFileArr);
fs.writeFile("file.json", '', "utf8",function(err,res){ //Making the file empty
if(!err){
fs.writeFile("file.json", newUsers, "utf8",function(err,res){ //Writing the whole object back
if(err)console.log(err);
console.info(res);
});
}else{
console.log(err);
}
});
}
})
}
I think this is better instead of using some, get the matching index and replace directly.
var jsonFileArr = JSON.parse(data); //Parse the data from JSON file
var foundId = jsonFileArr.findIndex(function (obj){ // Checks if the POST request have the same id as JSON file
return obj.id == POST.id;
});
if (foundId >= 0) {
jsonFileArr[foundId] =
{
id: POST.id,
name: POST.name,
phone: POST.phone,
}
}
.... and then write back to file

IndexedDB and Javascript: JSON and objects misunderstanding

I'm trying to obtain information from a JSON file download to the client through AJAX and I'm getting different results depending on the JSON format and I don't know how to fix the one with problem.
First case:
The json files looks like:
[{"name": "nick",
"age": 28},
{"name": "katie",
"age": 32}]
My AJAX .done method looks like:
.done(
function(data) {
addObjectsDB (data, "people");
})
This method calls a second one that iterates through data and stored correctly each object into IndexedDB.
Second case:
Now I have a JSON file with different format:
[
{
"husband": {
"name": "Jhon",
"age": 23 },
"wife": {
"name": "Marie",
"age": 24 }
}
]
Now my .done() AJAX method iterates through data and add each person, husband or wife to an array which is then sent to the DB with the same method than the first case:
.done(
function(data) {
var people = [];
$(data).each(function (key, value){
people.push(value.husband);
people.push(value.wife);
});
addObjectsDB (people, "people");
})
In this case the insertion into the database fails, if for example, instead of adding value.husband to people array I just add value to people array the insertion works, but I need each person stored separated in the DB.
The addObjectsDB method is:
function addObjectsDB (data, collection) {
var objectStore = db.transaction(collection, "readwrite").objectStore(collection);
$.each (data, function (key, value) {
var request = objectStore.add(value);
});
}
As I said the first case works perfectly but the second one inserts nothing and no error is showed...
I think the problem is that I don't understand javascript types adequately but I'm starting with it and I've spent a whole evening with it.

There's nothing wrong with your IDB code. Look for your answer in the code you haven't presented, particularily the AJAX response (is your JSON parsed the way you think it is?)
Be sure to attach event listeners for the error event. I'm positive that if your IDB "inserts nothing" then in fact it's not true that "no error is showed" and rather no error is seen due to callback mismanagement.
Here's a working implementation, modified from a previous answer I've given on this tag. This implementation doesn't have the uniqueness constraints you've put on your schema on purpose: it shows that your looping is fine. The entries below all look good.
var db_name = 'SO_22977915',
store_name = 'people';
$.ajax({
url: '/echo/json/',
type: 'post',
dataType: 'json',
data: {
json: JSON.stringify({
case1: [{
"name": "nick",
"age": 28
}, {
"name": "katie",
"age": 32
}],
case2: [{
"husband": {
"name": "Jhon",
"age": 23
},
"wife": {
"name": "Marie",
"age": 24
}
}]
})
},
success: function (data) {
var request,
upgrade = false,
doTx = function (db, entry) {
addData(db, entry, function () {
getData(db);
});
},
getData = function (db) {
db.transaction([store_name], "readonly").objectStore(store_name).openCursor(IDBKeyRange.lowerBound(0)).onsuccess = function (event) {
var cursor = event.target.result;
if (null !== cursor) {
console.log("entry", cursor.value);
cursor.continue();
}
};
},
addData = function (db, entry, finished) {
console.log('adding', entry);
var tx = db.transaction([store_name], "readwrite"),
people = [];
tx.addEventListener('complete', function (e) {
finished();
});
$.each(entry.case1, function (key, value) {
tx.objectStore(store_name).add(value);
});
$(entry.case2).each(function (key, value){
people.push(value.husband);
people.push(value.wife);
});
$.each(people, function (key, value) {
tx.objectStore(store_name).add(value);
});
};
request = window.indexedDB.open(db_name);
request.oncomplete = function (event) {
if (upgrade) {
doTx(request.result, data);
}
};
request.onsuccess = function (event) {
if (!upgrade) {
doTx(request.result, data);
}
};
request.onupgradeneeded = function (event) {
var db = event.target.result;
db.createObjectStore(store_name, {
keyPath: null,
autoIncrement: true
});
}
}
});
A cursor and console.log shows all entries as being added: