Develop Reference

JavaScript array of huge data (500K record) taking time to search and add

My function is called in loop which returns more than 500k record.
I have to insert that record in a JavaScript array. Before inserting records to array need to check existing array has duplicate records or not. If the record is duplicate then exclude the record.
When array size increases the run time of the function is very high. Please suggest me a way to optimize search.
function AddDataToArray(StdName, currObjectSTD, bufferObject, attributes, bufferSTD) {
var result = false;
var existingObjects = AllDataArray.find(item => {
item.OBJECTID==attributes.OBJECTID
&& item.name == bufferObject.name
&& item.StdName == StdName);
});
if (existingObjects.length == 0) {
var currentObject = {
"ID": 0,
"currObjectSTD": currObjectSTD,
"color": bufferObject.color,
"name": bufferObject.name,
"attributes": attributes,
"StdName": StdName,
"objectID": objectID,
"bufferSTD": bufferSTD,
"shape": null,
"shapeSTD": null
};
AllDataArray.push(currentObject);
result = true;
}
return result;
}

As a speedup workaround I suggest you coming up with some kind of hash map based on your array to avoid continuos looping through array
const dataHashMap = _(AllDataArray)
.keyBy(item => `${item.OBJECTID}-${item.name}-${item.StdName}`)
.mapValues(() => true)
.value();
var existingObjects = dataHashMap[`${attributes.OBJECTID}-${bufferObject.name}-${StdName}`]
or alternative solution
let groupedDataHashMap = {}
AllDataArray.forEach(item => {
_.set(
groupedDataHashMap,
[
item.OBJECTID,
item.name,
item.StdName
],
true
)
})
var existingObjects = _.get(
groupedDataHashMap,
[
attributes.OBJECTID,
bufferObject.name,
StdName
],
false
)
I used lodash methods but if you prefer using native array/object methods you can come up with your own implementation, but the idea is the same
P.S you need to create this hash map once you fetched your array and populate it with new items simultaneously with your array to keep it up-to-date with your array

getting two towers updated at the same time every time I make a move Towers of Hanoi js

I am trying to write a very simple implementation of the Towers of Hanoi puzzle to practice what I have just learned about js constructors and prototypes. I am currently having problems with what I have written so far because every time I move a 'disc' from tower[0] to let's say tower[1], my tower[2] also gets updated with the same disc. Also, when I try to make an invalid move, I still get the disc I tried to move, taken away from its tower. I have checked out my logic and I can't see anything wrong with it (I could be just biased at this point too). I am wondering if it is a problem with my constructor function or any of my methods?
Here my code :
function TowersOfHanoi(numberOfTowers){
let towersQuant = numberOfTowers || 3 , towers;
towers = Array(towersQuant).fill([]);
towers[0] = Array(towersQuant).fill(towersQuant).map((discNumber, idx) => discNumber - idx);
this.towers = towers;
}
TowersOfHanoi.prototype.displayTowers = function(){
return this.towers;
}
TowersOfHanoi.prototype.moveDisc = function(fromTower,toTower){
let disc = this.towers[fromTower].pop();
if(this.isValidMove(disc,toTower)){
this.towers[toTower].push(disc);
return 'disc moved!'
} else {
return 'disc couldn\'t be moved.'
}
}
TowersOfHanoi.prototype.isValidMove = function(disc,toTower){
if(this.towers[toTower][toTower.length-1] > disc || this.towers[toTower].length === 0){
return true;
}else {
return false;
}
}
this is what I am testing :
let game2 = new TowersOfHanoi();
console.log(game2.displayTowers());
console.log(game2.moveDisc(0,1));
console.log(game2.displayTowers());
console.log(game2.moveDisc(0, 2));
console.log(game2.displayTowers());
and my output :
[ [ 3, 2, 1 ], [], [] ]
disc moved!
[ [ 3, 2 ], [ 1 ], [ 1 ] ]
disc couldn't be moved.
[ [ 3 ], [ 1 ],[ 1 ] ]
I appreciate any guidance. I am not necessarily looking for code. just to understand. Thanks

This quote from the Array fill() documentation tells you the problem:
When the fill method gets passed an object, it will copy the reference and fill the array with references to that object.
towers = Array(towersQuant).fill([]);
Arrays are objects. So what you've done is copy a reference of the first Array to each of the other arrays. The whole thing is compounded when you iterate these references to modify them.
Update
Something like this will work:
function TowersOfHanoi(numberOfTowers){
let towersQuant = numberOfTowers || 3 , towers = [];
for(let i=1; i < towersQuant; i++){
towers.push([]);
towers[0].push(i);
}
towers[0].reverse();
this.towers = towers;
}

Efficient way to search within multiple values in json

I have multiple records like this,
name: John Doe aliases: John, Doe, JD unique_id: 1 ...
My question is how do I search efficiently within the aliases & full name.
If the search query is any of those 4 (John Doe, John, Doe, JD) I would like to find the unique id (in this case 1).
What I have done: I have a very straightforward implementation that loops through the entire data until it finds. It takes a long time since the number of fields is very high.
Note: I am using javascript if it helps. Also I have the permission to change the data format (permanently), if it will make the search more efficient. Most of the search queries tend to be one of the aliases rather than full name.
Sample Code: https://jsfiddle.net/nh7yqafh/
function SearchJSON(json, query) {
var champs = json.champs;
for (var i = 0; i < champs.length; ++i) {
if (query == champs[i].name)
return champs[i].unique_id;
for (var j = 0; j < champs[i].aliases.length; ++j) {
if (query == champs[i].aliases[j])
return champs[i].unique_id;
}
}
}
//Data format is similar to what vivick said
var json_string = '{"count":5,"champs":[{"name":"Abomination","aliases":["abomination","AB","ABO"],"unique_id":1},{"name":"Black Bolt","aliases":["blackbolt","BB","BBT"],"unique_id":2},{"name":"Black Panther","aliases":["blackpanther","BP","BPR"],"unique_id":3},{"name":"Captain America","aliases":["captainamerica","CA","CAP"],"unique_id":4}]}'
var json = JSON.parse(json_string);
query="CA";
alert( "id of "+query+" is "+SearchJSON(json, query));

I guess you have a structure similar to the following one :
[
{
"name": "xxx",
"aliases": ["x", "xx", "xxx"],
"unique_id": 1,
/* [...] */
},
/* [...] */
]
You can then do something like this :
const queryParam = /*search query*/;
const arr = /* get the JSON record */;
const IDs = arr
.filter( entry =>(entry.aliases.includes(queryParam) || queryParam===entry.name) )
.map(entry=>entry.uniqueId);
This will give you an array of IDs which are potential matches.
If you need either 0 or 1 result only :
const ID = IDs[0] || null;
This will simply retrieve the first matched ID if there's one, otherwise it will set ID to null.
NB:
If you use an object of objects instead of an array of object, there's just a little bit of modifications to do (mainly using Object.entries) but it still is trivial.
PS:
I would recommend to always add the full name in the aliases, this will ease the filtering part (no || would be required).

How to read this JSON with javascript & Get the value?

Here is my JSON code. I'm storing this json in an array.
total: {
limited: {
things: "451",
platforms: [
{
count: "358",
id: "Windows"
},
{
count: "44",
id: "X11"
},
{
count: "42",
id: "Macintosh"
},
{
count: "2",
id: "Linux"
},
{
count: "1",
id: "iPhone"
},
{
count: "1",
id: "iPod"
}
]
},
}
When i want to show the count of things in total > limited > things, I'm using the below code and it's working fine.
document.getElementById( "limited" ).value = arr.total.limited.things;
It's showing the 'things' value in 'limited' div area.
But I want to show the count of the particular id in platforms.
total > limited > platforms > id > windows.
How can i show the value of particular id from above json?
document.getElementById( "limited" ).value = arr.total.limited.platforms[0].count;
is showing the count but, the order of platforms always change, so we don't know where the windows is in the order exactly to use the above method.
How can we show the count of particular id from above json?
Also, how can we combine particular multiple id's count? for example, how to know all the count of Macintosh, iphone & ipod count combined?
Thanks.

You can filter the array to look for the Windows entry. Then, when you've got an array with only one element, access the count property of the the first element:
arr.total.limited.platforms.filter(
function(el) { return el.id == "Windows"; })[0].count
Getting the sum of counts for multiple platforms could be done like this by using the Array.map function:
// here, arr is the structure you describe in your question, and platf is an
// array of all desired platforms
function combinedCount(arr, platf) {
// for each element of the list of platforms in `arr`, we check
// if its id is inside the list of desired platforms,
// and return either its count or 0
var x = arr.total.limited.platforms.map(function(el) {
return (platf.indexOf(el.id) != -1) ? parseInt(el.count) : 0; });
// now x is an array of counts for the relevant platforms and
// 0 for all others, so we can just add its elements and return the sum
var count = 0;
for (var i = 0; i < x.length; i++) count += x[i];
return count;
}
You'd use it like this:
combinedCount(arr, ["Windows", "Linux"])
// returns 360

you can iterate over the elements of the platforms array, an look for the one you need.
for (var p in total.element.platform) {
if (total.element.platform[p].id == "iPhone") {
alert(total.element.platform[p].count)
}
}
If its possible you can structure your "platforms" as an object, that makes it possible to adress the platforms by key, this way you dont need to iterate over the entire array

A hacky solution to that could be to run arr.total.limited.things.platforms through a for loop and check the id tag to see if it's windows, if it is return value.
If you're using underscore.js ->
_.each(arr.total.limited.things.platforms, function(x) {
if (x.id === "Windows") {
document.getElementById( "limited" ).value = x.value;
}
});
or a good old fashioned for loop will work too! Using this method, you can change the if statement to check for the wanted iphone/etc. ids and then increment the value.
Another solution to this could be to change the way your JSON is stored. So instead of making platforms an array, you could make it an object:
platforms{
'windows: { id = '', value = 1 },
etc. and then just call by key!

Loop Through JSON, Insert Key/Value Between Objects?

UPDATE - Thanks for all the great answers and incredibly fast response. I've learned a great deal from the suggested solutions. I ultimately chose the answer I did because the outcome was exactly as I asked, and I was able to get it working in my application with minimal effort - including the search function. This site is an invaluable resource for developers.
Probably a simple task, but I can't seem to get this working nor find anything on Google. I am a Javascript novice and complex JSON confuses the hell out of me. What I am trying to do is make a PhoneGap Application (Phone Directory) for our company. I'll try to explain my reasoning and illustrate my attempts below.
I have JSON data of all of our employees in the following format:
[
{
"id":"1",
"firstname":"John",
"lastname":"Apple",
"jobtitle":"Engineer"
},
{
"id":"2",
"firstname":"Mark",
"lastname":"Banana",
"jobtitle":"Artist"
},
... and so on
]
The mobile framework (Framework 7) that I am using offers a "Virtual List" solution which I need to take advantage of as our directory is fairly large. The virtual list requires you to know the exact height of each list item, however, you can use a function to set a dynamic height.
What I am trying to do is create "headers" for the alphabetical listing based on their last name. The JSON data would have to be restructured as such:
[
{
"title":"A"
},
{
"id":"1",
"firstname":"John",
"lastname":"Apple",
"jobtitle":"Engineer"
},
{
"title":"B"
},
{
"id":"2",
"firstname":"Mark",
"lastname":"Banana",
"jobtitle":"Artist"
},
... and so on
]
I've been able to add key/value pairs to existing objects in the data using a for loop:
var letter, newLetter;
for(var i = 0; i < data.length; i++) {
newLetter = data[i].lastname.charAt(0);
if(letter != newLetter) {
letter = newLetter
data[i].title = letter;
}
}
This solution changes the JSON, thus outputting a title bar that is connected to the list item (the virtual list only accepts ONE <li></li> so the header bar is a div inside that bar):
{
"id":"1",
"firstname":"John",
"lastname":"Apple",
"jobtitle":"Engineer",
"title":"A"
},
{
"id":"1",
"firstname":"Mike",
"lastname":"Apricot",
"jobtitle":"Engineer",
"title":""
}
This solution worked until I tried implementing a search function to the listing. When I search, it works as expected but looks broken as the header titles ("A", "B", etc...) are connected to the list items that start the particular alphabetical section. For this reason, I need to be able to separate the titles from the existing elements and use them for the dynamic height / exclude from search results.
The question: How can I do a for loop that inserts [prepends] a NEW object (title:letter) at the start of a new letter grouping? If there is a better way, please enlighten me. As I mentioned, I am a JS novice and I'd love to become more efficient programming web applications.

var items = [
{ "lastname":"Apple" },
{ "lastname":"Banana" },
{ "lastname":"Box" },
{ "lastname":"Bump" },
{ "lastname":"Can" },
{ "lastname":"Switch" }
];
var lastC = null; //holds current title
var updated = []; //where the updated array will live
for( var i=0;i<items.length;i++) {
var val = items[i]; //get current item
var firstLetter = val.lastname.substr(0,1); //grab first letter
if (firstLetter!==lastC) { //if current title does not match first letter than add new title
updated.push({title:firstLetter}); //push title
lastC = firstLetter; //update heading
}
updated.push(val); //push current index
}
console.log(updated);

Well right now you have an array of objects - prefixing the title as its own object may be a bit confusing - a better structure may be:
[
{
title: "A",
contacts: [
{
"id":"1",
"firstname":"John",
"lastname":"Apple",
"jobtitle":"Engineer",
"title":"A"
}
]
Given your current structure, you could loop and push:
var nameIndexMap = {};
var newContactStructure = [];
for (var i = 0; i < data.length; i++) {
var letter = data[i].lastname.charAt(0);
if (nameIndexMap.hasOwnProperty(letter)) {
//push to existing
newContactStructure[nameIndexMap[letter]].contacts.push(data[i])
} else {
//Create new
nameIndexMap[letter] = newContactStructure.length;
newContactStructure.push({
title: letter,
contacts: [
data[i]
]
});
}
}
newContactStructure will now contain your sorted data.
Demo: http://jsfiddle.net/7s50k104/

Simple for loop with Array.prototype.splice will do the trick:
for (var i = 0; i < data.length; i++) {
if (i == 0 || data[i-1].lastname[0] !== data[i].lastname[0]) {
data.splice(i, 0, {title: data[i].lastname[0]});
i++;
}
}
Demo. Check the demo below.
var data = [
{"lastname":"Apple"},
{"lastname":"Banana"},
{"lastname":"Bob"},
{"lastname":"Car"},
{"lastname":"Christ"},
{"lastname":"Dart"},
{"lastname":"Dog"}
];
for (var i = 0; i < data.length; i++) {
if (i == 0 || data[i-1].lastname[0] !== data[i].lastname[0]) {
data.splice(i, 0, {title: data[i].lastname[0]});
i++;
}
}
alert(JSON.stringify( data, null, 4 ));