Can someone please help me to find best way to optimise the below code as it is taking a long time when I have thousand of records searching
var arr =[
{
children:[
{
children:[
{
children:[
{
name:'XYZZZZZ'
}
]
}
]
}
]
}
];
let list = [];
//Calculate column list
arr.forEach((obj0) => {
if (obj0.hasOwnProperty('children')) {
if (obj0.children.length > 0) {
let objchid1 = obj0.children;
objchid1.forEach((obj1) => {
if (obj1.hasOwnProperty('children')) {
if (obj1.children.length > 0) {
let objchid2 = obj1.children;
objchid2.forEach((obj2) => {
if (obj2.hasOwnProperty('children')) {
if (obj2.children.length > 0) {
let objchid3 = obj2.children;
objchid3.forEach((obj3) => {
if (obj3.name !== 'james') {
console.log('IN THREEE', obj3.name);
list.push(obj3.name);
}
});
}
}
});
}
}
});
}
}
});
I have tried searching a lot but no luck Thanks in advance.!!!
Optimize your data structure. Do not use nested arrays unless you really need to. NoSQL is so popular when it comes to WebDev because reads happen 100.000 times more than writes and saving on bandwidth (for you and the user) is worth more than saving on duplicate data in a database considering how cheap hardware is
You can save the elements of the deepest array as object keys (with the nested .name attribute in your case) and the index of the respective position in the array as the object value. This way you can do myArray[myElementsToIndexObject['elementIamLookingFor']] iterating only one single time over the nested array (for building myElementsToIndexObject)
If the data is from a JSON string, the search can be done during the parsing :
var list = [], json = '[{"child":[{"child":[{"child":[{"name":"XYZZZZZ"}]}]}]}]'
var arr = JSON.parse(json, (key, val) => (key === 'name' && list.push(val), val))
console.log(list)
console.log(arr)
Related
I recently wrote a solution for an interview question which was rejected. Please analyze my solution and advice? is it not efficient to handle large files?
Problem
Given a data file containing scored records, in your favorite programming language, write a program to output the N highest record IDs ordered by descending
score. The output should be well-formed JSON. Consider giving thought to the
resource efficiency of your solution.
DATA File
The input data file is a series of key-value pairs (one per line) with the format
:
3830591998918656: {"id":"2208ef95-355c-53a6-96bc-206a4831f2fe","data":"Tu"}
548113328635904: {"id":"d5887987-bf5d-5813-b524-722ffff11882","data":"Vubiteh hone na dabupkof tetnut."}
4322085113430016: {"id":"8f5330b9-0279-5a67-aee3-d6acf221a53a","data":"Losu nihpe upaveitpse teblujnis."}
6348702421614592: {"id":"1fef0dbc-c75b-5959-835f-80e5f15b6da1","data":"Riliv kaliku laiza zen daze ."}
can be upto 100k lines or more
Success Conditions
Upon successful running, your solution should exit with exit code 0. If the input
data file is not valid, your solution should exit with code 2, while if the input
file is not found, your solution should exit with code 1. Empty lines in the input
file should be ignored rather than treated as invalid input.
My solution
highest.js the intry point
{
let argv = require("yargs").argv;
const yargs = require("yargs");
const handler = require("./handler");
const fileName = argv._[0];
const numberOfEntries = argv._[1];
if (!fileName || !numberOfEntries) {
console.log("Command Incorrect : --use node highest <filename> <count>");
} else {
handler.getHighestScore(fileName, numberOfEntries);
}
}
Handler.js
{
const path = require("path");
const fs = require("fs");
const es = require("event-stream");
let ln = 0;
const scoreArray = [];
exports.getHighestScore = (fileName, numberOfEntries) => {
const filePath = path.join(path.dirname(require.main.filename), fileName);
fs.stat(filePath, function (err, stat) {
if (stat && stat.isFile()) {
let s = fs
.createReadStream(filePath)
.pipe(es.split())
.pipe(
es
.mapSync(function (line) {
s.pause();
if (ln < parseInt(numberOfEntries)) {
if (line.trim().length > 0) {
let score = line.slice(0, line.indexOf(":"));
let secondPart = line.slice(line.indexOf(":") + 1);
let jsonObject = JSON.parse(secondPart);
if (
jsonObject.hasOwnProperty("id") &&
jsonObject.id.trim().length > 0
) {
let outputObject = { score: score, id: jsonObject.id };
scoreArray.push(outputObject);
} else {
process.stdout.write(
"***********File Invalid - ID Mssing************* code(2)"
);
process.exit(2);
}
}
}
if (ln == parseInt(numberOfEntries)) {
s.end();
} else {
ln += 1;
}
s.resume();
})
.on("error", function (err) {
process.stdout.write(
"***********Error while reading file************* code(2)"
);
process.exit(2);
})
.on("end", function () {
let arr = scoreArray.sort((a, b) => (b.score > a.score ? 1 : -1));
process.stdout.write(
"TOTAL LINES READ = " +
ln +
" TOTAL OUTPUT = " +
arr.length +
"\n"
);
process.stdout.write(JSON.stringify(arr));
process.stdout.write("\n");
process.exit(0);
})
);
} else {
process.stdout.write("***********FILE NOT FOUND************* code(1)");
process.exit(1);
}
});
};
}
Any advice and criticism about solution is appreciated
You are reading the entire input file into memory and storing everything. A space efficient way to do things is to stream the input one line at a time, parse it, check to see if it contains an N-highest score. If so, add it to the N-highest data structure. If not, skip it. Only retain in memory the N-highest data as you go through the whole file. This seems to be the main point that your code misses.
For efficiency reasons, this does an insertion sort into the highest array so it isn't constantly resorting the entire array every time you add a value.
Here's an implementation of that in nodejs:
const readline = require('node:readline');
const fs = require('node:fs');
// sample input data per line
// 3830591998918656: { "id": "2208ef95-355c-53a6-96bc-206a4831f2fe", "data": "Tu" }
// 548113328635904: { "id": "d5887987-bf5d-5813-b524-722ffff11882", "data": "Vubiteh hone na dabupkof tetnut." }
// 4322085113430016: { "id": "8f5330b9-0279-5a67-aee3-d6acf221a53a", "data": "Losu nihpe upaveitpse teblujnis." }
// 6348702421614592: { "id": "1fef0dbc-c75b-5959-835f-80e5f15b6da1", "data": "Riliv kaliku laiza zen daze ." }
const scoreRegex = /^\s*(\d+):\s*/;
function parseLine(line) {
// ok to skip empty lines
if (line.trim() === "") return null;
const result = {};
// parse leading digits
const scoreMatch = line.match(scoreRegex);
if (!scoreMatch) throw new Error("Missing score at beginning of line");
result.score = BigInt(scoreMatch[1], 10);
const remainingLine = line.slice(scoreMatch[0].length);
result.info = JSON.parse(remainingLine);
if (typeof result.info.id !== "string" || result.info.id === "") {
throw new Error("Missing valid id value");
}
return result;
}
// input and output files
const fileInput = "input.txt";
const fileOutput = "output.txt";
const howManyHighest = 2;
const highestScores = [];
function getLowest() {
return highestScores[highestScores.length - 1].score;
}
// do an insertion sort into the highestScores array
// highest score record first
// maintain length at no more than howManyHighest
function insertHighest(val) {
let inserted = false;
// for performance reasons, only search through the highestScores
// list if this new score is higher than the lowest score we have in the list so far
if (highestScores.length && val.score > getLowest()) {
for (let [index, item] of highestScores.entries()) {
if (val.score > item.score) {
// insert it into this position in the array, pushing the others up
highestScores.splice(index, 0, val);
inserted = true;
break;
}
}
}
if (inserted) {
// trim overflow, if any
if (highestScores.length > howManyHighest) {
highestScores.pop();
}
} else {
// didn't insert it, see if we aren't full yet
if (highestScores.length < howManyHighest) {
highestScores.push(val);
}
}
}
const rl = readline.createInterface({
input: fs.createReadStream(fileInput),
crlfDelay: Infinity
});
rl.on('error', err => {
if (err.code === 'ENOENT') {
process.exit(1);
} else {
console.log(err);
// some sort of read error
process.exit(2);
}
}).on('line', line => {
try {
const data = parseLine(line);
if (data) {
insertHighest(data);
}
} catch (e) {
console.log(e);
console.log(`Invalid line: ${line}`);
process.exit(2);
}
}).on('close', () => {
// generate array of highest scores
const output = highestScores.map(item => item.info.id)
console.log(output);
fs.writeFile(fileOutput, JSON.stringify(output), err => {
if (err) {
console.log(err);
// output write error
process.exit(3);
} else {
// all done, successfully
process.exit(0);
}
});
});
I ran into a few unspecified implementation questions, which if you bring up during your answer will show you've fully understood the problem.
What is the max score value that can exist in this data? This determines if we can use a Javascript number type or whether BigInt needs to be used to handle arbitrarily large numbers (at a cost of some run-time performance). Since this was not specified, I've used BigInt parsing and comparisons here to make the score values limitless integers.
Is the output data supposed to be only an array of ids (sorted by highest score) that belong to the highest scores. Or is it supposed to be some sorted data structure that includes the score and the id and the other data? Your question does not make that entirely clear. Showing an example of the output data for N = 3 would make this clear. This code produces an output array of id values, sorted by id with highest score first.
It is not specified what the valid structure of an id property is. This code just tests for a non-empty string value.
It is not specified what should be output if there are high tie scores or if the highest N+1 scores are all the same (e.g. there's no unique N highest scores). In case of ties at the end of the high score list, this retains only the first ones encountered in the input file (up to the N we're outputing).
Here my code and what i tried :
filterPrestationsByServiceSelected(arrayOfServices) {
console.log(arrayOfServices); // ['Repassage', 'Couture']
this.filteredPrestationsByService = this.filteredPrestations.filter(item => item.service.name.includes(arrayOfServices.values()));
},
I want to filter all items of this.filteredPrestations where the service name contains values of the arrayOfServices.
Anyone have an idea of what i can do ?
Thank's !
Remove .values() it returns an iterator which you don't need
filterPrestationsByServiceSelected(arrayOfServices) {
console.log(arrayOfServices); // ['Repassage', 'Couture']
this.filteredPrestationsByService = this.filteredPrestations.filter(item => item.service.name.includes(arrayOfServices));
}
You have to compare the items of a list with another. So you would have to have a compare each element of one data structure with another. Since you are comparing arrays you should do that way:
filterPrestationsByServiceSelected(arrayOfServices) {
console.log(arrayOfServices); // ['Repassage', 'Couture']
this.filteredPrestationsByService = this.filteredPrestations.filter(item => arrayOfServices.find(e => e === item.service.name))
},
That way you could compare the elements one by one.
Can you try this code. I think this code will work.
filterPrestationsByServiceSelected(arrayOfServices) {
console.log(arrayOfServices); // ['Repassage', 'Couture']
this.filteredPrestationsByService = this.filteredPrestations.filter(item => arrayOfServices.includes(item.service.name));
},
i have a really big collection of objects that i want to search through.
The array have > 60.000 items and the search performance can be really slow from time to time.
One object in that array looks like this:
{
"title": "title"
"company": "abc company"
"rating": 13 // internal rating based on comments and interaction
...
}
I want to search for the title and the company info and order that by the rating of the items.
This is what my search currently look like:
onSearchInput(searchTerm) {
(<any>window).clearTimeout(this.searchInputTimeout);
this.searchInputTimeout = window.setTimeout(() => {
this.searchForFood(searchTerm);
}, 500);
}
searchForFood(searchTerm) {
if (searchTerm.length > 1) {
this.searchResults = [];
this.foodList.map(item => {
searchTerm.split(' ').map(searchTermPart => {
if (item.title.toLowerCase().includes(searchTermPart.toLowerCase())
|| item.company.toLowerCase().includes(searchTermPart.toLowerCase())) {
this.searchResults.push(item);
}
});
});
this.searchResults = this.searchResults.sort(function(a, b) {
return a.rating - b.rating;
}).reverse();
} else {
this.searchResults = [];
}
}
Question: Is there any way to improve the search logic and performance wise?
A bunch of hints:
It's a bit excessive to put searching through 60,000 items on the front-end. Any way you can perform part of the search on the back-end? If you really must do it on the front-end considering searching in chunks of e.g. 10,000 and then using a setImmediate() to perform the next part of the search so the user's browser won't completely freeze during processing time.
Do the splitting and lowercasing of the search term outside of the loop.
map() like you're using it is weird as you don't use the return value. Better to use forEach(). Better still, is use filter() to get the items that match.
When iterating over the search terms, use some() (as pointed out in the comments) as it's an opportunity to early return.
sort() mutates the original array so you don't need to re-assign it.
sort() with reverse() is usually a smell. Instead, swap the sides of your condition to be b - a.
At this scale, it may make sense to do performance tests with includes(), indexOf(), roll-your-own-for-loop, match() (can almost guarantee it will be slower though)
Alex's suggestions are good. My only suggestion would be, if you could afford to pre-process the data during idle time (preferably don't hold up first render or interaction) you could process the data into a modified prefix trie. That would let you search for the items in O(k) time where k is the length of the search term (right now you are searching in O(kn) time because you look at every item and then do an includes which takes k time (it's actually a little worse because of the toLowerCase's but I don't want to get into the weeds of it).
If you aren't familiar with what a trie is, hopefully the code below gives you the idea or you can search for information with your search engine of choice. It's basically a mapping of characters in a string in nested hash maps.
Here's some sample code of how you might construct the trie:
function makeTries(data){
let companyTrie = {};
let titleTrie = {};
data.forEach(item => {
addToTrie(companyTrie, item.company, item, 0);
addToTrie(titleTrie, item.title, item, 0);
});
return {
companyTrie,
titleTrie
}
}
function addToTrie(trie, str, item, i){
trie.data = trie.data || [];
trie.data.push(item);
if(i >= str.length)
return;
if(! trie[str[i]]){
trie[str[i]] = {};
}
addToTrie(trie[str[i]], str, item, ++i);
}
function searchTrie(trie, term){
if(trie == undefined)
return [];
if(term == "")
return trie.data;
return searchTrie(trie[term[0]], term.substring(1));
}
var testData = [
{
company: "abc",
title: "def",
rank: 5
},{
company: "abd",
title: "deg",
rank: 5
},{
company: "afg",
title: "efg",
rank: 5
},{
company: "afgh",
title: "efh",
rank: 5
},
];
const tries = makeTries(testData);
console.log(searchTrie(tries.companyTrie, "afg"));
This question is related to this other question.
I'm sorry but I can't find the solution. I need to group an array of objects given by dataObject in this fiddle by the property objetivo with the properties: id, indicadores, objetivo and perspetiva.
This is my code:
var res = dataObject.reduce(function(res, currentValue) {
if ( res.indexOf(currentValue.objetivo) === -1 ) {
res.push(currentValue.objetivo);
}
return res;
}, []).map(function(objetivo) {
return {
objetivo: objetivo,
indicadores: dataObject.filter(function(_el) {
return _el.objetivo === objetivo;
}).map(function(_el) { return _el.indicador; }),
perspetiva: dataObject.perspetiva,
id: dataObject.id
}
});
console.log(res);
Which is grouping by objetivo correctly, but returning undefined for perspetiva and id.
Thanks for helping and sorry if this is duplicating other questions.
You're trying to access dataObject.id and .perspetiva, but dataObject is your main array and doesn't have those fields.
After reduce has picked out the unique objetivas, go back and find the items they came from:
.map(function(objetivo) {
var items = dataObject.filter(function(_el) {
return _el.objetivo === objetivo;
});
Then you can find all the indicadors, and since it looks like id and perspetiva are consistent for items with the same objetiva, you can just pick from the first one.
return {
objetivo: objetivo,
indicadores: items.map(function(_el) { return _el.indicador; }),
perspetiva: items[0].perspetiva,
id: items[0].id
}
Here's a working fiddle.
I'm trying to select the last messages of an inbox and group them in a list by the topic_id. i want to display the last message of each topic.
the array looks like this:
[{
"id":"5",
"topic_id":"4",
"message_from":"24",
"message":"how do you do?",
"date":"2015-01-13 15:34:59"
},
{
"id":"6",
"topic_id":"1",
"message_from":"33",
"message":"go go go!!",
"date":"2015-01-13 13:35:06"
},
{
"id":"7",
"topic_id":"4",
"message_from":"33",
"message":"Je suis charlie",
"date":"2015-01-14 16:24:46"
},....
is there a solution to do it without a loop?
You can't do this without loops, but you can make this easier by breaking down the sequence of events into smaller functions. You might not like this approach, but it's the cleanest imo. Alternatively you can use a third-party library (underscore, perhaps?) that allows you to run groupings on data.
Basically, get a list of all the topic_ids for all records, loop over that topic_id array and pull out the last record for each and add that to an output array.
// Get a list of all the topic ids - no duplicates
function getTopicIds(arr) {
var out = [];
arr.forEach(function (el) {
if (out.indexOf(el.topic_id) === -1) out.push(el.topic_id);
});
return out;
}
// Given a topic_id, filter the array for only those records
// sort in desc order by id, and return the first record.
// Given that each record has a unique id, and we know that older
// messages will have higher ids, it's easier to sort by id than
// date here
function getLastMsg(id, arr) {
return arr.filter(function (el) {
return el.topic_id === id;
}).sort(function (a, b) { return +b.id - +a.id; })[0];
}
// return a array of the last messages for each topic_id
// in the records array
function getLastMsgs(arr) {
return getTopicIds(arr).map(function (id) {
return getLastMsg(id, arr);
});
}
var result = getLastMsgs(arr);
DEMO