Develop Reference

Reading very large data files using nodejs? Interview question

I recently wrote a solution for an interview question which was rejected. Please analyze my solution and advice? is it not efficient to handle large files?
Problem
Given a data file containing scored records, in your favorite programming language, write a program to output the N highest record IDs ordered by descending
score. The output should be well-formed JSON. Consider giving thought to the
resource efficiency of your solution.
DATA File
The input data file is a series of key-value pairs (one per line) with the format
:
3830591998918656: {"id":"2208ef95-355c-53a6-96bc-206a4831f2fe","data":"Tu"}
548113328635904: {"id":"d5887987-bf5d-5813-b524-722ffff11882","data":"Vubiteh hone na dabupkof tetnut."}
4322085113430016: {"id":"8f5330b9-0279-5a67-aee3-d6acf221a53a","data":"Losu nihpe upaveitpse teblujnis."}
6348702421614592: {"id":"1fef0dbc-c75b-5959-835f-80e5f15b6da1","data":"Riliv kaliku laiza zen daze ."}
can be upto 100k lines or more
Success Conditions
Upon successful running, your solution should exit with exit code 0. If the input
data file is not valid, your solution should exit with code 2, while if the input
file is not found, your solution should exit with code 1. Empty lines in the input
file should be ignored rather than treated as invalid input.
My solution
highest.js the intry point
{
let argv = require("yargs").argv;
const yargs = require("yargs");
const handler = require("./handler");
const fileName = argv._[0];
const numberOfEntries = argv._[1];
if (!fileName || !numberOfEntries) {
console.log("Command Incorrect : --use node highest <filename> <count>");
} else {
handler.getHighestScore(fileName, numberOfEntries);
}
}
Handler.js
{
const path = require("path");
const fs = require("fs");
const es = require("event-stream");
let ln = 0;
const scoreArray = [];
exports.getHighestScore = (fileName, numberOfEntries) => {
const filePath = path.join(path.dirname(require.main.filename), fileName);
fs.stat(filePath, function (err, stat) {
if (stat && stat.isFile()) {
let s = fs
.createReadStream(filePath)
.pipe(es.split())
.pipe(
es
.mapSync(function (line) {
s.pause();
if (ln < parseInt(numberOfEntries)) {
if (line.trim().length > 0) {
let score = line.slice(0, line.indexOf(":"));
let secondPart = line.slice(line.indexOf(":") + 1);
let jsonObject = JSON.parse(secondPart);
if (
jsonObject.hasOwnProperty("id") &&
jsonObject.id.trim().length > 0
) {
let outputObject = { score: score, id: jsonObject.id };
scoreArray.push(outputObject);
} else {
process.stdout.write(
"***********File Invalid - ID Mssing************* code(2)"
);
process.exit(2);
}
}
}
if (ln == parseInt(numberOfEntries)) {
s.end();
} else {
ln += 1;
}
s.resume();
})
.on("error", function (err) {
process.stdout.write(
"***********Error while reading file************* code(2)"
);
process.exit(2);
})
.on("end", function () {
let arr = scoreArray.sort((a, b) => (b.score > a.score ? 1 : -1));
process.stdout.write(
"TOTAL LINES READ = " +
ln +
" TOTAL OUTPUT = " +
arr.length +
"\n"
);
process.stdout.write(JSON.stringify(arr));
process.stdout.write("\n");
process.exit(0);
})
);
} else {
process.stdout.write("***********FILE NOT FOUND************* code(1)");
process.exit(1);
}
});
};
}
Any advice and criticism about solution is appreciated

You are reading the entire input file into memory and storing everything. A space efficient way to do things is to stream the input one line at a time, parse it, check to see if it contains an N-highest score. If so, add it to the N-highest data structure. If not, skip it. Only retain in memory the N-highest data as you go through the whole file. This seems to be the main point that your code misses.
For efficiency reasons, this does an insertion sort into the highest array so it isn't constantly resorting the entire array every time you add a value.
Here's an implementation of that in nodejs:
const readline = require('node:readline');
const fs = require('node:fs');
// sample input data per line
// 3830591998918656: { "id": "2208ef95-355c-53a6-96bc-206a4831f2fe", "data": "Tu" }
// 548113328635904: { "id": "d5887987-bf5d-5813-b524-722ffff11882", "data": "Vubiteh hone na dabupkof tetnut." }
// 4322085113430016: { "id": "8f5330b9-0279-5a67-aee3-d6acf221a53a", "data": "Losu nihpe upaveitpse teblujnis." }
// 6348702421614592: { "id": "1fef0dbc-c75b-5959-835f-80e5f15b6da1", "data": "Riliv kaliku laiza zen daze ." }
const scoreRegex = /^\s*(\d+):\s*/;
function parseLine(line) {
// ok to skip empty lines
if (line.trim() === "") return null;
const result = {};
// parse leading digits
const scoreMatch = line.match(scoreRegex);
if (!scoreMatch) throw new Error("Missing score at beginning of line");
result.score = BigInt(scoreMatch[1], 10);
const remainingLine = line.slice(scoreMatch[0].length);
result.info = JSON.parse(remainingLine);
if (typeof result.info.id !== "string" || result.info.id === "") {
throw new Error("Missing valid id value");
}
return result;
}
// input and output files
const fileInput = "input.txt";
const fileOutput = "output.txt";
const howManyHighest = 2;
const highestScores = [];
function getLowest() {
return highestScores[highestScores.length - 1].score;
}
// do an insertion sort into the highestScores array
// highest score record first
// maintain length at no more than howManyHighest
function insertHighest(val) {
let inserted = false;
// for performance reasons, only search through the highestScores
// list if this new score is higher than the lowest score we have in the list so far
if (highestScores.length && val.score > getLowest()) {
for (let [index, item] of highestScores.entries()) {
if (val.score > item.score) {
// insert it into this position in the array, pushing the others up
highestScores.splice(index, 0, val);
inserted = true;
break;
}
}
}
if (inserted) {
// trim overflow, if any
if (highestScores.length > howManyHighest) {
highestScores.pop();
}
} else {
// didn't insert it, see if we aren't full yet
if (highestScores.length < howManyHighest) {
highestScores.push(val);
}
}
}
const rl = readline.createInterface({
input: fs.createReadStream(fileInput),
crlfDelay: Infinity
});
rl.on('error', err => {
if (err.code === 'ENOENT') {
process.exit(1);
} else {
console.log(err);
// some sort of read error
process.exit(2);
}
}).on('line', line => {
try {
const data = parseLine(line);
if (data) {
insertHighest(data);
}
} catch (e) {
console.log(e);
console.log(`Invalid line: ${line}`);
process.exit(2);
}
}).on('close', () => {
// generate array of highest scores
const output = highestScores.map(item => item.info.id)
console.log(output);
fs.writeFile(fileOutput, JSON.stringify(output), err => {
if (err) {
console.log(err);
// output write error
process.exit(3);
} else {
// all done, successfully
process.exit(0);
}
});
});
I ran into a few unspecified implementation questions, which if you bring up during your answer will show you've fully understood the problem.
What is the max score value that can exist in this data? This determines if we can use a Javascript number type or whether BigInt needs to be used to handle arbitrarily large numbers (at a cost of some run-time performance). Since this was not specified, I've used BigInt parsing and comparisons here to make the score values limitless integers.
Is the output data supposed to be only an array of ids (sorted by highest score) that belong to the highest scores. Or is it supposed to be some sorted data structure that includes the score and the id and the other data? Your question does not make that entirely clear. Showing an example of the output data for N = 3 would make this clear. This code produces an output array of id values, sorted by id with highest score first.
It is not specified what the valid structure of an id property is. This code just tests for a non-empty string value.
It is not specified what should be output if there are high tie scores or if the highest N+1 scores are all the same (e.g. there's no unique N highest scores). In case of ties at the end of the high score list, this retains only the first ones encountered in the input file (up to the N we're outputing).

TurfJS: Efficiently locate the nearest point on a line string

Edit:
The quicksort algorithm isn't actually working...😳
CODESANDBOX
SITUATION
I have a line string with up to 2000 points. I am trying to find the nearest point on the line from the user's location. This operation is executed roughly every 5 seconds and failing to optimise it would unnecessarily drain the users battery.
The source code shows that the distance operation is conducted on every point on the line string - not good for my users.
I would like to implement a quick-sort style algorithm to get the nearest point and reduce the number of the operations. The example below is working quite well and produces a point on the line with far fewer operations (for a list of 1700 points it takes 14 operations to locate the closest point).
PROBLEM
I cannot determine an elegant way to track the index of the result. Once I have the nearest point, I do not want to have to search the original list again to find it's index.
first operation - 0 | median
second - fHalf (0 | median of fHalf) | sHalf (median of list | median of sHalf)
third - at this point it becomes a mess to track
What I have so far:
import distance from "#turf/distance";
import { point } from "#turf/helpers";
const user = point([-77.037076, 38.884017]);
function getMedian(list) {
return Math.ceil(list.length / 2);
}
function getHalves(list) {
const median = getMedian(list);
const firstHalf = list.slice(0, median);
const secondHalf = list.slice(-median);
return { firstHalf, secondHalf };
}
function getClosest(list) {
const to = point(list[0]);
const meters = distance(user, to, { units: "meters" });
return meters;
}
let operations = 0; // used in development to track operations
function selectHalf(list) {
operations += 1;
const { firstHalf, secondHalf } = getHalves(list);
const firstDistance = getClosest(firstHalf);
const secondDistance = getClosest(secondHalf);
const nextList = firstDistance < secondDistance ? firstHalf : secondHalf;
if (list.length === 1)
console.log("operations:", operations, "closest point:", nextList);
else selectHalf(nextList);
}
const morePoints = `-37.0467378013181,145.1634433308106
-37.04674949407303,145.1634394751351
-37.04676521014147,145.1634369605642
-37.04678021374815,145.1634352003645
-37.04679207414114,145.1634343621742
-37.04680510800057,145.1634334401648
// Full list is available in the codesandbox link below
`
.split("\n")
.map((p) => p.split(",").map((n) => parseFloat(n)));
selectHalf(morePoints);

Javascript circular calculation involving arrays (example: amortization schedule)

I'm fairly early on in building an app that can be used as an alternative to spreadsheets in a number of scenarios. It offers the user a non-tabular approach to creating and organizing the analysis. The user's instructions are transpiled to and executed by Javascript (using pure functions only). (The interface is between something like Google Blockly and straight coding in a text editor.) In place of cell ranges that you would typically use in a spreadsheet, this product uses Javascript arrays. A problem that I'm facing with this approach is (quasi-)circular calculations such as that found in a simple amortization schedule.
My (unsuccessful) attempts to resolve the issue so far involve:
lazy list evaluation (using https://github.com/dankogai/js-list-lazy)
wrapping all transpiled JS in functions to delay eval so that the user's content doesn't need to be topologically sorted (which it currently is).
Below I'll provide hopefully enough context to illustrate the issue, which can be pretty broadly extrapolated. Take the example of an amortization schedule for a mortgage:
Basically, the BOP ("Beginning of Period") Balance depends on the EOP ("End of Period") Balance from the previous period. And the EOP Balance depends on the BOP Balance from the same period. In a spreadsheet that uses ranges of contiguous cells, this isn't circular because every BOP Balance and EOP Balance is a discrete cell. However, if BOP Balance and EOP Balance (and all of the other time-series-based calcs) are arrays then there is a circular reference when trying to retrieve elements. Spreadsheet screenshots of the example are provided at the end of the post as a supplement.
An attempt to build this analysis in my app generates the following JS (which I've edited and reorganized for clarity). This code, if translated over to a spreadsheet, works just fine (see supplemental screenshots at the end of the post):
// Credit: Apache OpenOffice
function payment (rate, periods, present_value, future_value, type) {
var fv = future_value || 0
var t = type || 0
if (rate === 0) {
return -1 * ((present_value + fv) / periods)
} else {
var term = (1 + rate) ** periods // Transpiling with Babel; otherwise, use Math.pow(1 + rate, periods)
if (t === 1) {
return -1 * ((fv * rate / (term - 1) + present_value * rate / (1 - 1 / term)) / (1 + rate))
} else {
return -1 * (fv * rate / (term - 1) + present_value * rate / (1 - 1 / term))
}
}
}
var loan_principal = 1000000
var annual_interest_rate = 0.06
var interest_rate_monthly = annual_interest_rate / 12
var amortization_period_years = 25
var amortization_period_months = amortization_period_years * 12
var months_range = _.range(1, amortization_period_months + 1, 1) // See: http://underscorejs.org/#range [1, 2, 3, ... 298, 299, 300]
var bop_balance = months_range.map(function (current_element, current_list_position) {
if (current_list_position === 0) {
return loan_principal
} else {
// Along with eop_balance, this causes a graph cycle
return eop_balance[current_list_position - 1]
}
})
var monthly_payment = months_range.map(function (current_element, current_list_position) {
return payment(interest_rate_monthly, amortization_period_months, loan_principal)
})
var principal_payment = months_range.map(function (current_element, current_list_position) {
var current_mthly_pmt = monthly_payment[current_list_position]
var current_int_pmt = interest_payment[current_list_position]
return current_mthly_pmt - current_int_pmt
})
var interest_payment = months_range.map(function (current_element, current_list_position) {
if (current_list_position === 0) {
return loan_principal * interest_rate_monthly * -1
} else {
var previous_bal = eop_balance[current_list_position - 1]
return previous_bal * interest_rate_monthly * -1
}
})
var eop_balance = months_range.map(function (current_element, current_list_position) {
// This causes a graph cycle
var cur_bal = bop_balance[current_list_position]
var cur_prin_pmt = principal_payment[current_list_position]
return cur_bal + cur_prin_pmt
})
This code will not topologically sort because of the cycle between bop_balance and eop_balance. And it won't fully evaluate because of the circular reference.
Any suggestions on how to work around this general scenario? Thank you.
Supplemental Info:
Here are two views of the same spreadsheet representing the analysis:
The reliance on pure functions in the app is to try and minimize confusion for users coming from spreadsheets.
If seeing the actual app would help provide context, please feel free to visit https://www.getpinevale.com. I'm placing this at the end so it doesn't distract from the question.

You shouldn't do a double linked structure. I would suggest use a plain for loop and build up both arrays.
for(int i=0; i<range.length;i++){
if(i==0){
bop_balance[i]= loan_principal;
eop_balance[i]= somecalculatedamount(loan_principal);
}else{
bop_balance[i] = eop_balance[i-1];
eop_balance[i] = somecalculatedamount(bop_balance[i])
}
}
I don't know if my function is correct but the essential point I am trying to make:
don't link the data structures
and use a control structure that is outside of both arrays
Based on the comment below I'd make the suggestion of using reduce.
let range = [1,2,3,4];
let output = range.reduce(function(accumulator, currentValue, currentIndex) {
let balance = {};
if (currentIndex == 0) {
balance.bop = 0;
balance.eop = currentValue;
}else{
balance.bop = accumulator[currentIndex-1].eop;
balance.eop = balance.bop+5;
}
accumulator.push(balance);
return accumulator;
}, []);
console.log(output);

Creating millions of Objects in Javascript

Let me be the first to say that this isn't something I normally do, but out of curiousity, I'll see if anyone has a good idea on how to handle a problem like this.
The application I am working on is a simulated example of the game Let's make a Deal featuring the Monty Hall problem.
I won't go into details about my implementation, but it more or less allows a user to enter a number of how many games they want to simulate, and then if an option is toggled off, the player of those x games won't switch their choice, while if it is toggled on, they will switch their choice every single instance of the game.
My object generator looks like this:
const game = function(){
this[0] = null;
this[1] = null;
this[2] = null;
this.pick = Math.floor(Math.random() * 3);
this.correctpick = Math.floor(Math.random() * 3);
this[this.correctpick] = 1;
for (let i=0; i<3; i++){
if ((this[i] !== 1) && (i !== this.pick)){
this.eliminated = i;
break;
}
}
}
const games = arg => {
let ret = [];
for(let i=0; i<arg; i++){
ret.push(new game);
}
return ret;
}
This structure generates an array which i stringify later that looks like this:
[
{
"0": 1,
"1": null,
"2": null,
"pick": 2,
"correctpick": 0,
"eliminated": 1
},
{
"0": null,
"1": null,
"2": 1,
"pick": 2,
"correctpick": 2,
"eliminated": 0
}
]
As sloppy as the constructor for game looks, the reason is because I have refactored it into having as few function calls as possible, where now I'm literally only calling Math functions at the current time (I removed any helper functions that made the code easier to read, in opt for performance).
This app can be ran both in the browser and in node (cross platform), but I have clamped the arg a user can pass into the games function to 5 million. Any longer than that and the process (or window) freezes for longer than a few seconds, or even potentially crashes.
Is there anything else I can do to increase performance if a huge number is given by a user? Also, if you need more information, I will be happy to supply it!
Thanks!

The obvious performance optimisation would be not to create and store 5 million objects at all, relieving memory pressure. Instead you'd create the objects on the fly only when you need them and throw them away immediately after. I'm not sure what your app does, but it sounds like you want to re-use the same game instances when evaluating results with the different options. In that case, you need to store them of course - but I'd advise to re-think the design and consider immediately evaluating each game with all possible options, accumulating only the results for each choice of options but not keeping all games in memory.
Apart from that, I'd recommend to simplify a bit:
You can drop that loop completely and use some clever arithmetic to find the eliminated option: this.eliminated = this.pick == this.correctpick ? +!this.pick : 3 - this.pick - this.correctpick;. Or use a simple lookup table this.eliminated = [1, 2, 1, 2, 0, 0, 1, 0, 0][this.pick * 3 + this.correctpick].
I'd avoid changing the type of the array elements from null (reference) to 1 (number). Just keep them as integers and initialise your elements with 0 instead.
Don't store 6 properties in your object that are completely redundant. You only need 2 of them: pick and correctpick - everything else can be computed on the fly from them when you need it. Precomputing and storing it would only be advantageous if the computation was heavy and the result was used often. Neither of this is the case, but keeping a low memory footprint is important (However, don't expect much from this).

Not sure about your implementation, but do you really need an Array?
How about only using results (see snippet)?
If it's blocking the browser that worries you, maybe delegating the work to a web worker is the solution for that: see this jsFiddle for a web worker version of this snippet.
(() => {
document.querySelector("#doit")
.addEventListener("click", playMontyHall().handleRequest);
function playMontyHall() {
const result = document.querySelector("#result");
const timing = document.querySelector("#timing");
const nOfGames = document.querySelector("#nGames");
const switchDoors = document.querySelector("#switchyn");
// Create a game
const game = (doSwitch) => {
const doors = [0, 1, 2];
const pick = Math.floor(Math.random() * 3);
const correctPick = Math.floor(Math.random() * 3);
const eliminated = doors.filter(v => v !== pick && v !== correctPick)[0];
return {
correctpick: correctPick,
pick: doSwitch ? doors.filter(v => v !== pick && v !== eliminated)[0] : pick,
eliminated: eliminated,
};
};
const getWinner = game => ~~(game.correctpick === game.pick);
// Sum wins using a generator function
const winningGenerator = function* (doSwitch, n) {
let wins = 0;
while (n--) {
wins += getWinner(game(doSwitch));
yield wins;
}
};
// calculate the number of succeeded games
const calculateGames = (nGames, switchAlways) => {
const funNGames = winningGenerator(switchAlways, nGames);
let numberOfWins = 0;
while (nGames--) {
numberOfWins = funNGames.next().value;
}
return numberOfWins;
}
const cleanUp = playOut => {
result.textContent =
"Playing ... (it may last a few seconds)";
timing.textContent = "";
setTimeout(playOut, 0);
};
const report = results => {
timing.textContent = `This took ${
(performance.now() - results.startTime).toFixed(3)} milliseconds`;
result.innerHTML =
`<b>${!results.switchAlways ? "Never s" : "Always s"}witching doors</b>:
${results.winners} winners out of ${results.nGames} games
(${((results.winners/+results.nGames)*100).toFixed(2)}%)`;
};
// (public) handle button click
function clickHandle() {
cleanUp(() => {
const nGames = nOfGames.value || 5000000;
const switchAlways = switchDoors.checked;
report({
switchAlways: switchAlways,
startTime: performance.now(),
winners: calculateGames(nGames, switchAlways),
nGames: nGames
});
});
}
return {
handleRequest: clickHandle
};
}
})();
body {
margin: 2em;
font: normal 12px/15px verdana, arial;
}
#timing {
color: red;
}
<p><input type="number" id="nGames" value="5000000"> N of games</p>
<p><input type="checkbox" id="switchyn"> Always switch doors</p>
<p><button id="doit">Play</button>
<p id="result"></p>
<p id="timing"></p>

Not truly async?

I have a array of about 18 000 elements. I'm creating a map application where I want to add the elements when the user zooms in to a certain level.
So when the user zooms in under 9 I loop tru the array looking for elements that is in the view.
However, it does take some time looping thru the elements, causing the map application lag each time the user zooms out and in of "level 9". Even if there are no elements to add or not, so the bottleneck is the looping I guess.
I've tried to solve it by asyncing it like:
function SearchElements(elementArr) {
var ret = new Promise(resolve => {
var arr = [];
for (var i in elementArr) {
var distanceFromCenter = getDistanceFromLatLonInKm(view.center.latitude, view.center.longitude, dynamicsEntities[i].pss_latitude, dynamicsEntities[i].pss_longitude);
var viewWidthInKm = getSceneWidthInKm(view, true);
if (distanceFromCenter > viewWidthInKm) continue;
arr.push(elementArr[i]);
}
resolve(arr);
});
return ret;
}
SearchElements(myElementsArray).Then(arr => {
// ...
});
But its still not async, this method hangs while the for loop runs.

Because you still have a tight loop that loops through all the elements in one loop, you'll always have the responsiveness issues
One way to tackle the issue is to works on chunks of the data
Note: I'm assuming elementArr is a javascript Array
function SearchElements(elementArr) {
var sliceLength = 100; // how many elements to work on at a time
var totalLength = elementArr.length;
var slices = ((totalLength + sliceLength - 1) / sliceLength) | 0; // integer
return Array.from({length:slices})
.reduce((promise, unused, outerIndex) =>
promise.then(results =>
Promise.resolve(elementArr.slice(outerIndex * sliceLength, sliceLength).map((item, innerIndex) => {
const i = outerIndex * sliceLength + innerIndex;
const distanceFromCenter = getDistanceFromLatLonInKm(view.center.latitude, view.center.longitude, dynamicsEntities[i].pss_latitude, dynamicsEntities[i].pss_longitude);
const viewWidthInKm = getSceneWidthInKm(view, true);
if (distanceFromCenter <= viewWidthInKm) {
return item; // this is like your `push`
}
// if distanceFromCenter > viewWidthInKm, return value will be `undefined`, filtered out later - this is like your `continue`
})).then(batch => results.concat(batch)) // concatenate to results
), Promise.resolve([]))
.then(results => results.filter(v => v !== undefined)); // filter out the "undefined"
}
use:
SearchElements(yourDataArray).then(results => {
// all results available here
});
My other suggestion in the comment was Web Workers (I originally called it worker threads, not sure where I got that term from) - I'm not familiar enough with Web Workers to offer a solution, however https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers and https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API should get you going
To be honest, I think this sort of heavy task would be better suited to Web Workers