Develop Reference

Default Javascript objects are very slow when large?

I am doing a modified version of collecting word co-occurrences, so I wrote my own javascript, and I am tracking the occurrences in three objects. However, once the objects get large (~8 million, 3 million, and 172000) a function that took 5 seconds per 100000 sentences now takes minutes to do one sentence with 30 words (30 tokens). I am nowhere near my RAM cap (I have 12 more GBs of RAM it could be using, and the program is only using 2.2GB). Using Node.js v17.3.1.
Why does my function take so long when the objects get bigger (even though the sentences remain the same length)? Should I be using a different object besides Javascript's default object, or is there a way improve the speed of access and setting these objects when they are so big?
Code:
let posCounts = {};
let negCounts = {};
// the number of times each word occurs
let wordCounts = {};
let tokens = // some function that gets tokens;
for (let k = 0; k < tokens.length; k++) {
// count word occurences
if (tokens[k] in wordCounts) {
wordCounts[tokens[k]] += 1;
} else {
wordCounts[tokens[k]] = 1;
}
for(let tok = k + 1; tok < tokens.length; tok++) {
if (tok == k) {
// avoid word to self cooccurrence
// should no longer be possible
continue;
} else {
// check which form of the cooccurence exists already in either count
actual_tok = (tokens[k] + "-" + tokens[tok]);
if(actual_tok in posCounts || actual_tok in negCounts) {
// no-op
} else {
actual_tok = (tokens[tok] + "-" + tokens[k]);
}
// condition set before this block of code
if(condition) {
if (actual_tok in posCounts) {
posCounts[actual_tok] += 1;
} else {
posCounts[actual_tok] = 1;
}
} else {
if (actual_tok in negCounts) {
negCounts[actual_tok] += 1;
} else {
negCounts[actual_tok] = 1;
}
}
}
}
}
Update: I've tried increasing the heap size via node train_matrices.js --max-old-space-size=12288 and node train_matrices.js --max_old_space_size=12288 (underline instead of dash), and that didn't work either.

Probably not the main issue in your code, but you can reduce the number of lookups by changing this structure from this:
if (tokens[k] in wordCounts) {
wordCounts[tokens[k]] += 1;
} else {
wordCounts[tokens[k]] = 1;
}
to this:
let token = tokens[k];
let cnt = wordCounts[token] || 0;
wordCounts[token] = cnt + 1;
And, as I said in a comment, I've read that a Map object with .get() and .set() is better suited when there are lots of dynamically created keys whereas plain objects are better suited when you have lots of objects with all the same keys (as the JS compiler can sometimes make a C-like struct for it), but this can't be done when you're regularly adding new keys.

The answer was to both use the increase memory flag node <YOUR_FILE_NAME>.js --max-old-space-size=12288 and change to using a Map instead of an object - thanks to #jfriend00 and #Norman Breau for the suggestions. That said, maps have a max capacity of 2^24 items or 1 GB, so I ended up using a modified version of the BigMap from this stackoverflow (modified to limit the total number of items still - ended up running completely out of RAM).
Modified code (you can replace BigMap with Map if you want):
let posCounts = new BigMap();
let negCounts = new BigMap();
let wordCounts = new BigMap();
let actual_tok;
tokens = // some code
// mark every cooccurrence
for (let k = 0; k < tokens.length; k++) {
// count word occurences
if (wordCounts.has(tokens[k])) {
wordCounts.set(tokens[k], wordCounts.get(tokens[k]) + 1);
} else {
wordCounts.set(tokens[k], 1);
}
for(let tok = k + 1; tok < tokens.length; tok++) {
if (tok == k) {
// avoid word to self cooccurrence
// should no longer be possible
continue;
} else {
// check which form of the cooccurence exists already in either count
actual_tok = (tokens[k] + "-" + tokens[tok]);
if(posCounts.has(actual_tok) || negCounts.has(actual_tok)) {
// no-op
} else {
actual_tok = (tokens[tok] + "-" + tokens[k]);
}
if(condition) {
if (posCounts.has(actual_tok)) {
posCounts.set(actual_tok, posCounts.get(actual_tok) + 1);
} else {
posCounts.set(actual_tok, 1);
}
} else {
if (negCounts.has(actual_tok)) {
negCounts.set(actual_tok, negCounts.get(actual_tok) + 1);
} else {
negCounts.set(actual_tok, 1);
}
}
}
}
}
}

How do I use indexOf to find a substring and then if it finds it set a value to another part of the array?

I'm working with date-related data (the code parses the dates after this code snippet), but due to some inconsistencies in the data I want to assign a value to the different dates. So if the date is complete it is assigned '0', and then two other levels '1' and '2'. So first I insert three new columns to the array with a default value, which works. But then I want to use indexOf to search for the dates with '00'. The '=== 5' is because it is searching for a substring of the full date 'YYYY-00-DD' and so on. I want the code to loop through the array and find these instances and when it does to assign the relevant number value to uncertainty (the someData[i]["uncertainty"] = 0;). I've been going round in circles trying to get it right so any help is gratefully received.
d3.csv('minimal.csv', function(error, someData) {
if (error) throw error;
console.log(someData);
// 1. add properties 'vstart' and 'vend' for inferred dates
// and uncertainty property
for (let i = 0; i < someData.length; i++) {
someData[i]["vstart"] = null;
someData[i]["vend"] = null;
someData[i]["uncertainty"] = 0;
};
/* 2. add 'uncertainty' levels:
0: no uncertainty, e.g. 1898-01-23
1: uncertainty in days, e.g. 1914-07-00
2: uncertainty in months e.g. 1906-00-00
*/
var uncertainty = someData.uncertainty;
for (let i = 0; i < someData.length; i++) {
if (someData.indexOf("00") === 5 ) { someData.uncertainty[i] = someData.uncertainty[i].replace(/0/i, 2); }
else if (someData.indexOf("00") === 8 ) { someData.uncertainty[i] = someData.uncertainty[i].replace(/0/i, 1); }
else if (someData.indexOf("00") === -1 ) { someData.uncertainty[i] = someData.uncertainty[i].replace(/0/i, 0); }
};

Created demo for you. Please check it.
You are trying to access array someData and apply indexOf() method. But this method will work on array element. So, you have to access array element by someData[i] and then apply the indexOf() method.
/* 2. add 'uncertainty' levels:
0: no uncertainty, e.g. 1898-01-23
1: uncertainty in days, e.g. 1914-07-00
2: uncertainty in months e.g. 1906-00-00
*/
var someData = [];
someData[0] = "1898-01-23";
someData[1] = "1914-07-00";
for (let i = 0; i < someData.length; i++) {
if (someData[i].indexOf("00") == 5 ) {
console.log("called 5");
}
else if (someData[i].indexOf("00") == 8 ) { console.log("called 8"); }
else if (someData[i].indexOf("00") == -1 ) { console.log("called -1"); }
};

for (let i = 0; i < someData.length; i++) {
const regex = /[0-9]+-([0-9]+)-([0-9]+)/gm;
let m;
while ((match = regex.exec(someData[i]["start"])) !== null) {
// This is necessary to avoid infinite loops with zero-width matches
if (match.index === regex.lastIndex) {
regex.lastIndex++;
}
// month
if (match[1] === "00") {
someData[i]["uncertainty"] = someData[i]["uncertainty"].replace(/0/i, '1');
console.log('month', match[0])
}
else if (match[2] === "00") {
someData[i]["uncertainty"] = someData[i]["uncertainty"].replace(/0/i, '2');
console.log('day', match[0])
}
}
};

Javascript Comparison Loop Not Working Between an Integer and an Array of Integers

Based on the code I have below, the user_id_org should match with the first entry of the array in orgs. I had this working without issue but now cannot get it to work, am unsure what I changed, and can't find the issue. When I run the code below, it returns with "No! We did not intersection with 1 of multiple orgs!" when it should return with "Yes! We matched with 1 of multiple orgs!". My goal here is to understand why/what I am doing wrong as to why the numbers are not matching, along with getting a fix.
Additionally, if someone has advice on how I could improve this code, I'd love to hear it.
var orgs = [1234,12345,37463,47716,37463];
var org_id = '';
var user_search = 1;
var org_search = 5;
var user_id_org = 1234;
if (org_id == '') {
if (user_search == 1) { //User Known
if (org_search == 1) {
//Check the single org and see if the org ID is tied to the found user, if so attach user to the correct org.
//If no intersection, submit a ticket as the user and do (information to be discussed in the meeting)
var arrayLength = orgs.length;
for (var i = 0; i < arrayLength; i++) {
if (orgs[i] == user_id_org) {
var intersection = 'Yes! We matched with 1 org!'
} else {
var intersection = 'No! We did not intersection with 1 org!'
}
}
console.log(intersection)
} else if (org_search > 1){
//Loop through orgs and see if orgs any intersection the org ID tied to the found user, if so attach user to the correct org.
var arrayLength = orgs.length;
for (var i = 0; i < arrayLength; i++) {
if (orgs[i] == user_id_org) {
var intersection = 'Yes! We matched with 1 of multiple orgs!'
} else {
var intersection = 'No! We did not intersection with 1 of multiple orgs!'
}
}
console.log(intersection)
} else if (org_search == 0) {
//Create a ticket assigned to the user, and flag it (meeting)
console.log('We did find a user, but no orgs at all, off to the dedicated org we go.')
} else {
console.log('Matching Error')
}
} else if (user_search !== 1) {
if (org_search >= 1) {
console.log('We did not find a user but found multiple orgs! To the dedicated org we go.')
} else if (org_search == 1) {
console.log('We did not find a user but found a single org! Assign them to the org.')
} else if (org_search == 0) {
var intersection = 'No intersection because we did not find a user.'
} else {
console.log('No User Found Error')
}
console.log(intersection)
} else {
console.log('Error');
}
} else if (org_id !== '') {
if (user_search == 1) {
var intersection = 'We matched because we had an org ID and found a user.'
} else if (user_search !== 1) {
//Create user and attach them to the org_id.
var intersection = 'We received an org ID but no user, go create them!'
} else {
console.log('Org ID Provided Error')
}
console.log(intersection)
} else {
return 'Primary Org ID Sorting Error';
}

The issue is with these lines of code:
for (var i = 0; i < arrayLength; i++) {
if (orgs[i] == user_id_org) {
var intersection = 'Yes! We matched with 1 of multiple orgs!'
} else {
var intersection = 'No! We did not intersection with 1 of multiple orgs!'
}
}
The for loop continues from 0 to arrayLength - 1. On the 1st iteration (i.e. i is 0), the intersection variable is set to 'Yes!...'. On subsequent iterations, it is overwritten to 'No!...'. You should exit early using the break statement (or return if there's no more code in the function that's relevant).
for (var i = 0; i < arrayLength; i++) {
if (orgs[i] == user_id_org) {
var intersection = 'Yes! We matched with 1 of multiple orgs!'
break; // 'skip' the remaining iterations
} else {
var intersection = 'No! We did not intersection with 1 of multiple orgs!'
}
}
// break jumps here
Other advice,
learn to debug your using breakpoints or console.logs.
use === instead of == which has unexpected type casting rules.
use let and const instead of var
end statements with semicolons ;
just styling, but most JS avoids using {} around one-line blocks
prefer array.forEach to traditional for loops.

How to fix this "if" statement

I don't really know the correct format to this if statement. I want it to count the frequency each word in my txt file was used.
function countWords(array, word, index) {
var count = 0;
var value = " "
for (var i = 0; i < array.length; i++) {
if (array[i] == 0 && value == word)
count++;
}
}
if (getUserSelectionForm.problem.value == "pay") {
countWords(working2DArray, "pay", 0)
if (getUserSelectionForm.problem.value == "staffing") {
countWords(working2DArray, "staffing", 0)
if (getUserSelectionForm.problem.value == "hours") {
countWords(working2DArray, "hours", 0)
if (getUserSelectionForm.problem.value == "management") {
countWords(working2DArray, "management", 0)
console.log(countWords)
document.getElementById('section2').innerHTML = "The selected word appears " + countWords + " times in the array."
}

Try not to use multiple IF statements and use a switch statement instead. Makes code much clearer and cleaner.
E.g.
switch(expression) {
case x:
// code block
break;
case y:
// code block
break;
default:
// code block
}
See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/switch
So:
var p = getUserSelectionForm.problem.value;
switch (p) {
case 'pay':
countWords(working2DArray, "pay", 0);
break;
case 'staffing':
countWords(working2DArray, "staffing", 0);
}

You are making three mistakes in code:
You are missing some of the closing curly braces } of you if blocks.
You do not return anything from function. You should return count.
countWords is a function you don't need to display that. You need to display its result.
You can make your code much simpler. You don't need these if statements at all because you are passing the same value of getUserSelectionForm.problem.value to the function so directly pass it.
function countWords(array, word, index) {
var count = 0;
var value= " "
for(var i = 0; i < array.length; i++){
if(array[i] == 0 && value == word)
count++;
}
return count;
}
let word = getUserSelectionForm.problem.value
document.getElementById('section2').innerHTML = `The selected word appears ${countWords(working2DArray,word,0)} times in array`
If you want to only check for few items then use create a array of "pay","staffing"... and use includes() on it
let word = getUserSelectionForm.problem.value
if(["pay","staffing","hours","management"].includes(word)){
document.getElementById('section2').innerHTML = `The selected word appears ${countWords(working2DArray,word,0)} times in array`
}

In my understanding , you want to trigger the function whenever problem.value ==='pay'||'staffing'||'hours'||'management' ,here is clearer version for your reference:
var problemValue = getUserSelectionForm.problem.value;
var isProblemValueMatch = ["pay", "staffing" ,"hours", "management"].includes(problemValue);
if (isProblemValueMatch ) {
var countWords = working2DArray.filter(function(v) {
return v === problemValue;
}).length;
console.log(countWords)
document.getElementById('section2').innerHTML = "The selected word appears " + countWords + " times in the array."
}

Modifying Luhn checker function to check for Laser cards

I use the following script to validate the card details entered in a form. I would like to add a function to this so that a visitor using a Laser card is alerted that we don't accept them.
The start digits of the Laser are 6304, 6706, 6771 & 6709
function Calculate(Luhn)
{
var sum = 0;
for (i=0; i<Luhn.length; i++ )
{
sum += parseInt(Luhn.substring(i,i+1));
}
var delta = new Array (0,1,2,3,4,-4,-3,-2,-1,0);
for (i=Luhn.length-1; i>=0; i-=2 )
{
var deltaIndex = parseInt(Luhn.substring(i,i+1));
var deltaValue = delta[deltaIndex];
sum += deltaValue;
}
var mod10 = sum % 10;
mod10 = 10 - mod10;
if (mod10==10)
{
mod10=0;
}
return mod10;
}
function Validate(Luhn)
{
var LuhnDigit = parseInt(Luhn.substring(Luhn.length-1,Luhn.length));
var LuhnLess = Luhn.substring(0,Luhn.length-1);
if (Calculate(LuhnLess)==parseInt(LuhnDigit))
{
return true;
}
alert("\n\nError with your card number! \nPlease check and correct.\n\n")
return false;

I wouldn't modify these functions to check the card type - they have a specific goal. If you change the Validate(Luhn) function so that it fails a card that passes its Luhn check but that has specific starting digits, you're making trouble for yourself later. Instead, add a new function, something like this:
function checkCardType(CardNumber) {
var CardStart = CardNumber.substring(0, 4);
if ((CardStart == '6304') || (CardStart == '6706') || (CardStart == '6771') || (CardStart == '6709')) {
alert('We do not accept Laser cards.');
return false;
} else {
return true;
}
}

Could you just keep a blacklist? You'd need to have a process to keep it up-to-date (e.g. load it from a database, manage it there - keeping it in the source is more hassle in the long term), but for such a small number of items, it would be useful:
function LoadBlacklist() {
// this is the simplest example; you may want to load the array dynamically
var bad_prefixes = [ "6304", "6706", "6771", "6709" ];
return bad_prefixes;
}
function Validate(Luhn)
{
var blacklist = LoadBlacklist();
var luhn_prefix = Luhn.substring(0,4); // only check the first 4 characters
for (var bl_pos = blacklist.length - 1; bl_pos >= 0; bl_pos--) {
if (blacklist[bl_pos] == luhn_prefix) {
// this card matches one of the blacklist prefixes
return false;
}
}
// if we're here, no match was found
// go on with validation...
}