Develop Reference

Can I easily implement full text search on a static website on the client side without a database?

I found this script somewhere... might have been within npm's source code actually... not sure, all I know is I did not write it myself.. but looking at it I can't help but wonder if it or a similar refactor of the following code could allow a quick web crawl of a static site and return a list of url's that lead to the pages that have the most hits on the search term... I don't need anything fancy like fuzzy search nor am I asking anyone to write the code for me so much as I would like a second (or third) pair of eyes to look at this code and decide if there's any potential in this to implement simple full text search.
const fs = require("fs");
const path = require("path");
const npm = require("./npm.js");
const color = require("ansicolors");
const output = require("./utils/output.js");
const usageUtil = require("./utils/usage.js");
const { promisify } = require("util");
const glob = promisify(require("glob"));
const readFile = promisify(fs.readFile);
const didYouMean = require("./utils/did-you-mean.js");
const { cmdList } = require("./utils/cmd-list.js");
const usage = usageUtil("help-search", "npm help-search <text>");
const completion = require("./utils/completion/none.js");
const npmUsage = require("./utils/npm-usage.js");
const cmd = (args, cb) =>
helpSearch(args)
.then(() => cb())
.catch(cb);
const helpSearch = async (args) => {
if (!args.length) throw usage;
const docPath = path.resolve(__dirname, "..", "docs/content");
const files = await glob(`${docPath}/*/*.md`);
const data = await readFiles(files);
const results = await searchFiles(args, data, files);
// if only one result, then just show that help section.
if (results.length === 1) {
return npm.commands.help([path.basename(results[0].file, ".md")], (er) => {
if (er) throw er;
});
}
const formatted = formatResults(args, results);
if (!formatted.trim()) npmUsage(false);
else {
output(formatted);
output(didYouMean(args[0], cmdList));
}
};
const readFiles = async (files) => {
const res = {};
await Promise.all(
files.map(async (file) => {
res[file] = (await readFile(file, "utf8"))
.replace(/^---\n(.*\n)*?---\n/, "")
.trim();
})
);
return res;
};
const searchFiles = async (args, data, files) => {
const results = [];
for (const [file, content] of Object.entries(data)) {
const lowerCase = content.toLowerCase();
// skip if no matches at all
if (!args.some((a) => lowerCase.includes(a.toLowerCase()))) continue;
const lines = content.split(/\n+/);
// if a line has a search term, then skip it and the next line.
// if the next line has a search term, then skip all 3
// otherwise, set the line to null. then remove the nulls.
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const nextLine = lines[i + 1];
let match = false;
if (nextLine) {
match = args.some((a) =>
nextLine.toLowerCase().includes(a.toLowerCase())
);
if (match) {
// skip over the next line, and the line after it.
i += 2;
continue;
}
}
match = args.some((a) => line.toLowerCase().includes(a.toLowerCase()));
if (match) {
// skip over the next line
i++;
continue;
}
lines[i] = null;
}
// now squish any string of nulls into a single null
const pruned = lines.reduce((l, r) => {
if (!(r === null && l[l.length - 1] === null)) l.push(r);
return l;
}, []);
if (pruned[pruned.length - 1] === null) pruned.pop();
if (pruned[0] === null) pruned.shift();
// now count how many args were found
const found = {};
let totalHits = 0;
for (const line of pruned) {
for (const arg of args) {
const hit =
(line || "").toLowerCase().split(arg.toLowerCase()).length - 1;
if (hit > 0) {
found[arg] = (found[arg] || 0) + hit;
totalHits += hit;
}
}
}
const cmd = "npm help " + path.basename(file, ".md").replace(/^npm-/, "");
results.push({
file,
cmd,
lines: pruned,
found: Object.keys(found),
hits: found,
totalHits,
});
}
// sort results by number of results found, then by number of hits
// then by number of matching lines
// coverage is ignored here because the contents of results are
// nondeterministic due to either glob or readFiles or Object.entries
return results
.sort(
/* istanbul ignore next */ (a, b) =>
a.found.length > b.found.length
? -1
: a.found.length < b.found.length
? 1
: a.totalHits > b.totalHits
? -1
: a.totalHits < b.totalHits
? 1
: a.lines.length > b.lines.length
? -1
: a.lines.length < b.lines.length
? 1
: 0
)
.slice(0, 10);
};
const formatResults = (args, results) => {
const cols = Math.min(process.stdout.columns || Infinity, 80) + 1;
const out = results
.map((res) => {
const out = [res.cmd];
const r = Object.keys(res.hits)
.map((k) => `${k}:${res.hits[k]}`)
.sort((a, b) => (a > b ? 1 : -1))
.join(" ");
out.push(
" ".repeat(Math.max(1, cols - out.join(" ").length - r.length - 1))
);
out.push(r);
if (!npm.flatOptions.long) return out.join("");
out.unshift("\n\n");
out.push("\n");
out.push("-".repeat(cols - 1) + "\n");
res.lines.forEach((line, i) => {
if (line === null || i > 3) return;
if (!npm.color) {
out.push(line + "\n");
return;
}
const hilitLine = [];
for (const arg of args) {
const finder = line.toLowerCase().split(arg.toLowerCase());
let p = 0;
for (const f of finder) {
hilitLine.push(line.substr(p, f.length));
const word = line.substr(p + f.length, arg.length);
const hilit = color.bgBlack(color.red(word));
hilitLine.push(hilit);
p += f.length + arg.length;
}
}
out.push(hilitLine.join("") + "\n");
});
return out.join("");
})
.join("\n");
const finalOut =
results.length && !npm.flatOptions.long
? "Top hits for " +
args.map(JSON.stringify).join(" ") +
"\n" +
"—".repeat(cols - 1) +
"\n" +
out +
"\n" +
"—".repeat(cols - 1) +
"\n" +
"(run with -l or --long to see more context)"
: out;
return finalOut.trim();
};
module.exports = Object.assign(cmd, { usage, completion });

Depending on how your site is structured and generated, I don't see why a client-side text search wouldn't work. I wouldn't recommend crawling the site on the client-side, so it would probably be better to generate a data file at build time, and then basing the search off that.
If your static site is generated with a static site generator, you might be able to get the static site generator to create a JSON file with all the content. Otherwise, if it's just static assets, you could probably create a script to read your content and create the data file that way.
There are also plenty of libraries available that do searching of a JSON object, such as fuse.js.
The main concern with a client-side search is the amount of text to search. If you have lots of content, the client would have to load everything into memory, which may be a concern, though you'll have to test things out for your particular use case.

Double for loop in Javascript inner array length

I am trying to create a function that takes in a string and changes each letters value to a "(" if the character is not duplicated in the string, and a ")" if the character does have a duplicate present in the string. I have decided to go an unconventional route to solve this problem but I am running in to an issue with a double for loop. From what I understand, the inner for loop in javascript does not have access to the variables outside of the loop. I want to loop through every item in an array twice but I'm not sure what to set the inner loops length as.
Here is my code:
function sortAndChange(word) {
const splitter = word.toLowerCase().split("");
//let jSplitter = word.toLowerCase().split("").length;
let endResult = "";
let truthArray = [];
for(i = 0; i < splitter.length; i++){
for(j = 0; j < splitter.length; j++){
console.log(j);
if(splitter[i] == splitter[j]){
truthArray.push(true);
} else {
truthArray.push(false);
}
}
console.log(truthArray);
truthArray.every(item => item === false) ? endResult += "(" : endResult += ")";
truthArray = [];
}
console.log(endResult);
}
Expected Result:
sortAndChange("Success") //expected output: ")())())"
sortAndChange("easy") //expected output: "(((("

You can do that in following steps:
Convert string to array using split and use map() on it.
Compare the indexOf() and lastIndexOf() to check if its duplicate or not.
Return the ) or ( based on ur condition. And then at last join the array
function sortAndChange(str){
let arr = str.toLowerCase().split('')
return arr.map(x => {
//if its not duplicated
if(arr.indexOf(x) === arr.lastIndexOf(x)){
return '('
}
//If its duplicated
else{
return ')'
}
}).join('');
}
console.log(sortAndChange("Success")) //expected output: ")())())"
console.log(sortAndChange("easy")) //expected output: "(((("

You could take a object and keep a boolean value for later mapping the values.
This approach has two loops with O(2n)
function sortAndChange(word) {
word = word.toLowerCase();
var map = [...word].reduce((m, c) => (m[c] = c in m, m), {});
return Array
.from(word, c => '()'[+map[c]])
.join('');
}
console.log(sortAndChange("Success")); // )())())
console.log(sortAndChange("easy")); // ((((

This can easily be achieved using a combination of regex and the map construct in javascript:
const input = "this is a test";
const characters = input.toLowerCase().split('');
const transformed = characters.map(currentCharacter => {
const regexpression = new RegExp(currentCharacter, "g");
if (input.toLowerCase().match(regexpression || []).length > 1) return ')'
return '(';
}).join("");
console.log(transformed);

Look at the following snippet and comments
function sortAndChange(str) {
// we create an array containing the characters on the string
// so we can use Array.reduce
return str.split('').reduce((tmp, x, xi) => {
// we look if the character is duplicate in the string
// by looking for instance of the character
if (str.slice(xi + 1).includes(x.toLowerCase())) {
// Duplicate - we replace every occurence of the character
tmp = tmp.replace(new RegExp(x, 'gi'), ')');
} else {
// Not duplicate
tmp = tmp.replace(new RegExp(x, 'gi'), '(');
}
return tmp;
}, str);
}
console.log(sortAndChange('Success')); //expected output: ")())())"
console.log(sortAndChange('Easy')); //expected output: "(((("

1) use Array.from to convert to array of chars
2) use reduce to build object with key-value pairs as char in string and ( or ) as value based on repetition .
3) Now convert original string to result string using the chars from above object.
function sortAndChange(str) {
const str_arr = Array.from(str.toLowerCase());
const obj = str_arr.reduce(
(acc, char) => ((acc[char] = char in acc ? ")" : "("), acc),
{}
);
return str_arr.reduce((acc, char) => `${acc}${obj[char]}`, "");
}
console.log(sortAndChange("Success")); // ")())())"
console.log(sortAndChange("easy")); // ((((

Create a function to remove all white spaces in string in JavaScript?

I’m trying to build a function to remove white spaces from both ends of a string (including \n,\t) without using built in functions (i.e. trim(), replace(), split(), join())
Similar to the following code but without the .replace:
function myTrim(x)
{
return x.replace(/^\s+|\s+$/gm,'');
}
function myFunction()
{
var str = myTrim(" Hello World! \t ");
}

Here it is using Regexp.exec:
var re = /^\s*(\S[\S\s.]*\S)\s*$/gm;
function trim(str) {
var b = re.exec(str);
return (b !== null) ? (re.exec(str),b[1]) : '';
}
console.log('['+trim("Hello World!")+']')
console.log('['+trim(" Hello World!")+']')
console.log('['+trim("Hello World! \t ")+']')
console.log('['+trim(" Hello World! \t ")+']')
One thing to note is that you must re-call re.exec if the first result was non-null to clear the functions buffer.

If you want to avoid built-in functions, you will have to iterate over your string.
Here is a way to do it by iterating over the string 3 times:
A first time to remove the leading spaces
Then to remove the trailing spaces by iterating in reverse order
And then a last time to reverse the inverted string generated by the last step
function myTrim(str) {
const isSpace = c => c === ' ' || c === '\n' || c === '\r' || c === '\t';
const loop = (str, fn) => { for (const c of str) fn(c) };
const loopReverse = (str, fn) => { for (let i = str.length - 1; i >= 0; --i) fn(str[i]) };
let out = '';
let found = false;
loop(str, c => {
if (!isSpace(c) || found) {
found = true;
out += c;
}
});
found = false;
let reversed = '';
loopReverse(out, c => {
if (!isSpace(c) || found) {
found = true;
reversed += c;
}
});
out = '';
loopReverse(reversed, c => out += c);
return out;
}
console.log(`[${myTrim(' \n Hello World! \t ')}]`);
console.log(`[${myTrim('Hello World! \n \t ')}]`);
console.log(`[${myTrim('Hello World!')}]`);

If I understood correctly. Try this.
x.replace(/[\n\t ]/g, "");

Writing Interpreter Using Javascript Tagged Template Literals

Have you ever thought it'd be cool if you could use your favorite features from other languages in plain old Javascript. For example, I'd love to be able to use the F# pipeline operator '|>', but the proposal for the pipeline operator in JS is only stage 1 ATM.
For starters I've attempted to enable the use of post-fix notation i.e. '1 2 +' = 3. I figured that using ES6 Tagged Template Literals I might get somewhere, but I've barely scratched the surface here. This is just a proof of concept.
const Interpret = (strings, ...expressions) => {
const hasPlus = strings.map(s => s.trim()).join('').match(/[+]$/)
return hasPlus ? expressions.reduce((t, c) => t + c, 0) : null
}
const a = 1
const b = 2
const c = 3
const d = 4
const result = Interpret`${a} ${b} ${c} ${d} +`
console.log(result) // 10
Moreover, I've stumbled upon a big hurdle. The signature of tagged template literal function is as follows - myTag(strings: string[], ...expressions). Expressions are anything ranging from numbers, string, booleans functions and so on. The problem is that the original order of strings and expressions is apparently lost.
It'd be fabulous if you could have access to an array of strings and expression, with their original order preserved ( the order in which they appeared between the backticks ).
For that would enable me to evaluate the elements of the array like so: Did I get an expression, great, push it onto the stack ( just an array for starters )... Next did I get an expression, cool, push it too onto the stack... Next did I get an expression, no - it's the plus operator. Now I can take value 'a' and value 'b' which the first two expressions evaluated to and feed them to the plus operator ( could very well be a function ). Next push the return value onto the stack and so on.
Does anyone have an idea on how to take the next step or perhaps pivot in another direction?
This seems like a step forward, thanks mpm.
const Interpret = (strings, ...expressions) => {
const raw = strings
.map( (s, i) => s + (expressions[i] || '') )
.map(s => s.trim())
const plusPosition = raw.findIndex(s => s.match(/[+]/))
return raw.slice(0, plusPosition).reduce((t, c) => t + parseInt(c), 0)
}
const a = 1
const b = 2
const c = 3
const d = 4
const result = Interpret`${a} ${b} ${c} ${d} +`
console.log(result)

I'm pretty sure the order is not lost otherwise string templates would be useless, strings should return all the string parts and the expressions rest parameter has all the expression values. Just rebuild the order by putting each expression between 2 strings element
const i = (strings, ...expressions) => {
const list = [];
for (let i = 0; i < strings.length; i++) {
if (strings[i].trim() !== '') {
list.push(strings[i].trim())
}
if (i < expressions.length) {
list.push(expressions[i])
}
}
let stack = [];
let result = 0;
do {
const token = list.shift()
if (token === '+') {
result = stack.reduce(function(result, value) {
return result + value
}, result)
stack.length = 0;
} else if (token === '-') {
result = stack.reduce(function(result, value) {
return result - value
}, result)
stack.length = 0;
} else {
stack.push(token)
}
} while (list.length > 0);
return result;
}
const a = 2,
b = 1,
c = 4,
d = 3,
e = 4;
console.log("result", i `${a} ${b} ${c} ${d} + ${e} - `);

Filtering an array based on number of optional RegEx matches with priorities

Say I have an array of files, which have been matched from a bigger array of files with the expression in my code below. For simplicity I'll say the array of files is the following:
prefix_pt1_pt3_pt5_pt6
prefix_pt1_pt4_pt5_pt6
prefix_pt1_pt3_pt4_pt6
prefix_pt1_pt5_pt6
The file names are not necessarily sequential however.
I want to prioritise each capture group. The code I've come up with so far will only prioritise files until it doesn't match a capture group, so from the files above it will just select the first one. I want prefix_pt1_pt3_pt4_pt6 to be the result of my function.
const parts = ['p1', 'p2', 'p3', 'p4', 'p5', 'p6'];
const existsRegex = new RegExp(
regexEscape(params.folder) +
regexEscape(prefix) +
parts.reduce((result, part) => result + `(_${regexEscape(part)})?`, '')
);
const validFiles = scanPath(existsRegex);
if (validFiles.length) {
const chosenFile = validFiles.reduce((file, currentFile) => {
const matches = currentFile.match(existsRegex);
const killFrom = matches.indexOf(undefined);
if (killFrom > 0) matches.length = killFrom;
if (matches.length > file.length) return matches;
return file;
}, []);
}

Supposing your group of files has the structure you mention and you want to retrieve the file with more parts in sequential order (part 1 has more priority than part 3), you could do something like the following:
Order the array of the chosenFiles using as criteria the number of the parts. In order to do that transform the name of the files into numbers (the numbers correspond to the concatenation of each number of each part).
Store the relation number-file in a dictionary.
Retrieve the name of the file from the dictionary, the key will be the first element in the ordered array.
var chosenFiles = ['prefix_pt1_pt3_pt5_pt6',
'prefix_pt1_pt4_pt5_pt6',
'prefix_pt1_pt3_pt4_pt6',
'prefix_pt1_pt5_pt6'
],
dicc = {};
function getKey(arr) {
return arr.join('');
}
function addEntry(file, index) {
var numberArr = file.replace(/\D+/g, '-').split('-').splice(1).map(Number);
dicc[getKey(numberArr)] = file;
return numberArr;
}
function sortByFileNumber(a, b) {
var i = 0;
while (a[i] && b[i] && a[i] >= b[i++]) {}
return a === b ? 0 : a[i] ? -1 : 1;
}
function pickFirst(arr) {
return dicc[getKey(arr[0])];
}
var chosenFile = pickFirst(chosenFiles
.map(addEntry)
.sort(sortByFileNumber));
console.log(chosenFile);