Develop Reference

How to generate a fixed-length code from a set of integers of a specific bit count in JavaScript

Generate string from integer with arbitrary base in JavaScript received the following answer:
function parseInt(value, code) {
return [...value].reduce((r, a) => r * code.length + code.indexOf(a), 0);
}
function toString(value, code) {
var digit,
radix= code.length,
result = '';
do {
digit = value % radix;
result = code[digit] + result;
value = Math.floor(value / radix);
} while (value)
return result;
}
console.log(parseInt('dj', 'abcdefghijklmnopqrstuvwxyz0123456789+-'));
console.log(toString(123, 'abcdefghijklmnopqrstuvwxyz0123456789+-'));
console.log(parseInt('a', 'abcdefghijklmnopqrstuvwxyz0123456789+-'));
console.log(toString(0, 'abcdefghijklmnopqrstuvwxyz0123456789+-'));
I am interested something slightly different. Whereas this will generate the shortest code for the number, I would like to now generate a constant-length code based on the number of bits. I am not sure if this is also a complex radix solution as well.
Say I want to generate 8-bit codes using a 16-character alphabet. That means I should be able to take the first 4 bits to select 1 character, and the next 4 bits to select the second character. So I might end up with MV if my 16 character set was ABDHNMOPQRSTUVYZ. Likewise if I had a 16-bit range, I would have 4 character code, and 32-bit range would be an 8-character code. So calling code32(1, 'ABDHNMOPQRSTUVYZ') would give an 8 letter code, while code8(1, 'ABDHNMOPQRSTUVYZ') would give a 2 digit code.
How could that be implemented in JavaScript? Something along these lines?
code8(i, alpha) // 0 to 255 it accepts
code16(i, alpha) // 0 to 65535 it accepts
code32(i, alpha) // 0 to 2^32-1 it accepts
Likewise, how would you get the string code back into the original number (or bit sequence)?

This really comes down to changing toString so that:
It only accepts a code that has a length of a power of 2
It pads the result to a given number of "digits" (characters)
The actual number of digits you would use for a 16 bit number depends on the size of the code. If the code has 16 characters, then it can cover for 4 bits, and so an output of 4 characters would be needed. If however the code has 4 characters, then the output would need 8 characters. You can have cases where the match is not exact, like when you would have a code with 8 characters. Then the output would need 6 characters.
Here I have highlighted the changes to the toString method. My personal preference is to also put the value as last parameter to toString.
function toString(digitCount, code, value) { // <-- add argument digitCount
// Perform a sanity check: code must have a length that is power of 2
if (Math.log2(code.length) % 1) throw "code size not power of 2: " + code.length;
var digit,
radix = code.length,
result = '';
do {
digit = value % radix;
result = code[digit] + result;
value = Math.floor(value / radix);
} while (value)
return result.padStart(digitCount, code[0]); // Pad to the desired output size
}
console.log(toString(4, 'abcdefghijklmnop', 123));
console.log(toString(4, 'abcdefghijklmnop', 0));
console.log(toString(4, 'abcdefghijklmnop', 0xFFFF));
// You could define some more specific functions
const code8 = (code, value) => toString(Math.ceil(8 / Math.log2(code.length)), code, value);
const code16 = (code, value) => toString(Math.ceil(16 / Math.log2(code.length)), code, value);
console.log(code16('abcdefghijklmnop', 123));
console.log(code16('abcdefghijklmnop', 0));
console.log(code16('abcdefghijklmnop', 0xFFFF));
console.log(code8('abcdefghijklmnop', 123));
console.log(code8('abcdefghijklmnop', 0));
console.log(code8('abcdefghijklmnop', 0xFF));

EDIT: I just noticed that you required a decoder as well. It is easy to implement a non-optimal version too, while an optimal one can be implemented via go through each letter and accumulate their value times their weighs.
Is this what you want? I tested this code for bit=16 and bit=8, but when bit=32 the count of codewords becomes too large and hangs the devtools of the browser. It's only a demonstrative code and may need optimization if need to be applied in practical use...
function genCode(len, alpha){
let tmp = [...alpha];
for(let i = 1; i != len; ++i){
const ttmp = [];
tmp.forEach(te => {
[...alpha].forEach(e => {
ttmp.push(te + e);
});
});
tmp = ttmp;
}
return tmp;
}
function code(bits, i, alpha){
const len = Math.ceil(bits / Math.floor(Math.log2(alpha.length)));
return genCode(len, alpha)[i];
}
function decode(bits, c, alpha){
const len = Math.ceil(bits / Math.floor(Math.log2(alpha.length)));
const codes = genCode(len, alpha);
return codes.indexOf(c);
}
console.log(code(16, 2, "ABDHNMOPQRSTUVYZ"));
console.log(decode(16, "AAAD", "ABDHNMOPQRSTUVYZ"));
console.log(code(8, 255, "ABDHNMOPQRSTUVYZ"));
console.log(decode(8, "ZZ", "ABDHNMOPQRSTUVYZ"));

Explaining functions of Vigenère cipher

i found the following code at http://rosettacode.org for the Vigenère cipher and i would like to undestand it better.
Could someone explain me what the single lines of code in function ordA(a) and in function(a) do?
function ordA(a) {
return a.charCodeAt(0) - 65;
}
// vigenere
function vigenere2(text, key, decode) {
var i = 0, b;
key = key.toUpperCase().replace(/[^A-Z]/g, '');
return text.toUpperCase().replace(/[^A-Z]/g, '').replace(/[A-Z]/g, function(a) {
b = key[i++ % key.length];
return String.fromCharCode(((ordA(a) + (decode ? 26 - ordA(b) : ordA(b))) % 26 + 65));
});
}

I'm not sure if that is supposed to be example code, but it mainly shows how not to program. Smart decisions are being made, but obviously the problem decomposition, variable naming and documentation leave a lot to be desired. Repeated code, convoluted lines, unexplained code fragments, the list goes on. Decode is a boolean, but the opposite of encryption is decryption not decoding. This code was made to not understand what is going on; what it does on the Rosetta site is mind-boggling in that respect.
returns an index in the English alphabet or ABC, assuming uppercase characters, 0 to 25 instead of 1 to 26 (because you can do modular calculations with zero indexing, not with one based indexing)
return a.charCodeAt(0) - 65;
function definition that takes a plaintext or ciphertext, a key which may be smaller than the plaintext and a Boolean to indicate if we're encoding or decoding
function vigenere2(text, key, decode)
index in plaintext and variable b, which will hold a character of the key for the index
var i = 0, b;
converts the key to uppercase and removed all characters not in the uppercase alphabet as well
key = key.toUpperCase().replace(/[^A-Z]/g, '');
this line is too long obviously; it converts the text to uppercase and removes the non-alphabet characters again
then it replaces the characters in the string using the function defined in the second argument of replace
return text.toUpperCase().replace(/[^A-Z]/g, '').replace(/[A-Z]/g, function(a) {
take the next character of the key in round robin fashion, using the modulus operator, update the index afterwards
b = key[i++ % key.length];
too much going on here, very bad program decomposition; in order of execution:
(decode ? 26 - ordA(b) : ordA(b)): calculate a number in the range to update the index of the plaintext character; use the opposite value for decryption (wrongly called "decoding" here)
(ordA(a) + (decode ? 26 - ordA(b) : ordA(b))) % 26 perform the addition with the calculated number, reduce to 0 to 25 (i.e. when reaching Z continue with A and vice versa)
((ordA(a) + (decode ? 26 - ordA(b) : ordA(b))) % 26 + 65) add 65 so the index is converted back into the ASCII index of uppercase characters, using two completely spurious parentheses
finally, returns a string from one character code result, otherwise + will be addition instead of concatenation
return String.fromCharCode(((ordA(a) + (decode ? 26 - ordA(b) : ordA(b))) % 26 + 65));
well, it needed to end
});
}
Let's show another way of programming this, using well named variables, functions for reused code and regular expressions that badly need a name to explain what they do.
var ALPHABET_SIZE = 'Z'.charCodeAt(0) - 'A'.charCodeAt(0) + 1;
var encrypted = vigenere(false, "B", "Zaphod Breeblebox");
document.body.append('<div>' + encrypted + '</div>');
var decrypted = vigenere(true, "B", encrypted);
document.body.append('<div>' + decrypted + '</div>');
function vigenere(decrypt, key, text) {
key = toJustUppercase(key);
text = toJustUppercase(text);
var textOffset = 0;
// iterate over all characters, performing the function on each of them
return text.replace(/[A-Z]/g, function(textChar) {
var keyChar = key[textOffset++ % key.length];
var cryptedChar = substituteCharacter(decrypt, keyChar, textChar);
return cryptedChar;
});
}
function substituteCharacter(decrypt, keyChar, textChar) {
var keyIndex = charToABCIndex(keyChar);
if (decrypt) {
// create the opposite of the encryption key index
keyIndex = ALPHABET_SIZE - keyIndex;
}
var textIndex = charToABCIndex(textChar);
// the actual Vigenere substitution, the rest is just indexing and conversion
var substitutedIndex = (textIndex + keyIndex) % ALPHABET_SIZE;
var substitutedChar = abcIndexToChar(substitutedIndex);
return substitutedChar;
}
function toJustUppercase(text) {
return text.toUpperCase().replace(/[^A-Z]/g, '')
}
function charToABCIndex(charValue) {
return charValue.charCodeAt(0) - 'A'.charCodeAt(0);
}
function abcIndexToChar(index) {
return String.fromCharCode(index + 'A'.charCodeAt(0));
}
Too many functions you say? Not really, I've not implemented ord and chr, or vigenereEncrypt and viginereDecrypt to make it even easier to read.

compressing a string of 0's and 1's in js

Itroduction
I'm currently working on John Conway's Game of Life in js. I have the game working (view here) and i'm working on extra functionalities such as sharing your "grid / game" to your friends. To do this i'm extracting the value's of the grid (if the cell is alive or dead) into a long string of 0's and 1's.
This string has a variable length since the grid is not always the same size. for example:
grid 1 has a length and width of 30 => so the string's length is 900
grid 2 has a length and width of 50 => so the string's length is 2500
The problem
As you can see these string's of 0's and 1's are way too long to copy around and share.
However hard i try I don't seem to be able to come up with a code that would compress a string this long to a easy to handle one.
Any ideas on how to compress (and decompress) this?
I have considered simply writing down every possible grid option for the gird sizes 1x1 to 100x100 and giving them a key/reference to use as sharable code. Doing that by hand would be madness but maybe any of you has an idea on how to create an algorithm that can do this?
GitHub repository

In case it wasn't already obvious, the string you're trying to store looks like a binary string.
Counting systems
Binary is a number in base-2. This essentially means that there are two characters being used to keep count. Normally we are used to count with base-10 (decimal characters). In computer science the hexadecimal system (base-16) is also widely being used.
Since you're not storing the bits as bits but as bytes (use var a = 0b1100001; if you ever wish to store them like bits) the 'binary' you wish to store just takes as much space as any other random string with the same length.
Since you're using the binary system each position just has 2 possible values. When using the hexadecimal value a single position can hold up to 16 possible values. This is already a big improvement when it comes to storing the data compactly. As an example 0b11111111 and 0xff both represents the decimal number 255.
In your situation that'd shave 6 bytes of every 8 bytes you have to store. In the end you'd be stuck with a string just 1/4th of the length of the original string.
Javascript implementation
Essentially what we want to do is to interpret the string you store as binary and retrieve the hexadecimal value. Luckily JavaScript has built in functionality to achieve stuff like this:
var bin =
'1110101110100011' +
'0000101111100001' +
'1010010101011010' +
'0000110111011111' +
'1111111001010101' +
'0111000011100001' +
'1011010100110001' +
'0111111110010100' +
'0111110110100101' +
'0000111101100111' +
'1100001111011100' +
'0101011100001111' +
'0110011011001101' +
'1000110010001001' +
'1010100010000011' +
'0011110000000000';
var returnValue = '';
for (var i = 0; i < parseInt(bin.length / 8); i++) {
returnValue += parseInt(bin.substr(i*8, 8), 2).toString(16);
}
console.log(bin.length); // Will return 265
console.log(returnValue.length); // Will return 64
We're saying "parse this string and interpret it like a base-2 number and store it as a hexadecimal string".
Decoding is practically the same. Replace all occurrences of the number 8 in the example above with 2 and vice versa.
Please note
A prerequisite for this code to work correctly is that the binary length is dividable by 8. See the following example:
parseInt('00011110', 2).toString(16); // returns '1e'
parseInt('1e', 16).toString(2); // returns '11110'
// Technically both representations still have the same decimal value
When decoding you should add leading zeros until you have a full byte (8 bits).
In case the positions you have to store are not dividable by 8 you can, for example, add padding and add a number to the front of the output string to identify how much positions to strip.
Wait, there's more
To get even shorter strings you can build a lookup table with 265 characters in which you search for the character associated with the specific position. (This works because you're still storing the hexadecimal value as a string.) Sadly neither the ASCII nor the UTF-8 encodings are suited for this as there are blocks with values which have no characters defined.
It may look like:
// Go fill this array until you have 265 values within it.
var lookup = ['A', 'B', 'C', 'D'];
var smallerValue = lookup[0x00];
This way you can have 265 possible values at a single position, AND you have used your byte to the fullest.
Please note that no real compression is happening here. We're rather utilising data types to be used more efficiently for your current use case.

If we make the assumption than the grid contains much more 0's than 1's, you may want to try this simple compression scheme:
convert the binary string to an hexadecimal string
convert '00' sub-strings to 'z' symbol
convert 'zz' sub-strings to 'Z' symbol
we could go further, but let's stop here for the demo
Below is an example with a 16x16 grid:
var bin =
'0000000000000000' +
'0000001000000000' +
'0000011100000000' +
'0000001000000000' +
'0000000000000000' +
'0000000000111000' +
'0000100000111000' +
'0000000000111000' +
'0000000000000000' +
'0000000000000000' +
'0000000010000000' +
'0000000101000000' +
'0000000010000000' +
'0000000000000000' +
'0000100000000000' +
'0000000000000000';
var packed = bin
.match(/(.{4})/g)
.map(function(x) {
return parseInt(x, 2).toString(16);
})
.join('')
.replace(/00/g, 'z')
.replace(/zz/g, 'Z');
This will produce the string "Z02z07z02ZZ380838z38ZZz8z14z08Zz8Zz".
The unpacking process is doing the exact opposite:
var bin = packed
.replace(/Z/g, 'zz')
.replace(/z/g, '00')
.split('')
.map(function(x) {
return ('000' + parseInt(x, 16).toString(2)).substr(-4, 4);
})
.join('');
Note that this code will only work correctly if the length of the input string is a multiple of 4. If it's not the case, you'll have to pad the input and crop the output.
EDIT : 2nd method
If the input is completely random -- with roughly as many 0's as 1's and no specific repeating patterns -- the best you can do is probably to convert the binary string to a BASE64 string. It will be significantly shorter (this time with a fixed compression ratio of about 17%) and can still be copied/pasted by the user.
Packing:
var bin =
'1110101110100011' +
'0000101111100001' +
'1010010101011010' +
'0000110111011111' +
'1111111001010101' +
'0111000011100001' +
'1011010100110001' +
'0111111110010100' +
'0111110110100101' +
'0000111101100111' +
'1100001111011100' +
'0101011100001111' +
'0110011011001101' +
'1000110010001001' +
'1010100010000011' +
'0011110000000000';
var packed =
btoa(
bin
.match(/(.{8})/g)
.map(function(x) {
return String.fromCharCode(parseInt(x, 2));
})
.join('')
);
Will produce the string "66ML4aVaDd/+VXDhtTF/lH2lD2fD3FcPZs2MiaiDPAA=".
Unpacking:
var bin =
atob(packed)
.split('')
.map(function(x) {
return ('0000000' + x.charCodeAt(0).toString(2)).substr(-8, 8);
})
.join('');
Or if you want to go a step further, you can consider using something like base91 instead, for a reduced encoding overhead.

LZ-string
Using LZ-string I was able to compress the "code" quite a bit.
By simply compressing it to base64 like this:
var compressed = LZString.compressToBase64(string)
Decompressing is also just as simple as this:
var decompressed = LZString.decompressFromBase64(compressed)
However the length of this compressed string is still pretty long given that you have about as many 0s as 1s (not given in the example)
example
But the compression does work.

ANSWER
For any of you who are wondering how exactly I ended up doing it, here's how:
First I made sure every string passed in would be padded with leading 0s untill it was devidable by 8. (saving the amount of 0s used to pad, since they're needed while decompressing)
I used Corstian's answer and functions to compress my string (interpreted as binary) into a hexadecimal string. Although i had to make one slight alteration.
Not every binary substring with a lenght of 8 will return exactly 2 hex characters. so for those cases i ended up just adding a 0 in front of the substring. The hex substring will have the same value but it's length will now be 2.
Next up i used a functionality from Arnaulds answer. Taking every double character and replacing it with a single character (one not used in the hexadecimal alphabet to avoid conflict). I did this twice for every hexadecimal character.
For example:
the hex string 11 will become h and hh will become H
01101111 will become 0h0H
Since most grids are gonna have more dead cells then alive ones, I made sure the 0s would be able to compress even further, using Arnaulds method again but going a step further.
00 -> g | gg -> G | GG -> w | ww -> W | WW -> x | xx -> X | XX-> y | yy -> Y | YY -> z | zz -> Z
This resulted in Z representing 4096 (binary) 0s
The last step of the compression was adding the amount of leading 0s in front of the compressed string, so we can shave those off at the end of decompressing.
This is how the returned string looks in the end.
amount of leading 0s-compressed string so a 64*64 empty grid, will result in 0-Z
Decompressing is practically doing everything the other way around.
Firstly splitting the number that represents how many leading 0s we've used as padding from the compressed string.
Then using Arnaulds functionality, turning the further "compressed" characters back into hexadecimal code.
Taking this hex string and turning it back into binary code. Making sure, as Corstian pointed out, that every binary substring will have a length of 8. (ifnot we pad the substrings with leading 0s untill the do, exactly, have a length of 8)
And then the last step is to shave off the leading 0s we've used as padding to make the begin string devidable by 8.
The functions
Function I use to compress:
/**
* Compresses the a binary string into a compressed string.
* Returns the compressed string.
*/
Codes.compress = function(bin) {
bin = bin.toString(); // To make sure the binary is a string;
var returnValue = ''; // Empty string to add our data to later on.
// If the lenght of the binary string is not devidable by 8 the compression
// won't work correctly. So we add leading 0s to the string and store the amount
// of leading 0s in a variable.
// Determining the amount of 'padding' needed.
var padding = ((Math.ceil(bin.length/8))*8)-bin.length;
// Adding the leading 0s to the binary string.
for (var i = 0; i < padding; i++) {
bin = '0'+bin;
}
for (var i = 0; i < parseInt(bin.length / 8); i++) {
// Determining the substring.
var substring = bin.substr(i*8, 8)
// Determining the hexValue of this binary substring.
var hexValue = parseInt(substring, 2).toString(16);
// Not all binary values produce two hex numbers. For example:
// '00000011' gives just a '3' while what we wand would be '03'. So we add a 0 in front.
if(hexValue.length == 1) hexValue = '0'+hexValue;
// Adding this hexValue to the end string which we will return.
returnValue += hexValue;
}
// Compressing the hex string even further.
// If there's any double hex chars in the string it will take those and compress those into 1 char.
// Then if we have multiple of those chars these are compressed into 1 char again.
// For example: the hex string "ff will result in a "v" and "ffff" will result in a "V".
// Also: "11" will result in a "h" and "1111" will result in a "H"
// For the 0s this process is repeated a few times.
// (string with 4096 0s) (this would represent a 64*64 EMPTY grid)
// will result in a "Z".
var returnValue = returnValue.replace(/00/g, 'g')
.replace(/gg/g, 'G')
// Since 0s are probably more likely to exist in our binary and hex, we go a step further compressing them like this:
.replace(/GG/g, 'w')
.replace(/ww/g, 'W')
.replace(/WW/g, 'x')
.replace(/xx/g, 'X')
.replace(/XX/g, 'y')
.replace(/yy/g, 'Y')
.replace(/YY/g, 'z')
.replace(/zz/g, 'Z')
//Rest of the chars...
.replace(/11/g, 'h')
.replace(/hh/g, 'H')
.replace(/22/g, 'i')
.replace(/ii/g, 'I')
.replace(/33/g, 'j')
.replace(/jj/g, 'J')
.replace(/44/g, 'k')
.replace(/kk/g, 'K')
.replace(/55/g, 'l')
.replace(/ll/g, 'L')
.replace(/66/g, 'm')
.replace(/mm/g, 'M')
.replace(/77/g, 'n')
.replace(/nn/g, 'N')
.replace(/88/g, 'o')
.replace(/oo/g, 'O')
.replace(/99/g, 'p')
.replace(/pp/g, 'P')
.replace(/aa/g, 'q')
.replace(/qq/g, 'Q')
.replace(/bb/g, 'r')
.replace(/rr/g, 'R')
.replace(/cc/g, 's')
.replace(/ss/g, 'S')
.replace(/dd/g, 't')
.replace(/tt/g, 'T')
.replace(/ee/g, 'u')
.replace(/uu/g, 'U')
.replace(/ff/g, 'v')
.replace(/vv/g, 'V');
// Adding the number of leading 0s that need to be ignored when decompressing to the string.
returnValue = padding+'-'+returnValue;
// Returning the compressed string.
return returnValue;
}
The function I use to decompress:
/**
* Decompresses the compressed string back into a binary string.
* Returns the decompressed string.
*/
Codes.decompress = function(compressed) {
var returnValue = ''; // Empty string to add our data to later on.
// Splitting the input on '-' to seperate the number of paddin 0s and the actual hex code.
var compressedArr = compressed.split('-');
var paddingAmount = compressedArr[0]; // Setting a variable equal to the amount of leading 0s used while compressing.
compressed = compressedArr[1]; // Setting the compressed variable to the actual hex code.
// Decompressing further compressed characters.
compressed = compressed// Decompressing the further compressed 0s. (even further then the rest of the chars.)
.replace(/Z/g, 'zz')
.replace(/z/g, 'YY')
.replace(/Y/g, 'yy')
.replace(/y/g, 'XX')
.replace(/X/g, 'xx')
.replace(/x/g, 'WW')
.replace(/W/g, 'ww')
.replace(/w/g, 'GG')
.replace(/G/g, 'gg')
.replace(/g/g, '00')
// Rest of chars...
.replace(/H/g, 'hh')
.replace(/h/g, '11')
.replace(/I/g, 'ii')
.replace(/i/g, '22')
.replace(/J/g, 'jj')
.replace(/j/g, '33')
.replace(/K/g, 'kk')
.replace(/k/g, '44')
.replace(/L/g, 'll')
.replace(/l/g, '55')
.replace(/M/g, 'mm')
.replace(/m/g, '66')
.replace(/N/g, 'nn')
.replace(/n/g, '77')
.replace(/O/g, 'oo')
.replace(/o/g, '88')
.replace(/P/g, 'pp')
.replace(/p/g, '99')
.replace(/Q/g, 'qq')
.replace(/q/g, 'aa')
.replace(/R/g, 'rr')
.replace(/r/g, 'bb')
.replace(/S/g, 'ss')
.replace(/s/g, 'cc')
.replace(/T/g, 'tt')
.replace(/t/g, 'dd')
.replace(/U/g, 'uu')
.replace(/u/g, 'ee')
.replace(/V/g, 'vv')
.replace(/v/g, 'ff');
for (var i = 0; i < parseInt(compressed.length / 2); i++) {
// Determining the substring.
var substring = compressed.substr(i*2, 2);
// Determining the binValue of this hex substring.
var binValue = parseInt(substring, 16).toString(2);
// If the length of the binary value is not equal to 8 we add leading 0s (js deletes the leading 0s)
// For instance the binary number 00011110 is equal to the hex number 1e,
// but simply running the code above will return 11110. So we have to add the leading 0s back.
if (binValue.length != 8) {
// Determining how many 0s to add:
var diffrence = 8 - binValue.length;
// Adding the 0s:
for (var j = 0; j < diffrence; j++) {
binValue = '0'+binValue;
}
}
// Adding the binValue to the end string which we will return.
returnValue += binValue
}
var decompressedArr = returnValue.split('');
returnValue = ''; // Emptying the return variable.
// Deleting the not needed leading 0s used as padding.
for (var i = paddingAmount; i < decompressedArr.length; i++) {
returnValue += decompressedArr[i];
}
// Returning the decompressed string.
return returnValue;
}
URL shortener
I still found the "compressed" strings a little long for sharing / pasting around. So i used a simple URL shortener (view here) to make this process a little easier for the user.
Now you might ask, then why did you need to compress this string anyway?
Here's why:
First of all, my project is hosted on github pages (gh-pages). The info page of gh-pages tells us that the url can't be any longer than 2000 chars. This would mean that the max grid size would be the square root of 2000 - length of the base url, which isn't that big. By using this "compression" we are able to share much larger grids.
Now the second reason why is that, it's a challange. I find dealing with problems like these fun and also helpfull since you learn a lot.
Live
You can view the live version of my project here. and/or find the github repository here.
Thankyou
I want to thank everyone who helped me with this problem. Especially Corstian and Arnauld, since i ended up using their answers to reach my final functions.
Sooooo.... thanks guys! apriciate it!

In the Game of Life there is a board of ones and zeros. I want to back up to previous generation - size 4800 - save each 16 cells as hexadecimal = 1/4 the size. http://innerbeing.epizy.com/cwebgl/gameoflife.html [g = Go] [b = Backup]
function drawGen(n) {
stop(); var i = clamp(n,0,brw*brh-1), hex = gensave[i].toString();
echo(":",i, n,nGEN); nGEN = i; var str = '';
for (var i = 0; i < parseInt(hex.length / 4); i++)
str = str + pad(parseInt(hex.substr(i*4,4), 16).toString(2),16,'0');
for (var j=0;j<Board.length;j++) Board[j] = intr(str.substr(j,1));
drawBoard();
}
function Bin2Hex(n) {
var i = n.indexOf("1"); /// leading Zeros = NAN
if (i == -1) return "0000";
i = right(n,i*-1);
return pad(parseInt(i,2).toString(16),4,'0');
}
function saveGen(n) {
var b = Board.join(''), str = ''; /// concat array to string 10101
for (var i = 0; i < parseInt(b.length / 16); i++)
str = str + Bin2Hex(b.substr(i*16,16));
gensave[n] = str;
}
function right(st,n) {
var s = st.toString();
if (!n) return s;
if (n < 0) return s.substr(n * -1,s.length + n);
return s.substr(s.length - n,n);
}
function pad(str, l, padwith) {
var s = str;
while (s.length < l) s = padwith + s;
return s;
}

javascript and string manipulation w/ utf-16 surrogate pairs

I'm working on a twitter app and just stumbled into the world of utf-8(16). It seems the majority of javascript string functions are as blind to surrogate pairs as I was. I've got to recode some stuff to make it wide character aware.
I've got this function to parse strings into arrays while preserving the surrogate pairs. Then I'll recode several functions to deal with the arrays rather than strings.
function sortSurrogates(str){
var cp = []; // array to hold code points
while(str.length){ // loop till we've done the whole string
if(/[\uD800-\uDFFF]/.test(str.substr(0,1))){ // test the first character
// High surrogate found low surrogate follows
cp.push(str.substr(0,2)); // push the two onto array
str = str.substr(2); // clip the two off the string
}else{ // else BMP code point
cp.push(str.substr(0,1)); // push one onto array
str = str.substr(1); // clip one from string
}
} // loop
return cp; // return the array
}
My question is, is there something simpler I'm missing? I see so many people reiterating that javascript deals with utf-16 natively, yet my testing leads me to believe, that may be the data format, but the functions don't know it yet. Am I missing something simple?
EDIT:
To help illustrate the issue:
var a = "0123456789"; // U+0030 - U+0039 2 bytes each
var b = "𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡"; // U+1D7D8 - U+1D7E1 4 bytes each
alert(a.length); // javascript shows 10
alert(b.length); // javascript shows 20
Twitter sees and counts both of those as being 10 characters long.

Javascript uses UCS-2 internally, which is not UTF-16. It is very difficult to handle Unicode in Javascript because of this, and I do not suggest attempting to do so.
As for what Twitter does, you seem to be saying that it is sanely counting by code point not insanely by code unit.
Unless you have no choice, you should use a programming language that actually supports Unicode, and which has a code-point interface, not a code-unit interface. Javascript isn't good enough for that as you have discovered.
It has The UCS-2 Curse, which is even worse than The UTF-16 Curse, which is already bad enough. I talk about all this in OSCON talk, 🔫 Unicode Support Shootout: 👍 The Good, the Bad, & the (mostly) Ugly 👎.
Due to its horrible Curse, you have to hand-simulate UTF-16 with UCS-2 in Javascript, which is simply nuts.
Javascript suffers from all kinds of other terrible Unicode troubles, too. It has no support for graphemes or normalization or collation, all of which you really need. And its regexes are broken, sometimes due to the Curse, sometimes just because people got it wrong. For example, Javascript is incapable of expressing regexes like [𝒜-𝒵]. Javascript doesn’t even support casefolding, so you can’t write a pattern like /ΣΤΙΓΜΑΣ/i and have it correctly match στιγμας.
You can try to use the XRegEXp plugin, but you won’t banish the Curse that way. Only changing to a language with Unicode support will do that, and 𝒥𝒶𝓋𝒶𝓈𝒸𝓇𝒾𝓅𝓉 just isn’t one of those.

I've knocked together the starting point for a Unicode string handling object. It creates a function called UnicodeString() that accepts either a JavaScript string or an array of integers representing Unicode code points and provides length and codePoints properties and toString() and slice() methods. Adding regular expression support would be very complicated, but things like indexOf() and split() (without regex support) should be pretty easy to implement.
var UnicodeString = (function() {
function surrogatePairToCodePoint(charCode1, charCode2) {
return ((charCode1 & 0x3FF) << 10) + (charCode2 & 0x3FF) + 0x10000;
}
function stringToCodePointArray(str) {
var codePoints = [], i = 0, charCode;
while (i < str.length) {
charCode = str.charCodeAt(i);
if ((charCode & 0xF800) == 0xD800) {
codePoints.push(surrogatePairToCodePoint(charCode, str.charCodeAt(++i)));
} else {
codePoints.push(charCode);
}
++i;
}
return codePoints;
}
function codePointArrayToString(codePoints) {
var stringParts = [];
for (var i = 0, len = codePoints.length, codePoint, offset, codePointCharCodes; i < len; ++i) {
codePoint = codePoints[i];
if (codePoint > 0xFFFF) {
offset = codePoint - 0x10000;
codePointCharCodes = [0xD800 + (offset >> 10), 0xDC00 + (offset & 0x3FF)];
} else {
codePointCharCodes = [codePoint];
}
stringParts.push(String.fromCharCode.apply(String, codePointCharCodes));
}
return stringParts.join("");
}
function UnicodeString(arg) {
if (this instanceof UnicodeString) {
this.codePoints = (typeof arg == "string") ? stringToCodePointArray(arg) : arg;
this.length = this.codePoints.length;
} else {
return new UnicodeString(arg);
}
}
UnicodeString.prototype = {
slice: function(start, end) {
return new UnicodeString(this.codePoints.slice(start, end));
},
toString: function() {
return codePointArrayToString(this.codePoints);
}
};
return UnicodeString;
})();
var ustr = UnicodeString("f𝌆𝌆bar");
document.getElementById("output").textContent = "String: '" + ustr + "', length: " + ustr.length + ", slice(2, 4): " + ustr.slice(2, 4);
<div id="output"></div>

Here are a couple scripts that might be helpful when dealing with surrogate pairs in JavaScript:
ES6 Unicode shims for ES3+ adds the String.fromCodePoint and String.prototype.codePointAt methods from ECMAScript 6. The ES3/5 fromCharCode and charCodeAt methods do not account for surrogate pairs and therefore give wrong results.
Full 21-bit Unicode code point matching in XRegExp with \u{10FFFF} allows matching any individual code point in XRegExp regexes.

Javascript string iterators can give you the actual characters instead of the surrogate code points:
>>> [..."0123456789"]
["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
>>> [..."𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡"]
["𝟘", "𝟙", "𝟚", "𝟛", "𝟜", "𝟝", "𝟞", "𝟟", "𝟠", "𝟡"]
>>> [..."0123456789"].length
10
>>> [..."𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡"].length
10

This is along the lines of what I was looking for. It needs better support for the different string functions. As I add to it I will update this answer.
function wString(str){
var T = this; //makes 'this' visible in functions
T.cp = []; //code point array
T.length = 0; //length attribute
T.wString = true; // (item.wString) tests for wString object
//member functions
sortSurrogates = function(s){ //returns array of utf-16 code points
var chrs = [];
while(s.length){ // loop till we've done the whole string
if(/[\uD800-\uDFFF]/.test(s.substr(0,1))){ // test the first character
// High surrogate found low surrogate follows
chrs.push(s.substr(0,2)); // push the two onto array
s = s.substr(2); // clip the two off the string
}else{ // else BMP code point
chrs.push(s.substr(0,1)); // push one onto array
s = s.substr(1); // clip one from string
}
} // loop
return chrs;
};
//end member functions
//prototype functions
T.substr = function(start,len){
if(len){
return T.cp.slice(start,start+len).join('');
}else{
return T.cp.slice(start).join('');
}
};
T.substring = function(start,end){
return T.cp.slice(start,end).join('');
};
T.replace = function(target,str){
//allow wStrings as parameters
if(str.wString) str = str.cp.join('');
if(target.wString) target = target.cp.join('');
return T.toString().replace(target,str);
};
T.equals = function(s){
if(!s.wString){
s = sortSurrogates(s);
T.cp = s;
}else{
T.cp = s.cp;
}
T.length = T.cp.length;
};
T.toString = function(){return T.cp.join('');};
//end prototype functions
T.equals(str)
};
Test results:
// plain string
var x = "0123456789";
alert(x); // 0123456789
alert(x.substr(4,5)) // 45678
alert(x.substring(2,4)) // 23
alert(x.replace("456","x")); // 0123x789
alert(x.length); // 10
// wString object
x = new wString("𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡");
alert(x); // 𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡
alert(x.substr(4,5)) // 𝟜𝟝𝟞𝟟𝟠
alert(x.substring(2,4)) // 𝟚𝟛
alert(x.replace("𝟜𝟝𝟞","x")); // 𝟘𝟙𝟚𝟛x𝟟𝟠𝟡
alert(x.length); // 10

Detect difference between & and %26 in location.hash

Analyzing the location.hash with this simple javascript code:
<script type="text/javascript">alert(location.hash);</script>
I have a difficult time separating out GET variables that contain a & (encoded as %26) and a & used to separate variables.
Example one:
code=php&age=15d
Example two:
code=php%20%26%20code&age=15d
As you can see, example 1 has no problems, but getting javascript to know that "code=php & code" in example two is beyond my abilities:
(Note: I'm not really using these variable names, and changing them to something else will only work so long as a search term does not match a search key, so I wouldn't consider that a valid solution.)

There is no difference between %26 and & in a fragment identifier (‘hash’). ‘&’ is only a reserved character with special meaning in a query (‘search’) segment of a URI. Escaping ‘&’ to ‘%26’ need be given no more application-level visibility than escaping ‘a’ to ‘%61’.
Since there is no standard encoding scheme for hiding structured data within a fragment identifier, you could make your own. For example, use ‘+XX’ hex-encoding to encode a character in a component:
hxxp://www.example.com/page#code=php+20+2B+20php&age=15d
function encodeHashComponent(x) {
return encodeURIComponent(x).split('%').join('+');
}
function decodeHashComponent(x) {
return decodeURIComponent(x.split('+').join('%'));
}
function getHashParameters() {
var parts= location.hash.substring(1).split('&');
var pars= {};
for (var i= parts.length; i-->0;) {
var kv= parts[i].split('=');
var k= kv[0];
var v= kv.slice(1).join('=');
pars[decodeHashComponent(k)]= decodeHashComponent(v);
}
return pars;
}

Testing on Firefox 3.1, it looks as if the browser converts hex codes to the appropriate characters when populating the location.hash variable, so there is no way JavaScript can know how the original was a single character or a hex code.
If you're trying to encode a character like & inside of your hash variables, I would suggest replacing it with another string.
You can also parse the string in weird ways, like (JS 1.6 here):
function pairs(xs) {
return xs.length > 1 ? [[xs[0], xs[1]]].concat(pairs(xs.slice(2))) : []
}
function union(xss) {
return xss.length == 0 ? [] : xss[0].concat(union(xss.slice(1)));
}
function splitOnLast(s, sub) {
return s.indexOf(sub) == -1 ? [s] :
[s.substr(0, s.lastIndexOf(sub)),
s.substr(s.lastIndexOf(sub) + sub.length)];
}
function objFromPairs(ps) {
var o = {};
for (var i = 0; i < ps.length; i++) {
o[ps[i][0]] = ps[i][1];
}
return o;
}
function parseHash(hash) {
return objFromPairs(
pairs(
union(
location.hash
.substr(1)
.split("=")
.map(
function (s) splitOnLast(s, '&')))))
}
>>> location.hash
"#code=php & code&age=15d"
>>> parseHash(location.hash)
{ "code": "php & code", "age": "15d" }

Just do the same as you do with the first example, but after you have split on the & then call unescape() to convert the %26 to & and the %20 to a space.
Edit:
Looks like I'm a bit out of date and you should be using decodeURIComponent() now, though I don't see any clear explanation on what it does differently to unescape(), apart from a suggestion that it doesn't handle Unicode properly.

This worked fine for me:
var hash = [];
if (location.hash) {
hash = location.href.split('#')[1].split('&');
}