Develop Reference

Is there a way to take a unique string of 256 chars, reduce it's size with JS to a unique string of 50± chars, and retain guaranteed uniqueness?

I am working on building a parser for a text file to build into a JSON object.
Presuming I have a string somewhere in the line which is a unique value: 4E183437F3FAEBC107C333CD16657661598A2A523867A4A95F6DAD4D25E191722AFFB3FFE29E287D9EB21ED8035E7666D06E65EC064D5D4B337C034459BA7CD6F72437B439329B45F15D57176122965728B36DBB455D335AA904106A29B64694507B6A0F89FBF4A166BB3BBB4DDDC1674A1DB3B7D0FD332117877C4CE56465D8
I want to use this unique identifier as a key at some level, but it's much too long to suit my tastes.
I am wondering if there is a way to create a smaller 'hash':
but not lose and information
guarantee uniqueness
be converted back to it's original
Some library/plugin?

You can create a string of smaller length by turning the 16-bit hex string into a number (with BigInt), and then by creating a new string with base, say, 10000, so that a character with character code C represents the number 'C'.charCodeAt().
Then turn it back into the original string by doing the same process in reverse.
The string length is shorter, and you've saved some bits, but it's still long:
const encode = (input) => {
let n = 0n;
[...input].forEach((char, i) => {
const code = char.charCodeAt();
const value = BigInt(code <= 57 ? code - 48 : code - 55);
n += value * (16n ** BigInt(i));
});
const base10Str = String(n);
let output = '';
for (let i = 0; i < base10Str.length; i += 4) {
output += String.fromCharCode(base10Str.slice(i, i + 4));
}
return output;
};
const decode = (encoded) => {
let base10Str = '';
[...encoded].forEach((char) => {
base10Str += String(char.charCodeAt()).padStart(4, '0'); // todo: final char may need no padding
});
let n = BigInt(base10Str);
let inputStr = '';
for (let i = 255; i >= 0; i--) {
const thisHexMult = 16n ** BigInt(i);
const thisHexDigitBigInt = n / thisHexMult; // This will automatically effectively call Math.floor
n = n % thisHexMult;
const thisHexDigit = Number(thisHexDigitBigInt);
const charCode = thisHexDigit <= 9 ? thisHexDigit + 48 : thisHexDigit + 55;
inputStr = String.fromCharCode(charCode) + inputStr;
}
console.log(inputStr);
};
const encoded = encode('4E183437F3FAEBC107C333CD16657661598A2A523867A4A95F6DAD4D25E191722AFFB3FFE29E287D9EB21ED8035E7666D06E65EC064D5D4B337C034459BA7CD6F72437B439329B45F15D57176122965728B36DBB455D335AA904106A29B64694507B6A0F89FBF4A166BB3BBB4DDDC1674A1DB3B7D0FD332117877C4CE56465D8');
console.log('Encoded length:', encoded.length);
console.log(encoded);
decode(encoded);

Python - Javascript encrypt function for a known decrypt function

I found a streaming website that encrypts the iframe code with an interesting Javascript function. On the webpage is visible the decrypt function (obviously) but not the encryption one. This is the function:
function base64_decode(data) {
var b64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
var o1, o2, o3, h1, h2, h3, h4, bits, i = 0,
ac = 0,
dec = '',
tmp_arr = [];
if (!data) {
return data;
}
data += '';
do {
h1 = b64.indexOf(data.charAt(i++));
h2 = b64.indexOf(data.charAt(i++));
h3 = b64.indexOf(data.charAt(i++));
h4 = b64.indexOf(data.charAt(i++));
bits = h1 << 18 | h2 << 12 | h3 << 6 | h4;
o1 = bits >> 16 & 0xff;
o2 = bits >> 8 & 0xff;
o3 = bits & 0xff;
if (h3 == 64) {
tmp_arr[ac++] = String.fromCharCode(o1);
} else if (h4 == 64) {
tmp_arr[ac++] = String.fromCharCode(o1, o2);
} else {
tmp_arr[ac++] = String.fromCharCode(o1, o2, o3);
}
} while (i < data.length);
dec = tmp_arr.join('');
return dec.replace(/\0+$/, '');
}
function ord(string) {
var str = string + '',
code = str.charCodeAt(0);
if (0xD800 <= code && code <= 0xDBFF) {
var hi = code;
if (str.length === 1) {
return code;
}
var low = str.charCodeAt(1);
return ((hi - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000;
}
if (0xDC00 <= code && code <= 0xDFFF) {
return code;
}
return code;
}
function decrypt(sData, sKey) {
var sResult = "";
sData = base64_decode(sData);
var i = 0;
for (i = 0; i < sData.length; i++) {
var sChar = sData.substr(i, 1);
var sKeyChar = sKey.substr(i % sKey.length - 1, 1);
sChar = Math.floor(ord(sChar) - ord(sKeyChar));
sChar = String.fromCharCode(sChar);
sResult = sResult + sChar;
}
return sResult;
}
So this code:
decrypt('s+Dd6djk3Jfq6dq0md/r6+fqsaam5ufc5ePm2Nul2uam3OTZ3Numy83Zw87aqazMvbimmZfq2unm4+Pg5d60meXmmZfd6djk3Nnm6dvc6bSZp5mX7uDb69+0mainp5yZl9/c4N7f67SZqKennJmX2OPj5u7d7OPj6trp3NzltJnr6ezcmZfu3Nni4OvY4+Pm7t3s4+Pq2unc3OW0mevp7NyZl+Tm8djj4+bu3ezj4+ra6dzc5bSZ6+ns3Jm1s6bg3enY5Ny1', 'w')
will return:
<iframe src="https://openload.co/embed/TVbLWc25UFA/" scrolling="no" frameborder="0" width="100%" height="100%" allowfullscreen="true" webkitallowfullscreen="true" mozallowfullscreen="true"></iframe>
I translated the decrypt function in Python with math and base64 modules and it works well, but now I need the encrypt function (in Python) that starting from a string outputs encrypted string + key. Is it some kind of known encryption?

This seems to be a bad implementation of the Vigenère cipher, without the modulus as the resulting sChar may have higher values. It simply adds the value of a key character to each plain character, reusing the key if it is depleted. It will mainly function as something to confuse virus-scanners or packet-inspecting firewalls as the encryption itself is of course completely insecure. It won't have a name (and no self-respecting cryptographer will lend his name for it either).
There seems to be a bug in the code here:
sKey.substr(i % sKey.length - 1, 1);
I'm not sure why the - 1 is required or how this plays out in practice (this is why languages and API's should be strict in what is acceptable).
ord seems to have been implemented to avoid issues with 16-bit Unicode characters.
base64_decode simply implements base 64 decoding, nothing to see there.

Javascript AND C# Unicode De-/Encode with same results

I found this informative thread:
C# solution to de-/encode a unicode string:
How do you convert Byte Array to Hexadecimal String, and vice versa?
Javascript solution for de-/encode a unicode string:
Javascript: Unicode string to hex
But the solutions mix the chars.
Example Javascript (code 1:1 from link above):
var str = "그러하지";
hex = str.hexEncode(); // returns "adf8b7ecd558c9c0"
Example C# (tried 2 solutions, same results):
/// <summary>
/// Convert a string to hex value
/// </summary>
/// <param name="stringValue"></param>
/// <returns></returns>
public string HexEncode(string stringValue)
{
var ba = Encoding.Unicode.GetBytes(stringValue);
// SOLUTION 1
//var c = new char[ba.Length * 2];
//for (var i = 0; i < ba.Length; i++)
//{
// var b = ba[i] >> 4;
// c[i * 2] = (char)(55 + b + (((b - 10) >> 31) & -7));
// b = ba[i] & 0xF;
// c[i * 2 + 1] = (char)(55 + b + (((b - 10) >> 31) & -7));
//}
//return new string(c);
// SOLUTION 2
var hex = new StringBuilder(ba.Length * 2);
foreach (var b in ba)
hex.AppendFormat("{0:x2}", b);
return hex.ToString();
}
/// <summary>
/// Converts a hex value to a string
/// </summary>
/// <param name="hexString"></param>
/// <returns></returns>
public string HexDecode(string hexString)
{
if (hexString == null || (hexString.Length & 1) == 1) return "";
// SOLUTION 1
//hexString = hexString.ToUpper();
//var hexStringLength = hexString.Length;
//var b = new byte[hexStringLength / 2];
//for (var i = 0; i < hexStringLength; i += 2)
//{
// var topChar = (hexString[i] > 0x40 ? hexString[i] - 0x37 : hexString[i] - 0x30) << 4;
// var bottomChar = hexString[i + 1] > 0x40 ? hexString[i + 1] - 0x37 : hexString[i + 1] - 0x30;
// b[i / 2] = Convert.ToByte(topChar + bottomChar);
//}
// SOLUTION 2
var numberChars = hexString.Length;
var bytes = new byte[numberChars / 2];
for (var i = 0; i < numberChars; i += 2)
bytes[i / 2] = Convert.ToByte(hexString.Substring(i, 2), 16);
return Encoding.Unicode.GetString(bytes);
}
var hex = tools.HexEncode("그러하지");
var str = tools.HexDecode(hex); // f8adecb758d5c0c9
JS: adf8 b7ec d558 c9c0
C#: f8ad ecb7 58d5 c0c9
So the sequence is exchanged.
Both encode and decode works as long I am in the same environment. But I need to encode in JS and decode in C# and vice versa.
I do not know which one is the correct one, if correct can be defined here.
And how do I fix this?

Both values are correct. It's just that your javascript solution gives you unicode array in Big Endian notation, and C# - in Little Endian (MSDN article, see Remarks section).
To make C# byte array same like your javascript, define your encoding like this:
UnicodeEncoding bigEndianUnicode = new UnicodeEncoding(true, true);
And later use it like this:
var ba = bigEndianUnicode.GetBytes(stringValue);
Demo: .Net Fiddle

How to verify with PHP signature genrerated by cryptico.js

I try to sign messages in javascript before sending to a PHP application.
The PHP application must check the signature to be sure it's not false.
In javascript I use cryptico.js.
This is the js function for signing messages
var sign = function(passphrase, text) {
signingkey = cryptico.generateRSAKey(passphrase, 2048);
signString = cryptico.b16to64(signingkey.signString(text, "sha256"));
return signString;
}
This is the function for getting the public key:
var getPublicKey = function(passphrase) {
var rsaKey = cryptico.generateRSAKey(passphrase, 2048);
return = cryptico.publicKeyString(rsaKey);
}
For example, for the message "message" and the passphrase "test2" the public key and signature are
qH/J3/gvF/h5U02uPyC9Qzn/hHEV5DzB9nFfqk5zbQqHdInVe4sfL+npa+4fjLGrBU30Iuvcr+o9paEjzpH5dY48cq6JHqz1RyJ0CQIc2Jr5+sS4eL1ZIjxWlyN1pKMR+4aE2rlDAad56Ad1cytiaHuVvyK/gdtbKiuGroSQhJ1EVfZ60m3NIqnqmpi5Zdsnmzny4VH/d66BcGXxGaGaUaqFn0WTypuwIMZMMtzZEK7peKoaW4H4rfkfdrKcD8AaT+z9v5lLGkTl0NcZZ4LN9sSUzsHNfyAFK6cSXo/73z0tDAlGb5K+yWV6UHoYW1rcoIsxlNRZM6/6FYgMXbbfow==
XbF4O6v6oadEQGtdpQ7d54Q2JB9/ZEXEUH3S1FMn4E/PSqk7HLXjG4tNfuiUBa5eS8kYV49gwC8Yr+mn6YUAHt+K9lHPSsmltWoiHNOaas4aqai9nlyeft4TYYhP+GYbQfw+3n2TcO39s6M0vw0m0a8AX9JfF02JwCUhP4bu4dzG6Bl5dj000TbUkric14Jyurp8OHmmMvKW62TvXPhNOW39+wS1Qkfn9Bxmzi8UEVSVe3wP45JWZNgmgeGnpubDhD05FJEDErfVtZ/DRKD81q5YRd4X4cCkeDPDcJLgKW1jkCsA7yBqESXPDSkkrVUM06A9qMFUwk4mRI88fZ8ryQ==
I'm asking me how to verify it in php?
I tryed something like:
$rsa = new Crypt_RSA();
$rsa->loadKey('qH/J3/gvF/h5U02uPyC9Qzn/hHEV5DzB9nFfqk5zbQqHdInVe4sfL+npa+4fjLGrBU30Iuvcr+o9paEjzpH5dY48cq6JHqz1RyJ0CQIc2Jr5+sS4eL1ZIjxWlyN1pKMR+4aE2rlDAad56Ad1cytiaHuVvyK/gdtbKiuGroSQhJ1EVfZ60m3NIqnqmpi5Zdsnmzny4VH/d66BcGXxGaGaUaqFn0WTypuwIMZMMtzZEK7peKoaW4H4rfkfdrKcD8AaT+z9v5lLGkTl0NcZZ4LN9sSUzsHNfyAFK6cSXo/73z0tDAlGb5K+yWV6UHoYW1rcoIsxlNRZM6/6FYgMXbbfow=='); // public key
echo $rsa->verify('message', 'XbF4O6v6oadEQGtdpQ7d54Q2JB9/ZEXEUH3S1FMn4E/PSqk7HLXjG4tNfuiUBa5eS8kYV49gwC8Yr+mn6YUAHt+K9lHPSsmltWoiHNOaas4aqai9nlyeft4TYYhP+GYbQfw+3n2TcO39s6M0vw0m0a8AX9JfF02JwCUhP4bu4dzG6Bl5dj000TbUkric14Jyurp8OHmmMvKW62TvXPhNOW39+wS1Qkfn9Bxmzi8UEVSVe3wP45JWZNgmgeGnpubDhD05FJEDErfVtZ/DRKD81q5YRd4X4cCkeDPDcJLgKW1jkCsA7yBqESXPDSkkrVUM06A9qMFUwk4mRI88fZ8ryQ==') ? 'verified' : 'unverified';
I think the signature and/or public key are not formated correctly for php. Any idea?
Thank you in advance,
[EDIT]
I'm not sure the signature is correct. If I use the js function cryptico.b64to16(signature), the signature will be somethink like :
5db1783babfaa1a744406b5da50edde78436241f7f6445c4507dd2d45327e04fcf4aa93b1cb5e31b8b4d7ee89405ae5e4bc918578f60c02f18afe9a7e985001edf8af651cf4ac9a5b56a221cd39a6ace1aa9a8bd9e5c9e7ede1361884ff8661b41fc3ede7d9370edfdb3a334bf0d26d1af005fd25f174d89c025213f86eee1dcc6e81979763d34d136d492b89cd78272baba7c3879a632f296eb64ef5cf84d396dfdfb04b54247e7f41c66ce2f141154957b7c0fe3925664d82681e1a7a6e6c3843d3914910312b7d5b59fc344a0fcd6ae5845de17e1c0a47833c37092e0296d63902b00ef206a1125cf0d2924ad550cd3a03da8c154c24e26448f3c7d9f2bc9
I am not sure about the format of the param key of $rsa->verify. I tryed to add the prefix ssh-rsa. But it do not works better.
So I tryed the to signature format and the to key. The message is each time "unverified"

Thanks #neubert, it's du to PSS signature.
So here is the solution.
I use:
phpseclib : PHP lib used to validate message
jsrsasign : JS lib used to sign message
jsencrypt : JS lib to create private and public key
First, generate the keys in JS:
crypt = new JSEncrypt({default_key_size: 512});
var key = crypt.getKey();
var publicKey = key.getPublicKey();
var privateKey = key.getPrivateKey();
Secondly, create the signature:
var rsa = new RSAKey();
rsa.readPrivateKeyFromPEMString(privateKey);
var hSig = rsa.signStringPSS('message', 'sha1');
var signature = linebrk(hSig, 64);
console.log(signature);
By default the signature is not in the good format. We have to encode hSig in base 64 with the function base64Chars
var base64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
b16to64 = function(h) {
var i;
var c;
var ret = "";
if(h.length % 2 == 1)
{
h = "0" + h;
}
for (i = 0; i + 3 <= h.length; i += 3)
{
c = parseInt(h.substring(i, i + 3), 16);
ret += base64Chars.charAt(c >> 6) + base64Chars.charAt(c & 63);
}
if (i + 1 == h.length)
{
c = parseInt(h.substring(i, i + 1), 16);
ret += base64Chars.charAt(c << 2);
}
else if (i + 2 == h.length)
{
c = parseInt(h.substring(i, i + 2), 16);
ret += base64Chars.charAt(c >> 2) + base64Chars.charAt((c & 3) << 4);
}
while ((ret.length & 3) > 0) ret += "=";
return ret;
}
To finish we validate in PHP. We assume the signature and the public key are stored in the vars with the same name:
$rsa = new Crypt_RSA();
$rsa->loadKey($publickey); // public key;
echo $rsa->verify('message', base64_decode($signature)) ? 'verified' : 'unverified';

How to convert a very large hex number to decimal in javascript

I am trying without much success to convert a very large hex number to decimal.
My problem is that using deciaml = parseInt(hex, 16)
gives me errors in the number when I try to convert a hex number above 14 digits.
I have no problem with this in Java, but Javascript does not seem to be accurate above 14 digits of hex.
I have tried "BigNumber" but tis gives me the same erroneous result.
I have trawled the web to the best of my ability and found web sites that will do the conversion but cannot figure out how to do the conversion longhand.
I have tried getting each character in turn and multiplying it by its factor i.e. 123456789abcdef
15 * Math.pow(16, 0) + 14 * Math.pow(16, 1).... etc but I think (being a noob) that my subroutines may not hev been all they should be because I got a completely (and I mean really different!) answer.
If it helps you guys I can post what I have written so far for you to look at but I am hoping someone has simple answer for me.
<script>
function Hex2decimal(hex){
var stringLength = hex.length;
var characterPosition = stringLength;
var character;
var hexChars = new Array();
hexChars[0] = "0";
hexChars[1] = "1";
hexChars[2] = "2";
hexChars[3] = "3";
hexChars[4] = "4";
hexChars[5] = "5";
hexChars[6] = "6";
hexChars[7] = "7";
hexChars[8] = "8";
hexChars[9] = "9";
hexChars[10] = "a";
hexChars[11] = "b";
hexChars[12] = "c";
hexChars[13] = "d";
hexChars[14] = "e";
hexChars[15] = "f";
var index = 0;
var hexChar;
var result;
// document.writeln(hex);
while (characterPosition >= 0)
{
// document.writeln(characterPosition);
character = hex.charAt(characterPosition);
while (index < hexChars.length)
{
// document.writeln(index);
document.writeln("String Character = " + character);
hexChar = hexChars[index];
document.writeln("Hex Character = " + hexChar);
if (hexChar == character)
{
result = hexChar;
document.writeln(result);
}
index++
}
// document.write(character);
characterPosition--;
}
return result;
}
</script>
Thank you.
Paul

The New 'n' Easy Way
var hex = "7FDDDDDDDDDDDDDDDDDDDDDD";
if (hex.length % 2) { hex = '0' + hex; }
var bn = BigInt('0x' + hex);
var d = bn.toString(10);
BigInts are now available in most browsers (except IE).
Earlier in this answer:
BigInts are now available in both node.js and Chrome. Firefox shouldn't be far behind.
If you need to deal with negative numbers, that requires a bit of work:
How to handle Signed JS BigInts
Essentially:
function hexToBn(hex) {
if (hex.length % 2) {
hex = '0' + hex;
}
var highbyte = parseInt(hex.slice(0, 2), 16)
var bn = BigInt('0x' + hex);
if (0x80 & highbyte) {
// You'd think `bn = ~bn;` would work... but it doesn't
// manually perform two's compliment (flip bits, add one)
// (because JS binary operators are incorrect for negatives)
bn = BigInt('0b' + bn.toString(2).split('').map(function (i) {
return '0' === i ? 1 : 0
}).join('')) + BigInt(1);
bn = -bn;
}
return bn;
}

Ok, let's try this:
function h2d(s) {
function add(x, y) {
var c = 0, r = [];
var x = x.split('').map(Number);
var y = y.split('').map(Number);
while(x.length || y.length) {
var s = (x.pop() || 0) + (y.pop() || 0) + c;
r.unshift(s < 10 ? s : s - 10);
c = s < 10 ? 0 : 1;
}
if(c) r.unshift(c);
return r.join('');
}
var dec = '0';
s.split('').forEach(function(chr) {
var n = parseInt(chr, 16);
for(var t = 8; t; t >>= 1) {
dec = add(dec, dec);
if(n & t) dec = add(dec, '1');
}
});
return dec;
}
Test:
t = 'dfae267ab6e87c62b10b476e0d70b06f8378802d21f34e7'
console.log(h2d(t))
prints
342789023478234789127089427304981273408912349586345899239
which is correct (feel free to verify).

Notice that "0x" + "ff" will be considered as 255, so convert your hex value to a string and add "0x" ahead.
function Hex2decimal(hex)
{
return ("0x" + hex) / 1;
}

If you are using the '0x' notation for your Hex String, don't forget to add s = s.slice(2) to remove the '0x' prefix.

Keep in mind that JavaScript only has a single numeric type (double), and does not provide any separate integer types. So it may not be possible for it to store exact representations of your numbers.
In order to get exact results you need to use a library for arbitrary-precision integers, such as BigInt.js. For example, the code:
var x = str2bigInt("5061756c205768697465",16,1,1);
var s = bigInt2str(x, 10);
$('#output').text(s);
Correctly converts 0x5061756c205768697465 to the expected result of 379587113978081151906917.
Here is a jsfiddle if you would like to experiment with the code listed above.

The BigInt constructor can take a hex string as argument:
/** #param hex = "a83b01cd..." */
function Hex2decimal(hex) {
return BigInt("0x" + hex).toString(10);
}
Usage:
Hex2decimal("100");
Output:
256
A rip-off from the other answer, but without the meaningless 0 padding =P