I have base64 encoded binary data(~2.5MB). The data is consisting of c double value array. In c language it comes with no cost using this array - you just read the whole data into memory and use double* address to access any element.
However, now I need to use this data in javascript. I'm looking for most efficient way to be able to read this array from js code.
You could use this function in Node:
function doublesFromBase64(base64) {
let buff = Buffer.from(base64, 'base64');
let result = [];
for (let i = 0; i < buff.length; i += 8) {
result.push(buff.readDoubleLE(i));
}
return result;
}
To test it, I produced a sample base64 encoded string from the following piece of C code, which uses a function base64_encode found in this answer:
double arr[] = {15.9234, 2.05e30, -19};
size_t output_length;
char * result = base64_encode((unsigned char *) arr, 3* sizeof(double), &output_length);
printf("%s", result);
This outputs:
uECC4sfYL0Cjw3dB6N85RgAAAAAAADPA
I passed this string to the function at the top:
function doublesFromBase64(base64) {
let buff = Buffer.from(base64, 'base64');
let result = [];
for (let i = 0; i < buff.length; i += 8) {
result.push(buff.readDoubleLE(i));
}
return result;
}
let result = doublesFromBase64("uECC4sfYL0Cjw3dB6N85RgAAAAAAADPA");
console.log(result);
This outputs:
[ 15.9234, 2.05e+30, -19 ]
Related
Hi one of my trainer asked me to solve this mysterious question which is related to hashing, and this question is so uncommon that i'm not able to find any good source to solve and learn logic behind it.
Given a string of bytes, which when encoded in hexadecimal notation look like this:
f064b8b61422a3456cb273a474a1fb0cabb04200a6a82a9426bd01f56c97fbf8c4ef58634fd5cf21af29e7db3406de4f886fe71408696789f853af9932a84b79
Find a 4-byte prefix so that, a SHA256 hash of the prefix combined with the original string of bytes, has two last bytes as 0xca, 0xfe.
I tried an attempt. I work with strings I'm sure it's not the most efficient. But Let's see if it works.
async function sha256(message) {
const msgBuffer = new TextEncoder().encode(message);
const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer);
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
return hashHex;
}
function hexToString(hex) {
for (var bytes = [], c = 0; c < hex.length; c += 2) {
bytes.push(parseInt(hex.substr(c, 2), 16));
}
return bytes.map(function(byte) {
return String.fromCharCode(byte)
}).join("")
// return bytes;
}
function hex32(val) {
val &= 0xFFFFFFFF;
var hex = val.toString(16).toUpperCase();
return ("00000000" + hex).slice(-8);
}
var str = "f064b8b61422a3456cb273a474a1fb0cabb04200a6a82a9426bd01f56c97fbf8c4ef58634fd5cf21af29e7db3406de4f886fe71408696789f853af9932a84b79";
var original = hexToString(str);
var found = false;
async function search() {
for (var i = 0; i < 0xffffffff; i++) {
var prefix = hexToString(hex32(i))
await sha256(prefix + original).then(function(result) {
// console.log(result.slice(-4))
if (result.slice(-4) == 'cafe') {
console.log(`found a prefix: "${prefix}"`);
found = true;
}
});
if (found) {
break;
}
}
}
search();
I am working on building a parser for a text file to build into a JSON object.
Presuming I have a string somewhere in the line which is a unique value: 4E183437F3FAEBC107C333CD16657661598A2A523867A4A95F6DAD4D25E191722AFFB3FFE29E287D9EB21ED8035E7666D06E65EC064D5D4B337C034459BA7CD6F72437B439329B45F15D57176122965728B36DBB455D335AA904106A29B64694507B6A0F89FBF4A166BB3BBB4DDDC1674A1DB3B7D0FD332117877C4CE56465D8
I want to use this unique identifier as a key at some level, but it's much too long to suit my tastes.
I am wondering if there is a way to create a smaller 'hash':
but not lose and information
guarantee uniqueness
be converted back to it's original
Some library/plugin?
You can create a string of smaller length by turning the 16-bit hex string into a number (with BigInt), and then by creating a new string with base, say, 10000, so that a character with character code C represents the number 'C'.charCodeAt().
Then turn it back into the original string by doing the same process in reverse.
The string length is shorter, and you've saved some bits, but it's still long:
const encode = (input) => {
let n = 0n;
[...input].forEach((char, i) => {
const code = char.charCodeAt();
const value = BigInt(code <= 57 ? code - 48 : code - 55);
n += value * (16n ** BigInt(i));
});
const base10Str = String(n);
let output = '';
for (let i = 0; i < base10Str.length; i += 4) {
output += String.fromCharCode(base10Str.slice(i, i + 4));
}
return output;
};
const decode = (encoded) => {
let base10Str = '';
[...encoded].forEach((char) => {
base10Str += String(char.charCodeAt()).padStart(4, '0'); // todo: final char may need no padding
});
let n = BigInt(base10Str);
let inputStr = '';
for (let i = 255; i >= 0; i--) {
const thisHexMult = 16n ** BigInt(i);
const thisHexDigitBigInt = n / thisHexMult; // This will automatically effectively call Math.floor
n = n % thisHexMult;
const thisHexDigit = Number(thisHexDigitBigInt);
const charCode = thisHexDigit <= 9 ? thisHexDigit + 48 : thisHexDigit + 55;
inputStr = String.fromCharCode(charCode) + inputStr;
}
console.log(inputStr);
};
const encoded = encode('4E183437F3FAEBC107C333CD16657661598A2A523867A4A95F6DAD4D25E191722AFFB3FFE29E287D9EB21ED8035E7666D06E65EC064D5D4B337C034459BA7CD6F72437B439329B45F15D57176122965728B36DBB455D335AA904106A29B64694507B6A0F89FBF4A166BB3BBB4DDDC1674A1DB3B7D0FD332117877C4CE56465D8');
console.log('Encoded length:', encoded.length);
console.log(encoded);
decode(encoded);
i have such a code to generate password written in Java
MessageDigest messageDigestPassword = MessageDigest.getInstance("SHA1");
messageDigestPassword .reset();
byte[] password = "password".getBytes();
messageDigestPassword .update(password);
byte[] encryptedPassword = messageDigestPassword .digest();
String date = "2019-10-22T11:33:13.393Z";
byte[] dateBytes = date.getBytes(StandardCharsets.UTF_8);
int offset = 0;
byte[] outputBytes = new byte[dateBytes.length + encryptedPassword .length];
System.arraycopy(dateBytes, 0, outputBytes, offset, dateBytes.length);
offset += dateBytes.length;
System.arraycopy(encryptedPassword , 0, outputBytes, offset, encryptedPassword .length);
MessageDigest finalMessageDigeset = MessageDigest.getInstance("SHA-1");
finalMessageDigeset.reset();
finalMessageDigeset.update(outputBytes);
byte[] finalPasswordBytes= finalMessageDigeset .digest();
String finalBase64Password = new String(Base64.encode(finalPasswordBytes));
and im trying to rewrite it to JavaScript to use it in postman with - CryptoJS
So far i have :
function wordArrayToByteArray(wordArray, length) {
if (wordArray.hasOwnProperty("sigBytes") &&
wordArray.hasOwnProperty("words")) {
length = wordArray.sigBytes;
wordArray = wordArray.words;
}
var result = [],
bytes,
i = 0;
while (length > 0) {
bytes = wordToByteArray(wordArray[i], Math.min(4, length));
length -= bytes.length;
result.push(bytes);
i++;
}
return [].concat.apply([], result);
}
function stringToBytes ( str ) {
var ch, st, re = [];
for (var i = 0; i < str.length; i++ ) {
ch = str.charCodeAt(i); // get char
st = []; // set up "stack"
do {
st.push( ch & 0xFF ); // push byte to stack
ch = ch >> 8; // shift value down by 1 byte
}
while ( ch );
// add stack contents to result
// done because chars have "wrong" endianness
re = re.concat( st.reverse() );
}
// return an array of bytes
return re;
}
var dateFixed = "2019-10-22T11:33:13.393Z";
var fixedDateBytes = stringToBytes(dateFixed);
var sha1Password= CryptoJS.SHA1("password");
console.log("sha1Password",sha1Password.toString(CryptoJS.enc.Hex));
var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20);
var concatedBytes= fixedDateBytes.concat(sha1PasswordBytes);
var finalShaPassWords= CryptoJS.SHA1(concatedBytes);
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Hex));
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Base64));
However unfortunatelly Base64 representations written in those 2 languages doesnt match.
I have checked and bytes from date are equal. Bytes from hashed password are not. So hashing after concat fails in JavaScript.
I have checked first password hashing and generated bytes and both of them are the same. So my guess line var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20); causes that line var finalShaPassWords= CryptoJS.SHA1(concatedBytes); returns bad value.
Can someone give me some idea what is wrong? Mayby it should be written diffrent ?
Since you are using CryptoJS anyway, you can also use the CryptoJS encoders and the WordArray#concat-method, which considerably simplifies the code:
var CryptoJS = require("crypto-js");
// Input
var inPwd = "password";
var inDate = "2019-10-22T11:33:13.393Z";
// Processing
var pwdHash = CryptoJS.SHA1(inPwd); // hash and convert to WordArray
var date = CryptoJS.enc.Utf8.parse(inDate); // convert to WordArray
var joinedData = date.clone().concat(pwdHash); // join date and hashed password
var joinedDataHash = CryptoJS.SHA1(joinedData); // hash joined data
var joinedDataHashB64 = CryptoJS.enc.Base64.stringify(joinedDataHash); // convert to Base64 string
// Output
console.log("Result: " + joinedDataHashB64 ); // Output: D235TBTZMfpSyB/CDl5MHAjH5fI=
The output of this code is the same as the output of the Java-code: D235TBTZMfpSyB/CDl5MHAjH5fI=
I need to convert a 256-bit ArrayBuffer into 24 11-bit values and then back again.
Is there a simple code snippet to handle this type of operation.
I have this version, to convert it to 24 11 bit values.
var newBuffer = new Uint8Array(data);
var result = [];
for (var i =0, l = 24;i<l;i++){
var index = parseInt((i*11.0)/8.0);
var subBuffer;
if (i==23){
var proxyBuffer = new Uint8Array(2);
proxyBuffer.set(newBuffer.slice(index,index+1));
subBuffer = proxyBuffer;
}else{
subBuffer = newBuffer.slice(index,index+2);
}
var value = new Uint16Array(subBuffer.buffer);
value = value >> (i*3)%8;
value = value % 2048;
result.push(value);
}
console.log(result);
Using bit operations can simplify the conversion process - using parseInt and decimal arithmetic is not an easy approach.
The concept code below uses plain arrays of octet and 11 bit values. While Uint8Array and Uint16Array types may be a better choice, creating typed arrays and/or converting an arrayBuffer to and from a suitable array type is not included.
function ui8To11( buffer8) {
var buffer11 = [];
var acc = 0;
var accBits = 0;
function add( octet) {
acc = (octet << accBits) | acc;
accBits += 8;
if( accBits >=11) {
buffer11.push( acc & 0x7ff);
acc >>= 11;
accBits -= 11;
}
}
function flush() {
if( accBits) {
buffer11.push( acc);
}
}
buffer8.forEach( add);
flush();
return buffer11;
}
function ui11To8( buffer11) {
var buffer8 = [];
var acc = 0;
var accBits = 0;
function add( ui11) {
acc = (ui11 << accBits) | acc;
accBits += 11;
while( accBits >= 8) {
buffer8.push( acc & 0xff);
acc >>= 8;
accBits -= 8;
}
}
function flush() {
if( accBits) {
buffer8.push( acc);
}
}
buffer11.forEach( add);
flush();
return buffer8;
}
var buffer8 = [1,2,3]; // 8 bit values, least significant octet at index 0
console.log("octets: ", buffer8);
var buffer11 = ui8To11( buffer8);
console.log("undectets: ", buffer11);
var reconstructed = ui11To8( buffer11)
console.log("convertedBack", reconstructed);
There is an assumption here that the input array is little-endian, as in each entry in the input array is more significant than the previous entry.
Conversion between 8 and 11 bit values and back again follows a similar pattern, but pushing bits from the accumulator to the output array requires a loop when converting from a higher number of bits to a lower.
The example takes 3 x 8 bit values (24 bits in total) and produces 3 x 11 bit values (33 bits in total). Converting back 33 bits to uint8 integers produces 5 x 8 bit values ( 40 bits). You may need to add code to limit the number of integers pushed into output arrays within conversion routines or truncate output arrays returned as required.
There is a library called Uint1Array which makes everything much easier.
var arr = new Uint1Array(buffer);
for (let i=0, l=arr.length; i<l; i+=11){
var zero = new Uint1Array(16);
for (let index =0, length = 11; index<length;index++){
zero[index]=arr[i+index];
}
let bit16 = new Uint16Array(zero.buffer)[0];
outPut.push(bit16);
}
console.log(outPut);
var bit256 = new Uint1Array(256);
for (let i=0, l=outPut.length;i<l;i++){
var hold = new Uint16Array(1);
hold[0]=outPut[i];
let bit16 = new Uint1Array(hold.buffer);
let bit11 = bit16.slice(0,11);
for (let i2=0, l2=11;i2<l2;i2++){
bit256[(i*11)+i2]=bit11[i2];
}
}
var hex1 = "B1C85C061C98E713DEF0E2EDDDDB432738674C9F8962F09B75E943D55F9FB39F";
var hex2 = "121B0D3327A21B8048FC7CA6FD07AACC0D8DF59B99DB098686696573E3686E6C";
var result = hex1 ^ hex2; //XOR the values
console.log(result); // outputs: 0 which does not sound good.
Any ideas how to perform XOR operations on hex values?
Bitwise operations in JavaScript only work on numeric values.
You should parseInt(hexString, 16) your hex string before. Specifically in your case this wouldn't work because your hex is too big for a number. You would have to create your own customized XOR function.
Take a look at this link: How to convert hex string into a bytes array, and a bytes array in the hex string?
The resulting bytearray will be ellegible for a manual XOR. Byte by byte. Maybe this will help: Java XOR over two arrays.
If you are on Nodejs, you could transform the hex strings to Buffers then use map to build the resulting string.
function xor(hex1, hex2) {
const buf1 = Buffer.from(hex1, 'hex');
const buf2 = Buffer.from(hex2, 'hex');
const bufResult = buf1.map((b, i) => b ^ buf2[i]);
return bufResult.toString('hex');
}
str = 'abc';
c = '';
key = 'K';
for(i=0; i<str.length; i++) {
c += String.fromCharCode(str[i].charCodeAt(0).toString(10) ^ key.charCodeAt(0).toString(10)); // XORing with letter 'K'
}
return c;
Output of string 'abc':
"*)("
You can use a function like this.
function xor(a, b) {
if (!Buffer.isBuffer(a)) a = new Buffer(a)
if (!Buffer.isBuffer(b)) b = new Buffer(b)
var res = []
if (a.length > b.length) {
for (var i = 0; i < b.length; i++) {
res.push(a[i] ^ b[i])
}
} else {
for (var i = 0; i < a.length; i++) {
res.push(a[i] ^ b[i])
}
}
return new Buffer(res);
}
Source: https://github.com/czzarr/node-bitwise-xor
Below is function that takes in 2 strings like "041234FFFFFFFFFF" and "0000000709000003" (a classic example of pin block and card block)
Expected result from the above 2 strings is "041234F8F6FFFFFC"
function bitwiseXorHexString(pinBlock1, pinBlock2) {
var result = ''
for (let index = 0; index < 16; index++) {
const temp = (parseInt(pinBlock1.charAt(index), 16) ^ parseInt(pinBlock2.charAt(index), 16)).toString(16).toUpperCase()
result += temp
}
return result
}
Note: This was made to xor 2 strings of fixed length 16. You may modify it as per your needs.