We are trying to decrypt large files (1GB) in browsers. With AES-CTR it should be possible to decrypt chunk by chunk - where chunk must be the correct size and you also have to provide nonce + counter.
Does anyone have any examples or ideas how to do this in javascript?
What we tried so far:
var length = value.byteLength;
var chunkSize = 128;
var index = 0;
let chunks = [];
let aesCounter = byteArrayToLong(subtleIv);
do {
let newCount = aesCounter + index / 16;
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
index += chunkSize;
} while(index < length);
let newCount = aesCounter + index / 16;
decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
let decryptedAll = Buffer.concat(chunks);
function longToByteArray(/*long*/long) {
var byteArray = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
for ( var index = 0; index < byteArray.length; index ++ ) {
var byte = long & 0xff;
byteArray [ index ] = byte;
long = (long - byte) / 256 ;
}
return byteArray;
}
function byteArrayToLong(/*byte[]*/byteArray) {
var value = 0;
for ( var i = byteArray.length - 1; i >= 0; i--) {
value = (value * 256) + byteArray[i];
}
return value;
}
The only flaw in your implementation is actually the conversion between integer and byte array. Firstly, in JavaScript the maximum integer is 0x1FFFFFFFFFFFFF, see here, secondly, even with smaller numbers the little endian order is used, but the WebCrypto API applies the big endian order.
As a first step to a fix you could use e.g. the BigInt implementation of JavaScript and the here described conversion between BigInt and ArrayBuffer.
Since this implementation works with ArrayBuffer and Uint8Array respectively, an implementation for concatenation is needed, e.g. from here.
This changes your implementation slightly as follows (key, IV and ciphertext are imported hex encoded):
(async () => {
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var key = hex2ab(keyHex);
var subtleKey = await window.crypto.subtle.importKey(
"raw",
key,
{ name: "AES-CTR" },
false,
["encrypt", "decrypt"]
);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var subtleIv = hex2ab(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertext = hex2ab(ciphertextHex);
// Decrypt and concat
var length = ciphertext.byteLength;
var chunkSize = 128; // chunkSize in bytes
var index = 0;
var chunks = [];
var aesCounter = bufToBn(subtleIv);
do {
var newCount = aesCounter + BigInt(index / 16); // index / 16 = number of blocks
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: bnToBuf(newCount), length: 128}, subtleKey, ciphertext.slice(index, index+chunkSize)); // length in bits
chunks.push(new Uint8Array(decrypted));
index += chunkSize;
} while(index < length);
var mergedChunks = merge(chunks);
// Decode and output
var decrypted = String.fromCharCode.apply(null, mergedChunks);
console.log(decrypted);
// https://coolaj86.com/articles/convert-js-bigints-to-typedarrays/
function bnToBuf(bn) {
var hex = BigInt(bn).toString(16);
if (hex.length % 2) { hex = '0' + hex; }
var len = hex.length / 2;
var u8 = new Uint8Array(len);
var i = 0;
var j = 0;
while (i < len) {
u8[i] = parseInt(hex.slice(j, j+2), 16);
i += 1;
j += 2;
}
return u8;
}
function bufToBn(buf) {
var hex = [];
u8 = Uint8Array.from(buf);
u8.forEach(function (i) {
var h = i.toString(16);
if (h.length % 2) { h = '0' + h; }
hex.push(h);
});
return BigInt('0x' + hex.join(''));
}
// https://stackoverflow.com/a/49129872/9014097
function merge(chunks){
let size = 0;
chunks.forEach(item => {
size += item.length;
});
let mergedArray = new Uint8Array(size);
let offset = 0;
chunks.forEach(item => {
mergedArray.set(item, offset);
offset += item.length;
});
return mergedArray;
}
function hex2ab(hex){
return new Uint8Array(hex.match(/[\da-f]{2}/gi).map(function (h) {
return parseInt(h, 16)}));
}
})();
which successfully decrypts the ciphertext. Btw, the ciphertext was generated with CyberChef.
Unlike the WebCrypto API, CryptoJS supports progressive encryption, so the same logic can be implemented significantly easier with CryptoJS:
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var keyWA = CryptoJS.enc.Hex.parse(keyHex);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var ivWA = CryptoJS.enc.Hex.parse(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertextWA = CryptoJS.enc.Hex.parse(ciphertextHex);
// Decrypt and concat
var length = ciphertextWA.sigBytes;
var chunkSize = 128;
var index = 0;
var decryptedWA = CryptoJS.enc.Hex.parse("");
var aesDecryptor = CryptoJS.algo.AES.createDecryptor(keyWA, { iv: ivWA, mode: CryptoJS.mode.CTR, padding: CryptoJS.pad.NoPadding });
var chunk = null;
do {
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
index += chunkSize;
} while(index < length - chunkSize);
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
chunk.sigBytes = length - index;
chunk.clamp();
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
decryptedWA = decryptedWA.concat(aesDecryptor.finalize());
// Decode and output
var decrypted = decryptedWA.toString(CryptoJS.enc.Utf8);
console.log(decrypted);
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js"></script>
A drawback of the first variant is that the BigInt class should actually not be used in the context of cryptography, since the operations are not constant-time, which results in a vulnerability to timing attacks. So, here you would have to apply a cryptographically more secure JavaScript BigInteger implementation for production.
For such reasons, the use of an established library (as opposed to a custom implementation), such as CryptoJS, is generally more secure (although ultimately vulnerabilities cannot be ruled out here either).
You are well on your way; cutting the ciphertext in chunks of C-blocks and then updating the counter block according to the block count is exactly what you should be doing.
However, note that the counter block is 128 bits: the longToByteArray should be called createCounterBlock(nonce, counter). The nonce should be contained in the initial/leftmost (8?) bytes. The counter is encoded as unsigned big endian in the final/rightmost 8 bytes, making 16 bytes total. This you can then provide as an IV to decrypt the chunk.
Currently your counter is placed in the leftmost bytes, which will fail for most CTR implementations; I don't think that JS uses 128 bit integers. Officially the contents of the counter block is not defined, but generally a big endian 128 bit value is used.
I am from c# so know nothing about java script.
I have excel file (xlsx) that I red into byte array (with unity3d c# in webGL build) and want to send it into java script function that parse it into csv structure and return as string.
So the question part is only related to java script that received xlsx as byte array(or any type from memory stream) and return csv as string.
I need that function. What else (libs) do I need for that?
(Update)
The javascript code is
MyConverterXlsxToCsvReturn: function (array,size) {
var buffer = new ArrayBuffer(size);
for (var i = 0; i < size; i++)
buffer[i] = HEAPU8[array + i];
var txt = XLSX.utils.sheet_to_txt(buffer, {type: 'arraybuffer'});
window.alert(Pointer_stringify(txt));
window.alert(Pointer_stringify(txt.length));
var returnStr = Pointer_stringify(txt);
var bufferSize = lengthBytesUTF8(returnStr) + 1;
var buffer = _malloc(bufferSize);
stringToUTF8(returnStr, buffer, bufferSize);
return buffer;
},
I am trying to send byte[] and convert into arraybuffer but in search of correct way to do that.
For now that function return empty string.
I wanted to convert byte array that I received in C# and then red the array in javascript.
As solution I converted the the byte array into hex string with method:
private string ByteArrayToString(byte[] ba)
{
StringBuilder hex = new StringBuilder(ba.Length * 2);
foreach (byte b in ba)
hex.AppendFormat("{0:x2}", b);
return hex.ToString();
}
Then this string i sent to javascript function and converted to array. The rezult was returned as string:
ExcelHexToCSV: function (hexStr) {
console.log("javascript: ExcelHexToCSV");
console.log("javascript received: " + Pointer_stringify(hexStr));
// convert part
var str = Pointer_stringify(hexStr);
var a = [];
for (var i = 0, len = str.length; i < len; i += 2) {
a.push(parseInt(str.substr(i, 2), 16));
}
var data = new Uint8Array(a);
console.log("javascript hex_to_byte:" + data);
// excel part
var workbook = XLSX.read(data, {type: "array"});
var sheetname = workbook.SheetNames[0];
console.log("javascript sheetname: " + sheetname);
var sheetdata = XLSX.utils.sheet_to_csv(workbook.Sheets[sheetname]);
console.log("javascript sheetdata: = " + sheetdata);
var rezult = sheetdata;
var returnStr = rezult;
var bufferSize = lengthBytesUTF8(returnStr) + 1;
var buffer = _malloc(bufferSize);
stringToUTF8(returnStr, buffer, bufferSize);
return buffer;
},
Github link for the my example project
i have such a code to generate password written in Java
MessageDigest messageDigestPassword = MessageDigest.getInstance("SHA1");
messageDigestPassword .reset();
byte[] password = "password".getBytes();
messageDigestPassword .update(password);
byte[] encryptedPassword = messageDigestPassword .digest();
String date = "2019-10-22T11:33:13.393Z";
byte[] dateBytes = date.getBytes(StandardCharsets.UTF_8);
int offset = 0;
byte[] outputBytes = new byte[dateBytes.length + encryptedPassword .length];
System.arraycopy(dateBytes, 0, outputBytes, offset, dateBytes.length);
offset += dateBytes.length;
System.arraycopy(encryptedPassword , 0, outputBytes, offset, encryptedPassword .length);
MessageDigest finalMessageDigeset = MessageDigest.getInstance("SHA-1");
finalMessageDigeset.reset();
finalMessageDigeset.update(outputBytes);
byte[] finalPasswordBytes= finalMessageDigeset .digest();
String finalBase64Password = new String(Base64.encode(finalPasswordBytes));
and im trying to rewrite it to JavaScript to use it in postman with - CryptoJS
So far i have :
function wordArrayToByteArray(wordArray, length) {
if (wordArray.hasOwnProperty("sigBytes") &&
wordArray.hasOwnProperty("words")) {
length = wordArray.sigBytes;
wordArray = wordArray.words;
}
var result = [],
bytes,
i = 0;
while (length > 0) {
bytes = wordToByteArray(wordArray[i], Math.min(4, length));
length -= bytes.length;
result.push(bytes);
i++;
}
return [].concat.apply([], result);
}
function stringToBytes ( str ) {
var ch, st, re = [];
for (var i = 0; i < str.length; i++ ) {
ch = str.charCodeAt(i); // get char
st = []; // set up "stack"
do {
st.push( ch & 0xFF ); // push byte to stack
ch = ch >> 8; // shift value down by 1 byte
}
while ( ch );
// add stack contents to result
// done because chars have "wrong" endianness
re = re.concat( st.reverse() );
}
// return an array of bytes
return re;
}
var dateFixed = "2019-10-22T11:33:13.393Z";
var fixedDateBytes = stringToBytes(dateFixed);
var sha1Password= CryptoJS.SHA1("password");
console.log("sha1Password",sha1Password.toString(CryptoJS.enc.Hex));
var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20);
var concatedBytes= fixedDateBytes.concat(sha1PasswordBytes);
var finalShaPassWords= CryptoJS.SHA1(concatedBytes);
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Hex));
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Base64));
However unfortunatelly Base64 representations written in those 2 languages doesnt match.
I have checked and bytes from date are equal. Bytes from hashed password are not. So hashing after concat fails in JavaScript.
I have checked first password hashing and generated bytes and both of them are the same. So my guess line var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20); causes that line var finalShaPassWords= CryptoJS.SHA1(concatedBytes); returns bad value.
Can someone give me some idea what is wrong? Mayby it should be written diffrent ?
Since you are using CryptoJS anyway, you can also use the CryptoJS encoders and the WordArray#concat-method, which considerably simplifies the code:
var CryptoJS = require("crypto-js");
// Input
var inPwd = "password";
var inDate = "2019-10-22T11:33:13.393Z";
// Processing
var pwdHash = CryptoJS.SHA1(inPwd); // hash and convert to WordArray
var date = CryptoJS.enc.Utf8.parse(inDate); // convert to WordArray
var joinedData = date.clone().concat(pwdHash); // join date and hashed password
var joinedDataHash = CryptoJS.SHA1(joinedData); // hash joined data
var joinedDataHashB64 = CryptoJS.enc.Base64.stringify(joinedDataHash); // convert to Base64 string
// Output
console.log("Result: " + joinedDataHashB64 ); // Output: D235TBTZMfpSyB/CDl5MHAjH5fI=
The output of this code is the same as the output of the Java-code: D235TBTZMfpSyB/CDl5MHAjH5fI=
I writing a client which have to transmit files to a REST webservice.
I take the File object from the input control, then I use FileReader object to convert it in base64 (String). I noticed that the base64 string weight is 30-40% over the original file size.
If I convert that string in binary size literally explodes, so I was wandering if there is a more efficient way to convert in string this file and send over HTTP.
function toBase64() {
var file = document.querySelector('#base64').files[0];
var reader = new FileReader();
reader.addEventListener("load", function () {
var bin = base64toBin(reader.result);
var base64 = reader.result;
console.log(bin, base64);
});
if (file) {
reader.readAsDataURL(file);
}
}
function base64toBin(data) {
var binArray = []
var datEncode = "";
for (let i=0; i < data.length; i++) {
binArray.push(data[i].charCodeAt(0).toString(2));
}
for (let j=0; j < binArray.length; j++) {
var pad = padding_left(binArray[j], '0', 8);
datEncode += pad + ' ';
}
function padding_left(s, c, n) { if (! s || ! c || s.length >= n) {
return s;
}
var max = (n - s.length)/c.length;
for (var i = 0; i < max; i++) {
s = c + s; } return s;
}
return binArray;
}
Here is a poc:
jsfiddle
I have an array with bytes and need them to be in StringBase64 and I am using the following:
var base64String = btoa(String.fromCharCode.apply(null, new Uint8Array(arrayDigestion)));
console.log('digestionB64_2 .......:' + digestionB64);
with the following result:
digestionB64_2 .......:6d9310a8df39348ef2bbd8a0f04f65bba64180666848526a4c93e86aa69433e7
And I've also used the following code and get the same results no difference whatsoever:
function arrayBufferToString(buffer) {
var binary = '';
var bytes = new Uint8Array( buffer );
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode( bytes[ i ] );
}
return binary;
}
As I understand Strings in Base 64 must end with an = or == characters
Would it be valid if I add those characters manually at the end or is it valid to have a StringBase64 without them?