Develop Reference

How to decrypt large files (1 GB) with AES-CTR chunk by chunk in javascript (browsers)?

We are trying to decrypt large files (1GB) in browsers. With AES-CTR it should be possible to decrypt chunk by chunk - where chunk must be the correct size and you also have to provide nonce + counter.
Does anyone have any examples or ideas how to do this in javascript?
What we tried so far:
var length = value.byteLength;
var chunkSize = 128;
var index = 0;
let chunks = [];
let aesCounter = byteArrayToLong(subtleIv);
do {
let newCount = aesCounter + index / 16;
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
index += chunkSize;
} while(index < length);
let newCount = aesCounter + index / 16;
decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
let decryptedAll = Buffer.concat(chunks);
function longToByteArray(/*long*/long) {
var byteArray = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
for ( var index = 0; index < byteArray.length; index ++ ) {
var byte = long & 0xff;
byteArray [ index ] = byte;
long = (long - byte) / 256 ;
}
return byteArray;
}
function byteArrayToLong(/*byte[]*/byteArray) {
var value = 0;
for ( var i = byteArray.length - 1; i >= 0; i--) {
value = (value * 256) + byteArray[i];
}
return value;
}

The only flaw in your implementation is actually the conversion between integer and byte array. Firstly, in JavaScript the maximum integer is 0x1FFFFFFFFFFFFF, see here, secondly, even with smaller numbers the little endian order is used, but the WebCrypto API applies the big endian order.
As a first step to a fix you could use e.g. the BigInt implementation of JavaScript and the here described conversion between BigInt and ArrayBuffer.
Since this implementation works with ArrayBuffer and Uint8Array respectively, an implementation for concatenation is needed, e.g. from here.
This changes your implementation slightly as follows (key, IV and ciphertext are imported hex encoded):
(async () => {
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var key = hex2ab(keyHex);
var subtleKey = await window.crypto.subtle.importKey(
"raw",
key,
{ name: "AES-CTR" },
false,
["encrypt", "decrypt"]
);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var subtleIv = hex2ab(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertext = hex2ab(ciphertextHex);
// Decrypt and concat
var length = ciphertext.byteLength;
var chunkSize = 128; // chunkSize in bytes
var index = 0;
var chunks = [];
var aesCounter = bufToBn(subtleIv);
do {
var newCount = aesCounter + BigInt(index / 16); // index / 16 = number of blocks
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: bnToBuf(newCount), length: 128}, subtleKey, ciphertext.slice(index, index+chunkSize)); // length in bits
chunks.push(new Uint8Array(decrypted));
index += chunkSize;
} while(index < length);
var mergedChunks = merge(chunks);
// Decode and output
var decrypted = String.fromCharCode.apply(null, mergedChunks);
console.log(decrypted);
// https://coolaj86.com/articles/convert-js-bigints-to-typedarrays/
function bnToBuf(bn) {
var hex = BigInt(bn).toString(16);
if (hex.length % 2) { hex = '0' + hex; }
var len = hex.length / 2;
var u8 = new Uint8Array(len);
var i = 0;
var j = 0;
while (i < len) {
u8[i] = parseInt(hex.slice(j, j+2), 16);
i += 1;
j += 2;
}
return u8;
}
function bufToBn(buf) {
var hex = [];
u8 = Uint8Array.from(buf);
u8.forEach(function (i) {
var h = i.toString(16);
if (h.length % 2) { h = '0' + h; }
hex.push(h);
});
return BigInt('0x' + hex.join(''));
}
// https://stackoverflow.com/a/49129872/9014097
function merge(chunks){
let size = 0;
chunks.forEach(item => {
size += item.length;
});
let mergedArray = new Uint8Array(size);
let offset = 0;
chunks.forEach(item => {
mergedArray.set(item, offset);
offset += item.length;
});
return mergedArray;
}
function hex2ab(hex){
return new Uint8Array(hex.match(/[\da-f]{2}/gi).map(function (h) {
return parseInt(h, 16)}));
}
})();
which successfully decrypts the ciphertext. Btw, the ciphertext was generated with CyberChef.
Unlike the WebCrypto API, CryptoJS supports progressive encryption, so the same logic can be implemented significantly easier with CryptoJS:
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var keyWA = CryptoJS.enc.Hex.parse(keyHex);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var ivWA = CryptoJS.enc.Hex.parse(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertextWA = CryptoJS.enc.Hex.parse(ciphertextHex);
// Decrypt and concat
var length = ciphertextWA.sigBytes;
var chunkSize = 128;
var index = 0;
var decryptedWA = CryptoJS.enc.Hex.parse("");
var aesDecryptor = CryptoJS.algo.AES.createDecryptor(keyWA, { iv: ivWA, mode: CryptoJS.mode.CTR, padding: CryptoJS.pad.NoPadding });
var chunk = null;
do {
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
index += chunkSize;
} while(index < length - chunkSize);
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
chunk.sigBytes = length - index;
chunk.clamp();
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
decryptedWA = decryptedWA.concat(aesDecryptor.finalize());
// Decode and output
var decrypted = decryptedWA.toString(CryptoJS.enc.Utf8);
console.log(decrypted);
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js"></script>
A drawback of the first variant is that the BigInt class should actually not be used in the context of cryptography, since the operations are not constant-time, which results in a vulnerability to timing attacks. So, here you would have to apply a cryptographically more secure JavaScript BigInteger implementation for production.
For such reasons, the use of an established library (as opposed to a custom implementation), such as CryptoJS, is generally more secure (although ultimately vulnerabilities cannot be ruled out here either).

You are well on your way; cutting the ciphertext in chunks of C-blocks and then updating the counter block according to the block count is exactly what you should be doing.
However, note that the counter block is 128 bits: the longToByteArray should be called createCounterBlock(nonce, counter). The nonce should be contained in the initial/leftmost (8?) bytes. The counter is encoded as unsigned big endian in the final/rightmost 8 bytes, making 16 bytes total. This you can then provide as an IV to decrypt the chunk.
Currently your counter is placed in the leftmost bytes, which will fail for most CTR implementations; I don't think that JS uses 128 bit integers. Officially the contents of the counter block is not defined, but generally a big endian 128 bit value is used.

Java SHA-1 to javascript using CryptoJS

i have such a code to generate password written in Java
MessageDigest messageDigestPassword = MessageDigest.getInstance("SHA1");
messageDigestPassword .reset();
byte[] password = "password".getBytes();
messageDigestPassword .update(password);
byte[] encryptedPassword = messageDigestPassword .digest();
String date = "2019-10-22T11:33:13.393Z";
byte[] dateBytes = date.getBytes(StandardCharsets.UTF_8);
int offset = 0;
byte[] outputBytes = new byte[dateBytes.length + encryptedPassword .length];
System.arraycopy(dateBytes, 0, outputBytes, offset, dateBytes.length);
offset += dateBytes.length;
System.arraycopy(encryptedPassword , 0, outputBytes, offset, encryptedPassword .length);
MessageDigest finalMessageDigeset = MessageDigest.getInstance("SHA-1");
finalMessageDigeset.reset();
finalMessageDigeset.update(outputBytes);
byte[] finalPasswordBytes= finalMessageDigeset .digest();
String finalBase64Password = new String(Base64.encode(finalPasswordBytes));
and im trying to rewrite it to JavaScript to use it in postman with - CryptoJS
So far i have :
function wordArrayToByteArray(wordArray, length) {
if (wordArray.hasOwnProperty("sigBytes") &&
wordArray.hasOwnProperty("words")) {
length = wordArray.sigBytes;
wordArray = wordArray.words;
}
var result = [],
bytes,
i = 0;
while (length > 0) {
bytes = wordToByteArray(wordArray[i], Math.min(4, length));
length -= bytes.length;
result.push(bytes);
i++;
}
return [].concat.apply([], result);
}
function stringToBytes ( str ) {
var ch, st, re = [];
for (var i = 0; i < str.length; i++ ) {
ch = str.charCodeAt(i); // get char
st = []; // set up "stack"
do {
st.push( ch & 0xFF ); // push byte to stack
ch = ch >> 8; // shift value down by 1 byte
}
while ( ch );
// add stack contents to result
// done because chars have "wrong" endianness
re = re.concat( st.reverse() );
}
// return an array of bytes
return re;
}
var dateFixed = "2019-10-22T11:33:13.393Z";
var fixedDateBytes = stringToBytes(dateFixed);
var sha1Password= CryptoJS.SHA1("password");
console.log("sha1Password",sha1Password.toString(CryptoJS.enc.Hex));
var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20);
var concatedBytes= fixedDateBytes.concat(sha1PasswordBytes);
var finalShaPassWords= CryptoJS.SHA1(concatedBytes);
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Hex));
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Base64));
However unfortunatelly Base64 representations written in those 2 languages doesnt match.
I have checked and bytes from date are equal. Bytes from hashed password are not. So hashing after concat fails in JavaScript.
I have checked first password hashing and generated bytes and both of them are the same. So my guess line var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20); causes that line var finalShaPassWords= CryptoJS.SHA1(concatedBytes); returns bad value.
Can someone give me some idea what is wrong? Mayby it should be written diffrent ?

Since you are using CryptoJS anyway, you can also use the CryptoJS encoders and the WordArray#concat-method, which considerably simplifies the code:
var CryptoJS = require("crypto-js");
// Input
var inPwd = "password";
var inDate = "2019-10-22T11:33:13.393Z";
// Processing
var pwdHash = CryptoJS.SHA1(inPwd); // hash and convert to WordArray
var date = CryptoJS.enc.Utf8.parse(inDate); // convert to WordArray
var joinedData = date.clone().concat(pwdHash); // join date and hashed password
var joinedDataHash = CryptoJS.SHA1(joinedData); // hash joined data
var joinedDataHashB64 = CryptoJS.enc.Base64.stringify(joinedDataHash); // convert to Base64 string
// Output
console.log("Result: " + joinedDataHashB64 ); // Output: D235TBTZMfpSyB/CDl5MHAjH5fI=
The output of this code is the same as the output of the Java-code: D235TBTZMfpSyB/CDl5MHAjH5fI=

Equivalent of Java's getBytes in JavaScript for different encodings

I have a function in Java that I need to convert to JavaScript and that contains this line:
byte[] bytes = ttText.getBytes(Charset.forName("Cp1250"));
ttText is String. I need to do the same. I need to get the bytes of a string encoded in Cp1250 (windows-1250), modify those bytes and then convert it back to string. Is there a way how to do it in JavaScript?
I discovered for example TextEncoder and TextDecoder but the support for different encodings than UTF-8 was dropped some time ago.

var cp1250 = '€ ‚ „…†‡ ‰Š‹ŚŤŽŹ ‘’“”•–— ™š›śťžź ˇ˘Ł¤Ą¦§¨©Ş«¬­®Ż°±˛ł´µ¶·¸ąş»Ľ˝ľżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙';
function encodeCP1250(text) {
var buf = [];
for (var i = 0; i < text.length; i++) {
var code = cp1250.indexOf(text[i]);
if (code >= 0) {
code += 128;
} else {
code = text.charCodeAt(i);
}
buf.push(code > 255 ? 32 : code);
}
return buf;
}
function decodeCP1250(buf) {
var text = '';
for (var i = 0; i < buf.length; i++) {
var code = buf[i];
text += code > 127 ? cp1250[code - 128] : String.fromCharCode(code);
}
return text;
}
var buf = encodeCP1250('AÁÂĂÄ'); // [65, 193, 194, 195, 196]
var text = decodeCP1250(buf); // 'AÁÂĂÄ'
Upd: Chrome and Firefox have TextDecoder as experimental feature, but TextEncoder works only with UTF-8.

Try this.
https://mths.be/windows-1250
This looks promising. It provides support for both encoding and decoding.
All you need to do is add the library and use the methods.
var encodedData = windows1250.encode(text);

how to assign value using charCode?

I want to convert string to bytes it worked fine if i assign value to var str as $scope.event[0] it gives me bytes only for first index ,if i assign $scope.event[] it hrows error str.charCodeAt is not a function. How can i get total bytes of the array ?
ctrl.js
$scope.event = ["lorem ipsum", "lorem ipsum","lorem ipsum"]
function jsonToArray() {
var total = 0;
var str = $scope.event;
var bytes = []; // char codes
var bytesv2 = []; // char codes
for (var i = 0; i < str.length; ++i) {
var code = str.charCodeAt(i);
bytes = bytes.concat([code]);
bytesv2 = bytesv2.concat([code & 0xff, code / 256 >>> 0]);
}
var totalBytes = 0;
console.log('bytes', bytes.join(', '));
for ( var i = 0 ; i < bytes.length ; i++ ) {
totalBytes += bytes[i];
totalCurrentBytes.push(totalBytes);
}
console.log('Total bytes', totalBytes);
formatBytes(totalBytes);
}
jsonToArray();

You can map/reduce the array to convert the strings to char codes. Start by running map over your keywords (we want to convert one array into another), with each word, split the word into an array so you are working with each letter and reduce that array of characters into the sum of their charCodes:
$scope.event.map(word => word.split('').reduce((total,cur) => total + cur.charCodeAt(0), 0));
Non ES6
$scope.event.map(function(word) {
return word.split(' ').reduce(function(total, cur) {
return total + cur.charCodeAt(0);
}, 0);
});

Converting arraybuffer to string : Maximum call stack size exceeded

This is my code.
var xhr = new XMLHttpRequest();
xhr.open('GET',window.location.href, true);
xhr.responseType = "arraybuffer";
xhr.onload = function(event) {
debugger;
console.log(" coverting array buffer to string ");
alert(String.fromCharCode.apply(null, new Uint8Array(this.response)));
};
xhr.send();
That request is being made to a PDF URL which is around 3 MB in size. I have read a few threads with same error, Maximum call stack size exceeded, telling me that there must be some recursive call but I do not see any recursive call here. Can anyone help?

I had the same problem and I finally used this code:
function _arrayBufferToBase64( buffer ) {
var binary = '';
var bytes = new Uint8Array( buffer );
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode( bytes[ i ] );
}
return window.btoa( binary );
}

The error is caused by a limitation in the number of function arguments. See "RangeError: Maximum call stack size exceeded" Why?
Instead of String.fromCharCode.apply(), use e. g. a TextEncoder. See Uint8Array to string in Javascript

this.response.arrayBuffer()
.then((buf) => {
const uint8Array = new Uint8Array(buf);
const data = uint8Array.reduce((acc, i) => acc += String.fromCharCode.apply(null, [i]), '');
return data;
})

Thank you so much Anthony O, you saved my 10 days work.
small modification in my scenario is:
Angular 7 :
/** Convert Uint8Array to string */
private static arrayBufferToBase64( buffer: Uint8Array ) {
var binary = '';
var bytes = new Uint8Array( buffer );
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode( bytes[ i ] );
}
return binary;
}