Develop Reference

How to decrypt large files (1 GB) with AES-CTR chunk by chunk in javascript (browsers)?

We are trying to decrypt large files (1GB) in browsers. With AES-CTR it should be possible to decrypt chunk by chunk - where chunk must be the correct size and you also have to provide nonce + counter.
Does anyone have any examples or ideas how to do this in javascript?
What we tried so far:
var length = value.byteLength;
var chunkSize = 128;
var index = 0;
let chunks = [];
let aesCounter = byteArrayToLong(subtleIv);
do {
let newCount = aesCounter + index / 16;
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
index += chunkSize;
} while(index < length);
let newCount = aesCounter + index / 16;
decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
let decryptedAll = Buffer.concat(chunks);
function longToByteArray(/*long*/long) {
var byteArray = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
for ( var index = 0; index < byteArray.length; index ++ ) {
var byte = long & 0xff;
byteArray [ index ] = byte;
long = (long - byte) / 256 ;
}
return byteArray;
}
function byteArrayToLong(/*byte[]*/byteArray) {
var value = 0;
for ( var i = byteArray.length - 1; i >= 0; i--) {
value = (value * 256) + byteArray[i];
}
return value;
}

The only flaw in your implementation is actually the conversion between integer and byte array. Firstly, in JavaScript the maximum integer is 0x1FFFFFFFFFFFFF, see here, secondly, even with smaller numbers the little endian order is used, but the WebCrypto API applies the big endian order.
As a first step to a fix you could use e.g. the BigInt implementation of JavaScript and the here described conversion between BigInt and ArrayBuffer.
Since this implementation works with ArrayBuffer and Uint8Array respectively, an implementation for concatenation is needed, e.g. from here.
This changes your implementation slightly as follows (key, IV and ciphertext are imported hex encoded):
(async () => {
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var key = hex2ab(keyHex);
var subtleKey = await window.crypto.subtle.importKey(
"raw",
key,
{ name: "AES-CTR" },
false,
["encrypt", "decrypt"]
);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var subtleIv = hex2ab(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertext = hex2ab(ciphertextHex);
// Decrypt and concat
var length = ciphertext.byteLength;
var chunkSize = 128; // chunkSize in bytes
var index = 0;
var chunks = [];
var aesCounter = bufToBn(subtleIv);
do {
var newCount = aesCounter + BigInt(index / 16); // index / 16 = number of blocks
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: bnToBuf(newCount), length: 128}, subtleKey, ciphertext.slice(index, index+chunkSize)); // length in bits
chunks.push(new Uint8Array(decrypted));
index += chunkSize;
} while(index < length);
var mergedChunks = merge(chunks);
// Decode and output
var decrypted = String.fromCharCode.apply(null, mergedChunks);
console.log(decrypted);
// https://coolaj86.com/articles/convert-js-bigints-to-typedarrays/
function bnToBuf(bn) {
var hex = BigInt(bn).toString(16);
if (hex.length % 2) { hex = '0' + hex; }
var len = hex.length / 2;
var u8 = new Uint8Array(len);
var i = 0;
var j = 0;
while (i < len) {
u8[i] = parseInt(hex.slice(j, j+2), 16);
i += 1;
j += 2;
}
return u8;
}
function bufToBn(buf) {
var hex = [];
u8 = Uint8Array.from(buf);
u8.forEach(function (i) {
var h = i.toString(16);
if (h.length % 2) { h = '0' + h; }
hex.push(h);
});
return BigInt('0x' + hex.join(''));
}
// https://stackoverflow.com/a/49129872/9014097
function merge(chunks){
let size = 0;
chunks.forEach(item => {
size += item.length;
});
let mergedArray = new Uint8Array(size);
let offset = 0;
chunks.forEach(item => {
mergedArray.set(item, offset);
offset += item.length;
});
return mergedArray;
}
function hex2ab(hex){
return new Uint8Array(hex.match(/[\da-f]{2}/gi).map(function (h) {
return parseInt(h, 16)}));
}
})();
which successfully decrypts the ciphertext. Btw, the ciphertext was generated with CyberChef.
Unlike the WebCrypto API, CryptoJS supports progressive encryption, so the same logic can be implemented significantly easier with CryptoJS:
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var keyWA = CryptoJS.enc.Hex.parse(keyHex);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var ivWA = CryptoJS.enc.Hex.parse(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertextWA = CryptoJS.enc.Hex.parse(ciphertextHex);
// Decrypt and concat
var length = ciphertextWA.sigBytes;
var chunkSize = 128;
var index = 0;
var decryptedWA = CryptoJS.enc.Hex.parse("");
var aesDecryptor = CryptoJS.algo.AES.createDecryptor(keyWA, { iv: ivWA, mode: CryptoJS.mode.CTR, padding: CryptoJS.pad.NoPadding });
var chunk = null;
do {
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
index += chunkSize;
} while(index < length - chunkSize);
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
chunk.sigBytes = length - index;
chunk.clamp();
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
decryptedWA = decryptedWA.concat(aesDecryptor.finalize());
// Decode and output
var decrypted = decryptedWA.toString(CryptoJS.enc.Utf8);
console.log(decrypted);
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js"></script>
A drawback of the first variant is that the BigInt class should actually not be used in the context of cryptography, since the operations are not constant-time, which results in a vulnerability to timing attacks. So, here you would have to apply a cryptographically more secure JavaScript BigInteger implementation for production.
For such reasons, the use of an established library (as opposed to a custom implementation), such as CryptoJS, is generally more secure (although ultimately vulnerabilities cannot be ruled out here either).

You are well on your way; cutting the ciphertext in chunks of C-blocks and then updating the counter block according to the block count is exactly what you should be doing.
However, note that the counter block is 128 bits: the longToByteArray should be called createCounterBlock(nonce, counter). The nonce should be contained in the initial/leftmost (8?) bytes. The counter is encoded as unsigned big endian in the final/rightmost 8 bytes, making 16 bytes total. This you can then provide as an IV to decrypt the chunk.
Currently your counter is placed in the leftmost bytes, which will fail for most CTR implementations; I don't think that JS uses 128 bit integers. Officially the contents of the counter block is not defined, but generally a big endian 128 bit value is used.

Convert xlsx (byte array) to csv(string) with javascript

I am from c# so know nothing about java script.
I have excel file (xlsx) that I red into byte array (with unity3d c# in webGL build) and want to send it into java script function that parse it into csv structure and return as string.
So the question part is only related to java script that received xlsx as byte array(or any type from memory stream) and return csv as string.
I need that function. What else (libs) do I need for that?
(Update)
The javascript code is
MyConverterXlsxToCsvReturn: function (array,size) {
var buffer = new ArrayBuffer(size);
for (var i = 0; i < size; i++)
buffer[i] = HEAPU8[array + i];
var txt = XLSX.utils.sheet_to_txt(buffer, {type: 'arraybuffer'});
window.alert(Pointer_stringify(txt));
window.alert(Pointer_stringify(txt.length));
var returnStr = Pointer_stringify(txt);
var bufferSize = lengthBytesUTF8(returnStr) + 1;
var buffer = _malloc(bufferSize);
stringToUTF8(returnStr, buffer, bufferSize);
return buffer;
},
I am trying to send byte[] and convert into arraybuffer but in search of correct way to do that.
For now that function return empty string.

I wanted to convert byte array that I received in C# and then red the array in javascript.
As solution I converted the the byte array into hex string with method:
private string ByteArrayToString(byte[] ba)
{
StringBuilder hex = new StringBuilder(ba.Length * 2);
foreach (byte b in ba)
hex.AppendFormat("{0:x2}", b);
return hex.ToString();
}
Then this string i sent to javascript function and converted to array. The rezult was returned as string:
ExcelHexToCSV: function (hexStr) {
console.log("javascript: ExcelHexToCSV");
console.log("javascript received: " + Pointer_stringify(hexStr));
// convert part
var str = Pointer_stringify(hexStr);
var a = [];
for (var i = 0, len = str.length; i < len; i += 2) {
a.push(parseInt(str.substr(i, 2), 16));
}
var data = new Uint8Array(a);
console.log("javascript hex_to_byte:" + data);
// excel part
var workbook = XLSX.read(data, {type: "array"});
var sheetname = workbook.SheetNames[0];
console.log("javascript sheetname: " + sheetname);
var sheetdata = XLSX.utils.sheet_to_csv(workbook.Sheets[sheetname]);
console.log("javascript sheetdata: = " + sheetdata);
var rezult = sheetdata;
var returnStr = rezult;
var bufferSize = lengthBytesUTF8(returnStr) + 1;
var buffer = _malloc(bufferSize);
stringToUTF8(returnStr, buffer, bufferSize);
return buffer;
},
Github link for the my example project

decrypting CyproJS AES encryption in Cryptodome python

I am trying to decrypt message (in python) that has been encoded using CryptoJS in JavaScript. I have created an API in python, to post data I am using postman pre-request script.
The Error I am getting:
ValueError: Data must be padded to 16 byte boundary in CBC mode
JavaScript code for encryption
var data = {"feature_0": 0,
"feature_1": 0,
"feature_2": 0,
"feature_3": 0,
"feature_4": 0,
"feature_5": 0
};
let password = "lazydog";
let salt = "salt";
let iterations = 128;
data = JSON.stringify(data);
let len = 16 - ((data.length) % 16);
data += len.toString().repeat(len); --> removed (as suggested)
let bytes = CryptoJS.PBKDF2(password, salt, { keySize: 48, iterations: iterations });
let iv = CryptoJS.enc.Hex.parse(bytes.toString().slice(0, 32));
let key = CryptoJS.enc.Hex.parse(bytes.toString().slice(32, 96));
let encrypted = CryptoJS.AES.encrypt(data, key, {iv: iv}); //, mode: CryptoJS.mode.CBC
//encrypted = btoa(encrypted); --> removed (as suggested)
encrypted = encrypted.toString() -->added (as suggested)
postman.setGlobalVariable("data", encrypted);
python code for decryption:
def decode(encrypted):
data = b64decode(encrypted)
byte = PBKDF2("lazydog".encode("utf-8"), "salt".encode("utf-8"), 48, 128)
iv = byte[0:16]
key = byte[16:48]
cipher = AES.new(key, AES.MODE_CBC, iv)
text = cipher.decrypt(data) ## error is at this line
text = text[:-text[-1]].decode("utf-8")
return text
As the error said padding problem I added padding in JS code. Still I am not getting good results.
What's wrong I am doing here?

The encrypted string has posted to API which has written in python.
I don't know why but when encrypted passed to python '+' chars are being replaced with ' '(space). By replacing the spaces with '+' char I resolved the problem.
code
var data = {"feature_0": 0,
"feature_1": 0,
"feature_2": 0,
"feature_3": 0,
"feature_4": 0,
"feature_5": 0
};
let password = "lazydog";
let salt = "salt";
let iterations = 128;
data = JSON.stringify(data);
let bytes = CryptoJS.PBKDF2(password, salt, { keySize: 48, iterations:
iterations });
let iv = CryptoJS.enc.Hex.parse(bytes.toString().slice(0, 32));
let key = CryptoJS.enc.Hex.parse(bytes.toString().slice(32, 96));
let encrypted = CryptoJS.AES.encrypt(data, key, {iv: iv});
encrypted = encrypted.toString()
postman.setGlobalVariable("data", encrypted);
python code
def decode(encrypted):
encrypted = encrypted.replace(' ', '+') --> this line is added
data = b64decode(encrypted)
byte = PBKDF2("lazydog".encode("utf-8"), "salt".encode("utf-8"), 48, 128)
iv = byte[0:16]
key = byte[16:48]
cipher = AES.new(key, AES.MODE_CBC, iv)
text = cipher.decrypt(data) ## error is at this line
text = text[:-text[-1]].decode("utf-8")
return text
As CryptoJs pads data implicitly custom padding has been removed. And removed btoa (which is not required). Then encrypted data is converted to String. suggested by #Topaco in the comments

Java SHA-1 to javascript using CryptoJS

i have such a code to generate password written in Java
MessageDigest messageDigestPassword = MessageDigest.getInstance("SHA1");
messageDigestPassword .reset();
byte[] password = "password".getBytes();
messageDigestPassword .update(password);
byte[] encryptedPassword = messageDigestPassword .digest();
String date = "2019-10-22T11:33:13.393Z";
byte[] dateBytes = date.getBytes(StandardCharsets.UTF_8);
int offset = 0;
byte[] outputBytes = new byte[dateBytes.length + encryptedPassword .length];
System.arraycopy(dateBytes, 0, outputBytes, offset, dateBytes.length);
offset += dateBytes.length;
System.arraycopy(encryptedPassword , 0, outputBytes, offset, encryptedPassword .length);
MessageDigest finalMessageDigeset = MessageDigest.getInstance("SHA-1");
finalMessageDigeset.reset();
finalMessageDigeset.update(outputBytes);
byte[] finalPasswordBytes= finalMessageDigeset .digest();
String finalBase64Password = new String(Base64.encode(finalPasswordBytes));
and im trying to rewrite it to JavaScript to use it in postman with - CryptoJS
So far i have :
function wordArrayToByteArray(wordArray, length) {
if (wordArray.hasOwnProperty("sigBytes") &&
wordArray.hasOwnProperty("words")) {
length = wordArray.sigBytes;
wordArray = wordArray.words;
}
var result = [],
bytes,
i = 0;
while (length > 0) {
bytes = wordToByteArray(wordArray[i], Math.min(4, length));
length -= bytes.length;
result.push(bytes);
i++;
}
return [].concat.apply([], result);
}
function stringToBytes ( str ) {
var ch, st, re = [];
for (var i = 0; i < str.length; i++ ) {
ch = str.charCodeAt(i); // get char
st = []; // set up "stack"
do {
st.push( ch & 0xFF ); // push byte to stack
ch = ch >> 8; // shift value down by 1 byte
}
while ( ch );
// add stack contents to result
// done because chars have "wrong" endianness
re = re.concat( st.reverse() );
}
// return an array of bytes
return re;
}
var dateFixed = "2019-10-22T11:33:13.393Z";
var fixedDateBytes = stringToBytes(dateFixed);
var sha1Password= CryptoJS.SHA1("password");
console.log("sha1Password",sha1Password.toString(CryptoJS.enc.Hex));
var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20);
var concatedBytes= fixedDateBytes.concat(sha1PasswordBytes);
var finalShaPassWords= CryptoJS.SHA1(concatedBytes);
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Hex));
console.log("finalShaPassWords",finalShaPassWords.toString(CryptoJS.enc.Base64));
However unfortunatelly Base64 representations written in those 2 languages doesnt match.
I have checked and bytes from date are equal. Bytes from hashed password are not. So hashing after concat fails in JavaScript.
I have checked first password hashing and generated bytes and both of them are the same. So my guess line var sha1PasswordBytes= wordArrayToByteArray(sha1Password, 20); causes that line var finalShaPassWords= CryptoJS.SHA1(concatedBytes); returns bad value.
Can someone give me some idea what is wrong? Mayby it should be written diffrent ?

Since you are using CryptoJS anyway, you can also use the CryptoJS encoders and the WordArray#concat-method, which considerably simplifies the code:
var CryptoJS = require("crypto-js");
// Input
var inPwd = "password";
var inDate = "2019-10-22T11:33:13.393Z";
// Processing
var pwdHash = CryptoJS.SHA1(inPwd); // hash and convert to WordArray
var date = CryptoJS.enc.Utf8.parse(inDate); // convert to WordArray
var joinedData = date.clone().concat(pwdHash); // join date and hashed password
var joinedDataHash = CryptoJS.SHA1(joinedData); // hash joined data
var joinedDataHashB64 = CryptoJS.enc.Base64.stringify(joinedDataHash); // convert to Base64 string
// Output
console.log("Result: " + joinedDataHashB64 ); // Output: D235TBTZMfpSyB/CDl5MHAjH5fI=
The output of this code is the same as the output of the Java-code: D235TBTZMfpSyB/CDl5MHAjH5fI=

Converting Java's PBEWithMD5AndDES to JavaScript

I'm trying to replicate the Java code in JavaScript.
below is my Java code:
public static String encrypt(String input)
final byte[] SALT= { (byte) 0x21, (byte) 0x21, (byte) 0xF0, (byte) 0x55, (byte) 0xC3, (byte) 0x9F, (byte) 0x5A, (byte) 0x75 };
final int ITERATION_COUNT = 31;
{
if (input == null)
{
throw new IllegalArgumentException();
}
try
{
KeySpec keySpec = new PBEKeySpec(null, SALT, ITERATION_COUNT);
AlgorithmParameterSpec paramSpec = new PBEParameterSpec(SALT, ITERATION_COUNT);
SecretKey key = SecretKeyFactory.getInstance("PBEWithMD5AndDES").generateSecret(keySpec);
Cipher ecipher = Cipher.getInstance(key.getAlgorithm());
ecipher.init(Cipher.ENCRYPT_MODE, key, paramSpec);
byte[] enc = ecipher.doFinal(input.getBytes());
String res = new String(Base64.encodeBase64(enc));
// escapes for url
res = res.replace('+', '-').replace('/', '_').replace("%", "%25").replace("\n", "%0A");
LOGGER.info("String Encrypted Successfully");
return res;
}
catch (Exception e)
{
LOGGER.error("encrypt Exception: "+e.getMessage());
}
return "";
}
and the JavaScript code, so far hammed up is below:
var encrypt = function(){
var iterations = 31;
var key = CryptoJS.MD5("PBEWithMD5AndDES");
var salt = CryptoJS.enc.Hex.parse('0021002100f0005500C3009F005A0075');
var options = {
mode: CryptoJS.mode.CBC,
iv: salt
};
var hashedPassword = CryptoJS.MD5($scope.data.webPassword);
var encryptedPassword = CryptoJS.DES.encrypt(hashedPassword, key,options).toString();
var result = encryptedPassword.toString(CryptoJS.enc.Base64);
}
but with both the encryption the encoded string I'm getting is different.

PBEwithMD5andDES is obsolete technology and should not be used nowadays. This answer is only provided for demonstration purposes.
PBEwithMD5andDES is defined in PKCS#5 v1.5 which is nothing more than deriving key+IV using PBKDF1 (with MD5) and encrypting with DES.
var password = CryptoJS.enc.Utf8.parse("test");
var salt = CryptoJS.enc.Hex.parse("2121F055C39F5A75");
var iterations = 31;
// PBE according to PKCS#5 v1.5 (in other words: PBKDF1)
var md5 = CryptoJS.algo.MD5.create();
md5.update(password);
md5.update(salt);
var result = md5.finalize();
md5.reset();
for(var i = 1; i < iterations; i++) {
md5.update(result);
result = md5.finalize();
md5.reset();
}
// splitting key and IV
var key = CryptoJS.lib.WordArray.create(result.words.slice(0, 2));
var iv = CryptoJS.lib.WordArray.create(result.words.slice(2, 4));
var encrypted = CryptoJS.DES.encrypt("test", key, {
iv: iv
});
enchex.innerHTML = encrypted.ciphertext.toString();
encbase64.innerHTML = encrypted.ciphertext.toString(CryptoJS.enc.Base64);
<script src="https://cdn.rawgit.com/CryptoStore/crypto-js/3.1.2/build/rollups/tripledes.js"></script>
<script src="https://cdn.rawgit.com/CryptoStore/crypto-js/3.1.2/build/rollups/md5.js"></script>
<div>Hex: <span id="enchex"></span></div>
<div>Base64: <span id="encbase64"></span></div>
Here is a jsFiddle to experiment with and here is the example Java code. Both produce the same result in Hex: aa8101a7d63093c6.
Security considerations:
PBEwithMD5andDES should not be used and there are better alternatives like PBEWithHmacSHA256AndAES_128 which require a slightly different approach.
The number of iterations must be large (a thousand to a million) in order to make it hard to brute-force the password. DES only provides 56 bits of security, so it is even possible to brute-force the key directly with today's means.
The salt must be randomly generated in order to achieve semantic security. The salt itself doesn't need to be secret. Since it has a known length it can be simply prepended to the ciphertext and sliced off before decryption.