We are trying to decrypt large files (1GB) in browsers. With AES-CTR it should be possible to decrypt chunk by chunk - where chunk must be the correct size and you also have to provide nonce + counter.
Does anyone have any examples or ideas how to do this in javascript?
What we tried so far:
var length = value.byteLength;
var chunkSize = 128;
var index = 0;
let chunks = [];
let aesCounter = byteArrayToLong(subtleIv);
do {
let newCount = aesCounter + index / 16;
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
index += chunkSize;
} while(index < length);
let newCount = aesCounter + index / 16;
decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
let decryptedAll = Buffer.concat(chunks);
function longToByteArray(/*long*/long) {
var byteArray = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
for ( var index = 0; index < byteArray.length; index ++ ) {
var byte = long & 0xff;
byteArray [ index ] = byte;
long = (long - byte) / 256 ;
}
return byteArray;
}
function byteArrayToLong(/*byte[]*/byteArray) {
var value = 0;
for ( var i = byteArray.length - 1; i >= 0; i--) {
value = (value * 256) + byteArray[i];
}
return value;
}
The only flaw in your implementation is actually the conversion between integer and byte array. Firstly, in JavaScript the maximum integer is 0x1FFFFFFFFFFFFF, see here, secondly, even with smaller numbers the little endian order is used, but the WebCrypto API applies the big endian order.
As a first step to a fix you could use e.g. the BigInt implementation of JavaScript and the here described conversion between BigInt and ArrayBuffer.
Since this implementation works with ArrayBuffer and Uint8Array respectively, an implementation for concatenation is needed, e.g. from here.
This changes your implementation slightly as follows (key, IV and ciphertext are imported hex encoded):
(async () => {
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var key = hex2ab(keyHex);
var subtleKey = await window.crypto.subtle.importKey(
"raw",
key,
{ name: "AES-CTR" },
false,
["encrypt", "decrypt"]
);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var subtleIv = hex2ab(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertext = hex2ab(ciphertextHex);
// Decrypt and concat
var length = ciphertext.byteLength;
var chunkSize = 128; // chunkSize in bytes
var index = 0;
var chunks = [];
var aesCounter = bufToBn(subtleIv);
do {
var newCount = aesCounter + BigInt(index / 16); // index / 16 = number of blocks
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: bnToBuf(newCount), length: 128}, subtleKey, ciphertext.slice(index, index+chunkSize)); // length in bits
chunks.push(new Uint8Array(decrypted));
index += chunkSize;
} while(index < length);
var mergedChunks = merge(chunks);
// Decode and output
var decrypted = String.fromCharCode.apply(null, mergedChunks);
console.log(decrypted);
// https://coolaj86.com/articles/convert-js-bigints-to-typedarrays/
function bnToBuf(bn) {
var hex = BigInt(bn).toString(16);
if (hex.length % 2) { hex = '0' + hex; }
var len = hex.length / 2;
var u8 = new Uint8Array(len);
var i = 0;
var j = 0;
while (i < len) {
u8[i] = parseInt(hex.slice(j, j+2), 16);
i += 1;
j += 2;
}
return u8;
}
function bufToBn(buf) {
var hex = [];
u8 = Uint8Array.from(buf);
u8.forEach(function (i) {
var h = i.toString(16);
if (h.length % 2) { h = '0' + h; }
hex.push(h);
});
return BigInt('0x' + hex.join(''));
}
// https://stackoverflow.com/a/49129872/9014097
function merge(chunks){
let size = 0;
chunks.forEach(item => {
size += item.length;
});
let mergedArray = new Uint8Array(size);
let offset = 0;
chunks.forEach(item => {
mergedArray.set(item, offset);
offset += item.length;
});
return mergedArray;
}
function hex2ab(hex){
return new Uint8Array(hex.match(/[\da-f]{2}/gi).map(function (h) {
return parseInt(h, 16)}));
}
})();
which successfully decrypts the ciphertext. Btw, the ciphertext was generated with CyberChef.
Unlike the WebCrypto API, CryptoJS supports progressive encryption, so the same logic can be implemented significantly easier with CryptoJS:
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var keyWA = CryptoJS.enc.Hex.parse(keyHex);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var ivWA = CryptoJS.enc.Hex.parse(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertextWA = CryptoJS.enc.Hex.parse(ciphertextHex);
// Decrypt and concat
var length = ciphertextWA.sigBytes;
var chunkSize = 128;
var index = 0;
var decryptedWA = CryptoJS.enc.Hex.parse("");
var aesDecryptor = CryptoJS.algo.AES.createDecryptor(keyWA, { iv: ivWA, mode: CryptoJS.mode.CTR, padding: CryptoJS.pad.NoPadding });
var chunk = null;
do {
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
index += chunkSize;
} while(index < length - chunkSize);
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
chunk.sigBytes = length - index;
chunk.clamp();
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
decryptedWA = decryptedWA.concat(aesDecryptor.finalize());
// Decode and output
var decrypted = decryptedWA.toString(CryptoJS.enc.Utf8);
console.log(decrypted);
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js"></script>
A drawback of the first variant is that the BigInt class should actually not be used in the context of cryptography, since the operations are not constant-time, which results in a vulnerability to timing attacks. So, here you would have to apply a cryptographically more secure JavaScript BigInteger implementation for production.
For such reasons, the use of an established library (as opposed to a custom implementation), such as CryptoJS, is generally more secure (although ultimately vulnerabilities cannot be ruled out here either).
You are well on your way; cutting the ciphertext in chunks of C-blocks and then updating the counter block according to the block count is exactly what you should be doing.
However, note that the counter block is 128 bits: the longToByteArray should be called createCounterBlock(nonce, counter). The nonce should be contained in the initial/leftmost (8?) bytes. The counter is encoded as unsigned big endian in the final/rightmost 8 bytes, making 16 bytes total. This you can then provide as an IV to decrypt the chunk.
Currently your counter is placed in the leftmost bytes, which will fail for most CTR implementations; I don't think that JS uses 128 bit integers. Officially the contents of the counter block is not defined, but generally a big endian 128 bit value is used.
I'm trying to convert a hexadecimal string to string with Windows-1251 encoding. I need to use JavaScript. I've tried using this sample that someone posted:
var win1251 = new TextDecoder("windows-1251");
for (var i = 0x00; i < 0xFF; i++) {
var hex = (i <= 0x0F ? "0" : "") + i.toString(16).toUpperCase();
decodeMap[hex] = win1251.decode(Uint8Array.from([i]));
}
but I can't seem to make it work. Can someone please help out?
Propably you're using it in incorrect way.
The code in you post creates dictionary (hex => windows-1251).
So, to translate hex string to windows-1251 string you have to:
split your hex string into array with 2-hex elements
Translate every element to windows-1251
Join result array into string
var decodeMap = {};
var win1251 = new TextDecoder("windows-1251");
for (var i = 0x00; i < 0xFF; i++) {
var hex = (i <= 0x0F ? "0" : "") + i.toString(16).toUpperCase();
decodeMap[hex] = win1251.decode(Uint8Array.from([i]));
}
var number = "3230313830363131313535303435303831";
var hexarr = number.toUpperCase().match(/[0-9A-F]{2}/g)
var win1251arr = hexarr.map(el=> decodeMap[el]);
console.log(win1251arr.join(""))
You can also do it in much shorter way (directly translate your hex string without creating dictionary):
var win1251 = new TextDecoder("windows-1251");
var number = "3230313830363131313535303435303831";
var arr = number.toUpperCase().match(/[0-9A-F]{2}/g).map(el=>"0x"+el);
var uarr = Uint8Array.from(arr);
console.log(win1251.decode(uarr));
If your browser doesn't support TextDecoder, you can use dictionary from first snippet:
var decodeMap = getHexToWin1251Dictionary();
var number = "3230313830363131313535303435303831";
var hexarr = number.toUpperCase().match(/[0-9A-F]{2}/g)
var win1251arr = hexarr.map(el=> decodeMap[el]);
console.log(win1251arr.join(""))
// function below returns content of decodeMap from first snippet
function getHexToWin1251Dictionary(){
return {"10":"\u0010","11":"\u0011","12":"\u0012","13":"\u0013","14":"\u0014","15":"\u0015","16":"\u0016","17":"\u0017","18":"\u0018","19":"\u0019","20":" ","21":"!","22":"\"","23":"#","24":"$","25":"%","26":"&","27":"'","28":"(","29":")","30":"0","31":"1","32":"2","33":"3","34":"4","35":"5","36":"6","37":"7","38":"8","39":"9","40":"#","41":"A","42":"B","43":"C","44":"D","45":"E","46":"F","47":"G","48":"H","49":"I","50":"P","51":"Q","52":"R","53":"S","54":"T","55":"U","56":"V","57":"W","58":"X","59":"Y","60":"`","61":"a","62":"b","63":"c","64":"d","65":"e","66":"f","67":"g","68":"h","69":"i","70":"p","71":"q","72":"r","73":"s","74":"t","75":"u","76":"v","77":"w","78":"x","79":"y","80":"Ђ","81":"Ѓ","82":"‚","83":"ѓ","84":"„","85":"…","86":"†","87":"‡","88":"€","89":"‰","90":"ђ","91":"‘","92":"’","93":"“","94":"”","95":"•","96":"–","97":"—","98":"","99":"™","00":"\u0000","01":"\u0001","02":"\u0002","03":"\u0003","04":"\u0004","05":"\u0005","06":"\u0006","07":"\u0007","08":"\b","09":"\t","0A":"\n","0B":"\u000b","0C":"\f","0D":"\r","0E":"\u000e","0F":"\u000f","1A":"\u001a","1B":"\u001b","1C":"\u001c","1D":"\u001d","1E":"\u001e","1F":"\u001f","2A":"*","2B":"+","2C":",","2D":"-","2E":".","2F":"/","3A":":","3B":";","3C":"<","3D":"=","3E":">","3F":"?","4A":"J","4B":"K","4C":"L","4D":"M","4E":"N","4F":"O","5A":"Z","5B":"[","5C":"\\","5D":"]","5E":"^","5F":"_","6A":"j","6B":"k","6C":"l","6D":"m","6E":"n","6F":"o","7A":"z","7B":"{","7C":"|","7D":"}","7E":"~","7F":"","8A":"Љ","8B":"‹","8C":"Њ","8D":"Ќ","8E":"Ћ","8F":"Џ","9A":"љ","9B":"›","9C":"њ","9D":"ќ","9E":"ћ","9F":"џ","A0":" ","A1":"Ў","A2":"ў","A3":"Ј","A4":"¤","A5":"Ґ","A6":"¦","A7":"§","A8":"Ё","A9":"©","AA":"Є","AB":"«","AC":"¬","AD":"","AE":"®","AF":"Ї","B0":"°","B1":"±","B2":"І","B3":"і","B4":"ґ","B5":"µ","B6":"¶","B7":"·","B8":"ё","B9":"№","BA":"є","BB":"»","BC":"ј","BD":"Ѕ","BE":"ѕ","BF":"ї","C0":"А","C1":"Б","C2":"В","C3":"Г","C4":"Д","C5":"Е","C6":"Ж","C7":"З","C8":"И","C9":"Й","CA":"К","CB":"Л","CC":"М","CD":"Н","CE":"О","CF":"П","D0":"Р","D1":"С","D2":"Т","D3":"У","D4":"Ф","D5":"Х","D6":"Ц","D7":"Ч","D8":"Ш","D9":"Щ","DA":"Ъ","DB":"Ы","DC":"Ь","DD":"Э","DE":"Ю","DF":"Я","E0":"а","E1":"б","E2":"в","E3":"г","E4":"д","E5":"е","E6":"ж","E7":"з","E8":"и","E9":"й","EA":"к","EB":"л","EC":"м","ED":"н","EE":"о","EF":"п","F0":"р","F1":"с","F2":"т","F3":"у","F4":"ф","F5":"х","F6":"ц","F7":"ч","F8":"ш","F9":"щ","FA":"ъ","FB":"ы","FC":"ь","FD":"э","FE":"ю"};
}
i have this pseudo code and i would like to convert it into working code:
string constructSignature(string timestamp, string UID, string secretKey) {
baseString = timestamp + "_" + UID; // Construct a "base string" for signing
binaryBaseString = ConvertUTF8ToBytes(baseString); // Convert the base string into a binary array
binaryKey = ConvertFromBase64ToBytes(secretKey); // Convert secretKey from BASE64 to a binary array
binarySignature = hmacsha1(binaryKey, baseString); // Use the HMAC-SHA1 algorithm to calculate the signature
signature = ConvertToBase64(binarySignature); // Convert the signature to a BASE64
return signature;
}
any idea?
thanks
Some ideas:
Don't calculate BinaryBaseString, given that you never use it
Don't refer to UTF8 for no reason at all -- strings in JavaScript are Unicode, which is only distantly connected to UTF.
Don't put implementation details in your pseudocode -- especially ones you don't actually need (like assuming the secret key should be in "bytes", whatever that means, when in fact, the standard library takes and returns strings).
Don't write comments that just restate what the code is doing.
Which leaves us with:
var constructSignature = function(timestamp, UID, secretKey) {
return Crypto.HMAC(Crypto.SHA1, timestamp + "_" + UID, secretKey,
{ asString: true });
};
Here is a full implementation of your code:
let atob = function(a){
return new Buffer(a, 'base64').toString('binary');
}
function ConvertUTF8ToBytes(str) {
var utf8 = unescape(encodeURIComponent(str));
var arr = [];
for (var i = 0; i < utf8.length; i++) {
arr.push(utf8.charCodeAt(i));
}
return new Uint8Array(arr);
}
let ConvertFromBase64ToBytes = function (b64Data){
var byteCharacters = atob(b64Data);
var byteNumbers = new Array(byteCharacters.length);
for (var i = 0; i < byteCharacters.length; i++) {
byteNumbers[i] = byteCharacters.charCodeAt(i);
}
return new Uint8Array(byteNumbers);
}
let constructSignature = function(timestamp, UID, secretKey) {
let baseString = timestamp + "_" + UID;
let binaryBaseString = ConvertUTF8ToBytes(baseString)
let binaryKey = ConvertFromBase64ToBytes(secretKey)
let binarySignature = Crypto.createHmac('sha1', binaryKey)
.update( binaryBaseString )
.digest('base64');
return binarySignature;
};