I writing a client which have to transmit files to a REST webservice.
I take the File object from the input control, then I use FileReader object to convert it in base64 (String). I noticed that the base64 string weight is 30-40% over the original file size.
If I convert that string in binary size literally explodes, so I was wandering if there is a more efficient way to convert in string this file and send over HTTP.
function toBase64() {
var file = document.querySelector('#base64').files[0];
var reader = new FileReader();
reader.addEventListener("load", function () {
var bin = base64toBin(reader.result);
var base64 = reader.result;
console.log(bin, base64);
});
if (file) {
reader.readAsDataURL(file);
}
}
function base64toBin(data) {
var binArray = []
var datEncode = "";
for (let i=0; i < data.length; i++) {
binArray.push(data[i].charCodeAt(0).toString(2));
}
for (let j=0; j < binArray.length; j++) {
var pad = padding_left(binArray[j], '0', 8);
datEncode += pad + ' ';
}
function padding_left(s, c, n) { if (! s || ! c || s.length >= n) {
return s;
}
var max = (n - s.length)/c.length;
for (var i = 0; i < max; i++) {
s = c + s; } return s;
}
return binArray;
}
Here is a poc:
jsfiddle
Related
We are trying to decrypt large files (1GB) in browsers. With AES-CTR it should be possible to decrypt chunk by chunk - where chunk must be the correct size and you also have to provide nonce + counter.
Does anyone have any examples or ideas how to do this in javascript?
What we tried so far:
var length = value.byteLength;
var chunkSize = 128;
var index = 0;
let chunks = [];
let aesCounter = byteArrayToLong(subtleIv);
do {
let newCount = aesCounter + index / 16;
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
index += chunkSize;
} while(index < length);
let newCount = aesCounter + index / 16;
decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: Buffer.from(longToByteArray(newCount)), length: chunkSize}, subtleKey, value.slice(index, index+chunkSize));
chunks.push(Buffer.from(decrypted));
let decryptedAll = Buffer.concat(chunks);
function longToByteArray(/*long*/long) {
var byteArray = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
for ( var index = 0; index < byteArray.length; index ++ ) {
var byte = long & 0xff;
byteArray [ index ] = byte;
long = (long - byte) / 256 ;
}
return byteArray;
}
function byteArrayToLong(/*byte[]*/byteArray) {
var value = 0;
for ( var i = byteArray.length - 1; i >= 0; i--) {
value = (value * 256) + byteArray[i];
}
return value;
}
The only flaw in your implementation is actually the conversion between integer and byte array. Firstly, in JavaScript the maximum integer is 0x1FFFFFFFFFFFFF, see here, secondly, even with smaller numbers the little endian order is used, but the WebCrypto API applies the big endian order.
As a first step to a fix you could use e.g. the BigInt implementation of JavaScript and the here described conversion between BigInt and ArrayBuffer.
Since this implementation works with ArrayBuffer and Uint8Array respectively, an implementation for concatenation is needed, e.g. from here.
This changes your implementation slightly as follows (key, IV and ciphertext are imported hex encoded):
(async () => {
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var key = hex2ab(keyHex);
var subtleKey = await window.crypto.subtle.importKey(
"raw",
key,
{ name: "AES-CTR" },
false,
["encrypt", "decrypt"]
);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var subtleIv = hex2ab(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertext = hex2ab(ciphertextHex);
// Decrypt and concat
var length = ciphertext.byteLength;
var chunkSize = 128; // chunkSize in bytes
var index = 0;
var chunks = [];
var aesCounter = bufToBn(subtleIv);
do {
var newCount = aesCounter + BigInt(index / 16); // index / 16 = number of blocks
var decrypted = await window.crypto.subtle.decrypt({name: "AES-CTR", counter: bnToBuf(newCount), length: 128}, subtleKey, ciphertext.slice(index, index+chunkSize)); // length in bits
chunks.push(new Uint8Array(decrypted));
index += chunkSize;
} while(index < length);
var mergedChunks = merge(chunks);
// Decode and output
var decrypted = String.fromCharCode.apply(null, mergedChunks);
console.log(decrypted);
// https://coolaj86.com/articles/convert-js-bigints-to-typedarrays/
function bnToBuf(bn) {
var hex = BigInt(bn).toString(16);
if (hex.length % 2) { hex = '0' + hex; }
var len = hex.length / 2;
var u8 = new Uint8Array(len);
var i = 0;
var j = 0;
while (i < len) {
u8[i] = parseInt(hex.slice(j, j+2), 16);
i += 1;
j += 2;
}
return u8;
}
function bufToBn(buf) {
var hex = [];
u8 = Uint8Array.from(buf);
u8.forEach(function (i) {
var h = i.toString(16);
if (h.length % 2) { h = '0' + h; }
hex.push(h);
});
return BigInt('0x' + hex.join(''));
}
// https://stackoverflow.com/a/49129872/9014097
function merge(chunks){
let size = 0;
chunks.forEach(item => {
size += item.length;
});
let mergedArray = new Uint8Array(size);
let offset = 0;
chunks.forEach(item => {
mergedArray.set(item, offset);
offset += item.length;
});
return mergedArray;
}
function hex2ab(hex){
return new Uint8Array(hex.match(/[\da-f]{2}/gi).map(function (h) {
return parseInt(h, 16)}));
}
})();
which successfully decrypts the ciphertext. Btw, the ciphertext was generated with CyberChef.
Unlike the WebCrypto API, CryptoJS supports progressive encryption, so the same logic can be implemented significantly easier with CryptoJS:
// Key import
var keyHex = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
var keyWA = CryptoJS.enc.Hex.parse(keyHex);
// IV import
var ivHex = "404142434445464748494a4b4c4d4e4f";
var ivWA = CryptoJS.enc.Hex.parse(ivHex);
// Ciphertext import
var ciphertextHex = "ef11ad5afa7ad39fe00e0fe7e934dd38c2556dfadcce052cee9d91ee33393b428013d78ed995d5248cadd7be2d855e1adc932167d779923170447505c164eb46b59efd59e695de56512366738072afee57c16a71583346e0eac4a52dbb423b86e1c931ed7bdc3bbc17e5c662ad9cf676a7053ed435eb0968e6b1108531e2f686f491a8e2c02e43edda8162407b9e774f517e8cc8c683bada7044b1573d501a2ac54022ca1e98e26fa0f6ab60485124adb76472af0a5780a0fc2c3332cceed5395339aef3c818996bd24dd5a8d3573eab4646de859b318810dee23fb4558be8932ab790bd87d5f66531943a8bf7c70ea21b44aca6285e1e48a5852fcfa2beda61cd9f0745b8e6c10161678743b307e4ccfcb49e4c44216c32dd7e65a9f408e0aca457a9a92223e14d5d48c7855db0f7cf97e1dd176391beb0c4ecc466c9a6c4cdb211540cfd0448f4cc35b9719f31c9caf440d2aab66a42f92c65993b216449cef809ca65152bd0b509de4a7f859d630e4a5cb5c17eb6815ed1291379fe547801c7ab22501d2da6fd73111697d275b4086b455e66a36e9e8ad62f1910a832e9461606d88c407e6969f2044ff34417d391d0f6c97480264fd3c7e1b45acc";
var ciphertextWA = CryptoJS.enc.Hex.parse(ciphertextHex);
// Decrypt and concat
var length = ciphertextWA.sigBytes;
var chunkSize = 128;
var index = 0;
var decryptedWA = CryptoJS.enc.Hex.parse("");
var aesDecryptor = CryptoJS.algo.AES.createDecryptor(keyWA, { iv: ivWA, mode: CryptoJS.mode.CTR, padding: CryptoJS.pad.NoPadding });
var chunk = null;
do {
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
index += chunkSize;
} while(index < length - chunkSize);
chunk = CryptoJS.lib.WordArray.create(ciphertextWA.words.slice(index/4, (index + chunkSize)/4));
chunk.sigBytes = length - index;
chunk.clamp();
decryptedWA = decryptedWA.concat(aesDecryptor.process(chunk));
decryptedWA = decryptedWA.concat(aesDecryptor.finalize());
// Decode and output
var decrypted = decryptedWA.toString(CryptoJS.enc.Utf8);
console.log(decrypted);
<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/4.1.1/crypto-js.min.js"></script>
A drawback of the first variant is that the BigInt class should actually not be used in the context of cryptography, since the operations are not constant-time, which results in a vulnerability to timing attacks. So, here you would have to apply a cryptographically more secure JavaScript BigInteger implementation for production.
For such reasons, the use of an established library (as opposed to a custom implementation), such as CryptoJS, is generally more secure (although ultimately vulnerabilities cannot be ruled out here either).
You are well on your way; cutting the ciphertext in chunks of C-blocks and then updating the counter block according to the block count is exactly what you should be doing.
However, note that the counter block is 128 bits: the longToByteArray should be called createCounterBlock(nonce, counter). The nonce should be contained in the initial/leftmost (8?) bytes. The counter is encoded as unsigned big endian in the final/rightmost 8 bytes, making 16 bytes total. This you can then provide as an IV to decrypt the chunk.
Currently your counter is placed in the leftmost bytes, which will fail for most CTR implementations; I don't think that JS uses 128 bit integers. Officially the contents of the counter block is not defined, but generally a big endian 128 bit value is used.
I am from c# so know nothing about java script.
I have excel file (xlsx) that I red into byte array (with unity3d c# in webGL build) and want to send it into java script function that parse it into csv structure and return as string.
So the question part is only related to java script that received xlsx as byte array(or any type from memory stream) and return csv as string.
I need that function. What else (libs) do I need for that?
(Update)
The javascript code is
MyConverterXlsxToCsvReturn: function (array,size) {
var buffer = new ArrayBuffer(size);
for (var i = 0; i < size; i++)
buffer[i] = HEAPU8[array + i];
var txt = XLSX.utils.sheet_to_txt(buffer, {type: 'arraybuffer'});
window.alert(Pointer_stringify(txt));
window.alert(Pointer_stringify(txt.length));
var returnStr = Pointer_stringify(txt);
var bufferSize = lengthBytesUTF8(returnStr) + 1;
var buffer = _malloc(bufferSize);
stringToUTF8(returnStr, buffer, bufferSize);
return buffer;
},
I am trying to send byte[] and convert into arraybuffer but in search of correct way to do that.
For now that function return empty string.
I wanted to convert byte array that I received in C# and then red the array in javascript.
As solution I converted the the byte array into hex string with method:
private string ByteArrayToString(byte[] ba)
{
StringBuilder hex = new StringBuilder(ba.Length * 2);
foreach (byte b in ba)
hex.AppendFormat("{0:x2}", b);
return hex.ToString();
}
Then this string i sent to javascript function and converted to array. The rezult was returned as string:
ExcelHexToCSV: function (hexStr) {
console.log("javascript: ExcelHexToCSV");
console.log("javascript received: " + Pointer_stringify(hexStr));
// convert part
var str = Pointer_stringify(hexStr);
var a = [];
for (var i = 0, len = str.length; i < len; i += 2) {
a.push(parseInt(str.substr(i, 2), 16));
}
var data = new Uint8Array(a);
console.log("javascript hex_to_byte:" + data);
// excel part
var workbook = XLSX.read(data, {type: "array"});
var sheetname = workbook.SheetNames[0];
console.log("javascript sheetname: " + sheetname);
var sheetdata = XLSX.utils.sheet_to_csv(workbook.Sheets[sheetname]);
console.log("javascript sheetdata: = " + sheetdata);
var rezult = sheetdata;
var returnStr = rezult;
var bufferSize = lengthBytesUTF8(returnStr) + 1;
var buffer = _malloc(bufferSize);
stringToUTF8(returnStr, buffer, bufferSize);
return buffer;
},
Github link for the my example project
I have an array with bytes and need them to be in StringBase64 and I am using the following:
var base64String = btoa(String.fromCharCode.apply(null, new Uint8Array(arrayDigestion)));
console.log('digestionB64_2 .......:' + digestionB64);
with the following result:
digestionB64_2 .......:6d9310a8df39348ef2bbd8a0f04f65bba64180666848526a4c93e86aa69433e7
And I've also used the following code and get the same results no difference whatsoever:
function arrayBufferToString(buffer) {
var binary = '';
var bytes = new Uint8Array( buffer );
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode( bytes[ i ] );
}
return binary;
}
As I understand Strings in Base 64 must end with an = or == characters
Would it be valid if I add those characters manually at the end or is it valid to have a StringBase64 without them?
I am trying to write a script that generates random numbers these number then are converted to groups of 4Bytes each then these 4-bytes-groups are put into an Uint8Array finally I try to save the array to binary file. Here is my code:
<html>
<head>
<title>Raandom Number Generator</title>
</head>
<body>
<script src="http://code.jquery.com/jquery-1.11.0.min.js"></script>
<script src="filesaver.min.js"></script>
<script type="text/javascript">
$(function() {
if (!Date.now) {
Date.now = function() {
return new Date().getTime();
};
}
alert("started");
var currentMousePos = {
x : -1,
y : -1
};
var randomData = [];
var bytes = new Int8Array(4);
$(document).mousemove(function(event) {
if(randomData.length>=1231){
$(document).unbind('mousemove');
console.log("Finished generating numbers ... trying to save file");
randomData = new Uint8Array(randomData);
var blob = new Blob(randomData, {type: "application/octet-stream"});
saveAs(blob, "rand.bin");
return;
}
currentMousePos.x = event.pageX;
currentMousePos.y = event.pageY;
var longRandomNumber = Date.now() * (event.pageX + 1)
* (event.pageY + 1);
for ( var i = 3; i >= 0; i--) {
bytes[i] = longRandomNumber & (255);
longRandomNumber = longRandomNumber >> 8
}
for ( var i = 0; i < 4; i++) {
randomData.push(bytes[i]);
}
console.log(randomData.length);
});
})
</script>
</body>
</html>
The problem is that the generated file contains numbers plain numbers for example an element in the Uint8Array may be 65, in my understanding to binary this value should be saved as capital letter A but it is stored as 65 instead
Note
filesaver.min.js is a library that I use to save files from JS (Link on GitHub)
The Blob constructor does expect an array of typed arrays to be concatenated, but you are passing only a single Uint8Array into it. This will probably be interpreted as (should I say, "casted to"?) an array of DOM-strings - that's where your numbers are coming from.
A quickfix would be to use
new Blob([randomData], {type: "application/octet-stream"})
// ^ ^
but I would suggest to either do
var randomData = [];
// while randomData.length < 308
var bytes = new Uint8Array(4);
for (var i=4; i--; ) {
bytes[i] = longRandomNumber & (255);
longRandomNumber = longRandomNumber >> 8
}
randomData.push(bytes);
var blob = new Blob(randomData, {type: "application/octet-stream"});
or not use those 4-byte bytes arrays at all:
var randomData = new Uint8Array(1232),
count = 0;
// while count < randomData.length
for (var i=4; i--; ) {
randomData[count++] = longRandomNumber & (255);
longRandomNumber = longRandomNumber >> 8
}
var blob = new Blob([randomData], {type: "application/octet-stream"});
i´m trying to bring a simple text parser from Java to Javascript.
The requierement is to transform a given csv file in to another format. The original file list a number of values according to one id in certain lines:
for example:
11111; 12; 23; 23 ;....
11111; 32; 12; 12 ;....
So the first value is an Id and the other values are according to this Id.
Now I need the same file with alle the values according to the one Id in a single line.
the result should be something like:
11111;12; 23; 23; 32; 12; 12 ;....
I already achieved this with a simple Java class:
public static void main(String[] args) throws Exception {
PrintWriter writer = new PrintWriter("t2_lines.csv", "UTF-8");
BufferedReader br = new BufferedReader(new FileReader("t2.csv"));
String previousId="";
String line;
while ((line = br.readLine()) != null) {
String [] words = line.split(";");
String id = words[0];
if (previousId.equals(id)){
// the loop starts at 4 to cut out some unneded values
for(int i=4;i<words.length;i++) {
writer.print(words[i]+";");
}
}else{
writer.println("");
for(String word : words)
writer.print(word+";");
previousId = id;
}
}
br.close();
writer.close();
}
and now I try to rebuild this thing in Javascript by read in a file from the client and present the result in a textfield - but unfortunately i´ve never implemented anything in Javascript before...
This is my approach so far:
window.onload = function () {
var fileInput = document.getElementById('fileInput');
var origFileDisplayArea = document.getElementById('origFileDisplayArea');
var reformatFileDisplayArea= document.getElementById('reformatFileDisplayArea');
fileInput.addEventListener('change', function (e) {
var file = fileInput.files[0];
var textType = /text.*/;
if (file.type.match(textType)) {
var reader = new FileReader();
reader.readAsText(file);
reader.onload = function (e) {
var result = reader.result;
var table = parse(result);
origFileDisplayArea.innerText = table;
}
} else {
origFileDisplayArea.innerText = "File not supported!"
}
});
}
function parse(input) {
var previousId = "";
var table = "";
if (typeof input !== "undefined")
var lines = input.split("\n");
for (var i = 0; i <= lines.length; i++) {
var line = lines[i];
if (typeof line !== "undefined")
var words = line.split(";");
console.log("words length: ", words.length);
for (var j = 0; j <= words.length; j++ ) {
var word = words[j];
if (typeof word !== "undefined") {
word.toString();
var id = words[0];
if (previousId === id) {
for (var jj = 4; jj <=words.length; jj++){
console.log("jj: " + jj)
table += words[jj]+";";
}
}else {
table += "\n";
for (var word in words) {
table += word + ";";
previousId = id;
}
}
}
}
}
return table;
}
But unfortunately i´m stucked now with undefined values and the whole thing took ages to run.
So any hints/help would be greatly appreciated.
Thanks in advance
Yes for the FileReader, I can't see a way to avoid that in this context. That doesn't look like where you have the problem.
As for parse, the split method can use up a lot of memory so I'd avoid using it on the whole file, and for..in is not designed for looping over an Array.
function parse(str_in) {
var i = -1, j = -1,
str_out = '',
last_id = '',
words;
str_in += '\n'; // not sure if necessary - let the last line pass `while`
// loop by seeking out the next new line
// i = old_index + 1
// j = next \n after old_index
// .slice(i, j) gives just the line
while (-1 !== (j = str_in.indexOf('\n', i = j + 1))) {
words = str_in.slice(i, j).split(';')
// loop words to trim whitespace here if you want
if (last_id === words[0]) // throw away first item if on the same id
words = words.slice(1);
else {
last_id = words[0];
if (str_out.length) // lazy prevent first char newline
str_out += '\n';
}
str_out += words.join(';'); // if you trimmed witespace, re-add here
// if you don't have a final semicolon, add it too
}
return str_out;
}
Now
parse('11111; 12; 23; 23 ;\n11111; 32; 12; 12 ;');
// "11111; 12; 23; 23 ; 32; 12; 12 ;"
Alternatively, you might find it easier to write methods similar to what you're used to in Java so you can work with minimal changes, e.g.
function ReadLineGenerator(text) {
var start = -1, end = -1;
return function readLine() {
if (end < start) {
start = end = -1;
return null;
}
start = end + 1;
end = text.indexOf('\n', start);
if (end !== -1)
return text.slice(start, end);
else
return text.slice(start);
};
}
// example usage
var str = 'a\nb\nc',
f = ReadLineGenerator(str),
line;
while (null !== (line = f()))
console.log(line);
// "a", "b", "c" logged
// line === null