Related
I have some UTF-8 encoded data living in a range of Uint8Array elements in Javascript. Is there an efficient way to decode these out to a regular javascript string (I believe Javascript uses 16 bit Unicode)? I dont want to add one character at the time as the string concaternation would become to CPU intensive.
TextEncoder and TextDecoder from the Encoding standard, which is polyfilled by the stringencoding library, converts between strings and ArrayBuffers:
var uint8array = new TextEncoder().encode("someString");
var string = new TextDecoder().decode(uint8array);
This should work:
// http://www.onicos.com/staff/iz/amuse/javascript/expert/utf.txt
/* utf.js - UTF-8 <=> UTF-16 convertion
*
* Copyright (C) 1999 Masanao Izumo <iz#onicos.co.jp>
* Version: 1.0
* LastModified: Dec 25 1999
* This library is free. You can redistribute it and/or modify it.
*/
function Utf8ArrayToStr(array) {
var out, i, len, c;
var char2, char3;
out = "";
len = array.length;
i = 0;
while(i < len) {
c = array[i++];
switch(c >> 4)
{
case 0: case 1: case 2: case 3: case 4: case 5: case 6: case 7:
// 0xxxxxxx
out += String.fromCharCode(c);
break;
case 12: case 13:
// 110x xxxx 10xx xxxx
char2 = array[i++];
out += String.fromCharCode(((c & 0x1F) << 6) | (char2 & 0x3F));
break;
case 14:
// 1110 xxxx 10xx xxxx 10xx xxxx
char2 = array[i++];
char3 = array[i++];
out += String.fromCharCode(((c & 0x0F) << 12) |
((char2 & 0x3F) << 6) |
((char3 & 0x3F) << 0));
break;
}
}
return out;
}
It's somewhat cleaner as the other solutions because it doesn't use any hacks nor depends on Browser JS functions, e.g. works also in other JS environments.
Check out the JSFiddle demo.
Also see the related questions: here and here
Here's what I use:
var str = String.fromCharCode.apply(null, uint8Arr);
In Node "Buffer instances are also Uint8Array instances", so buf.toString() works in this case.
In NodeJS, we have Buffers available, and string conversion with them is really easy. Better, it's easy to convert a Uint8Array to a Buffer. Try this code, it's worked for me in Node for basically any conversion involving Uint8Arrays:
let str = Buffer.from(uint8arr.buffer).toString();
We're just extracting the ArrayBuffer from the Uint8Array and then converting that to a proper NodeJS Buffer. Then we convert the Buffer to a string (you can throw in a hex or base64 encoding if you want).
If we want to convert back to a Uint8Array from a string, then we'd do this:
let uint8arr = new Uint8Array(Buffer.from(str));
Be aware that if you declared an encoding like base64 when converting to a string, then you'd have to use Buffer.from(str, "base64") if you used base64, or whatever other encoding you used.
This will not work in the browser without a module! NodeJS Buffers just don't exist in the browser, so this method won't work unless you add Buffer functionality to the browser. That's actually pretty easy to do though, just use a module like this, which is both small and fast!
Found in one of the Chrome sample applications, although this is meant for larger blocks of data where you're okay with an asynchronous conversion.
/**
* Converts an array buffer to a string
*
* #private
* #param {ArrayBuffer} buf The buffer to convert
* #param {Function} callback The function to call when conversion is complete
*/
function _arrayBufferToString(buf, callback) {
var bb = new Blob([new Uint8Array(buf)]);
var f = new FileReader();
f.onload = function(e) {
callback(e.target.result);
};
f.readAsText(bb);
}
The solution given by Albert works well as long as the provided function is invoked infrequently and is only used for arrays of modest size, otherwise it is egregiously inefficient. Here is an enhanced vanilla JavaScript solution that works for both Node and browsers and has the following advantages:
• Works efficiently for all octet array sizes
• Generates no intermediate throw-away strings
• Supports 4-byte characters on modern JS engines (otherwise "?" is substituted)
var utf8ArrayToStr = (function () {
var charCache = new Array(128); // Preallocate the cache for the common single byte chars
var charFromCodePt = String.fromCodePoint || String.fromCharCode;
var result = [];
return function (array) {
var codePt, byte1;
var buffLen = array.length;
result.length = 0;
for (var i = 0; i < buffLen;) {
byte1 = array[i++];
if (byte1 <= 0x7F) {
codePt = byte1;
} else if (byte1 <= 0xDF) {
codePt = ((byte1 & 0x1F) << 6) | (array[i++] & 0x3F);
} else if (byte1 <= 0xEF) {
codePt = ((byte1 & 0x0F) << 12) | ((array[i++] & 0x3F) << 6) | (array[i++] & 0x3F);
} else if (String.fromCodePoint) {
codePt = ((byte1 & 0x07) << 18) | ((array[i++] & 0x3F) << 12) | ((array[i++] & 0x3F) << 6) | (array[i++] & 0x3F);
} else {
codePt = 63; // Cannot convert four byte code points, so use "?" instead
i += 3;
}
result.push(charCache[codePt] || (charCache[codePt] = charFromCodePt(codePt)));
}
return result.join('');
};
})();
Uint8Array to String
let str = Buffer.from(key.secretKey).toString('base64');
String to Uint8Array
let uint8arr = new Uint8Array(Buffer.from(data,'base64'));
I was frustrated to see that people were not showing how to go both ways or showing that things work on none trivial UTF8 strings. I found a post on codereview.stackexchange.com that has some code that works well. I used it to turn ancient runes into bytes, to test some crypo on the bytes, then convert things back into a string. The working code is on github here. I renamed the methods for clarity:
// https://codereview.stackexchange.com/a/3589/75693
function bytesToSring(bytes) {
var chars = [];
for(var i = 0, n = bytes.length; i < n;) {
chars.push(((bytes[i++] & 0xff) << 8) | (bytes[i++] & 0xff));
}
return String.fromCharCode.apply(null, chars);
}
// https://codereview.stackexchange.com/a/3589/75693
function stringToBytes(str) {
var bytes = [];
for(var i = 0, n = str.length; i < n; i++) {
var char = str.charCodeAt(i);
bytes.push(char >>> 8, char & 0xFF);
}
return bytes;
}
The unit test uses this UTF-8 string:
// http://kermitproject.org/utf8.html
// From the Anglo-Saxon Rune Poem (Rune version)
const secretUtf8 = `ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ
ᛋᚳᛖᚪᛚ᛫ᚦᛖᚪᚻ᛫ᛗᚪᚾᚾᚪ᛫ᚷᛖᚻᚹᛦᛚᚳ᛫ᛗᛁᚳᛚᚢᚾ᛫ᚻᛦᛏ᛫ᛞᚫᛚᚪᚾ
ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬`;
Note that the string length is only 117 characters but the byte length, when encoded, is 234.
If I uncomment the console.log lines I can see that the string that is decoded is the same string that was encoded (with the bytes passed through Shamir's secret sharing algorithm!):
Do what #Sudhir said, and then to get a String out of the comma seperated list of numbers use:
for (var i=0; i<unitArr.byteLength; i++) {
myString += String.fromCharCode(unitArr[i])
}
This will give you the string you want,
if it's still relevant
If you can't use the TextDecoder API because it is not supported on IE:
You can use the FastestSmallestTextEncoderDecoder polyfill recommended by the Mozilla Developer Network website;
You can use this function also provided at the MDN website:
function utf8ArrayToString(aBytes) {
var sView = "";
for (var nPart, nLen = aBytes.length, nIdx = 0; nIdx < nLen; nIdx++) {
nPart = aBytes[nIdx];
sView += String.fromCharCode(
nPart > 251 && nPart < 254 && nIdx + 5 < nLen ? /* six bytes */
/* (nPart - 252 << 30) may be not so safe in ECMAScript! So...: */
(nPart - 252) * 1073741824 + (aBytes[++nIdx] - 128 << 24) + (aBytes[++nIdx] - 128 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
: nPart > 247 && nPart < 252 && nIdx + 4 < nLen ? /* five bytes */
(nPart - 248 << 24) + (aBytes[++nIdx] - 128 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
: nPart > 239 && nPart < 248 && nIdx + 3 < nLen ? /* four bytes */
(nPart - 240 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
: nPart > 223 && nPart < 240 && nIdx + 2 < nLen ? /* three bytes */
(nPart - 224 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
: nPart > 191 && nPart < 224 && nIdx + 1 < nLen ? /* two bytes */
(nPart - 192 << 6) + aBytes[++nIdx] - 128
: /* nPart < 127 ? */ /* one byte */
nPart
);
}
return sView;
}
let str = utf8ArrayToString([50,72,226,130,130,32,43,32,79,226,130,130,32,226,135,140,32,50,72,226,130,130,79]);
// Must show 2H₂ + O₂ ⇌ 2H₂O
console.log(str);
Try these functions,
var JsonToArray = function(json)
{
var str = JSON.stringify(json, null, 0);
var ret = new Uint8Array(str.length);
for (var i = 0; i < str.length; i++) {
ret[i] = str.charCodeAt(i);
}
return ret
};
var binArrayToJson = function(binArray)
{
var str = "";
for (var i = 0; i < binArray.length; i++) {
str += String.fromCharCode(parseInt(binArray[i]));
}
return JSON.parse(str)
}
source: https://gist.github.com/tomfa/706d10fed78c497731ac, kudos to Tomfa
I'm using this function, which works for me:
function uint8ArrayToBase64(data) {
return btoa(Array.from(data).map((c) => String.fromCharCode(c)).join(''));
}
For ES6 and UTF8 string
decodeURIComponent(escape(String.fromCharCode(...uint8arrData)))
By far the easiest way that has worked for me is:
//1. Create or fetch the Uint8Array to use in the example
const bufferArray = new Uint8Array([10, 10, 10])
//2. Turn the Uint8Array into a regular array
const array = Array.from(bufferArray);
//3. Stringify it (option A)
JSON.stringify(array);
//3. Stringify it (option B: uses #serdarsenay code snippet to decode each item in array)
let binArrayToString = function(binArray) {
let str = "";
for (let i = 0; i < binArray.length; i++) {
str += String.fromCharCode(parseInt(binArray[i]));
}
return str;
}
binArrayToString(array);
class UTF8{
static encode(str:string){return new UTF8().encode(str)}
static decode(data:Uint8Array){return new UTF8().decode(data)}
private EOF_byte:number = -1;
private EOF_code_point:number = -1;
private encoderError(code_point) {
console.error("UTF8 encoderError",code_point)
}
private decoderError(fatal, opt_code_point?):number {
if (fatal) console.error("UTF8 decoderError",opt_code_point)
return opt_code_point || 0xFFFD;
}
private inRange(a:number, min:number, max:number) {
return min <= a && a <= max;
}
private div(n:number, d:number) {
return Math.floor(n / d);
}
private stringToCodePoints(string:string) {
/** #type {Array.<number>} */
let cps = [];
// Based on http://www.w3.org/TR/WebIDL/#idl-DOMString
let i = 0, n = string.length;
while (i < string.length) {
let c = string.charCodeAt(i);
if (!this.inRange(c, 0xD800, 0xDFFF)) {
cps.push(c);
} else if (this.inRange(c, 0xDC00, 0xDFFF)) {
cps.push(0xFFFD);
} else { // (inRange(c, 0xD800, 0xDBFF))
if (i == n - 1) {
cps.push(0xFFFD);
} else {
let d = string.charCodeAt(i + 1);
if (this.inRange(d, 0xDC00, 0xDFFF)) {
let a = c & 0x3FF;
let b = d & 0x3FF;
i += 1;
cps.push(0x10000 + (a << 10) + b);
} else {
cps.push(0xFFFD);
}
}
}
i += 1;
}
return cps;
}
private encode(str:string):Uint8Array {
let pos:number = 0;
let codePoints = this.stringToCodePoints(str);
let outputBytes = [];
while (codePoints.length > pos) {
let code_point:number = codePoints[pos++];
if (this.inRange(code_point, 0xD800, 0xDFFF)) {
this.encoderError(code_point);
}
else if (this.inRange(code_point, 0x0000, 0x007f)) {
outputBytes.push(code_point);
} else {
let count = 0, offset = 0;
if (this.inRange(code_point, 0x0080, 0x07FF)) {
count = 1;
offset = 0xC0;
} else if (this.inRange(code_point, 0x0800, 0xFFFF)) {
count = 2;
offset = 0xE0;
} else if (this.inRange(code_point, 0x10000, 0x10FFFF)) {
count = 3;
offset = 0xF0;
}
outputBytes.push(this.div(code_point, Math.pow(64, count)) + offset);
while (count > 0) {
let temp = this.div(code_point, Math.pow(64, count - 1));
outputBytes.push(0x80 + (temp % 64));
count -= 1;
}
}
}
return new Uint8Array(outputBytes);
}
private decode(data:Uint8Array):string {
let fatal:boolean = false;
let pos:number = 0;
let result:string = "";
let code_point:number;
let utf8_code_point = 0;
let utf8_bytes_needed = 0;
let utf8_bytes_seen = 0;
let utf8_lower_boundary = 0;
while (data.length > pos) {
let _byte = data[pos++];
if (_byte == this.EOF_byte) {
if (utf8_bytes_needed != 0) {
code_point = this.decoderError(fatal);
} else {
code_point = this.EOF_code_point;
}
} else {
if (utf8_bytes_needed == 0) {
if (this.inRange(_byte, 0x00, 0x7F)) {
code_point = _byte;
} else {
if (this.inRange(_byte, 0xC2, 0xDF)) {
utf8_bytes_needed = 1;
utf8_lower_boundary = 0x80;
utf8_code_point = _byte - 0xC0;
} else if (this.inRange(_byte, 0xE0, 0xEF)) {
utf8_bytes_needed = 2;
utf8_lower_boundary = 0x800;
utf8_code_point = _byte - 0xE0;
} else if (this.inRange(_byte, 0xF0, 0xF4)) {
utf8_bytes_needed = 3;
utf8_lower_boundary = 0x10000;
utf8_code_point = _byte - 0xF0;
} else {
this.decoderError(fatal);
}
utf8_code_point = utf8_code_point * Math.pow(64, utf8_bytes_needed);
code_point = null;
}
} else if (!this.inRange(_byte, 0x80, 0xBF)) {
utf8_code_point = 0;
utf8_bytes_needed = 0;
utf8_bytes_seen = 0;
utf8_lower_boundary = 0;
pos--;
code_point = this.decoderError(fatal, _byte);
} else {
utf8_bytes_seen += 1;
utf8_code_point = utf8_code_point + (_byte - 0x80) * Math.pow(64, utf8_bytes_needed - utf8_bytes_seen);
if (utf8_bytes_seen !== utf8_bytes_needed) {
code_point = null;
} else {
let cp = utf8_code_point;
let lower_boundary = utf8_lower_boundary;
utf8_code_point = 0;
utf8_bytes_needed = 0;
utf8_bytes_seen = 0;
utf8_lower_boundary = 0;
if (this.inRange(cp, lower_boundary, 0x10FFFF) && !this.inRange(cp, 0xD800, 0xDFFF)) {
code_point = cp;
} else {
code_point = this.decoderError(fatal, _byte);
}
}
}
}
//Decode string
if (code_point !== null && code_point !== this.EOF_code_point) {
if (code_point <= 0xFFFF) {
if (code_point > 0)result += String.fromCharCode(code_point);
} else {
code_point -= 0x10000;
result += String.fromCharCode(0xD800 + ((code_point >> 10) & 0x3ff));
result += String.fromCharCode(0xDC00 + (code_point & 0x3ff));
}
}
}
return result;
}
`
Using base64 as the encoding format works quite well. This is how it was implemented for passing secrets via urls in Firefox Send. You will need the base64-js package. These are the functions from the Send source code:
const b64 = require("base64-js")
function arrayToB64(array) {
return b64.fromByteArray(array).replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "")
}
function b64ToArray(str) {
return b64.toByteArray(str + "===".slice((str.length + 3) % 4))
}
With vanilla, browser side, recording from microphone, base64 functions worked for me (I had to implement an audio sending function to a chat).
const ui8a = new Uint8Array(e.target.result);
const string = btoa(ui8a);
const ui8a_2 = atob(string).split(',');
Full code now. Thanks to Bryan Jennings & breakspirit#py4u.net for the code.
https://medium.com/#bryanjenningz/how-to-record-and-play-audio-in-javascript-faa1b2b3e49b
https://www.py4u.net/discuss/282499
index.html
<html>
<head>
<title>Record Audio Test</title>
<meta name="encoding" charset="utf-8" />
</head>
<body>
<h1>Audio Recording Test</h1>
<script src="index.js"></script>
<button id="action" onclick="start()">Start</button>
<button id="stop" onclick="stop()">Stop</button>
<button id="play" onclick="play()">Listen</button>
</body>
</html>
index.js:
const recordAudio = () =>
new Promise(async resolve => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const mediaRecorder = new MediaRecorder(stream);
const audioChunks = [];
mediaRecorder.addEventListener("dataavailable", event => {
audioChunks.push(event.data);
});
const start = () => mediaRecorder.start();
const stop = () =>
new Promise(resolve => {
mediaRecorder.addEventListener("stop", () => {
const audioBlob = new Blob(audioChunks);
const audioUrl = URL.createObjectURL(audioBlob);
const audio = new Audio(audioUrl);
const play = () => audio.play();
resolve({ audioBlob, audioUrl, play });
});
mediaRecorder.stop();
});
resolve({ start, stop });
});
let recorder = null;
let audio = null;
const sleep = time => new Promise(resolve => setTimeout(resolve, time));
const start = async () => {
recorder = await recordAudio();
recorder.start();
}
const stop = async () => {
audio = await recorder.stop();
read(audio.audioUrl);
}
const play = ()=> {
audio.play();
}
const read = (blobUrl)=> {
var xhr = new XMLHttpRequest;
xhr.responseType = 'blob';
xhr.onload = function() {
var recoveredBlob = xhr.response;
const reader = new FileReader();
// This fires after the blob has been read/loaded.
reader.addEventListener('loadend', (e) => {
const ui8a = new Uint8Array(e.target.result);
const string = btoa(ui8a);
const ui8a_2 = atob(string).split(',');
playByteArray(ui8a_2);
});
// Start reading the blob as text.
reader.readAsArrayBuffer(recoveredBlob);
};
// get the blob through blob url
xhr.open('GET', blobUrl);
xhr.send();
}
window.onload = init;
var context; // Audio context
var buf; // Audio buffer
function init() {
if (!window.AudioContext) {
if (!window.webkitAudioContext) {
alert("Your browser does not support any AudioContext and cannot play back this audio.");
return;
}
window.AudioContext = window.webkitAudioContext;
}
context = new AudioContext();
}
function playByteArray(byteArray) {
var arrayBuffer = new ArrayBuffer(byteArray.length);
var bufferView = new Uint8Array(arrayBuffer);
for (i = 0; i < byteArray.length; i++) {
bufferView[i] = byteArray[i];
}
context.decodeAudioData(arrayBuffer, function(buffer) {
buf = buffer;
play2();
});
}
// Play the loaded file
function play2() {
// Create a source node from the buffer
var source = context.createBufferSource();
source.buffer = buf;
// Connect to the final output node (the speakers)
source.connect(context.destination);
// Play immediately
source.start(0);
}
var decodedString = decodeURIComponent(escape(String.fromCharCode(...new Uint8Array(err))));
var obj = JSON.parse(decodedString);
I am using this Typescript snippet:
function UInt8ArrayToString(uInt8Array: Uint8Array): string
{
var s: string = "[";
for(var i: number = 0; i < uInt8Array.byteLength; i++)
{
if( i > 0 )
s += ", ";
s += uInt8Array[i];
}
s += "]";
return s;
}
Remove the type annotations if you need the JavaScript version.
Hope this helps!
I need to change this code from JavaScript to Java from this website:
https://jsfiddle.net/3L6L2fmv/?utm_source=website&utm_medium=embed&utm_campaign=3L6L2fmv
This is the call from another function that generates random numbers but has a 1 in 51 chance of the number being zero: (where "hash" is a 64-bit hexadecimal number)
// In 1 of 51 games the game crashes instantly.
if (divisible(hash, 51))
return 0;
And this is the function: (where mod is always 51)
function divisible(hash, mod) {
// So ABCDEFGHIJ should be chunked like AB CDEF GHIJ
var val = 0;
var o = hash.length % 4;
for (var i = o > 0 ? o - 4 : 0; i < hash.length; i += 4) {
val = ((val << 16) + parseInt(hash.substring(i, i+4), 16)) % mod;
}
return val === 0;
}
So basically when ever the JavaScript version returns a zero for a certain hash, I need the Java version to also return a zero for the exact same hash.
Here is my first attempt and pretty much as far as I can get due to my poor knowledge of JavaScript.
public static double divisible(String hash, int mod)
{
int val = 0;
int o = hash.length() % 4;
int i = 0;
if(o > 0) {i = o - 4;}
if(o < 0) {i = 0;}
for ( int a = i; a < hash.length(); a += 4)
{
val = ((val << 16) + Integer.parseInt(hash.substring(i, i+4), 16)) % mod;
}
return val;
}
Looks like you just confused the variables a bit in your Java code, other than that, it seems fine.
public static double divisible(String hash, int mod)
{
int val = 0;
int o = hash.length() % 4;
int i = 0;
if(o > 0) {i = o - 4;}
if(o < 0) {i = 0;}
for ( int a = i; a < hash.length(); a += 4)
{
val = ((val << 16) + Integer.parseInt(hash.substring(a, a+4), 16)) % mod;
}
return val;
}
The values in the substring should be a not i.
You probably assumed i = index.
An improvement would be the following. You do not need to create a new variable in Java for loops.
public static double divisible(String hash, int mod)
{
int val = 0;
int o = hash.length() % 4;
int i = 0;
if(o > 0) {i = o - 4;}
if(o < 0) {i = 0;}
for (; i < hash.length(); i += 4)
{
val = ((val << 16) + Integer.parseInt(hash.substring(i, i+4), 16)) % mod;
}
return val;
}
Ive made a Websocket server in C# and a HTML UI together with JS.
I can get the server to communicate and do the handshake properly, but the information being sent is absolute gibberish and looks something like this:
???=u??X?G?
I have tried encoding it, but the result is not very different.
My JS looks like this:
// the user clicked the big red button
$('#bigredbutton_send').click(function () {
ws_send($('#console_input').val());
});
$('#console_input').keyup(function (e) {
if(e.keyCode == 13) // enter is pressed
ws_send($('#console_input').val());
});
CSS:
<input type="image" src="button.png" name="bigredbutton_send" id="bigredbutton_send" value="VALUE" />
<input type="text" name="console_input" id="console_input" value="Hello123" />
and c# module that recieves the information looks like this:
private void Read(IAsyncResult ar)
{
int sizeOfReceivedData = ConnectionSocket.EndReceive(ar);
if (sizeOfReceivedData > 0)
{
int start = 0, end = dataBuffer.Length - 1;
// if we are not already reading something, look for the start byte as specified in the protocol
if (!readingData)
{
for (start = 0; start < dataBuffer.Length - 1; start++)
{
if (dataBuffer[start] == (byte)WrapperBytes.Start)
{
readingData = true; // we found the begining and can now start reading
start++; // we dont need the start byte. Incrementing the start counter will walk us past it
break;
}
}
} // no else here, the value of readingData might have changed
// if a begining was found in the buffer, or if we are continuing from another buffer
if (readingData)
{
bool endIsInThisBuffer = false;
// look for the end byte in the received data
for (end = start; end < sizeOfReceivedData; end++)
{
byte currentByte = dataBuffer[end];
if (dataBuffer[end] == (byte)WrapperBytes.End)
{
endIsInThisBuffer = true; // we found the ending byte
break;
}
}
// the end is in this buffer, which means that we are done reading
if (endIsInThisBuffer == true)
{
// we are no longer reading data
readingData = false;
// put the data into the string builder
dataString.Append(Encoding.UTF8.GetString(dataBuffer, start, end - start));
// trigger the event
int size = Encoding.UTF8.GetBytes(dataString.ToString().ToCharArray()).Length;
recievedData = dataString.ToString();
OnDataReceived(new DataReceivedEventArgs(size, dataString.ToString()));
dataString = null;
dataString = new StringBuilder();
}
else // if the end is not in this buffer then put everyting from start to the end of the buffer into the datastring and keep on reading
{
dataString.Append(Encoding.UTF8.GetString(dataBuffer, start, end - start));
}
}
// continue listening for more data
Listen();
}
else // the socket is closed
{
if (Disconnected != null)
Disconnected(this, EventArgs.Empty);
}
// Testing to see if readable
ReadRecievedData(Convert.ToString(dataString));
}
And they all return something, however, they always return this raw, gibberishlike, data that looks like this:
???=u??X?G?
I understand that it lacks encoding, and I have tried to encode it several times - but the information just looks weirder and actually never returns anything of what I want it to. Any help would be greatly appreciated.
Update
The dataBuffer is called when a new connection is invoked.
public WebSocketConnection(Socket socket, int bufferSize)
{
ConnectionSocket = socket;
dataBuffer = new byte[bufferSize];
dataString = new StringBuilder();
GUID = System.Guid.NewGuid();
Listen();
}
Listen() creates this:
private void Listen()
{
ConnectionSocket.BeginReceive(dataBuffer, 0, dataBuffer.Length, 0, Read, null);
}
SOLVED!
I looked over it again yesterday and I solved the problem. I wasnt parsing the bits correct - So I created this instead:
byte b = dataBuffer[1];
int dataLength = 0;
int totalLength = 0;
int keyIndex = 0;
int length = dataBuffer.Length;
if (b - 128 <= 125)
{
dataLength = b - 128;
keyIndex = 2;
totalLength = dataLength + 6;
}
if (b - 128 == 126)
{
dataLength = BitConverter.ToInt16(new byte[] { dataBuffer[3], dataBuffer[2] }, 0);
keyIndex = 4;
totalLength = dataLength + 8;
}
if (b - 128 == 127)
{
dataLength = (int)BitConverter.ToInt64(new byte[] { dataBuffer[9], dataBuffer[8], dataBuffer[7], dataBuffer[6], dataBuffer[5], dataBuffer[4],
dataBuffer[3], dataBuffer[2] }, 0);
keyIndex = 10;
totalLength = dataLength + 14;
}
if (totalLength > length)
throw new Exception("The buffer length is small than the data length");
byte[] key = new byte[] { dataBuffer[keyIndex], dataBuffer[keyIndex + 1], dataBuffer[keyIndex + 2], dataBuffer[keyIndex + 3] };
int dataIndex = keyIndex + 4;
int count = 0;
for (int i = dataIndex; i < totalLength; i++)
{
dataBuffer[i] = (byte)(dataBuffer[i] ^ key[count % 4]);
count++;
}
ReadRecievedData(Encoding.ASCII.GetString(dataBuffer, dataIndex, dataLength));
It builds on the solution over hereHow to (de)construct data frames in WebSockets hybi 08+?
WebSocket data is framed, so you have to read frame by frame and extract the data from it.
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-------+-+-------------+-------------------------------+
|F|R|R|R| opcode|M| Payload len | Extended payload length |
|I|S|S|S| (4) |A| (7) | (16/64) |
|N|V|V|V| |S| | (if payload len==126/127) |
| |1|2|3| |K| | |
+-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - +
| Extended payload length continued, if payload len == 127 |
+ - - - - - - - - - - - - - - - +-------------------------------+
| |Masking-key, if MASK set to 1 |
+-------------------------------+-------------------------------+
| Masking-key (continued) | Payload Data |
+-------------------------------- - - - - - - - - - - - - - - - +
: Payload Data continued ... :
+ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
| Payload Data continued ... |
+---------------------------------------------------------------+
Writing a WebSocket server in C#
Writing WebSocket servers
Please try this
private void Read(IAsyncResult ar)
{
int sizeOfReceivedData = ConnectionSocket.EndReceive(ar);
if (sizeOfReceivedData > 0)
{
int start = 0, end = sizeOfReceivedData - 1;
var bufferList = dataBuffer.ToList();
bool endIsInThisBuffer = dataBuffer.Contains(255); // 255 = end
if (endIsInThisBuffer)
{
end = bufferList.IndexOf(255);
end--; // we dont want to include this byte
}
bool startIsInThisBuffer = dataBuffer.Contains(0); // 0 = start
if (startIsInThisBuffer)
{
var zeroPos = bufferList.IndexOf(0);
if (zeroPos < end) // we might be looking at one of the bytes in the end of the array that hasn't been set
{
start = zeroPos;
start++; // we dont want to include this byte
}
}
dataString.Append(Encoding.UTF8.GetString(dataBuffer, start, (end - start) + 1));
if (endIsInThisBuffer)
{
var data = dataString.ToString();
OnDataReceived(new DataReceivedEventArgs(data.Length, data));
// Testing to see if readable
ReadRecievedData(data);
dataString = new StringBuilder();
}
Listen();
}
else // the socket is closed
{
if (Disconnected != null)
Disconnected(this, EventArgs.Empty);
}
}
SOLVED!
I looked over it again yesterday and I solved the problem. I wasnt parsing the bits correct - So I created this instead:
byte b = dataBuffer[1];
int dataLength = 0;
int totalLength = 0;
int keyIndex = 0;
int length = dataBuffer.Length;
if (b - 128 <= 125)
{
dataLength = b - 128;
keyIndex = 2;
totalLength = dataLength + 6;
}
if (b - 128 == 126)
{
dataLength = BitConverter.ToInt16(new byte[] { dataBuffer[3], dataBuffer[2] }, 0);
keyIndex = 4;
totalLength = dataLength + 8;
}
if (b - 128 == 127)
{
dataLength = (int)BitConverter.ToInt64(new byte[] { dataBuffer[9], dataBuffer[8], dataBuffer[7], dataBuffer[6], dataBuffer[5], dataBuffer[4],
dataBuffer[3], dataBuffer[2] }, 0);
keyIndex = 10;
totalLength = dataLength + 14;
}
if (totalLength > length)
throw new Exception("The buffer length is small than the data length");
byte[] key = new byte[] { dataBuffer[keyIndex], dataBuffer[keyIndex + 1], dataBuffer[keyIndex + 2], dataBuffer[keyIndex + 3] };
int dataIndex = keyIndex + 4;
int count = 0;
for (int i = dataIndex; i < totalLength; i++)
{
dataBuffer[i] = (byte)(dataBuffer[i] ^ key[count % 4]);
count++;
}
ReadRecievedData(Encoding.ASCII.GetString(dataBuffer, dataIndex, dataLength));
It builds on the solution over hereHow to (de)construct data frames in WebSockets hybi 08+?
I have a file in the following format:
utf-8 encoded text block
separator
binary data block
I use JavaScript's FileReader to read the file as a binary string using
FileReader.readAsBinaryString like so:
var reader = new FileReader();
reader.onload = function(evt){
// Here I use the separator position to divide the file content into
// header and binary
...
console.log(header);
};
FileReader.onerror = function (evt) {
onFailure(evt.target.error.code);
}
reader.readAsBinaryString(blobFile);
The header is not parsed as UTF-8. I know that FileReader.readAsText takes the encoding of the file into account while FileReader.readAsBinaryString reads the file byte by byte.
How do I convert the header to utf8? reading the file twice, once as binary string to read the binary data and again as text to get the first block as utf8 encoded don't appeal to me.
I found the answer on http://snipplr.com/view/31206/:
I have tested it on French characters and it converts then to utf8 without any issues.
function readUTF8String(bytes) {
var ix = 0;
if (bytes.slice(0, 3) == "\xEF\xBB\xBF") {
ix = 3;
}
var string = "";
for (; ix < bytes.length; ix++) {
var byte1 = bytes[ix].charCodeAt(0);
if (byte1 < 0x80) {
string += String.fromCharCode(byte1);
} else if (byte1 >= 0xC2 && byte1 < 0xE0) {
var byte2 = bytes[++ix].charCodeAt(0);
string += String.fromCharCode(((byte1 & 0x1F) << 6) + (byte2 & 0x3F));
} else if (byte1 >= 0xE0 && byte1 < 0xF0) {
var byte2 = bytes[++ix].charCodeAt(0);
var byte3 = bytes[++ix].charCodeAt(0);
string += String.fromCharCode(((byte1 & 0xFF) << 12) + ((byte2 & 0x3F) << 6) + (byte3 & 0x3F));
} else if (byte1 >= 0xF0 && byte1 < 0xF5) {
var byte2 = bytes[++ix].charCodeAt(0);
var byte3 = bytes[++ix].charCodeAt(0);
var byte4 = bytes[++ix].charCodeAt(0);
var codepoint = ((byte1 & 0x07) << 18) + ((byte2 & 0x3F) << 12) + ((byte3 & 0x3F) << 6) + (byte4 & 0x3F);
codepoint -= 0x10000;
string += String.fromCharCode(
(codepoint >> 10) + 0xD800, (codepoint & 0x3FF) + 0xDC00
);
}
}
return string;
}
the result is a atring so you can iterate and convert every byte to its ascii representation using String.fromCharCode something like.
var cursor=0
var header="";
while(cursor!=blob.length && blob[cursor]!=/*separator code*/){
header+=String.fromCharCode(blob[cursor]);
cursor+=1;
}
or
var pos=blob.indexOf(/*separator*/);
var header=String.fromCharCode.apply(this,blob.substr(0,pos).split(' '))
The error in the title is thrown only in Google Chrome, according to my tests. I'm base64 encoding a big XML file so that it can be downloaded:
this.loader.src = "data:application/x-forcedownload;base64,"+
btoa("<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+"<"+this.gamesave.tagName+">"
+this.xml.firstChild.innerHTML
+"</"+this.gamesave.tagName+">");
this.loader is hidden iframe.
This error is actually quite a change because normally, Google Chrome would crash upon btoa call. Mozilla Firefox has no problems here, so the issue is browser related.
I'm not aware of any strange characters in file. Actually I do believe there are no non-ascii characters.
Q:
How do I find the problematic characters and replace them so that Chrome stops complaining?
I have tried to use Downloadify to initiate the download, but it does not work. It's unreliable and throws no errors to allow debug.
If you have UTF8, use this (actually works with SVG source), like:
btoa(unescape(encodeURIComponent(str)))
example:
var imgsrc = 'data:image/svg+xml;base64,' + btoa(unescape(encodeURIComponent(markup)));
var img = new Image(1, 1); // width, height values are optional params
img.src = imgsrc;
If you need to decode that base64, use this:
var str2 = decodeURIComponent(escape(window.atob(b64)));
console.log(str2);
Example:
var str = "äöüÄÖÜçéèñ";
var b64 = window.btoa(unescape(encodeURIComponent(str)))
console.log(b64);
var str2 = decodeURIComponent(escape(window.atob(b64)));
console.log(str2);
Note: if you need to get this to work in mobile-safari, you might need to strip all the white-space from the base64 data...
function b64_to_utf8( str ) {
str = str.replace(/\s/g, '');
return decodeURIComponent(escape(window.atob( str )));
}
2017 Update
This problem has been bugging me again.
The simple truth is, atob doesn't really handle UTF8-strings - it's ASCII only.
Also, I wouldn't use bloatware like js-base64.
But webtoolkit does have a small, nice and very maintainable implementation:
/**
*
* Base64 encode / decode
* http://www.webtoolkit.info
*
**/
var Base64 = {
// private property
_keyStr: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
// public method for encoding
, encode: function (input)
{
var output = "";
var chr1, chr2, chr3, enc1, enc2, enc3, enc4;
var i = 0;
input = Base64._utf8_encode(input);
while (i < input.length)
{
chr1 = input.charCodeAt(i++);
chr2 = input.charCodeAt(i++);
chr3 = input.charCodeAt(i++);
enc1 = chr1 >> 2;
enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);
enc4 = chr3 & 63;
if (isNaN(chr2))
{
enc3 = enc4 = 64;
}
else if (isNaN(chr3))
{
enc4 = 64;
}
output = output +
this._keyStr.charAt(enc1) + this._keyStr.charAt(enc2) +
this._keyStr.charAt(enc3) + this._keyStr.charAt(enc4);
} // Whend
return output;
} // End Function encode
// public method for decoding
,decode: function (input)
{
var output = "";
var chr1, chr2, chr3;
var enc1, enc2, enc3, enc4;
var i = 0;
input = input.replace(/[^A-Za-z0-9\+\/\=]/g, "");
while (i < input.length)
{
enc1 = this._keyStr.indexOf(input.charAt(i++));
enc2 = this._keyStr.indexOf(input.charAt(i++));
enc3 = this._keyStr.indexOf(input.charAt(i++));
enc4 = this._keyStr.indexOf(input.charAt(i++));
chr1 = (enc1 << 2) | (enc2 >> 4);
chr2 = ((enc2 & 15) << 4) | (enc3 >> 2);
chr3 = ((enc3 & 3) << 6) | enc4;
output = output + String.fromCharCode(chr1);
if (enc3 != 64)
{
output = output + String.fromCharCode(chr2);
}
if (enc4 != 64)
{
output = output + String.fromCharCode(chr3);
}
} // Whend
output = Base64._utf8_decode(output);
return output;
} // End Function decode
// private method for UTF-8 encoding
,_utf8_encode: function (string)
{
var utftext = "";
string = string.replace(/\r\n/g, "\n");
for (var n = 0; n < string.length; n++)
{
var c = string.charCodeAt(n);
if (c < 128)
{
utftext += String.fromCharCode(c);
}
else if ((c > 127) && (c < 2048))
{
utftext += String.fromCharCode((c >> 6) | 192);
utftext += String.fromCharCode((c & 63) | 128);
}
else
{
utftext += String.fromCharCode((c >> 12) | 224);
utftext += String.fromCharCode(((c >> 6) & 63) | 128);
utftext += String.fromCharCode((c & 63) | 128);
}
} // Next n
return utftext;
} // End Function _utf8_encode
// private method for UTF-8 decoding
,_utf8_decode: function (utftext)
{
var string = "";
var i = 0;
var c, c1, c2, c3;
c = c1 = c2 = 0;
while (i < utftext.length)
{
c = utftext.charCodeAt(i);
if (c < 128)
{
string += String.fromCharCode(c);
i++;
}
else if ((c > 191) && (c < 224))
{
c2 = utftext.charCodeAt(i + 1);
string += String.fromCharCode(((c & 31) << 6) | (c2 & 63));
i += 2;
}
else
{
c2 = utftext.charCodeAt(i + 1);
c3 = utftext.charCodeAt(i + 2);
string += String.fromCharCode(((c & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63));
i += 3;
}
} // Whend
return string;
} // End Function _utf8_decode
}
https://www.fileformat.info/info/unicode/utf8.htm
For any character equal to or below 127 (hex 0x7F), the UTF-8
representation is one byte. It is just the lowest 7 bits of the full
unicode value. This is also the same as the ASCII value.
For characters equal to or below 2047 (hex 0x07FF), the UTF-8
representation is spread across two bytes. The first byte will have
the two high bits set and the third bit clear (i.e. 0xC2 to 0xDF). The
second byte will have the top bit set and the second bit clear (i.e.
0x80 to 0xBF).
For all characters equal to or greater than 2048 but less than 65535
(0xFFFF), the UTF-8 representation is spread across three bytes.
Use a library instead
We don't have to reinvent the wheel. Just use a library to save the time and headache.
js-base64
https://github.com/dankogai/js-base64 is good and I confirm it supports unicode very well.
Base64.encode('dankogai'); // ZGFua29nYWk=
Base64.encode('小飼弾'); // 5bCP6aO85by+
Base64.encodeURI('小飼弾'); // 5bCP6aO85by-
Base64.decode('ZGFua29nYWk='); // dankogai
Base64.decode('5bCP6aO85by+'); // 小飼弾
// note .decodeURI() is unnecessary since it accepts both flavors
Base64.decode('5bCP6aO85by-'); // 小飼弾
Using btoa with unescape and encodeURIComponent didn't work for me. Replacing all the special characters with XML/HTML entities and then converting to the base64 representation was the only way to solve this issue for me. Some code:
base64 = btoa(str.replace(/[\u00A0-\u2666]/g, function(c) {
return '&#' + c.charCodeAt(0) + ';';
}));
I just thought I should share how I actually solved the problem and why I think this is the right solution (provided you don't optimize for old browser).
Converting data to dataURL (data: ...)
var blob = new Blob(
// I'm using page innerHTML as data
// note that you can use the array
// to concatenate many long strings EFFICIENTLY
[document.body.innerHTML],
// Mime type is important for data url
{type : 'text/html'}
);
// This FileReader works asynchronously, so it doesn't lag
// the web application
var a = new FileReader();
a.onload = function(e) {
// Capture result here
console.log(e.target.result);
};
a.readAsDataURL(blob);
Allowing user to save data
Apart from obvious solution - opening new window with your dataURL as URL you can do two other things.
1. Use fileSaver.js
File saver can create actual fileSave dialog with predefined filename. It can also fallback to normal dataURL approach.
2. Use (experimental) URL.createObjectURL
This is great for reusing base64 encoded data. It creates a short URL for your dataURL:
console.log(URL.createObjectURL(blob));
//Prints: blob:http://stackoverflow.com/7c18953f-f5f8-41d2-abf5-e9cbced9bc42
Don't forget to use the URL including the leading blob prefix. I used document.body again:
You can use this short URL as AJAX target, <script> source or <a> href location. You're responsible for destroying the URL though:
URL.revokeObjectURL('blob:http://stackoverflow.com/7c18953f-f5f8-41d2-abf5-e9cbced9bc42')
As an complement to Stefan Steiger answer: (as it doesn't look nice as a comment)
Extending String prototype:
String.prototype.b64encode = function() {
return btoa(unescape(encodeURIComponent(this)));
};
String.prototype.b64decode = function() {
return decodeURIComponent(escape(atob(this)));
};
Usage:
var str = "äöüÄÖÜçéèñ";
var encoded = str.b64encode();
console.log( encoded.b64decode() );
NOTE:
As stated in the comments, using unescape is not recommended as it may be removed in the future:
Warning: Although unescape() is not strictly deprecated (as in "removed from the Web standards"), it is defined in Annex B of the ECMA-262 standard, whose introduction states:
… All of the language features and behaviours specified in this annex have one or more undesirable characteristics and in the absence of legacy usage would be removed from this specification.
Note: Do not use unescape to decode URIs, use decodeURI or decodeURIComponent instead.
btoa() only support characters from String.fromCodePoint(0) up to String.fromCodePoint(255). For Base64 characters with a code point 256 or higher you need to encode/decode these before and after.
And in this point it becomes tricky...
Every possible sign are arranged in a Unicode-Table. The Unicode-Table is divided in different planes (languages, math symbols, and so on...). Every sign in a plane has a unique code point number. Theoretically, the number can become arbitrarily large.
A computer stores the data in bytes (8 bit, hexadecimal 0x00 - 0xff, binary 00000000 - 11111111, decimal 0 - 255). This range normally use to save basic characters (Latin1 range).
For characters with higher codepoint then 255 exist different encodings. JavaScript use 16 bits per sign (UTF-16), the string called DOMString. Unicode can handle code points up to 0x10fffff. That means, that a method must be exist to store several bits over several cells away.
String.fromCodePoint(0x10000).length == 2
UTF-16 use surrogate pairs to store 20bits in two 16bit cells. The first higher surrogate begins with 110110xxxxxxxxxx, the lower second one with 110111xxxxxxxxxx. Unicode reserved own planes for this: https://unicode-table.com/de/#high-surrogates
To store characters in bytes (Latin1 range) standardized procedures use UTF-8.
Sorry to say that, but I think there is no other way to implement this function self.
function stringToUTF8(str)
{
let bytes = [];
for(let character of str)
{
let code = character.codePointAt(0);
if(code <= 127)
{
let byte1 = code;
bytes.push(byte1);
}
else if(code <= 2047)
{
let byte1 = 0xC0 | (code >> 6);
let byte2 = 0x80 | (code & 0x3F);
bytes.push(byte1, byte2);
}
else if(code <= 65535)
{
let byte1 = 0xE0 | (code >> 12);
let byte2 = 0x80 | ((code >> 6) & 0x3F);
let byte3 = 0x80 | (code & 0x3F);
bytes.push(byte1, byte2, byte3);
}
else if(code <= 2097151)
{
let byte1 = 0xF0 | (code >> 18);
let byte2 = 0x80 | ((code >> 12) & 0x3F);
let byte3 = 0x80 | ((code >> 6) & 0x3F);
let byte4 = 0x80 | (code & 0x3F);
bytes.push(byte1, byte2, byte3, byte4);
}
}
return bytes;
}
function utf8ToString(bytes, fallback)
{
let valid = undefined;
let codePoint = undefined;
let codeBlocks = [0, 0, 0, 0];
let result = "";
for(let offset = 0; offset < bytes.length; offset++)
{
let byte = bytes[offset];
if((byte & 0x80) == 0x00)
{
codeBlocks[0] = byte & 0x7F;
codePoint = codeBlocks[0];
}
else if((byte & 0xE0) == 0xC0)
{
codeBlocks[0] = byte & 0x1F;
byte = bytes[++offset];
if(offset >= bytes.length || (byte & 0xC0) != 0x80) { valid = false; break; }
codeBlocks[1] = byte & 0x3F;
codePoint = (codeBlocks[0] << 6) + codeBlocks[1];
}
else if((byte & 0xF0) == 0xE0)
{
codeBlocks[0] = byte & 0xF;
for(let blockIndex = 1; blockIndex <= 2; blockIndex++)
{
byte = bytes[++offset];
if(offset >= bytes.length || (byte & 0xC0) != 0x80) { valid = false; break; }
codeBlocks[blockIndex] = byte & 0x3F;
}
if(valid === false) { break; }
codePoint = (codeBlocks[0] << 12) + (codeBlocks[1] << 6) + codeBlocks[2];
}
else if((byte & 0xF8) == 0xF0)
{
codeBlocks[0] = byte & 0x7;
for(let blockIndex = 1; blockIndex <= 3; blockIndex++)
{
byte = bytes[++offset];
if(offset >= bytes.length || (byte & 0xC0) != 0x80) { valid = false; break; }
codeBlocks[blockIndex] = byte & 0x3F;
}
if(valid === false) { break; }
codePoint = (codeBlocks[0] << 18) + (codeBlocks[1] << 12) + (codeBlocks[2] << 6) + (codeBlocks[3]);
}
else
{
valid = false; break;
}
result += String.fromCodePoint(codePoint);
}
if(valid === false)
{
if(!fallback)
{
throw new TypeError("Malformed utf-8 encoding.");
}
result = "";
for(let offset = 0; offset != bytes.length; offset++)
{
result += String.fromCharCode(bytes[offset] & 0xFF);
}
}
return result;
}
function decodeBase64(text, binary)
{
if(/[^0-9a-zA-Z\+\/\=]/.test(text)) { throw new TypeError("The string to be decoded contains characters outside of the valid base64 range."); }
let codePointA = 'A'.codePointAt(0);
let codePointZ = 'Z'.codePointAt(0);
let codePointa = 'a'.codePointAt(0);
let codePointz = 'z'.codePointAt(0);
let codePointZero = '0'.codePointAt(0);
let codePointNine = '9'.codePointAt(0);
let codePointPlus = '+'.codePointAt(0);
let codePointSlash = '/'.codePointAt(0);
function getCodeFromKey(key)
{
let keyCode = key.codePointAt(0);
if(keyCode >= codePointA && keyCode <= codePointZ)
{
return keyCode - codePointA;
}
else if(keyCode >= codePointa && keyCode <= codePointz)
{
return keyCode + 26 - codePointa;
}
else if(keyCode >= codePointZero && keyCode <= codePointNine)
{
return keyCode + 52 - codePointZero;
}
else if(keyCode == codePointPlus)
{
return 62;
}
else if(keyCode == codePointSlash)
{
return 63;
}
return undefined;
}
let codes = Array.from(text).map(character => getCodeFromKey(character));
let bytesLength = Math.ceil(codes.length / 4) * 3;
if(codes[codes.length - 2] == undefined) { bytesLength = bytesLength - 2; } else if(codes[codes.length - 1] == undefined) { bytesLength--; }
let bytes = new Uint8Array(bytesLength);
for(let offset = 0, index = 0; offset < bytes.length;)
{
let code1 = codes[index++];
let code2 = codes[index++];
let code3 = codes[index++];
let code4 = codes[index++];
let byte1 = (code1 << 2) | (code2 >> 4);
let byte2 = ((code2 & 0xf) << 4) | (code3 >> 2);
let byte3 = ((code3 & 0x3) << 6) | code4;
bytes[offset++] = byte1;
bytes[offset++] = byte2;
bytes[offset++] = byte3;
}
if(binary) { return bytes; }
return utf8ToString(bytes, true);
}
function encodeBase64(bytes) {
if (bytes === undefined || bytes === null) {
return '';
}
if (bytes instanceof Array) {
bytes = bytes.filter(item => {
return Number.isFinite(item) && item >= 0 && item <= 255;
});
}
if (
!(
bytes instanceof Uint8Array ||
bytes instanceof Uint8ClampedArray ||
bytes instanceof Array
)
) {
if (typeof bytes === 'string') {
const str = bytes;
bytes = Array.from(unescape(encodeURIComponent(str))).map(ch =>
ch.codePointAt(0)
);
} else {
throw new TypeError('bytes must be of type Uint8Array or String.');
}
}
const keys = [
'A',
'B',
'C',
'D',
'E',
'F',
'G',
'H',
'I',
'J',
'K',
'L',
'M',
'N',
'O',
'P',
'Q',
'R',
'S',
'T',
'U',
'V',
'W',
'X',
'Y',
'Z',
'a',
'b',
'c',
'd',
'e',
'f',
'g',
'h',
'i',
'j',
'k',
'l',
'm',
'n',
'o',
'p',
'q',
'r',
's',
't',
'u',
'v',
'w',
'x',
'y',
'z',
'0',
'1',
'2',
'3',
'4',
'5',
'6',
'7',
'8',
'9',
'+',
'/'
];
const fillKey = '=';
let byte1;
let byte2;
let byte3;
let sign1 = ' ';
let sign2 = ' ';
let sign3 = ' ';
let sign4 = ' ';
let result = '';
for (let index = 0; index < bytes.length; ) {
let fillUpAt = 0;
// tslint:disable:no-increment-decrement
byte1 = bytes[index++];
byte2 = bytes[index++];
byte3 = bytes[index++];
if (byte2 === undefined) {
byte2 = 0;
fillUpAt = 2;
}
if (byte3 === undefined) {
byte3 = 0;
if (!fillUpAt) {
fillUpAt = 3;
}
}
// tslint:disable:no-bitwise
sign1 = keys[byte1 >> 2];
sign2 = keys[((byte1 & 0x3) << 4) + (byte2 >> 4)];
sign3 = keys[((byte2 & 0xf) << 2) + (byte3 >> 6)];
sign4 = keys[byte3 & 0x3f];
if (fillUpAt > 0) {
if (fillUpAt <= 2) {
sign3 = fillKey;
}
if (fillUpAt <= 3) {
sign4 = fillKey;
}
}
result += sign1 + sign2 + sign3 + sign4;
if (fillUpAt) {
break;
}
}
return result;
}
let base64 = encodeBase64("\u{1F604}"); // unicode code point escapes for smiley
let str = decodeBase64(base64);
console.log("base64", base64);
console.log("str", str);
document.body.innerText = str;
how to use it: decodeBase64(encodeBase64("\u{1F604}"))
demo: https://jsfiddle.net/qrLadeb8/
Another solution for browser without using unescape:
function ToBinary(str)
{
let result="";
str=encodeURIComponent(str);
for(let i=0;i<str.length;i++)
if(str[i]=="%")
{
result+=String.fromCharCode(parseInt(str.substring(i+1,i+3),16));
i+=2;
}
else
result+=str[i];
return result;
}
btoa(ToBinary("тест"));//0YLQtdGB0YI=
I just ran into this problem myself.
First, modify your code slightly:
var download = "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+"<"+this.gamesave.tagName+">"
+this.xml.firstChild.innerHTML
+"</"+this.gamesave.tagName+">";
this.loader.src = "data:application/x-forcedownload;base64,"+
btoa(download);
Then use your favorite web inspector, put a breakpoint on the line of code that assigns this.loader.src, then execute this code:
for (var i = 0; i < download.length; i++) {
if (download[i].charCodeAt(0) > 255) {
console.warn('found character ' + download[i].charCodeAt(0) + ' "' + download[i] + '" at position ' + i);
}
}
Depending on your application, replacing the characters that are out of range may or may not work, since you'll be modifying the data. See the note on MDN about unicode characters with the btoa method:
https://developer.mozilla.org/en-US/docs/Web/API/window.btoa
A solution that converts the string to utf-8, which is slightly shorter than the utf-16 or URLEncoded versions many of the other answers suggest. It's also more compatible with how other languages like python and PHP would decode the strings:
Encode
function btoa_utf8(value) {
return btoa(
String.fromCharCode(
...new TextEncoder('utf-8')
.encode(hash_value)
)
);
}
Decode
function atob_utf8(value) {
const value_latin1 = atob(value);
return new TextDecoder('utf-8').decode(
Uint8Array.from(
{ length: value_latin1.length },
(element, index) => value_latin1.charCodeAt(index)
)
)
}
You can replace the 'utf-8' string in either of these with a different character encoding if you prefer.
Note This depends on the TextEncoder class. This is supported in most browsers nowadays but if you need to target older browsers check if it's available.