I'm working on a REST interface to a library system that uses the SIP2 protocol (https://en.wikipedia.org/wiki/Standard_Interchange_Protocol) and was able to get things working on a system that doesn't require error correction without a problem. However, my code is now talking to another system that requires checksums, described as so in the specification:
"To calculate the checksum add each character as an unsigned binary number, take the lower 16 bits of the total and perform a 2's complement. The checksum field is the result represented by four hex digits."
I've taken a few runs at this but not matter what I do I can't get a checksum back that matches my example message. I'm probably making this harder than it should be (seems like it would be easier in a lower-level language with proper binary types, etc.). Here's my latest attempt:
var checksum = 0;
var message = "63AOAA21221021780249|AD9999|AY0AZ";
// add each character as an unsigned binary number
for(var i=0;i<message.length;i++){
checksum += message[i].charCodeAt();
}
console.log("character sum: " + checksum);
// take the lower 16 bits of the total
checksum = checksum.toString(2);
console.log("character sum binary representation: " + checksum);
while(checksum.length < 16){
checksum = "0" + checksum;
}
checksum = checksum.substr(0,16);
console.log("lower 16 bits of character total: " + checksum);
// convert to dec
checksum = parseInt(checksum,2);
console.log("checksum dec: " + checksum);
// perform 2's complement
checksum = (checksum & 0xFFFF) * -1;
console.log("2s complement: " + checksum.toString(2));
// convert to 4 hex digits
checksum = dec2hex(checksum);
console.log("checksum hex: " + checksum);
function dec2hex(i) {
return (i+0x10000).toString(16).substr(-4).toUpperCase();
}
The expected checksum for the string above is "F39A".
I found a way to get there, it's probably not the most elegant approach, and certainly not high-performance, but it works reliably.
Here's the code should some other unfortunate soul find themselves looking to answer this question. I'm plan to bundle this up with some other SIP2-related bits in a library, but for now here's a function that will generate the checksum.
function sip2_checksum(message){
var checksum_int = 0;
var checksum_binary_string = "";
var checksum_binary_string_inverted = "";
var checksum_binary_string_inverted_plus1 = "";
var checksum_hex_string = "";
// add each character as an unsigned binary number
for(var i=0;i<message.length;i++){
checksum_int += message[i].charCodeAt();
}
// convert integer to binary representation stored in a string
while(checksum_int > 0){
checksum_binary_string = (checksum_int % 2).toString() + checksum_binary_string;
checksum_int = Math.floor(checksum_int / 2);
}
// pad binary string to 16 bytes
while(checksum_binary_string.length < 16){
checksum_binary_string = "0" + checksum_binary_string;
}
// invert the binary string
for(var i=0;i<checksum_binary_string.length;i++){
var inverted_value = "X"; // something weird to make mistakes jump out
if(checksum_binary_string[i] == "1"){
inverted_value = "0";
} else {
inverted_value = "1";
}
checksum_binary_string_inverted += inverted_value;
}
// add 1 to the binary string
var carry_bit = true;
for(var i=checksum_binary_string_inverted.length - 1;i>=0;i--){
if(carry_bit){
if(checksum_binary_string_inverted[i] === "0"){
checksum_binary_string_inverted_plus1 = "1" + checksum_binary_string_inverted_plus1;
carry_bit = false;
} else {
checksum_binary_string_inverted_plus1 = "0" + checksum_binary_string_inverted_plus1;
carry_bit = true;
}
} else {
checksum_binary_string_inverted_plus1 = checksum_binary_string_inverted[i] + checksum_binary_string_inverted_plus1;
}
}
// convert binary string to hex string and uppercase it because that's what the gateway likes
checksum_hex_string = parseInt(checksum_binary_string_inverted_plus1,2).toString(16).toUpperCase();
return checksum_hex_string;
}
Related
I need to generate a 256 bits random unsigned number as a decimal string (with nearly zero probability of collision with anything generated earlier).
How to do this in JavaScript (in a browser)?
You can use the new JavaScript tools if you are not very limited:
function rnd256() {
const bytes = new Uint8Array(32);
// load cryptographically random bytes into array
window.crypto.getRandomValues(bytes);
// convert byte array to hexademical representation
const bytesHex = bytes.reduce((o, v) => o + ('00' + v.toString(16)).slice(-2), '');
// convert hexademical value to a decimal string
return BigInt('0x' + bytesHex).toString(10);
}
console.log( rnd256() );
This code uses a loop to generate a 256 character long string of random binary digits, then converts it to a BigInt. You can then convert it to a string if you like, or whatever else you please.
var temp = '0b';
for (let i = 0; i < 256; i++) {
temp += Math.round(Math.random());
}
const randomNum = BigInt(temp);
console.log(randomNum.toString());
I need to cut the string - basically if the string if longer that 1 MB I should cut it to this size.
I am using these functions to check the string size
function __to_mb(bytes) {
return bytes / Math.pow(1024, 2)
}
function __size_mb(str) {
return __to_mb(Buffer.byteLength(str, 'utf8'))
}
Then I check the size of string like this
if (__size_mb(str) > 1) { /* do something */ }
But how to cut it?
A Javascript string consists of 16-bit sequences, with some characters using one 16-bit sequence and others needing two 16-bit sequences.
There is no easy way to just take an amount of bytes and consider it done - there might be a 2x 16-bit character at both sides of the cut-off location, which would then be cut in half.
To make a safe cut, we can use str.codePointAt(index) which was introduced in ES2015. It knows which characters are 16-bit and which are 2x 16-bit. It combines either 1 or 2 of these 16-bit values into an integer result value.
If codePointAt() returns a value <= 2^16-1 then we have a 16-bit character at offset index.
If codePointAt() returns a value >= 2^16 then we have a 2x 16-bit character at offsets index and index+1.
Unfortunately this means going through the entire string to assess each index. This may seem awkward, and it may even be slow, but I am not aware of a faster or smarter way of doing this.
Demo:
var str = "abç🔥😂déΩf👍g😏h"; // string of 13 characters
console.log("str.length = " + str.length); // shows 17 because of double-width chars
console.log("size in bytes = " + str.length * 2); // length * 2 gives size in bytes
var maxByteLengths = [8, 16, 24, 32, 40];
for (var maxBytes of maxByteLengths) {
var data = safeCutOff(str, maxBytes);
console.log(maxBytes + " bytes -> " + data.text + " (" + data.bytes + " bytes)");
}
function safeCutOff(str, maxBytes) {
let widthInBytes = 0;
for (var index = 0; index < str.length; /* index is incremented below */ ) {
let positionsUsed = str.codePointAt(index) <= 0xFFFF ? 1 : 2;
newWidthInBytes = widthInBytes + 2 * positionsUsed;
if (newWidthInBytes > maxBytes)
break;
index += positionsUsed;
widthInBytes = newWidthInBytes;
}
return { text: str.substring(0, index), bytes: widthInBytes };
}
I have a 64 bit unsigned integer I need to represent in PostgreSQL. I've broken it down into two 32 bit unsigned integers, high and low. To allow Postgres to accept it, I need to convert high and low to a string representing a signed 64 bit integer.
How can I go about converting two 32 bit unsigned integers to a string representing in decimal a signed 64 bit integer?
I've done exactly this in Javascript in a quick'n'dirty-but-works'n'fast manner at: Int64HighLowToFromString, using 53-bit mantissa double precision arithmetic and 32-bit bit operations, specialized for decimal input/output.
function Int64HiLoToString(hi,lo){
hi>>>=0;lo>>>=0;
var sign="";
if(hi&0x80000000){
sign="-";
lo=(0x100000000-lo)>>>0;
hi=0xffffffff-hi+ +(lo===0);
}
var dhi=~~(hi/0x5af4),dhirem=hi%0x5af4;
var dlo=dhirem*0x100000000+dhi*0xef85c000+lo;
dhi += ~~(dlo/0x5af3107a4000);
dlo%=0x5af3107a4000;
var slo=""+dlo;
if(dhi){
slo="000000000000000000".slice(0,14-slo.length)+dlo;
return sign+dhi+slo;
}else{
return sign+slo;
}
}
Most likely this is what you needed.
I adapted the base conversion code from https://codegolf.stackexchange.com/questions/1620/arbitrary-base-conversion. Mistakes are mine, clevernesses are theirs.
I also had to add a bunch of code to deal with negative numbers (twos complement).
This code is ecmascript5, and will need slight reworking to work in older browsers.
function convert(hi, lo) {
function invertBit(bit) {
return bit == "0" ? "1" : "0";
}
function binaryInvert(binaryString) {
return binaryString.split("").map(invertBit).join("");
}
function binaryIncrement(binaryString) {
var idx = binaryString.lastIndexOf("0");
return binaryString.substring(0, idx) + "1" + binaryInvert(binaryString.substring(idx + 1));
}
function binaryDecrement(binaryString) {
var idx = binaryString.lastIndexOf("1");
return binaryString.substring(0, idx) + binaryInvert(binaryString.substring(idx));
}
function binaryAbs(binaryString) {
if (binaryString[0] === "1") {
return invertBits(binaryDecrement(binaryString));
}
return binaryString;
}
function to32Bits(val) {
var binaryString = val.toString(2);
if (binaryString[0] === "-") {
binaryString = Array(33 - (binaryString.length - 1)).join("1") + binaryInvert(binaryString.substr(1));
return binaryIncrement(binaryString);
}
return Array(33 - binaryString.length).join("0") + binaryString;
}
var fullBinaryNumber = to32Bits(hi) + to32Bits(lo);
var isNegative = fullBinaryNumber[0] === "1";
fullBinaryNumber = binaryAbs(fullBinaryNumber);
var result = "";
while (fullBinaryNumber.length > 0) {
var remainingToConvert = "", resultDigit = 0;
for (var position = 0; position < fullBinaryNumber.length; ++position) {
var currentValue = Number(fullBinaryNumber[position]) + resultDigit * 2;
var remainingDigitToConvert = Math.floor(currentValue / 10);
resultDigit = currentValue % 10;
if (remainingToConvert.length || remainingDigitToConvert) {
remainingToConvert += remainingDigitToConvert;
}
}
fullBinaryNumber = remainingToConvert;
result = resultDigit + result;
}
return (isNegative?"-":"") + result;
}
Examples:
> // largest negative number -2^63 (just the most significant bit set)
> convert(1 << 31, 0)
'-9223372036854775808'
> // largest positive number
> convert(0x7fffffff, 0xffffffff)
'9223372036854775807'
> // -1 is all bits set.
> convert(0xffffffff, 0xffffffff)
'-1'
According to JavaScript can't handle 64-bit integers, can it?, native numbers in Javascript have 53 bits of mantissa, so JS can't deal with 64 bits integers unless using specialized libraries.
Whatever the datatype and implementation limits, I assume you want to compute the Two's complement of the initial 64 bits unsigned number, to convert it from the [0 ... 2^64-1] range into the [-2^63 ... 2^63-1] range.
high is presumably the initial unsigned 64 bits number divided by 2^32, and low is the remainder.
The conversion to a signed 64 bits should go like this:
if high>=2^63 then
s64 = -(2^64-(high*2^32+low))
else
s64 = high*2^32+low;
In a PostgreSQL function, this can be done using the exact-precision numeric type to avoid overflows in intermediate multiplications, and downcast the final result to bigint (signed 64 bits):
create function concat64(bigint, bigint) returns bigint
as $$
select (case when $1>=2147483648
then -(18446744073709551616::numeric-($1*4294967296::numeric+$2))
else $1*4294967296::numeric+$2 end)::bigint;
$$ language sql;
The input arguments have to be bigint (64 bits) because postgres doesn't have unsigned types.
They're assumed to be in the [0..4294967296] range and the output should be in the [-9223372036854775808..9223372036854775807] range.
Not for any practical purpose, of course, but I tried to generate a secure random floating point number using the Node's crypto module. Essentially, the following:
var crypto = require('crypto');
var buf = crypto.randomBytes(4);
var float = buf.readFloatBE();
This doesn't work, as far as the following test can tell, in an average of 0.40% of cases. Instead of getting a float, I get NaN.
var colors = require('colors');
var crypto = require('crypto');
var buf = new Buffer(4);
var fails = 0, tries = 100000;
var failures = [];
for (var i = 0; i < tries; i++) {
var num = crypto.randomBytes(4).readFloatBE(0);
try {
buf.writeFloatBE(num, 0);
} catch (e) {
fails++;
failures.push(buf.readUInt32BE(0).toString(2));
}
}
var percent = 100 * fails / tries;
if (fails)
percent = (percent.toFixed(2) + "%").red;
else
percent = '0.00%'.blue.bold;
console.log('Test ' + 'complete'.green.bold + ', ' + percent + ": " + fails + " / " + tries);
fails && console.log('Failed'.red + ' values:', failures.join(', '));
I'm guessing this is due to the IEEE single precision floating point number specification, but I'm not familiar with exactly how the float is stored as binary.
Why does this happen, exactly, and apart from simply generating floats until I get a valid number, how can I circumvent this?
EDIT: when looking at the filtered binary data, it appears they all follow the same pattern: the first 8 bits are all set. Everything else is random.
As is apparent from the current version of node, the buffer library's source specifies, on line 906, that the noAssert also checks to see if the Number provided is not NaN, so simply specifying noAssert allows writing of Infinity, NaN, and -Infinity to the buffer.
Ive been wondering for some time. Is there a good (And fast) way to make an number random while its seeded?
is there a good algorithm to convert one number into a seemingly random number.
A little illustration:
specialrand(1) = 8
specialrand(2) = 5
specialrand(3) = 2
specialrand(4) = 5
specialrand(5) = 1
specialrand(1) = 8
specialrand(4) = 5
specialrand(1) = 8
It would be very nice if the output could also be huge numbers.
As a note: I don't want to fill a array and randomize the numbers because I want to be able to feed it huge difference of numbers because I want the same output whenever I restart the program
You're not looking for a seeded random number. Instead what I think you're looking for is a hashing function. If you put in the same input and get the same output, that's not random.
If you're looking to generate a sequence of random numbers for a run, but have the same sequence generate from run to run, you can use a random number generator that generates the same sequence given the same seed value.
Thats how most basic pRNG's work. There are more cryptographically secure RNG's out there, but your standard Math.rand() should work to accomplish your needs.
Maybe pseudorandom number generators are what you are looking for.
For example the XORshift.
uint32_t xor128(void) {
static uint32_t x = 123456789;
static uint32_t y = 362436069;
static uint32_t z = 521288629;
static uint32_t w = 88675123;
uint32_t t;
t = x ^ (x << 11);
x = y; y = z; z = w;
return w = w ^ (w >> 19) ^ (t ^ (t >> 8));
}
You could create something like this:
take a seed
specialrand(5) is a function which takes the fifth random number from this seed
or specialrand(5) is a function which gets the first random number from the seed+5
Maybe this is enough for your purpose.
Try setting a key or set of keys then writing a function with an equation to return a new number based on that key:
a very basic example would be:
function specialrand(value) {
key = array (1,2,4,6,8);
for (k in key) {
if (k%2 === 0) {
value -= key[k] * value;
} else {
value += key[k] / value;
}
}
return value;
}
however you could create a highly complex equation to generate your 'random' number and ensure you return the same number each time.
You can use Date functionality
Math.valueOfSeed = function(n)
{
return Number(new Date(n%9999, n%12, n%30, n%24, n%60, n%60, n%1000));
};
alert(Math.valueOfSeed(1) + " = " + Math.valueOfSeed(1));
alert(Math.valueOfSeed(2) + " = " + Math.valueOfSeed(2));
alert(Math.valueOfSeed(15) + " = " + Math.valueOfSeed(15));
alert(Math.valueOfSeed(5555) + " = " + Math.valueOfSeed(5555));
alert(Math.valueOfSeed(21212121) + " = " + Math.valueOfSeed(21212121));
alert(Math.valueOfSeed(6554654654) + " = " + Math.valueOfSeed(6554654654));
test is here