So I have ~12600 subnets:
eg. 123.123.208.0/20
and an IP.
I can use a SQLite Database or an array or whatever
There was a similar question asked about a month ago, however I am not looking for checking one IP against one subnet but a bunch of subnets (obviously the most efficient way, hopefully not O(total subnets)) :)
How can I check that the IP is one of in one of these subnets, I need true or false not the subnet if that helps optimisation.
There are similar subnets in the current list eg.:
(actual extract)
123.123.48.0/22 <-- not a typo
123.123.48.0/24 <-- not a typo
123.123.90.0/24
123.123.91.0/24
123.123.217.0/24
In total they range from 4.x.y.z to 222.x.y.z
The best approach is IMO making use of bitwise operators. For example, 123.123.48.0/22 represents (123<<24)+(123<<16)+(48<<8)+0 (=2071670784; this might be a negative number) as a 32 bit numeric IP address, and -1<<(32-22) = -1024 as a mask. With this, and likewise, your test IP address converted to a number, you can do:
(inputIP & testMask) == testIP
For example, 123.123.49.123 is in that range, as 2071671163 & -1024 is 2071670784
So, here are some tool functions:
function IPnumber(IPaddress) {
var ip = IPaddress.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/);
if(ip) {
return (+ip[1]<<24) + (+ip[2]<<16) + (+ip[3]<<8) + (+ip[4]);
}
// else ... ?
return null;
}
function IPmask(maskSize) {
return -1<<(32-maskSize)
}
test:
(IPnumber('123.123.49.123') & IPmask('22')) == IPnumber('123.123.48.0')
yields true.
In case your mask is in the format '255.255.252.0', then you can use the IPnumber function for the mask, too.
Try this:
var ip2long = function(ip){
var components;
if(components = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/))
{
var iplong = 0;
var power = 1;
for(var i=4; i>=1; i-=1)
{
iplong += power * parseInt(components[i]);
power *= 256;
}
return iplong;
}
else return -1;
};
var inSubNet = function(ip, subnet)
{
var mask, base_ip, long_ip = ip2long(ip);
if( (mask = subnet.match(/^(.*?)\/(\d{1,2})$/)) && ((base_ip=ip2long(mask[1])) >= 0) )
{
var freedom = Math.pow(2, 32 - parseInt(mask[2]));
return (long_ip > base_ip) && (long_ip < base_ip + freedom - 1);
}
else return false;
};
Usage:
inSubNet('192.30.252.63', '192.30.252.0/22') => true
inSubNet('192.31.252.63', '192.30.252.0/22') => false
I managed to solve this by using the node netmask module.
You can check if an IP belongs to a subnet by making something like this:
import { Netmask } from 'netmask'
const block = new Netmask('123.123.208.0/20')
const ip = '123.123.208.0'
console.log(block.contains(ip))
Will here print true.
You can install it by using:
npm i --save netmask
Convert the lower ip and the upper ip in the range to integers and store the range in the db then make sure both columns are indexed.
Off the top of my head (pseudo code):
function ipmap(w,x,y,z) {
return 16777216*w + 65536*x + 256*y + z;
}
var masks = array[ipmap(128,0,0,0), ipmap(196,0,0,0), ..., ipmap(255,255,255,255)]
function lowrange(w, x, y, z, rangelength) {
return ipmap(w, x, y, z) & masks[rangelength]
}
function hirange(w, x, y, z, rangelength) {
return lowrange(w, x, y, z, ,rangelength) + ipmap(255,255,255,255) - masks[rangelength];
}
That ought to do it.
To find whether a particular ip falls in any of the ranges, convert it to an integer and do:
SELECT COUNT(*) FROM ipranges WHERE lowrange <= 1234567 AND 1234567 <= highrange
The query optimizer should be able to speed this up greatly.
Functions IPnumber and IPmask are nice, however I would rather test like:
(IPnumber('123.123.49.123') & IPmask('22')) == (IPnumber('123.123.48.0') & IPmask('22'))
Because for each address, you only need to take into account the network part of the address. Hence doing IPmask('22') will zero-out the computer part of the address and you should do the same with the network address.
Keywords: Binary searching, preprocessing, sorting
I had a similar problem and binary search appears to be very efficient if you can pre-process your subnet list and sort it. Then you can achieve an asymptotic time complexity of O(log n).
Here's my code (MIT License, original location: https://github.com/iBug/pac/blob/854289a674578d096f60241804f5893a3fa17523/code.js):
function belongsToSubnet(host, list) {
var ip = host.split(".").map(Number);
ip = 0x1000000 * ip[0] + 0x10000 * ip[1] + 0x100 * ip[2] + ip[3];
if (ip < list[0][0])
return false;
// Binary search
var x = 0, y = list.length, middle;
while (y - x > 1) {
middle = Math.floor((x + y) / 2);
if (list[middle][0] < ip)
x = middle;
else
y = middle;
}
// Match
var masked = ip & list[x][1];
return (masked ^ list[x][0]) == 0;
}
And an example usage:
function isLan(host) {
return belongsToSubnet(host, LAN);
}
var LAN = [
[0x0A000000, 0xFF000000], // 10.0.0.0/8
[0x64400000, 0xFFC00000], // 100.64.0.0/10
[0x7F000000, 0xFF000000], // 127.0.0.0/8
[0xA9FE0000, 0xFFFF0000], // 169.254.0.0/16
[0xAC100000, 0xFFF00000], // 172.16.0.0/12
[0xC0A80000, 0xFFFF0000] // 192.168.0.0/16
];
isLan("127.12.34.56"); // => true
isLan("8.8.8.8"); // => false (Google's Public DNS)
You can get a PAC script* and see how it performs (it loads a China IP list from somewhere else, sorts them and formats them appropriately) against 5000s of subnets. In practice its speed is surprisingly satisfactory.
The preprocessing code can be inspected using F12 Dev Tools on the above page. In short, you need to convert 1.2.3.4/16 to [0x01020304, 0xFFFF0000], i.e. 32-bit unsigned integer for the IP address and network mask.
* Link goes to my personal website.
Related
I was expecting to find hundreds of examples of functions to convert to and from CIDR and NETMASK for javascript, but was unable to find any.
I need to convert to and from CIDR and NETMASKS on a nodejs page which sets and retrieves the IP address for a machine using NETCTL.
Any easy solutions to do this using javascript / nodejs ??
This code could provide a solution:
var mask = "255.255.248.0";
var maskNodes = mask.match(/(\d+)/g);
var cidr = 0;
for(var i in maskNodes)
{
cidr += (((maskNodes[i] >>> 0).toString(2)).match(/1/g) || []).length;
}
return cidr;
Here's one that doesn't check if the netmask is valid:
const netmaskToCidr = n => n
.split('.')
.reduce((c, o) => c - Math.log2(256 - +o), 32)
NETMASK BINARY CIDR
255.255.248.0 11111111.11111111.11111000.00000000 /21
255.255.0.0 11111111.11111111.00000000.00000000 /16
255.192.0.0 11111111.11000000.00000000.00000000 /10
This how calculate CIDR.. So , it is the occurrences of 1 in the second cloumn. Thus , I design a readable algorithm as below :
const masks = ['255.255.255.224', '255.255.192.0', '255.0.0.0'];
/**
* Count char in string
*/
const countCharOccurences = (string , char) => string.split(char).length - 1;
const decimalToBinary = (dec) => (dec >>> 0).toString(2);
const getNetMaskParts = (nmask) => nmask.split('.').map(Number);
const netmask2CIDR = (netmask) =>
countCharOccurences(
getNetMaskParts(netmask)
.map(part => decimalToBinary(part))
.join(''),
'1'
);
masks.forEach((mask) => {
console.log(`Netmask =${mask}, CIDR = ${netmask2CIDR(mask)}`)
})
I know it's been long since this question was asked, but I just wanted to add checks to ensure that the netmask is valid:
function mask2cidr(mask){
var cidr = ''
for (m of mask.split('.')) {
if (parseInt(m)>255) {throw 'ERROR: Invalid Netmask'} // Check each group is 0-255
if (parseInt(m)>0 && parseInt(m)<128) {throw 'ERROR: Invalid Netmask'}
cidr+=(m >>> 0).toString(2)
}
// Condition to check for validity of the netmask
if (cidr.substring(cidr.search('0'),32).search('1') !== -1) {
throw 'ERROR: Invalid Netmask ' + mask
}
return cidr.split('1').length-1
}
As the mask is only valid when the bits in 1 go from left to right, the condition checks that no bit is 1 after the first bit in 0. It also checks each group is 0 or 128-255
The method of conversion is mostly the same as the other answers
Given that you have mentioned using node.js to implement this, I'm assuming you're looking for a way to run this server side in javascript, as opposed to client side. If that's correct, does the netmask npm module cover what you need to do?
I am working with an application called TEC-IT Wedge to interface with a grocery scale (for those not in the know a grocery scale is a glorified scale with a scanner.
The scale is connected to a serial port on the PC and thw wedge app monitors that port, the code below handles the scan of the barcode then requests the weight, all of this is working.
Where I am running into the issue is formating the returning weight from the scale.
the data returned from the scale is in the format of ASCII "S11####" so S110004 would be for an item weighing 0.04 pounds and this is how I need to send that wait to the application that we are using the scale to enter data with. For the code below a return of S110004 it returns 00.4
I think I am loosing 0's in the grab of the digits for some reason to place the decimal poit in the correct position.
var cartonverify = Left(DATARAW, 5);
var weightverify = Left(DATARAW, 3);
if (cartonverify == "S08B3")
{
ActivateWindow("Untitled - Notepad");
SendKeyStrokes(DATARAW.replace(/^S08B3+/i, '') );
DATA = "";
WriteToDevice("S11\x0d", 1000);
}
if (weightverify == "S11")
{
ActivateWindow("Untitled - Notepad");
var cartwieght = Right(DATA, 4);
if (cartwieght.length == 4)
{
var cartounces = Right(cartwieght, 2);
var cartpounds = Left(cartwieght, 2);
var WMWeight = cartpounds + "." + cartounces;
SendKeyStrokes(WMWeight);
DATA = "";
}
}
Re-form your string to something that can be parsed as a number as desired.
var str = 'S110004';
str = str.replace(/S11(\d\d)(\d\d)/, '$1.$2'); // "00.04"
var num = parseFloat(str); // 0.04
Putting the decimal place in at the RegExp step also saves you from having to use division.
Recently I run into the well known floating point precision errors of Javascript. Usually I would avoid floating point calculations on the thin client & rather leave it to the back-end.
I started using the big.js library created by Michael Mclaughlin. Though it has a square-root method/function, it does not have a nth-root methods/function nor does the power function support fraction values as arguments.
So I was wondering if anyone using the library has extended it to have such a function or at least use it to calculate accurate nth-root results.
Michael Mclaughlin suggested that I implement such a function similar in structure to the square-root function. However my attempts at understanding the logic proofed my maths-disability, resulting in simple calculations yielding very wrong results.
Using the algorithm on Rosetta Code also yields incorrect results.
So I was wondering if anyone using the library has extended it to have such a function or at least use it to calculate accurate nth-root results.
Here is the code to my last attempt:
P['nthrt'] = P['nthroot'] = function (n, prec)
{
var negate, r,
x = this,
xc = x['c'],
i = x['s'],
e = x['e'];
// Argument defaults
n = n || 2;
prec = prec || 12;
// Zero?
if ( !xc[0] ) {
return new Big(x)
}
// Negative?
negate = ( n % 2 == 1 && i < 0 );
// Estimate.
r = new Big(1); // Initial guess.
for (var i = 0; i < prec; i++) {
r = (ONE.div(n)).times(r.times(n-1).plus(x.div(r.pow(n-1))));
}
if (negate) r['s'] = -1;
return r;
};
It does not even get obvious results correct like the 4th root of 81 = 3, instead it gets 3.00000000xxx
Newton's method only gives an approximation for the root, so 3.0000xxx should be expected. If you know that the answer should be an integer, you can round r down (Newton's method overestimates the root) and check that r^n=x.
You can use big-numbers library to solve your problem. They support sqrt, pow, exp and many other features.
The pow method accept positive, negative, integer and floating point numbers:
var bn = new BigNumber();
var value = bn.of('81');
var xRoot = value.pow(0.25);
console.log('Result: ' + bn.format(xRoot));
You can use Basenumber.js to perform nth root. Documentation here.
E.g.
// Set precision decimals required
Base.setDecimals(25);
let x = Base("1e+10");
console.log(x.root(10).toString());
<script src='https://cdn.jsdelivr.net/gh/AlexSp3/Basenumber.js#main/BaseNumber.min.js'></script>
For my web application (in JavaScript) I want to generate short guids (for different objects - that are actually different types - strings and arrays of strings)
I want something like "aX4j9Z" for my uids (guids).
So these uids should be lightweight enough for web transfer and js string processing and quite unique for not a huge structure (not more than 10k elements). By saying "quite unique" I mean that after the generation of the uid I could check whether this uid does already exist in the structure and regenerate it if it does.
See #Mohamed's answer for a pre-packaged solution (the shortid package). Prefer that instead of any other solutions on this page if you don't have special requirements.
A 6-character alphanumeric sequence is pretty enough to randomly index a 10k collection (366 = 2.2 billion and 363 = 46656).
function generateUID() {
// I generate the UID from two parts here
// to ensure the random number provide enough bits.
var firstPart = (Math.random() * 46656) | 0;
var secondPart = (Math.random() * 46656) | 0;
firstPart = ("000" + firstPart.toString(36)).slice(-3);
secondPart = ("000" + secondPart.toString(36)).slice(-3);
return firstPart + secondPart;
}
UIDs generated randomly will have collision after generating ~ √N numbers (birthday paradox), thus 6 digits are needed for safe generation without checking (the old version only generates 4 digits which would have a collision after 1300 IDs if you don't check).
If you do collision checking, the number of digits can be reduced 3 or 4, but note that the performance will reduce linearly when you generate more and more UIDs.
var _generatedUIDs = {};
function generateUIDWithCollisionChecking() {
while (true) {
var uid = ("0000" + ((Math.random() * Math.pow(36, 4)) | 0).toString(36)).slice(-4);
if (!_generatedUIDs.hasOwnProperty(uid)) {
_generatedUIDs[uid] = true;
return uid;
}
}
}
Consider using a sequential generator (e.g. user134_item1, user134_item2, …) if you require uniqueness and not unpredictability. You could "Hash" the sequentially generated string to recover unpredictability.
UIDs generated using Math.random is not secure (and you shouldn't trust the client anyway). Do not rely on its uniqueness or unpredictability in mission critical tasks.
Update 08/2020:
shortid has been deprecated in favor of nanoid which is smaller and faster:
Small. 108 bytes (minified and gzipped). No dependencies. Size Limit controls the size.
Fast. It is 40% faster than UUID.
Safe. It uses cryptographically strong random APIs. Can be used in clusters.
Compact. It uses a larger alphabet than UUID (A-Za-z0-9_-). So ID size was reduced from 36 to 21 symbols.
Portable. Nano ID was ported to 14 programming languages.
import { nanoid } from 'nanoid'
// 21 characters (default)
// ~149 billion years needed, in order to have a 1% probability of at least one collision.
console.log(nanoid()) //=> "V1StGXR8_Z5jdHi6B-myT"
// 11 characters
// ~139 years needed, in order to have a 1% probability of at least one collision.
console.log(nanoid(11)) //=> "bdkjNOkq9PO"
More info here : https://zelark.github.io/nano-id-cc/
Old answer
There is also an awesome npm package for this : shortid
Amazingly short non-sequential url-friendly unique id generator.
ShortId creates amazingly short non-sequential url-friendly unique ids. Perfect for url shorteners, MongoDB and Redis ids, and any other id users might see.
By default 7-14 url-friendly characters: A-Z, a-z, 0-9, _-
Non-sequential so they are not predictable.
Supports cluster (automatically), custom seeds, custom alphabet.
Can generate any number of ids without duplicates, even millions per day.
Perfect for games, especially if you are concerned about cheating so you don't want an easily guessable id.
Apps can be restarted any number of times without any chance of repeating an id.
Popular replacement for Mongo ID/Mongoose ID.
Works in Node, io.js, and web browsers.
Includes Mocha tests.
Usage
var shortid = require('shortid');
console.log(shortid.generate()); //PPBqWA9
Here is a one liner, but it gives only lowercase letters and numbers:
var uuid = Math.random().toString(36).slice(-6);
console.log(uuid);
Get a simple counter to start from 100000000, convert the number into radix 36.
(100000000).toString(36); //1njchs
(2100000000).toString(36); //yqaadc
You can comfortably have 2 billion elegant unique ids, just like YouTube
The following generates 62^3 (238,328) unique values of 3 characters provided case sensitivity is unique and digits are allowed in all positions. If case insensitivity is required, remove either upper or lower case characters from chars string and it will generate 35^3 (42,875) unique values.
Can be easily adapted so that first char is always a letter, or all letters.
No dobut it can be optimised, and could also refuse to return an id when the limit is reached.
var nextId = (function() {
var nextIndex = [0,0,0];
var chars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('');
var num = chars.length;
return function() {
var a = nextIndex[0];
var b = nextIndex[1];
var c = nextIndex[2];
var id = chars[a] + chars[b] + chars[c];
a = ++a % num;
if (!a) {
b = ++b % num;
if (!b) {
c = ++c % num;
}
}
nextIndex = [a, b, c];
return id;
}
}());
var letters = 'abcdefghijklmnopqrstuvwxyz';
var numbers = '1234567890';
var charset = letters + letters.toUpperCase() + numbers;
function randomElement(array) {
with (Math)
return array[floor(random()*array.length)];
}
function randomString(length) {
var R = '';
for(var i=0; i<length; i++)
R += randomElement(charset);
return R;
}
This is an old question and there are some good answers, however I notice that we are in 2022 and we can use ES6 and if you don't like to depend on 3rd party libs. Here is a solution for you.
I implemented a very simple generator using the build-in functions that JavaScript offers to us these days. We will use Crypto.getRandomValues() and Uint8Array() so check the code below
const hashID = size => {
const MASK = 0x3d
const LETTERS = 'abcdefghijklmnopqrstuvwxyz'
const NUMBERS = '1234567890'
const charset = `${NUMBERS}${LETTERS}${LETTERS.toUpperCase()}`.split('')
const bytes = new Uint8Array(size)
crypto.getRandomValues(bytes)
return bytes.reduce((acc, byte) => `${acc}${charset[byte & MASK]}`, '')
}
console.log({id: hashID(6)})
This implementation uses these characters: [A-Z], [a-z], [0-9] in total they are 62 characters, if we add _ and - it will complete up to 64 characters like this:
const hashID = size => {
const MASK = 0x3d
const LETTERS = 'abcdefghijklmnopqrstuvwxyz'
const NUMBERS = '1234567890'
const charset = `${NUMBERS}${LETTERS}${LETTERS.toUpperCase()}_-`.split('')
const bytes = new Uint8Array(size)
crypto.getRandomValues(bytes)
return bytes.reduce((acc, byte) => `${acc}${charset[byte & MASK]}`, '')
}
console.log(`id: ${hashID(6)}`)
Note:
It will take around 2 days in order to have a 1% probability of at least one collision for 1000 IDs generated per hour with ID length of 6 characters. Keep this in mind when it is implemented into your project.
This will generate a sequence of unique values. It improves on RobG's answer by growing the string length when all values have been exhaused.
var IdGenerator = (function () {
var defaultCharset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890!##$%^&*()_-+=[]{};:?/.>,<|".split("");
var IdGenerator = function IdGenerator(charset) {
this._charset = (typeof charset === "undefined") ? defaultCharset : charset;
this.reset();
};
IdGenerator.prototype._str = function () {
var str = "",
perm = this._perm,
chars = this._charset,
len = perm.length,
i;
for (i = 0; i < len; i++) {
str += chars[perm[i]];
}
return str;
};
IdGenerator.prototype._inc = function () {
var perm = this._perm,
max = this._charset.length - 1,
i;
for (i = 0; true; i++) {
if (i > perm.length - 1) {
perm.push(0);
return;
} else {
perm[i]++;
if (perm[i] > max) {
perm[i] = 0;
} else {
return;
}
}
}
};
IdGenerator.prototype.reset = function () {
this._perm = [];
};
IdGenerator.prototype.current = function () {
return this._str();
};
IdGenerator.prototype.next = function () {
this._inc();
return this._str();
};
return IdGenerator;
}).call(null);
Usage:
var g = new IdGenerator(),
i;
for (i = 0; i < 100; i++) {
console.log(g.next());
}
This gist contains the above implementation and a recursive version.
just randomly generate some strings:
function getUID(len){
var chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789',
out = '';
for(var i=0, clen=chars.length; i<len; i++){
out += chars.substr(0|Math.random() * clen, 1);
}
// ensure that the uid is unique for this page
return getUID.uids[out] ? getUID(len) : (getUID.uids[out] = out);
}
getUID.uids = {};
You can shorten a GUID to 20 printable ASCII characters without losing information or the uniqueness of the GUID.
Jeff Atwood blogged about that years ago:
Equipping our ASCII Armor
This solution combines Math.random() with a counter.
Math.random() should give about 53 bits of entropy (compared with UUIDv4's 128), but when combined with a counter should give plenty enough uniqueness for a temporary ID.
let _id_counter = 0
function id() {
return '_' + (_id_counter++).toString(36) + '_' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36)
}
console.log(Array.from({length: 100}).map(() => id()))
Features:
Simple implementation
Output of about 13 chars
Case-insensitive
Safe for use as HTML id and React key
Not suitable for database storage
You can use the md5 algorithm for generating a random string. md5 is the node package
var randomChars = Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 2);
var shortUrl = md5(originalUrl + randomChars + new Date()).substring(0, 5).toString();
console.log(shortUrl);
This will generate unique string every time.
Since localStorage (currently) only supports strings as values, and in order to do that the objects need to be stringified (stored as JSON-string) before they can be stored, is there a defined limitation regarding the length of the values.
Does anyone know if there is a definition which applies to all browsers?
Quoting from the Wikipedia article on Web Storage:
Web storage can be viewed simplistically as an improvement on cookies, providing much greater storage capacity (10 MB per origin in Google Chrome(https://plus.google.com/u/0/+FrancoisBeaufort/posts/S5Q9HqDB8bh), Mozilla Firefox, and Opera; 10 MB per storage area in Internet Explorer) and better programmatic interfaces.
And also quoting from a John Resig article [posted January 2007]:
Storage Space
It is implied that, with DOM Storage,
you have considerably more storage
space than the typical user agent
limitations imposed upon Cookies.
However, the amount that is provided
is not defined in the specification,
nor is it meaningfully broadcast by
the user agent.
If you look at the Mozilla source code
we can see that 5120KB is the default
storage size for an entire domain.
This gives you considerably more space
to work with than a typical 2KB
cookie.
However, the size of this storage area
can be customized by the user (so a
5MB storage area is not guaranteed,
nor is it implied) and the user agent
(Opera, for example, may only provide
3MB - but only time will tell.)
Actually Opera doesn't have 5MB limit. It offers to increase limit as applications requires more. User can even choose "Unlimited storage" for a domain.
You can easily test localStorage limits/quota yourself.
Here's a straightforward script for finding out the limit:
if (localStorage && !localStorage.getItem('size')) {
var i = 0;
try {
// Test up to 10 MB
for (i = 250; i <= 10000; i += 250) {
localStorage.setItem('test', new Array((i * 1024) + 1).join('a'));
}
} catch (e) {
localStorage.removeItem('test');
localStorage.setItem('size', i - 250);
}
}
Here's the gist, JSFiddle and blog post.
The script will test setting increasingly larger strings of text until the browser throws and exception. At that point it’ll clear out the test data and set a size key in localStorage storing the size in kilobytes.
Find the maximum length of a single string that can be stored in localStorage
This snippet will find the maximum length of a String that can be stored in localStorage per domain.
//Clear localStorage
for (var item in localStorage) delete localStorage[item];
window.result = window.result || document.getElementById('result');
result.textContent = 'Test running…';
//Start test
//Defer running so DOM can be updated with "test running" message
setTimeout(function () {
//Variables
var low = 0,
high = 2e9,
half;
//Two billion may be a little low as a starting point, so increase if necessary
while (canStore(high)) high *= 2;
//Keep refining until low and high are equal
while (low !== high) {
half = Math.floor((high - low) / 2 + low);
//Check if we can't scale down any further
if (low === half || high === half) {
console.info(low, high, half);
//Set low to the maximum possible amount that can be stored
low = canStore(high) ? high : low;
high = low;
break;
}
//Check if the maximum storage is no higher than half
if (storageMaxBetween(low, half)) {
high = half;
//The only other possibility is that it's higher than half but not higher than "high"
} else {
low = half + 1;
}
}
//Show the result we found!
result.innerHTML = 'The maximum length of a string that can be stored in localStorage is <strong>' + low + '</strong> characters.';
//Functions
function canStore(strLen) {
try {
delete localStorage.foo;
localStorage.foo = Array(strLen + 1).join('A');
return true;
} catch (ex) {
return false;
}
}
function storageMaxBetween(low, high) {
return canStore(low) && !canStore(high);
}
}, 0);
<h1>LocalStorage single value max length test</h1>
<div id='result'>Please enable JavaScript</div>
Note that the length of a string is limited in JavaScript; if you want to view the maximum amount of data that can be stored in localStorage when not limited to a single string, you can use the code in this answer.
Edit: Stack Snippets don't support localStorage, so here is a link to JSFiddle.
Results
Chrome (45.0.2454.101): 5242878 characters
Firefox (40.0.1): 5242883 characters
Internet Explorer (11.0.9600.18036): 16386 122066 122070 characters
I get different results on each run in Internet Explorer.
Don't assume 5MB is available - localStorage capacity varies by browser, with 2.5MB, 5MB and unlimited being the most common values.
Source: http://dev-test.nemikor.com/web-storage/support-test/
I wrote this simple code that is testing localStorage size in bytes.
https://github.com/gkucmierz/Test-of-localStorage-limits-quota
const check = bytes => {
try {
localStorage.clear();
localStorage.setItem('a', '0'.repeat(bytes));
localStorage.clear();
return true;
} catch(e) {
localStorage.clear();
return false;
}
};
Github pages:
https://gkucmierz.github.io/Test-of-localStorage-limits-quota/
I have the same results on desktop Google chrome, opera, firefox, brave and mobile chrome which is ~10Mbytes
And half smaller result in safari ~4Mbytes
You don't want to stringify large objects into a single localStorage entry. That would be very inefficient - the whole thing would have to be parsed and re-encoded every time some slight detail changes. Also, JSON can't handle multiple cross references within an object structure and wipes out a lot of details, e.g. the constructor, non-numerical properties of arrays, what's in a sparse entry, etc.
Instead, you can use Rhaboo. It stores large objects using lots of localStorage entries so you can make small changes quickly. The restored objects are much more accurate copies of the saved ones and the API is incredibly simple. E.g.:
var store = Rhaboo.persistent('Some name');
store.write('count', store.count ? store.count+1 : 1);
store.write('somethingfancy', {
one: ['man', 'went'],
2: 'mow',
went: [ 2, { mow: ['a', 'meadow' ] }, {} ]
});
store.somethingfancy.went[1].mow.write(1, 'lawn');
BTW, I wrote it.
I've condensed a binary test into this function that I use:
function getStorageTotalSize(upperLimit/*in bytes*/) {
var store = localStorage, testkey = "$_test"; // (NOTE: Test key is part of the storage!!! It should also be an even number of characters)
var test = function (_size) { try { store.removeItem(testkey); store.setItem(testkey, new Array(_size + 1).join('0')); } catch (_ex) { return false; } return true; }
var backup = {};
for (var i = 0, n = store.length; i < n; ++i) backup[store.key(i)] = store.getItem(store.key(i));
store.clear(); // (you could iterate over the items and backup first then restore later)
var low = 0, high = 1, _upperLimit = (upperLimit || 1024 * 1024 * 1024) / 2, upperTest = true;
while ((upperTest = test(high)) && high < _upperLimit) { low = high; high *= 2; }
if (!upperTest) {
var half = ~~((high - low + 1) / 2); // (~~ is a faster Math.floor())
high -= half;
while (half > 0) high += (half = ~~(half / 2)) * (test(high) ? 1 : -1);
high = testkey.length + high;
}
if (high > _upperLimit) high = _upperLimit;
store.removeItem(testkey);
for (var p in backup) store.setItem(p, backup[p]);
return high * 2; // (*2 because of Unicode storage)
}
It also backs up the contents before testing, then restores them.
How it works: It doubles the size until the limit is reached or the test fails. It then stores half the distance between low and high and subtracts/adds a half of the half each time (subtract on failure and add on success); honing into the proper value.
upperLimit is 1GB by default, and just limits how far upwards to scan exponentially before starting the binary search. I doubt this will even need to be changed, but I'm always thinking ahead. ;)
On Chrome:
> getStorageTotalSize();
> 10485762
> 10485762/2
> 5242881
> localStorage.setItem("a", new Array(5242880).join("0")) // works
> localStorage.setItem("a", new Array(5242881).join("0")) // fails ('a' takes one spot [2 bytes])
IE11, Edge, and FireFox also report the same max size (10485762 bytes).
You can use the following code in modern browsers to efficiently check the storage quota (total & used) in real-time:
if ('storage' in navigator && 'estimate' in navigator.storage) {
navigator.storage.estimate()
.then(estimate => {
console.log("Usage (in Bytes): ", estimate.usage,
", Total Quota (in Bytes): ", estimate.quota);
});
}
I'm doing the following:
getLocalStorageSizeLimit = function () {
var maxLength = Math.pow(2,24);
var preLength = 0;
var hugeString = "0";
var testString;
var keyName = "testingLengthKey";
//2^24 = 16777216 should be enough to all browsers
testString = (new Array(Math.pow(2, 24))).join("X");
while (maxLength !== preLength) {
try {
localStorage.setItem(keyName, testString);
preLength = testString.length;
maxLength = Math.ceil(preLength + ((hugeString.length - preLength) / 2));
testString = hugeString.substr(0, maxLength);
} catch (e) {
hugeString = testString;
maxLength = Math.floor(testString.length - (testString.length - preLength) / 2);
testString = hugeString.substr(0, maxLength);
}
}
localStorage.removeItem(keyName);
// Original used this.storageObject in place of localStorage. I can only guess the goal is to check the size of the localStorage with everything but the testString given that maxLength is then added.
maxLength = JSON.stringify(localStorage).length + maxLength + keyName.length - 2;
return maxLength;
};
I really like cdmckay's answer, but it does not really look good to check the size in a real time: it is just too slow (2 seconds for me). This is the improved version, which is way faster and more exact, also with an option to choose how big the error can be (default 250,000, the smaller error is - the longer the calculation is):
function getLocalStorageMaxSize(error) {
if (localStorage) {
var max = 10 * 1024 * 1024,
i = 64,
string1024 = '',
string = '',
// generate a random key
testKey = 'size-test-' + Math.random().toString(),
minimalFound = 0,
error = error || 25e4;
// fill a string with 1024 symbols / bytes
while (i--) string1024 += 1e16;
i = max / 1024;
// fill a string with 'max' amount of symbols / bytes
while (i--) string += string1024;
i = max;
// binary search implementation
while (i > 1) {
try {
localStorage.setItem(testKey, string.substr(0, i));
localStorage.removeItem(testKey);
if (minimalFound < i - error) {
minimalFound = i;
i = i * 1.5;
}
else break;
} catch (e) {
localStorage.removeItem(testKey);
i = minimalFound + (i - minimalFound) / 2;
}
}
return minimalFound;
}
}
To test:
console.log(getLocalStorageMaxSize()); // takes .3s
console.log(getLocalStorageMaxSize(.1)); // takes 2s, but way more exact
This works dramatically faster for the standard error; also it can be much more exact when necessary.
Once I developed Chrome (desktop browser) extension and tested Local Storage real max size for this reason.
My results:
Ubuntu 18.04.1 LTS (64-bit)
Chrome 71.0.3578.98 (Official Build) (64-bit)
Local Storage content size 10240 KB (10 MB)
More than 10240 KB usage returned me the error:
Uncaught DOMException: Failed to execute 'setItem' on 'Storage': Setting the value of 'notes' exceeded the quota.
Edit on Oct 23, 2020
For a Chrome extensions available chrome.storage API. If you declare the "storage" permission in manifest.js:
{
"name": "My extension",
...
"permissions": ["storage"],
...
}
You can access it like this:
chrome.storage.local.QUOTA_BYTES // 5242880 (in bytes)