I have a database that has got a month full of datasets in 10min intervals. (So a dataset for every 10min)
Now I want to show that data on three graphs: last 24 hours, last 7 days and last 30 days.
The data looks like this:
{ "data" : 278, "date" : ISODate("2016-08-31T01:51:05.315Z") }
{ "data" : 627, "date" : ISODate("2016-08-31T01:51:06.361Z") }
{ "data" : 146, "date" : ISODate("2016-08-31T01:51:07.938Z") }
// etc
For the 24h graph I simply output the data for the last 24h, that's easy.
For the other graphs I thin the data:
const data = {}; //data from database
let newData = [];
const interval = 7; //for 7 days the interval is 7, for 30 days it's 30
for( let i = 0; i < data.length; i += interval ) {
newData.push( data[ i ] );
};
This works fine but extreme events where data is 0 or differs greatly from the other values average, can be lost depending on what time you search the data. Not thinning out the data however will result in a large sum of data points that are sent over the pipe and have to be processed on the front end. I'd like to avoid that.
Now to my question
How can I reduce the data for a 7 day period while keeping extremes in it? What's the most efficient way here?
Additions:
In essence I think I'm trying to simplify a graph to reduce points but keep the overall shape. (If you look at it from a pure image perspective)
Something like an implementation of Douglas–Peucker algorithm in node?
As you mention in the comments, the Ramer-Douglas-Peucker (RDP) algorithm is used to process data points in 2D figures but you want to use it for graph data where X values are fixed. I modified this Javascript implementation of the algorithm provided by M Oehm to consider only the vertical (Y) distance in the calculations.
On the other hand, data smoothing is often suggested to reduce the number of data points in a graph (see this post by csgillespie).
In order to compare the two methods, I made a small test program. The Reset button creates new test data. An algorithm can be selected and applied to obtain a reduced number of points, separated by the specified interval. In the case of the RDP algorithm however, the resulting points are not evenly spaced. To get the same number of points as for the specified interval, I run the calculations iteratively, adjusting the espilon value each time until the correct number of points is reached.
From my tests, the RDP algorithm gives much better results. The only downside is that the spacing between points varies. I don't think that this can be avoided, given that we want to keep the extreme points which are not evenly distributed in the original data.
Here is the code snippet, which is better seen in Full Page mode:
var svgns = 'http://www.w3.org/2000/svg';
var graph = document.getElementById('graph1');
var grpRawData = document.getElementById('grpRawData');
var grpCalculatedData = document.getElementById('grpCalculatedData');
var btnReset = document.getElementById('btnReset');
var cmbMethod = document.getElementById('cmbMethod');
var btnAddCalculated = document.getElementById('btnAddCalculated');
var btnClearCalculated = document.getElementById('btnClearCalculated');
var data = [];
var calculatedCount = 0;
var colors = ['black', 'red', 'green', 'blue', 'orange', 'purple'];
var getPeriod = function () {
return parseInt(document.getElementById('txtPeriod').value, 10);
};
var clearGroup = function (grp) {
while (grp.lastChild) {
grp.removeChild(grp.lastChild);
}
};
var showPoints = function (grp, pts, markerSize, color) {
var i, point;
for (i = 0; i < pts.length; i++) {
point = pts[i];
var marker = document.createElementNS(svgns, 'circle');
marker.setAttributeNS(null, 'cx', point.x);
marker.setAttributeNS(null, 'cy', point.y);
marker.setAttributeNS(null, 'r', markerSize);
marker.setAttributeNS(null, 'fill', color);
grp.appendChild(marker);
}
};
// Create and display test data
var showRawData = function () {
var i, x, y;
var r = 0;
data = [];
for (i = 1; i < 500; i++) {
x = i;
r += 15.0 * (Math.random() * Math.random() - 0.25);
y = 150 + 30 * Math.sin(x / 200) * Math.sin((x - 37) / 61) + 2 * Math.sin((x - 7) / 11) + r;
data.push({ x: x, y: y });
}
showPoints(grpRawData, data, 1, '#888');
};
// Gaussian kernel smoother
var createGaussianKernelData = function () {
var i, x, y;
var r = 0;
var result = [];
var period = getPeriod();
for (i = Math.floor(period / 2) ; i < data.length; i += period) {
x = data[i].x;
y = gaussianKernel(i);
result.push({ x: x, y: y });
}
return result;
};
var gaussianKernel = function (index) {
var halfRange = Math.floor(getPeriod() / 2);
var distance, factor;
var totalValue = 0;
var totalFactor = 0;
for (i = index - halfRange; i <= index + halfRange; i++) {
if (0 <= i && i < data.length) {
distance = Math.abs(i - index);
factor = Math.exp(-Math.pow(distance, 2));
totalFactor += factor;
totalValue += data[i].y * factor;
}
}
return totalValue / totalFactor;
};
// Ramer-Douglas-Peucker algorithm
var ramerDouglasPeuckerRecursive = function (pts, first, last, eps) {
if (first >= last - 1) {
return [pts[first]];
}
var slope = (pts[last].y - pts[first].y) / (pts[last].x - pts[first].x);
var x0 = pts[first].x;
var y0 = pts[first].y;
var iMax = first;
var max = -1;
var p, dy;
// Calculate vertical distance
for (var i = first + 1; i < last; i++) {
p = pts[i];
y = y0 + slope * (p.x - x0);
dy = Math.abs(p.y - y);
if (dy > max) {
max = dy;
iMax = i;
}
}
if (max < eps) {
return [pts[first]];
}
var p1 = ramerDouglasPeuckerRecursive(pts, first, iMax, eps);
var p2 = ramerDouglasPeuckerRecursive(pts, iMax, last, eps);
return p1.concat(p2);
}
var internalRamerDouglasPeucker = function (pts, eps) {
var p = ramerDouglasPeuckerRecursive(data, 0, pts.length - 1, eps);
return p.concat([pts[pts.length - 1]]);
}
var createRamerDouglasPeuckerData = function () {
var finalPointCount = Math.round(data.length / getPeriod());
var epsilon = getPeriod();
var pts = internalRamerDouglasPeucker(data, epsilon);
var iteration = 0;
// Iterate until the correct number of points is obtained
while (pts.length != finalPointCount && iteration++ < 20) {
epsilon *= Math.sqrt(pts.length / finalPointCount);
pts = internalRamerDouglasPeucker(data, epsilon);
}
return pts;
};
// Event handlers
btnReset.addEventListener('click', function () {
calculatedCount = 0;
clearGroup(grpRawData);
clearGroup(grpCalculatedData);
showRawData();
});
btnClearCalculated.addEventListener('click', function () {
calculatedCount = 0;
clearGroup(grpCalculatedData);
});
btnAddCalculated.addEventListener('click', function () {
switch (cmbMethod.value) {
case "Gaussian":
showPoints(grpCalculatedData, createGaussianKernelData(), 2, colors[calculatedCount++]);
break;
case "RDP":
showPoints(grpCalculatedData, createRamerDouglasPeuckerData(), 2, colors[calculatedCount++]);
return;
}
});
showRawData();
div
{
margin-bottom: 6px;
}
<div>
<button id="btnReset">Reset</button>
<select id="cmbMethod">
<option value="RDP">Ramer-Douglas-Peucker</option>
<option value="Gaussian">Gaussian kernel</option>
</select>
<label for="txtPeriod">Interval: </label>
<input id="txtPeriod" type="text" style="width: 36px;" value="7" />
</div>
<div>
<button id="btnAddCalculated">Add calculated points</button>
<button id="btnClearCalculated">Clear calculated points</button>
</div>
<svg id="svg1" width="765" height="450" viewBox="0 0 510 300">
<g id="graph1" transform="translate(0,300) scale(1,-1)">
<rect width="500" height="300" stroke="black" fill="#eee"></rect>
<g id="grpRawData"></g>
<g id="grpCalculatedData"></g>
</g>
</svg>
Im looking to align the following array of colours as precisely as possible.
After searching & trying many solutions suggested on Stackoverflow, the pusher.color library has the best solution, however, it too is far from perfect. I would like to hear solutions on how we can align them perfectly.
JSFIDDLE LINK : http://jsfiddle.net/dxux7y3e/
Code:
var coloursArray=['#FFE9E9','#B85958','#FFB1AE','#FFC2BF','#C55E58','#FFC7C4','#FF9A94','#FF9D96','#FA9790','#A78B88','#A78B88','#CE675B','#DB8073','#FF9D90','#FF7361','#FFD6D1','#F9A092','#FF7B67','#EBACA2','#FF806D','#DD6D5B','#D16654','#ED8673','#FFC4B8','#E2725B','#ED7A64','#8F3926','#BD492F','#9D3C27','#AD533E','#BF4024','#FFC9BC','#6B6766','#E1CDC8','#C2654C','#B3978F','#FFC7B8','#CE2B00','#C2654C','#A24D34','#FF926D','#E78667','#FFB198','#8C756D','#9E6D5B','#FFC7B0','#FFBEA4','#D2B9AF','#FFB193','#632710','#B26746','#976854','#F44900','#E79873','#EFA27F','#532510','#BC866B','#FDE5D9','#FF5B00','#D18C67','#FF5B00','#9E4312','#763713','#BB6B39','#B5622E','#CC7742','#6D4227','#B56B38','#FF7518','#F3B080','#995C30','#995C30','#FF6A00','#D89769','#71472A','#EDAC7B','#EEAB79','#EBCFB9','#FBE3D1','#E19255','#5E381B','#FFDCC1','#FFF0E4','#F68D39','#7B5B40','#FF8313','#FFCEA4','#AA8667','#975414','#CB9867','#8C5B2B','#FFCE9E','#7B4714','#FFF3E7','#FFA449','#CEAF90','#CDB69E','#EFD6BC','#DDA66B','#B27737','#B88A57','#CE9B61','#F4C38B','#543817','#BC9C78','#DBB07A','#FF8E04','#F6EADB','#DBC2A4','#C49B64','#CBA26B','#80551E','#FF9200','#FFECD3','#FFC87C','#FFB755','#DBB680','#D2D0CD','#EFDBBE','#E5C18B','#FFE5BC','#F2EADB','#885F12','#FFE7B6','#825A08','#906712','#F2D18E','#C8C6C2','#FFB000','#FFC243','#C6BEAD','#D0C3A4','#916800','#8C6700','#F4E9CA','#FFF0C5','#FFE080','#FFEBA8','#846600','#FFE692','#F5F0DB','#433F2F','#BBB394','#FFEFAA','#FFE76D','#FFFAE0','#3E3B28','#554900','#E1E0D8','#74725C','#605F54','#F8F7DD','#A5A467','#DDDDDA','#FFFFEE','#A3A39D','#E0E0D7','#BEBEB9','#E8E8E5','#454531','#ACACAA','#E9E9DF','#FFFFDC','#EBEBE7','#979831','#C5C6BE','#B9C866','#898D72','#F3FAD1','#616452','#CED5B0','#A1A787','#595C4E','#B0BB8C','#EEFFB6','#ACB78E','#8FA359','#858F6C','#86916E','#374912','#AEB0AA','#79904C','#627739','#747F60','#9FA98E','#E7F9CB','#E1F9BE','#495637','#8A9978','#4E5F39','#86996E','#C3CEB7','#78866B','#CEDDC1','#B5CEA2','#536149','#D6E6CC','#D6E6CC','#809873','#4F564C','#4F6C45','#555F52','#4F7942','#5F705B','#D0DFCD','#2B3929','#F0F7EF','#AAD5A4','#99BC95','#B6D4B4','#869E86','#618661','#006700','#E9EEE9','#739E73','#005B06','#EDF7EE','#D0E0D2','#809784','#ABCEB1','#C0E0C8','#3A5241','#435549','#E6ECE8','#E3EAE6','#3B604C','#00602F','#92B7A5','#2F5B49','#318061','#30745B','#316955','#00A275','#C2D1CE','#80A7A0','#00A082','#C2D1CF','#5C6E6C','#607473','#EDF7F7','#1E8285','#D5E7E8','#AADEE1','#188086','#107F87','#566364','#007B86','#66949A','#CAE2E5','#18656F','#004F61','#0C5B6C','#668E98','#BBD0DA','#91B4C5','#AFC3CD','#738A99','#3A5467','#476174','#244967','#556C80','#667A8C','#516D87','#1E4263','#7C8791','#849CB6','#738CAA','#1E3A5F','#1E3655','#9EB0CE','#B6BAC2','#67738D','#BEC1CD','#555559','#616180','#000049','#000031','#F8F8FC','#938BA4','#47375D','#F7F6F8','#3D0067','#514C53','#9566A2','#7F5482','#A279A4','#6D1261','#A06492','#925582','#945B80','#CE94BA','#ECCFE1','#A20058','#A6005B','#BC0061','#BB0061','#F3CEE1','#B3005B','#AB165F','#8A184D','#AA185B','#F3DAE4','#DB3779','#E71261','#E74F86','#FFD6E5','#BE9BA7','#D0396A','#DB1855','#F798B6','#9C294A','#D62B5B','#DE3969','#BC1641','#E7547A','#D52756','#9C7D85','#DB244F','#A1354F','#C22443','#FFBDCA','#8B6D73','#DC3D5B','#FF738C','#F13154','#BC4055','#FED4DB','#FFCFD6','#CB4E61','#ED455A','#F36C7B','#C94F5B','#F3959D','#A8444C','#FFCCD0','#735B5D','#D15D67','#B44B52','#FD868D','#FFD5D8','#C3767B','#FF8087','#C8242B','#FFEAEB','#F95A61','#E96D73','#E6656B','#FF6D73','#FF555B','#A35A5B','#FFD3D4','#B84B4D'];
var body=document.getElementsByTagName('body')[0];
function hexToRgb(hex) {
hex = hex.substring(1, hex.length);
var r = parseInt((hex).substring(0, 2), 16);
var g = parseInt((hex).substring(2, 4), 16);
var b = parseInt((hex).substring(4, 6), 16);
return r + "," + g + "," + b;
}
function rgbToHex(r, g, b) {
return "#" + ((1 << 24) + (r << 16) + (g << 8) + b).toString(16).slice(1);
}
var rgbArr=new Array();
var div=document.createElement('div');
div.id='Original';
body.appendChild(div);
for(var color in coloursArray){
color=coloursArray[color];
displayColor(color,div);
rgbArr.push(hexToRgb(color));
}
var hslArr=new Array();
for(var i=0;i<rgbArr.length;i++){
//Transforming rgb to hsl
//`hslArr[i][1]` (`i`) is a reference to the rgb color, in order to retrieve it later
hslArr[i]=[rgbToHsl(rgbArr[i]),i];
}
var sortedHslArr=new Array();
//Sorting `hslArr` into `sortedHslArr`
outerloop:
for(var i=0;i<hslArr.length;i++){
for(var j=0;j<sortedHslArr.length;j++){
if(sortedHslArr[j][0][0]>hslArr[i][0][0]){
sortedHslArr.splice(j,0,hslArr[i]);
continue outerloop;
}
}
sortedHslArr.push(hslArr[i]);
}
var sortedRgbArr=new Array();
//Retrieving rgb colors
for(var i=0;i<sortedHslArr.length;i++){
sortedRgbArr[i]=rgbArr[sortedHslArr[i][1]];
}
function displayColor(color,parent){
var div;
div=document.createElement('div');
div.style.backgroundColor=color;
div.style.width='22px';
div.style.height='22px';
div.style.cssFloat='left';
div.style.position='relative';
parent.appendChild(div);
}
var finalArray=new Array();
var div=document.createElement('div');
div.id='Sorted';
body.appendChild(div);
for(var color in sortedRgbArr){
color=sortedRgbArr[color];
color=color.split(',');
color=rgbToHex(parseInt(color[0]),parseInt(color[1]),parseInt(color[2]));
displayColor(color,div);
finalArray.push(color);
}
function rgbToHsl(c){
var r = c[0]/255, g = c[1]/255, b = c[2]/255;
var max = Math.max(r, g, b), min = Math.min(r, g, b);
var h, s, l = (max + min) / 2;
if(max == min){
h = s = 0; // achromatic
}else{
var d = max - min;
s = l > 0.5 ? d / (2 - max - min) : d / (max + min);
switch(max){
case r: h = (g - b) / d + (g < b ? 6 : 0); break;
case g: h = (b - r) / d + 2; break;
case b: h = (r - g) / d + 4; break;
}
h /= 6;
}
return new Array(h * 360, s * 100, l * 100);
}
var sorted = coloursArray.sort(function(colorA, colorB) {
return pusher.color(colorA).hue() - pusher.color(colorB).hue();
});
// console.log(sorted);
var div=document.createElement('div');
div.id='Pusher';
body.appendChild(div);
for(var color in sorted){
color=sorted[color];
displayColor(color,div);
}
var div=document.createElement('div');
body.appendChild(div);
var str='';
for(var color in sorted){
color=sorted[color];
str+='\''+color+'\',';
}
div.innerHTML=str;
function sorthueColors (colors) {
for (var c = 0; c < colors.length; c++) {
/* Get the hex value without hash symbol. */
var hex = colors[c].substring(1);
//var hex = colors[c].hex.substring(1);
/* Get the RGB values to calculate the Hue. */
var r = parseInt(hex.substring(0,2),16)/255;
var g = parseInt(hex.substring(2,4),16)/255;
var b = parseInt(hex.substring(4,6),16)/255;
/* Getting the Max and Min values for Chroma. */
var max = Math.max.apply(Math, [r,g,b]);
var min = Math.min.apply(Math, [r,g,b]);
/* Variables for HSV value of hex color. */
var chr = max-min;
var hue = 0;
var val = max;
var sat = 0;
if (val > 0) {
/* Calculate Saturation only if Value isn't 0. */
sat = chr/val;
if (sat > 0) {
if (r == max) {
hue = 60*(((g-min)-(b-min))/chr);
if (hue < 0) {hue += 360;}
} else if (g == max) {
hue = 120+60*(((b-min)-(r-min))/chr);
} else if (b == max) {
hue = 240+60*(((r-min)-(g-min))/chr);
}
}
}
/* Modifies existing objects by adding HSV values. */
colors[c].hue = hue;
colors[c].sat = sat;
colors[c].val = val;
}
/* Sort by Hue. */
return colors.sort(function(a,b){return a.hue - b.hue;});
}
Problem is, sorting requires a well-defined order - in other words, you need to map all colors to one dimension. While there are some approaches to display the color space in two dimensions, I am not aware of any that would display it in one dimension and still make sense to the human eye.
However, if you don't insist on some universal ordering and you merely want to place a given list of colors in a way that looks nice, then some clustering approach might produce better results. I tried the naive approach, the idea here is to put similar colors into the same cluster and to merge these clusters until you only have one. Here is the code I've got:
function colorDistance(color1, color2) {
// This is actually the square of the distance but
// this doesn't matter for sorting.
var result = 0;
for (var i = 0; i < color1.length; i++)
result += (color1[i] - color2[i]) * (color1[i] - color2[i]);
return result;
}
function sortColors(colors) {
// Calculate distance between each color
var distances = [];
for (var i = 0; i < colors.length; i++) {
distances[i] = [];
for (var j = 0; j < i; j++)
distances.push([colors[i], colors[j], colorDistance(colors[i], colors[j])]);
}
distances.sort(function(a, b) {
return a[2] - b[2];
});
// Put each color into separate cluster initially
var colorToCluster = {};
for (var i = 0; i < colors.length; i++)
colorToCluster[colors[i]] = [colors[i]];
// Merge clusters, starting with lowest distances
var lastCluster;
for (var i = 0; i < distances.length; i++) {
var color1 = distances[i][0];
var color2 = distances[i][1];
var cluster1 = colorToCluster[color1];
var cluster2 = colorToCluster[color2];
if (!cluster1 || !cluster2 || cluster1 == cluster2)
continue;
// Make sure color1 is at the end of its cluster and
// color2 at the beginning.
if (color1 != cluster1[cluster1.length - 1])
cluster1.reverse();
if (color2 != cluster2[0])
cluster2.reverse();
// Merge cluster2 into cluster1
cluster1.push.apply(cluster1, cluster2);
delete colorToCluster[color1];
delete colorToCluster[color2];
colorToCluster[cluster1[0]] = cluster1;
colorToCluster[cluster1[cluster1.length - 1]] = cluster1;
lastCluster = cluster1;
}
// By now all colors should be in one cluster
return lastCluster;
}
Complete fiddle
The colorDistance() function works with RGB colors that are expressed as arrays with three elements. There are certainly much better approaches to do this and they might produce results that look better. Note also that this isn't the most efficient algorithm because it calculates the distance between each and every color (O(n2)) so if you have lots of colors you might want to optimize it.
Assuming there is no best solution. Here is how i would find what i like the most.
First i would use color distance function:
var balance = [10, 0, 0.01];
function colorDistance(color1, color2) {
var result = 0;
color1 = rgbToHsl(color1[0], color1[1], color1[2]);
color2 = rgbToHsl(color2[0], color2[1], color2[2]);
for (var i = 0; i < color1.length; i++)
result += (color1[i] - color2[i]) * (color1[i] - color2[i]) * balance[i];
return result;
}
I would use balance function :
hue distance var balance = [1, 0, 0];
saturation distance var balance = [0, 1, 0];
lightness distance var balance = [0, 0, 1];
After that you can just experiment and pick what you like. Note that values are not normalized to appear between 0 to 1 but other then that it should work well. You can also you another distance function this is just most trivial and well known one that gets decent results.
This approach does not give you the best solution, but describes a way to fiddle with it and get result you where looking for. Have fun.
Using the Hue of a color is usually a pleasant way to map the color on a one-dimensional scale. You could convert the rgb color to hsv using the answers here: RGB to HSV color in javascript?
This is an improved code based on pusher.color lib.
https://jsfiddle.net/kt1zv6g4/3/