I'm new to D3 and trying to do a moving average of previous and next values on my data in order to smooth it out.
Currently, I have it working using the 2 previous values + the current value. It does work but 1) how would I also use the next values, and 2) what if I wanted to use the 15 previous and 15 next values? (it would be crazy to have 30 individual vars for storing all of them)
I'm used to traditional javascript but lost as to how to traverse the data this way in D3.
Hope someone can enlighten me, thanks.
See the whole code on bl.ocks.org:
http://bl.ocks.org/jcnesci/7439277
Or just the data parsing code here:
d3.json("by_date_mod.json", function(error, data) {
// Setup each row of data by formatting the Date for X, and by converting to a number for Y.
data = data.rows;
data.forEach(function(d) {
d.key = parseDate(String(d.key));
d.value = +d.value;
});
x.domain(d3.extent(data, function(d) { return d.key; }));
y.domain([0, d3.max(data, function(d) { return d.value; })]);
// Setup the moving average calculation.
// Currently is a hacky way of doing it by manually storing and using the previous 3 values for averaging.
// Looking for another way to address previous values so we can make the averaging window much larger (like 15 previous values).
var prevPrevVal = 0;
var prevVal = 0;
var curVal = 0
var movingAverageLine = d3.svg.line()
.x(function(d,i) { return x(d.key); })
.y(function(d,i) {
if (i == 0) {
prevPrevVal = y(d.value);
prevVal = y(d.value);
curVal = y(d.value);
} else if (i == 1) {
prevPrevVal = prevVal;
prevVal = curVal;
curVal = (prevVal + y(d.value)) / 2.0;
} else {
prevPrevVal = prevVal;
prevVal = curVal;
curVal = (prevPrevVal + prevVal + y(d.value)) / 3.0;
}
return curVal;
})
.interpolate("basis");
// Draw the moving average version of the data, as a line.
graph1.append("path")
.attr("class", "average")
.attr("d", movingAverageLine(data));
// Draw the raw data as an area.
graph1.append("path")
.datum(data)
.attr("class", "area")
.attr("d", area);
// Draw the X-axis of the graph.
graph1.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis);
// Draw the Y-axis of the graph.
graph1.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Value");
});
You need a function to calculate the moving average:
var movingWindowAvg = function (arr, step) { // Window size = 2 * step + 1
return arr.map(function (_, idx) {
var wnd = arr.slice(idx - step, idx + step + 1);
var result = d3.sum(wnd) / wnd.length;
// Check for isNaN, the javascript way
result = (result == result) ? result : _;
return result;
});
};
var avgData = movingWindowAvg(avg, 7); // 15 step moving window.
Note that this functions fudges the values a bit at the borders of the original array, when a complete window cannot be extracted.
Update: If the result is NaN, convert the result to the present number in the beginning. Checking result == result is the recommended way of testing for NaNs in Javascript.
If you really don't need a variable size window, this cumulative average might be a faster option without the slicing overhead:
function cumAvg(objects, accessor) {
return objects.reduce(
function(avgs, currObj, i) {
if (i == 1) {
return [ accessor(currObj) ];
} else {
var lastAvg = avgs[i - 2]; // reduce idxs are 1-based, arrays are 0
avgs.push( lastAvg + ( (accessor(currObj) - lastAvg) / i) );
return avgs;
}
}
}
Related
I am creating a horizontal bar chart that shows data from a CSV file dynamically. I am getting an extra space where another value would be, but without any data. I only want the top 10 values(students) plus keys(counties) and it looks like it adds one.
As you can see it adds a space and a small rect at the top. I just want to remove both.
// Parse Data
d3.csv("FA18_geographic.csv", function(data) {
data.sort(function(a,b) {
return d3.descending(+a.students, +b.students);
});
// find max county
var max = d3.max(data, function(d) { return +d.students;} );
var map = d3.map(data, function(d){return d.students; });
pmax = map.get(max);
// Add X axis
var x = d3.scaleLinear()
.domain([0, +pmax.students+500])
.range([ 0, width]);
svg.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.attr("transform", "translate(-10,0)rotate(-45)")
.style("text-anchor", "end");
// Y axis
var count = 0
var y = d3.scaleBand()
.range([ 0, height ])
.domain(
data.map(function(d) { //only show top 10 counties
count = count+1
if (count<=10){
return d.county;
}})
)
.padding(.3);
svg.append("g")
.call(d3.axisLeft(y));
//Bars //Something must be wrong in here???
svg.selectAll("rect.bar")
.data(data)
.enter()
.append("rect")
.attr("x", x(0) )
.attr("y", function(d) { return y(d.county); })
.attr("width", function(d) { return x(d.students); })
.attr("height", y.bandwidth() )
.attr("fill", "#79200D");
});
</script>
Your problem lies here:
.domain(data.map(function(d) { //only show top 10 counties
count = count + 1
if (count <= 10) {
return d.county;
}
}))
The issue here has nothing to do with D3, that's a JavaScript issue: you cannot skip interactions using Array.prototype.map.
Let's show this with the basic demo below, in which we'll try to return the item only if count is less than 5:
const data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
let count = -1;
const domain = data.map(function(d) {
count += 1;
if (count < 5) {
return d
};
});
console.log(domain)
As you see, it will return several undefined. You only see one empty tick because D3 band scale treats all those values as a single undefined (domain values in band scales are unique).
There are several solutions here. You can, for instance, use Array.prototype.reduce:
const data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
let count = -1;
const domain = data.reduce(function(acc, curr) {
count += 1;
if (count < 5) {
acc.push(curr)
};
return acc;
}, []);
console.log(domain)
Alternatively, you can use a filter after your map, or even a forEach to populate the array.
Finally, some tips:
You don't need a count, since Array.prototype.map has an index (the second argument), just like Array.prototype.reduce (the third argument).
You don't need to do count = count + 1, just do count += 1;
You are skipping the first element, because a JavaScript array is zero-based. So, start count as -1 instead of 0.
I have a d3 pie chart. It reads data from a json file (in this example it's just an array variable).
Some of the key names are not very helpful, so I am trying to write a map function to pass some different strings, but other key names are fine, so I want to skip over those or return them as normal.
If a key name is not declared my function returns undefined. Also the names are not being added to the legend.
I do know all of the key names, but I would rather continue over the ones I have not defined as they can be returned as they are, if this is possible (I thought an else/if with a continue condition). This code would be more useful if new keys are added to the data.
I have tested a version with all key names declared, it stops throwing an undefined variable, but is not appending the new key names to the legend.
There is a full jsfiddle here: https://jsfiddle.net/lharby/ugweordj/6/
Here is my js function (truncated).
var w = 400,
h = 400,
r = 180,
inner = 70,
color = d3.scale.category20c();
data = [{"label":"OOS", "value":194},
{"label":"TH10", "value":567},
{"label":"OK", "value":1314},
{"label":"KO", "value":793},
{"label":"Spark", "value":1929}];
mapData = data.map(function(d){
if(d.label == 'OOS'){
return 'Out of scope';
}else if(d.label == 'TH10'){
return 'Threshold 10';
}else{
// I don't care about the other keys, they can map as they are.
continue;
}
});
var total = d3.sum(data, function(d) {
return d3.sum(d3.values(d));
});
var vis = d3.select("#chart")
.append("svg:svg")
.data([data])
.attr("width", w)
.attr("height", h)
.append("svg:g")
.attr("transform", "translate(" + r * 1.1 + "," + r * 1.1 + ")")
var arc = d3.svg.arc()
.innerRadius(inner)
.outerRadius(r);
var pie = d3.layout.pie()
.value(function(d) { return d.value; });
var arcs = vis.selectAll("g.slice")
.data(pie)
.enter()
///
};
arcs.append("svg:path")
.attr("fill", function(d, i) { return color(i); } )
.attr("d", arc);
var legend = d3.select("#chart").append("svg")
.attr("class", "legend")
.attr("width", r)
.attr("height", r * 2)
.selectAll("g")
.data(mapData) // returning mapData up until undefined variables
.enter().append("g")
.attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("width", 18)
.attr("height", 18)
.style("fill", function(d, i) { return color(i); });
legend.append("text")
.attr("x", 24)
.attr("y", 9)
.attr("dy", ".35em")
.text(function(d) { return d.label; }); // should this point to a reference in mapData?
The HTML is a simple element with an id of #chart.
You can use .forEach() to iterate over your data and change just the labels of interest while keeping all others unchanged:
data = [{"label":"OOS", "value":194},
{"label":"TH10", "value":567},
{"label":"OK", "value":1314},
{"label":"KO", "value":793},
{"label":"Spark", "value":1929}];
data.forEach(d => {
if(d.label == 'OOS'){
d.label = 'Out of scope';
} else if(d.label == 'TH10'){
d.label = 'Threshold 10';
}
});
console.log(data);
You cannot skip elements using map. However, you can skip them using reduce:
var data = [
{"label":"OOS", "value":194},
{"label":"TH10", "value":567},
{"label":"OK", "value":1314},
{"label":"KO", "value":793},
{"label":"Spark", "value":1929}
];
var mapData = data.reduce(function(result, d) {
if (d.label == 'OOS') {
result.push('Out of scope');
} else if (d.label == 'TH10') {
result.push('Threshold 10');
};
return result;
}, []);
console.log(mapData)
EDIT: after your comment your desired outcome is clearer. You ca do this using map, just return the property's value:
var data = [{"label":"OOS", "value":194},
{"label":"TH10", "value":567},
{"label":"OK", "value":1314},
{"label":"KO", "value":793},
{"label":"Spark", "value":1929}];
var mapData = data.map(function(d) {
if (d.label == 'OOS') {
return 'Out of scope';
} else if (d.label == 'TH10') {
return 'Threshold 10';
} else {
return d.label
}
});
console.log(mapData);
The scatterplot I'm making has a correct y axis going from 0 to around 5000, but the nodes that are being drawn come way short of their value.
The problem (I think) is domain for y scale. I've tried using extent(), min and max, and hard coding values in. (these attempts are there and commented out). I've debugged the values getting fed into the y scale and they seem to be ok, they should be reaching the top of this chart, but they barely come half way up.
The only way to get the nodes to the top of the chart is if I hardcode the max of y scale domain to be 3000, which isn't even close to the max y value which is around 4600.
picture of chart
<!-- STACK OVERFLOW This code at the top is simple HTML/HTTP request stuff, you can
scroll down till my other comments to start looking at the problem.
I left this code here so people can run the chart on their browser.-->
<meta charset="utf-8">
<title>Template Loading D3</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.12/d3.min.js"></script>
<style media="screen">
.axis path,
.axis line {
fill: none;
stroke: black;
shape-rendering: crispEdges;
}
</style>
</head>
<body>
<h1>Bitcoin BTC</h1>
<script type="text/javascript">
var xhr = new XMLHttpRequest();
function makeRequest(){
xhr.open("GET", "https://api.coindesk.com/v1/bpi/historical/close.json?start=2010-07-17&end=2017-09-11", true);
xhr.send();
xhr.onreadystatechange = processRequest;
}
function processRequest(){
console.log("testing, state: ", xhr.readyState)
if(xhr.readyState == 4 && xhr.status == 200){
let response = JSON.parse(xhr.responseText);
makeChart(response);
}
}
// STACK OVERFLOW -- code above this can be ignored since it's just making HTTP request.
// I'm leaving it here so people can serve this code on their own machine and
// see it working in their browser. D3 code is below
// When the HTTP request is finished, this function gets called to draw my chart,
// this is where D3.js starts.
function makeChart(response){
var w = window.innerWidth - 100;
var h = window.innerHeight - 100;
var padding = 20;
var Lpadding = 45;
// makeDatesAndValues is not part of my problem, this formats the dates
// coming from HTTP request into something easier to feed into D3.js X Scale
var makeDatesAndValues = function(input){
let dates = [];
let values = [];
let returnData = [];
for(prop in input){
values.push(input[prop])
let counter = 0;
let year = [];
let month = [];
let day = [];
for( var j = 0; j < prop.length; j++){
if(lastDate[j] !== "-" && counter == 0){
year.push(prop[j]);
} else if(prop[j] == "-"){
counter++;
} else if(prop[j] !== "-" && counter == 1){
month.push(prop[j])
} else if(prop[j] !== "-" && counter == 2){
day.push(prop[j])
}
}
dates.push([Number(year.join("")), Number(month.join("")), Number(day.join(""))]);
returnData.push(
{
date: [Number(year.join("")), Number(month.join("")), Number(day.join(""))],
value: input[prop]
}
)
}
return returnData;
}
var inputData = makeDatesAndValues(response.bpi);
// HERE WE GO, this is where I think the problem is, drawing the actual chart.
var svg = d3.select("body")
.append("svg")
.attr("width", w)
.attr("height", h);
// Here is the problem child, both "y" and the commented out "yScale" are attempts to
// get the correct y-coordinates for values fed into the scale
// Both of these will make a correct y Axis, but then screw up the y-coordinates of my data
// getting fed in, as described on SO
var y = d3.scale.linear()
.domain([d3.min(inputData, function(d){ return d.value; }), d3.max(inputData, function(d){ return d.value; })])
// .domain([d3.extent(d3.values(inputData, function(d){ return d.value}))])
// .domain([0, 3000])
.range([h - padding, padding])
// var yScale = d3.scale.linear()
// // .domain([0, 5000])
// .domain([d3.min(inputData, function(d){ return d.value; }), d3.max(inputData, function(d){ return d.value; })])
// // .domain([d3.extent(d3.values(inputData, function(d){ return d.value}))])
// .range([0, h])
// X scale works fine, no problems here
var x = d3.time.scale()
.domain([new Date(2010, 7, 18), new Date(2017, 7, 18)])
.range([Lpadding, w]);
// Both Axes seem to be working fine
var xAxis = d3.svg.axis()
.scale(x)
.orient("bottom")
.ticks(8)
var yAxis = d3.svg.axis()
.scale(y)
.orient("left")
// .tickSize(-w, 0, 0)
.ticks(8)
svg.selectAll("circle")
.data(inputData)
.enter()
.append("circle")
.attr("cx", function(d){
let thisDate = x(new Date(d.date[0], d.date[1], d.date[2]))
return thisDate;
})
// And here is the other side of the problem. There are different attempts
// to plot they y-coordinate of my values commented out.
// None of these are working and I don't understand why it works correctly
// for the Y Axis, but my plotted values come way short of where they should be.
.attr("cy", function(d, i){
console.log("this is with yScale: ", y(d.value))
console.log("Without: ", d.value)
// return y(( d.value) - padding)
// return y(d.value);
// return y(d.value) - (h - padding)
return h - padding - y(d.value);
})
.attr("r", function(d){
return 1;
})
.attr("fill", "red")
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(" + 0 + ", " + (h - padding) + ")")
.call(xAxis);
svg.append("g")
.attr("class", "y axis")
.attr("transform", "translate(" + Lpadding + ", 0)")
.call(yAxis);
}
makeRequest();
</script>
</body>
I suggest you may need to consider people's feeling when you post such long codes with poor comments.
Your problem is you are not awareness the coordinate system of SVG where the y points to bottom. The following is a little modification:
svg.selectAll("circle")
.data(inputData)
.enter()
.append("circle")
.attr("cx", function(d){
let thisDate = x(new Date(d.date[0], d.date[1], d.date[2]));
return thisDate;
})
// this is the KEY part you should change:
.attr("cy", function(d, i){ return h - padding - y(d.value); })
.attr("r", 1)
.attr("fill", "red");
Since you don't offer detail data, I can't check it. hope it will help!
I am plotting points on a UK map using D3 off a live data stream. When the data points exceed 10,000 the browser becomes sluggish and the animation is no longer smooth. So I modify the dataPoints array to keep only the last 5000 points.
However when I modify the dataPoints the first time using splice() D3 stops rendering any new points. The old points gradually disappear (due to a transition) but there are no new points. I am not sure what I am doing wrong here.
I have simulated the problem by loading data of a CSV as well storing it in memory and plotting them at a rate of 1 point every 100ms. Once the number of dots goes above 10 I splice to retain the last 5 points. I see the same behaviour. Can someone review the code and let me know what I am doing wrong?
Setup and the plotting function:
var width = 960,
height = 1160;
var dataPoints = []
var svg = d3.select("#map").append("svg")
.attr("width", width)
.attr("height", height);
var projection = d3.geo.albers()
.center([0, 55.4])
.rotate([4.4, 0])
.parallels([40, 70])
.scale(5000)
.translate([width / 2, height / 2]);
function renderPoints() {
var points = svg.selectAll("circle")
.data(dataPoints)
points.enter()
.append("circle")
.attr("cx", function (d) {
prj = projection([d.longitude, d.latitude])
return prj[0];
})
.attr("cy", function (d) {
prj = projection([d.longitude, d.latitude])
return prj[1];
})
.attr("r", "4px")
.attr("fill", "blue")
.attr("fill-opacity", ".4")
.transition()
.delay(5000)
.attr("r", "0px")
}
/* JavaScript goes here. */
d3.json("uk.json", function(error, uk) {
if (error) return console.error(error);
console.log(uk);
var subunits = topojson.feature(uk, uk.objects.subunits);
var path = d3.geo.path()
.projection(projection);
svg.selectAll(".subunit")
.data(subunits.features)
.enter().append("path")
.attr("class", function(d) { return "subunit " + d.id })
.attr("d", path);
svg.append("path")
.datum(topojson.mesh(uk, uk.objects.subunits, function(a,b) {return a!== b && a.id !== 'IRL';}))
.attr("d", path)
.attr("class", "subunit-boundary")
svg.append("path")
.datum(topojson.mesh(uk, uk.objects.subunits, function(a,b) {return a=== b && a.id === 'IRL';}))
.attr("d", path)
.attr("class", "subunit-boundary IRL")
svg.selectAll(".place-label")
.attr("x", function(d) { return d.geometry.coordinates[0] > -1 ? 6 : -6; })
.style("text-anchor", function(d) { return d.geometry.coordinates[0] > -1 ? "start": "end"; });
svg.selectAll(".subunit-label")
.data(topojson.feature(uk, uk.objects.subunits).features)
.enter().append("text")
.attr("class", function(d) { return "subunit-label " + d.id })
.attr("transform", function(d) { return "translate(" + path.centroid(d) + ")"; })
.attr("dy", ".35em")
.text(function(d) { return d.properties.name; })
// function applyProjection(d) {
// console.log(d);
// prj = projection(d)
// console.log(prj);
// return prj;
// }
lon = -4.6
lat = 55.45
dataPoints.push([lon,lat])
renderPoints()
});
Function to cleanup old points
var cleanupDataPoints = function() {
num_of_elements = dataPoints.length
console.log("Pre:" + num_of_elements)
if(num_of_elements > 10) {
dataPoints = dataPoints.splice(-5, 5)
}
console.log("Post:" + dataPoints.length)
}
Loading data from CSV and plotting at a throttled rate
var bufferedData = null
var ptr = 0
var renderNext = function() {
d = bufferedData[ptr]
console.log(d)
dataPoints.push(d)
ptr++;
renderPoints()
cleanupDataPoints()
if(ptr < bufferedData.length)
setTimeout(renderNext, 100)
}
d3.csv('test.csv', function (error, data) {
bufferedData = data
console.log(data)
setTimeout(renderNext, 100)
})
In the lines
points = svg.selectAll("circle")
.data(dataPoints)
points.enter() (...)
d3 maps each element in dataPoints (indexed from 0 to 5000) to the circle elements (of which there should be 5000 eventually). So from its point of view, there is no enter'ing data: there are enough circles to hold all your points.
To make sure that the same data point is mapped to the same html element after it changed index in its array, you need to use an id field of some sort attached to each of your data point, and tell d3 to use this id to map the data to elements, instead of their index.
points = svg.selectAll("circle")
.data(dataPoints, function(d){return d.id})
If the coordinates are a good identifier for your point, you can directly use:
points = svg.selectAll("circle")
.data(dataPoints, function(d){return d.longitude+" "+d.latitude})
See https://github.com/mbostock/d3/wiki/Selections#data for more details.
I'm sure the solution is straight forward, but I'm trying to find out how to limit the range of radius when I plot circles onto a geomap. I have values that range in size significantly, and the larger values end up covering a significant amount of the map.
d3.csv("getdata.php", function(parsedRows) {
data = parsedRows
for (i = 0; i < data.length; i++) {
var mapCoords = this.xy([data[i].long, data[i].lat])
data[i].lat = mapCoords[0]
data[i].long = mapCoords[1]
}
vis.selectAll("circle")
.data(data)
.enter().append("svg:circle")
.attr("cx", function(d) { return d.lat })
.attr("cy", function(d) { return d.long })
.attr("stroke-width", "none")
.attr("fill", function() { return "rgb(255,148,0)" })
.attr("fill-opacity", .4)
.attr("r", function(d) { return Math.sqrt(d.count)})
})
This is what I have right now.
You'll probably want to use d3 scales by setting the domain (min/max input values) and the range (min/max output allowed values). To make this easy, don't hesitate to use d3.min and d3.max to setup the domain's values.
d3.scale will return a function that you can use when assigning the r attribute value. For example:
var scale = d3.scale.linear.domain([ inputMin, inputMax ]).range([ outputMin, outputMax ]);
vis.selectAll("circle")
// etc...
.attr( 'r', function(d) { return scale(d.count) });