Related
I am currently working on own siem system, where windows logs(in csv format) automatically collecting every hour and displaying in a Highcharts. The backend is Django.
But ran into problems:
There's too many data that GPU of browser is load.
I set auto parsing to per 1 hour but site not displaying. Other parts of code are working
Is it ok if I write logs in database?
Can anyone give me advices how to do it right?
Here is my Javascript code in template/dashboard
_categories = {{categories|safe}};
_values = {{values|safe}};
_data = {{data|safe}};
Highcharts.chart('pie', {
colors: ['#558dfa', '#a1c1fe', '#4c72bc', '#3a61ad'],
title: {
text: ''
},
chart: {
type: 'pie'
},
tooltip: {
valueSuffix: '%'
},
plotOptions: {
pie: {
allowPointSelect: false,
cursor: 'pointer',
dataLabels: {
enabled: false
},
showInLegend: false
}
},
series: [{
name: 'Percentage',
colorByPoint: true,
data: [
{
name: 'Critical',
y: {{range.0}}
},
{
name: 'Error',
y: {{range.1}}
},
{
name: 'Information',
y: {{range.2}}
},
{
name: 'Warning',
y: {{range.3}}
},
]
}]
});
Highcharts.stockChart('container', {
chart: {
alignTicks: false
},
rangeSelector: {
selected: 0
},
series: [{
type: 'column',
data: _data,
}],
dataGrouping: {
units: [[
'week', // unit name
[1] // allowed multiples
], [
'month',
[1, 2, 3, 4, 6]
]]
}
});
Here is my views.py
def dashboard_req(path, request):
df = pd.read_csv(path)
rs = df.groupby('Level')['Event ID'].count()
r = df.groupby('Time Generated')['Event ID'].count()
categories = list(rs.index)
values = list(rs.values)
count = len(df.index)
date = list(r.index)
values2 = list(r.values)
data = []
for i in range(len(date)):
date_format = datetime.datetime.strptime(date[i], "%Y-%m-%d %H:%M:%S")
unix_time = datetime.datetime.timestamp(date_format)
data.append([int(unix_time)*1000, values2[i]])
ranges = []
for i in values:
percent = i / count * 100
ranges.append(round(percent,1))
table_content = df.to_html(index=None)
table_content = table_content.replace("<thead>","<thead class='thead-dark'>")
table_content = table_content.replace('class="dataframe"',"class='table table-striped'")
table_content = table_content.replace('border="1"',"")
context = {"categories": categories, 'values': values, 'table_data':table_content, 'count':count, 'range':ranges, 'data':data}
# print('Done')
return render(request, './dashboard.html', context=context)
def do_something(scheduler, request):
scheduler.enter(3600, 1, do_something, (scheduler, request)) # sec
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data\log.csv')
server = 'localhost'
logtype = 'Application'
hand = win32evtlog.OpenEventLog(server,logtype)
flags = win32evtlog.EVENTLOG_BACKWARDS_READ|win32evtlog.EVENTLOG_SEQUENTIAL_READ
total = win32evtlog.GetNumberOfEventLogRecords(hand)
head = ['ID','Time Generated', 'Source Name', 'Event ID', 'Event Category', 'Level', 'Event Data']
id_count = 1
event_types = ['Critical', 'Error', 'Warning', 'Information']
with open(path, 'w', encoding="utf-8") as file:
writer = csv.writer(file)
writer.writerow(head)
while True:
events = win32evtlog.ReadEventLog(hand, flags,0)
if events:
for event in events:
data = event.StringInserts
if data:
for msg in data:
log_row = [id_count, event.TimeGenerated, event.SourceName, event.EventID, event.EventCategory, event_types[event.EventType-1], msg]
else:
log_row = [id_count, event.TimeGenerated, event.SourceName, event.EventID, event.EventCategory, event_types[event.EventType-1]]
writer.writerow(log_row)
id_count+=1
else:
break
dashboard_req(path, request)
def dashboard(request):
my_scheduler = sched.scheduler(time.time, time.sleep)
my_scheduler.enter(3600, 1, do_something, (my_scheduler, request))
my_scheduler.run()
I'm trying to create a candlestick chart with volume.
The candlestick chart works perfectly, the problem is with the volume.
What I have:
What I need:
datachart.php -> It sends the JSON data.
I feel that the error is in $data[], because the script index.htm doesn't recognize the volume data.
<?php
include '../dbh.php';//It connects to the database
$sql = "SELECT * FROM table ORDER BY date ASC";
$result = $conn->query($sql);
$data = array();
$count = 0;
while ($row = mysqli_fetch_array($result))
{
$newdate = strtotime($row['date']) * 1000;
$data[] = array($newdate, (float)$row['open'], (float)$row['high'], (float)$row['low'], (float)$row['close'], (float)$row['volume']);
$count++;
}
echo json_encode($data);
?>
index.htm
$.getJSON('datachart.php', function (data) {
// split the data set into ohlc and volume
var ohlc = [],
volume = [],
dataLength = data.length,
// set the allowed units for data grouping
groupingUnits = [[
'week', // unit name
[1] // allowed multiples
], [
'month',
[1, 2, 3, 4, 6]
]],
i = 0;
for (i; i < dataLength; i += 1) {
ohlc.push([
data[i][0], // the date
data[i][1], // open
data[i][2], // high
data[i][3], // low
data[i][4] // close
]);
volume.push([
data[i][0], // the date
data[i][5] // the volume
]);
}
// create the chart
Highcharts.stockChart('container', {
rangeSelector: {
selected: 1
},
title: {
text: 'Exchange Market'
},
yAxis: [{
labels: {
align: 'right',
x: -3
},
title: {
text: 'OHLC'
},
height: '60%',
lineWidth: 2,
resize: {
enabled: true
}
}, {
labels: {
align: 'right',
x: -3
},
title: {
text: 'Volume'
},
top: '65%',
height: '35%',
offset: 0,
lineWidth: 2
}],
tooltip: {
split: true
},
series: [{
type: 'candlestick',
name: 'AAPL',
data: ohlc,
dataGrouping: {
units: groupingUnits
}
}, {
type: 'column',
name: 'Volume',
data: volume,
yAxis: 1,
dataGrouping: {
units: groupingUnits
}
}]
});
});
If we take only this part of the index.htm code:
volume.push([
data[i][0], // the date
data[i][5] // the volume
]);
If I change the "5" to 1,2,3 or 4, it draws the volume graph. Then, why doesn't it draw the graph with "5" if the volume is in position 5?
I tried the code like this with many small restructuration and modification but without success.
Here is the code:
$(function () {
$.get('data.csv', function(data) {
// split the data set into ohlc and volume
var ohlc = [],
volume = [],
dataLength = data.length,
// set the allowed units for data grouping
groupingUnits = [[
'week', // unit name
[1] // allowed multiples
], [
'month',
[1, 2, 3, 4, 6]
]],
i = 0;
for (i; i < dataLength; i += 1) {
ohlc.push([
data[i][0], // the date
data[i][1], // open
data[i][2], // high
data[i][3], // low
data[i][4] // close
]);
volume.push([
data[i][0], // the date
data[i][5] // the volume
]);
}
$('#chart').highcharts({
rangeSelector: {
selected: 1
},
title: {
text: 'AAPL Historical'
},
yAxis: [{
labels: {
align: 'right',
x: -3
},
title: {
text: 'OHLC'
},
height: '60%',
lineWidth: 2
}, {
labels: {
align: 'right',
x: -3
},
title: {
text: 'Volume'
},
top: '65%',
height: '35%',
offset: 0,
lineWidth: 2
}],
data: {
csv: data
},
series: [{
type: 'candlestick',
name: 'AAPL',
data: ohlc,
dataGrouping: {
units: groupingUnits
}
}, {
type: 'column',
name: 'Volume',
data: volume,
yAxis: 1,
dataGrouping: {
units: groupingUnits
}
}]
});
});
});
Here is data.csv:
Date,Open,High,Low,Close,Volume
2013-12-20,9371.08,9413.09,9352.98,9400.18,161686900
2013-12-19,9279.68,9351.9,9257.24,9335.74,98276500
2013-12-18,9145.35,9190.73,9122.05,9181.75,82342700
2013-12-17,9142.75,9161.8,9085.12,9085.12,72207500
2013-12-16,9004.62,9187.78,8997.75,9163.56,99105600
2013-12-13,9016.78,9046.63,8990.58,9006.46,67761700
2013-12-12,9032.67,9060.54,8984.28,9017,75120200
2013-12-11,9093.26,9153.14,9065.51,9077.11,64845800
2013-12-10,9180.29,9223.73,9091.97,9114.44,74363400
Can you help me to figure out the problem or purpose new approch please ?
What is my goal ?
Is to be able to load a CSV file inside the chart instead of using JSON file.
Why ?
Because modifing CSV file is more easier for me using PHP than JSON, and it's for performance too.
Thank's
When you do data.length, you are getting length of the csv file string. What you need to do is split the data with the newline delimiter.
// sample from data
var data = `Date,Open,High,Low,Close,Volume
2013-12-20,9371.08,9413.09,9352.98,9400.18,161686900
2013-12-19,9279.68,9351.9,9257.24,9335.74,98276500`;
// split by \n (new line)
data = data.split('\n'); // now data is an array of rows
var finalObj = [];
// iterate over the rows
data.map(function(row){
var obj = {};
// row is a string separated by ','
row = row.split(','); // now row is an array
obj['date'] = row[0];
obj['open'] = row[1];
obj['high'] = row[2];
obj['low'] = row[3];
obj['close'] = row[4];
obj['volume'] = row[5];
finalObj.push(obj);
})
console.log(finalObj);
Output:
[
{
date:'Date',
open:'Open',
high:'High',
low:'Low',
close:'Close',
volume:'Volume'
},
{
date:'2013-12-20',
open:'9371.08',
high:'9413.09',
low:'9352.98',
close:'9400.18',
volume:'161686900'
},
{
date:'2013-12-19',
open:'9279.68',
high:'9351.9',
low:'9257.24',
close:'9335.74',
volume:'98276500'
}
]
Im building a chart system that will show me all data entries. I retrieve my data using ajax and I loop trough the data and group the results by colors (red, blue and yellow) and then divide them by months.
I setup base objects (dateCounts_Red, dateCounts_Blue and dateCounts_Yellow) so that by default it starts all months at 0. A counter would then add when it finds a match tot he apropriate color and month.
When I output my dateCounts I get:
{"2015":{"2015-12":1,"2015-10":null,"2015-08":null,"2015-11":null}}
{"2015":{"2015-12":0,"2015-10":null}}
{"2015":{"2015-12":0,"2015-10":null}}
Here is the code I have so far:
var dateCounts_Red = {"2015":{"2015-01":0,"2015-02":0,"2015-03":0,"2015-04":0},"2015":{"2015-05":0},"2015":{"2015-06":0},"2015":{"2015-07":0},"2015":{"2015-08":0},"2015":{"2015-09":0},"2015":{"2015-10":0},"2015":{"2015-11":0},"2015":{"2015-12":0}};
var dateCounts_Blue = {"2015":{"2015-01":0,"2015-02":0,"2015-03":0,"2015-04":0},"2015":{"2015-05":0},"2015":{"2015-06":0},"2015":{"2015-07":0},"2015":{"2015-08":0},"2015":{"2015-09":0},"2015":{"2015-10":0},"2015":{"2015-11":0},"2015":{"2015-12":0}};
var dateCounts_Yellow = {"2015":{"2015-01":0,"2015-02":0,"2015-03":0,"2015-04":0},"2015":{"2015-05":0},"2015":{"2015-06":0},"2015":{"2015-07":0},"2015":{"2015-08":0},"2015":{"2015-09":0},"2015":{"2015-10":0},"2015":{"2015-11":0},"2015":{"2015-12":0}};
data.d.results.forEach(function(element) {
var date = element.created_date.slice(0, 7);
var yr = date.slice(0, 4);
var Color = element.colorvalue;
if(Color == "red") {
dateCounts_Red[yr][date]++;
}
if(Color == "blue"){
dateCounts_Blue[yr][date]++;
}
if(Color == "yellow"){
dateCounts_Yellow[yr][date]++;
}
});
Red_yr_2015_data = [dateCounts_Red['2015']['2015-01'], dateCounts_Red['2015']['2015-02'], dateCounts_Red['2015']['2015-03'], dateCounts_Red['2015']['2015-04'], dateCounts_Red['2015']['2015-05'], dateCounts_Red['2015']['2015-06'], dateCounts_Red['2015']['2015-07'], dateCounts_Red['2015']['2015-08'], dateCounts_Red['2015']['2015-09'], dateCounts_Red['2015']['2015-10'], dateCounts_Red['2015']['2015-11'], dateCounts_Red['2015']['2015-12']];
Blue_yr_2015_data = [dateCounts_Blue['2015']['2015-01'], dateCounts_Blue['2015']['2015-02'], dateCounts_Blue['2015']['2015-03'], dateCounts_Blue['2015']['2015-04'], dateCounts_Blue['2015']['2015-05'], dateCounts_Blue['2015']['2015-06'], dateCounts_Blue['2015']['2015-07'], dateCounts_Blue['2015']['2015-08'], dateCounts_Blue['2015']['2015-09'], dateCounts_Blue['2015']['2015-10'], dateCounts_Blue['2015']['2015-11'], dateCounts_Blue['2015']['2015-12']];
Yellow_yr_2015_data = [dateCounts_Yellow['2015']['2015-01'], dateCounts_Yellow['2015']['2015-02'], dateCounts_Yellow['2015']['2015-03'], dateCounts_Yellow['2015']['2015-04'], dateCounts_Yellow['2015']['2015-05'], dateCounts_Yellow['2015']['2015-06'], dateCounts_Yellow['2015']['2015-07'], dateCounts_Yellow['2015']['2015-08'], dateCounts_Yellow['2015']['2015-09'], dateCounts_Yellow['2015']['2015-10'], dateCounts_Yellow['2015']['2015-11'], dateCounts_Yellow['2015']['2015-12']];
Im currently getting the following error from my Highcharts js:
Uncaught TypeError: Cannot set property 'index' of undefined
THis is preventing the chart system to work correctly the data returned is not being returned with it's expected data.
Here a full example to the issue https://jsfiddle.net/awo5aaqb/21/
Would anyone know what im missing?
Your date count objects have major structural flaw.
When you prettify them they look like:
var dateCounts_Blue = {
"2015": {
"2015-01": 0,
"2015-02": 0,
"2015-03": 0,
"2015-04": 0
},
"2015": {
"2015-05": 0
},
"2015": {
"2015-06": 0
},
"2015": {
"2015-07": 0
},
......
Object keys must be unique so these are clearly being repeated and the compiler will over write duplicates.
Fix the pattern that breaks away from the intended pattern grouping at the beginning
var dateCounts_Red = {
"2015":
{
"2015-01":0,
"2015-02":0,
"2015-03":0,
"2015-04":0,
"2015-05":0,
"2015-06":0,
"2015-07":0,
"2015-08":0,
"2015-09":0,
"2015-10":0,
"2015-11":0,
"2015-12":0
},
};
var dateCounts_Blue = {
"2015":{
"2015-01":0,
"2015-02":0,
"2015-03":0,
"2015-04":0,
"2015-05":0,
"2015-06":0,
"2015-07":0,
"2015-08":0,
"2015-09":0,
"2015-10":0,
"2015-11":0,
"2015-12":0
}
};
var dateCounts_Yellow = {
"2015":{
"2015-01":0,
"2015-02":0,
"2015-03":0,
"2015-04":0,
"2015-05":0,
"2015-06":0,
"2015-07":0,
"2015-08":0,
"2015-09":0,
"2015-10":0,
"2015-11":0,
"2015-12":0}
};
Your data structure is flawed and such comparing values when doing the foreach loop becomes inconsistent because it compares it to multiple values, the above JSON is the fix for your problem.
Not quite codereview.stackexchange.com, but I heavily modified your javascript to make it work a bit better
$.ajax({
url: basePath,
dataType: 'json',
cache: false,
success: function(data) {
var counts = {};
data.d.results.forEach(function(element) {
// If you know it's all the same year, you could totally ignore this
var yr = element.created_date.slice(0, 4);
var month = parseInt(element.created_date.slice(5,7));
var color = element.colorvalue;
if (counts[color] === undefined) {
counts[color] = {};
}
if (counts[color][yr] === undefined) {
counts[color][yr] = {};
}
current_value = counts[color][yr][month];
if (current_value === undefined) {
// Doesnt exist yet, so add it
counts[color][yr][month] = 1;
} else {
// Exists, so increment by 1
counts[color][yr][month] = current_value + 1;
}
});
console.log(JSON.stringify(counts));
console.log(transform_series(counts['red']['2015']));
console.log(transform_series(counts['blue']['2015']));
console.log(transform_series(counts['yellow']['2015']));
var Options = {
chart: {
renderTo: 'myfirstchart',
type: 'column',
margin: 75,
options3d: {
enabled: true,
alpha: 25,
beta: 0,
depth: 70
}
},
title: {
text: "Test Highcharts"
},
subtitle: {
text: 'Test charts'
},
plotOptions: {
column: {
depth: 25
}
},
xAxis: {
categories: ["Janvier", "Février", "Mars", "Avril", "Mai", "Juin", "Juillet", "Août", "Septembre", "Octobre", "Novembre", "Décembre"]
},
yAxis: {
title: {
text: "Number of entries"
}
},
tooltip: {
headerFormat: '<b>{point.key}</b><br>',
pointFormat: '<span style="color:{series.color}">\u25CF</span> {series.name}: {point.y} / {point.stackTotal}'
},
plotOptions: {
column: {
stacking: 'normal',
depth: 40
}
},
series: [{
name: 'Red',
color: 'red',
data: transform_series(counts['red']['2015']),
stack: '2015'
}, {
name: 'Blue',
color: 'blue',
data: transform_series(counts['blue']['2015']),
stack: '2015'
}, {
name: 'Yellow',
color: 'yellow',
data: transform_series(counts['yellow']['2015']),
stack: '2015'
}]
};
return new Highcharts.Chart(Options);
}
});
// this transforms the hash {10: 5, 11:1, 12:1} to get you all 12 months
// and returns an array of values [ 0, 0, 0, 0, 0 ... 5, 1, 1] that
// can be used in high charts
function transform_series(series) {
return Array.apply(null, Array(13)).map(function (_, i) {return (series[i] === undefined) ? 0 : series[i];}).slice(1,13);
}
Does anybody know if it's possible to create a chart using TWO PANES and, inside of a pane, MULTIPLE SERIES?
I did a screenshot of what I need:
Does anybody have a example in JFiddler?
I already tried to create one but I didn't have success...
Regards,
Marcelo
Yes, I did it!
Bellow follow the source code that I've created:
HTML
<script src="http://code.highcharts.com/stock/highstock.js"></script>
<script src="http://code.highcharts.com/stock/modules/exporting.js"></script>
<div id="container" style="height: 500px; min-width: 310px"></div>
Javascript Code
$(function() {
$.getJSON('http://www.highcharts.com/samples/data/jsonp.php?filename=aapl-ohlcv.json&callback=?', function(data) {
var ohlc = [],
ohlc2 = [],
volume = [],
dataLength = data.length,
groupingUnits = [
['week', [1]],
['month', [1, 2, 3, 4, 6]]
],
i = 0;
for (i; i < dataLength; i += 1) {
ohlc.push([
data[i][0], // the date
data[i][1], // open
data[i][2], // high
data[i][3], // low
data[i][4] // close
]);
ohlc2.push([
data[i][0], // the date
data[i][1] - (Math.ceil(Math.random()*100)+100), // open
data[i][2] - (Math.ceil(Math.random()*100)+100), // high
data[i][3] - (Math.ceil(Math.random()*100)+100), // low
data[i][4] - (Math.ceil(Math.random()*100)+100) // close
]);
volume.push([
data[i][0], // the date
data[i][5] // the volume
]);
}
// create the chart
$('#container').highcharts('StockChart', {
rangeSelector: {
inputEnabled: $('#container').width() > 480,
selected: 1
},
title: {
text: 'Two Panes & Multiple Series'
},
yAxis: [{
labels: {
align: 'right',
x: -3
},
title: {
text: 'OHLC'
},
height: '60%',
lineWidth: 2
}, {
labels: {
align: 'right',
x: -3
},
title: {
text: 'Volume'
},
top: '65%',
height: '35%',
offset: 0,
lineWidth: 2
}],
series: [{
name: 'AAPL',
data: ohlc,
dataGrouping: {
units: groupingUnits
}
}, {
name: 'AAPL2',
data: ohlc2,
dataGrouping: {
units: groupingUnits
}
}, {
name: 'Volume',
data: volume,
yAxis: 1,
dataGrouping: {
units: groupingUnits
}
}]
});
});
});
And the link in JSFiddler:
http://jsfiddle.net/marcelojuventino/tc1a78ma/1/
Thanks to everybody!