I am currently working on own siem system, where windows logs(in csv format) automatically collecting every hour and displaying in a Highcharts. The backend is Django.
But ran into problems:
There's too many data that GPU of browser is load.
I set auto parsing to per 1 hour but site not displaying. Other parts of code are working
Is it ok if I write logs in database?
Can anyone give me advices how to do it right?
Here is my Javascript code in template/dashboard
_categories = {{categories|safe}};
_values = {{values|safe}};
_data = {{data|safe}};
Highcharts.chart('pie', {
colors: ['#558dfa', '#a1c1fe', '#4c72bc', '#3a61ad'],
title: {
text: ''
},
chart: {
type: 'pie'
},
tooltip: {
valueSuffix: '%'
},
plotOptions: {
pie: {
allowPointSelect: false,
cursor: 'pointer',
dataLabels: {
enabled: false
},
showInLegend: false
}
},
series: [{
name: 'Percentage',
colorByPoint: true,
data: [
{
name: 'Critical',
y: {{range.0}}
},
{
name: 'Error',
y: {{range.1}}
},
{
name: 'Information',
y: {{range.2}}
},
{
name: 'Warning',
y: {{range.3}}
},
]
}]
});
Highcharts.stockChart('container', {
chart: {
alignTicks: false
},
rangeSelector: {
selected: 0
},
series: [{
type: 'column',
data: _data,
}],
dataGrouping: {
units: [[
'week', // unit name
[1] // allowed multiples
], [
'month',
[1, 2, 3, 4, 6]
]]
}
});
Here is my views.py
def dashboard_req(path, request):
df = pd.read_csv(path)
rs = df.groupby('Level')['Event ID'].count()
r = df.groupby('Time Generated')['Event ID'].count()
categories = list(rs.index)
values = list(rs.values)
count = len(df.index)
date = list(r.index)
values2 = list(r.values)
data = []
for i in range(len(date)):
date_format = datetime.datetime.strptime(date[i], "%Y-%m-%d %H:%M:%S")
unix_time = datetime.datetime.timestamp(date_format)
data.append([int(unix_time)*1000, values2[i]])
ranges = []
for i in values:
percent = i / count * 100
ranges.append(round(percent,1))
table_content = df.to_html(index=None)
table_content = table_content.replace("<thead>","<thead class='thead-dark'>")
table_content = table_content.replace('class="dataframe"',"class='table table-striped'")
table_content = table_content.replace('border="1"',"")
context = {"categories": categories, 'values': values, 'table_data':table_content, 'count':count, 'range':ranges, 'data':data}
# print('Done')
return render(request, './dashboard.html', context=context)
def do_something(scheduler, request):
scheduler.enter(3600, 1, do_something, (scheduler, request)) # sec
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data\log.csv')
server = 'localhost'
logtype = 'Application'
hand = win32evtlog.OpenEventLog(server,logtype)
flags = win32evtlog.EVENTLOG_BACKWARDS_READ|win32evtlog.EVENTLOG_SEQUENTIAL_READ
total = win32evtlog.GetNumberOfEventLogRecords(hand)
head = ['ID','Time Generated', 'Source Name', 'Event ID', 'Event Category', 'Level', 'Event Data']
id_count = 1
event_types = ['Critical', 'Error', 'Warning', 'Information']
with open(path, 'w', encoding="utf-8") as file:
writer = csv.writer(file)
writer.writerow(head)
while True:
events = win32evtlog.ReadEventLog(hand, flags,0)
if events:
for event in events:
data = event.StringInserts
if data:
for msg in data:
log_row = [id_count, event.TimeGenerated, event.SourceName, event.EventID, event.EventCategory, event_types[event.EventType-1], msg]
else:
log_row = [id_count, event.TimeGenerated, event.SourceName, event.EventID, event.EventCategory, event_types[event.EventType-1]]
writer.writerow(log_row)
id_count+=1
else:
break
dashboard_req(path, request)
def dashboard(request):
my_scheduler = sched.scheduler(time.time, time.sleep)
my_scheduler.enter(3600, 1, do_something, (my_scheduler, request))
my_scheduler.run()
Related
I have one Json with multiple array and foreach array I want to create Pie chart, but I don't know how to do it.
This is the array thet I have. And this is what I tried :
function Pie() {
$.getJSON("/Admin/Attivita/OreOggi", function (data) {
console.log(data);
var oreTecico = [];
var oreTecico = [];
var oreMalatia = [];
var oreStraordinario = [];
var oreInfortunio = [];
var oreFerie = [];
for (var i = 0; i < data.length; i++) {
nomeTecnico.push(data[i].nome);
oreTecico.push(data[i].odinario);
oreMalatia.push(data[i].malatia);
oreStraordinario.push(data[i].straordinario);
oreInfortunio.push(data[i].infortunio);
oreFerie.push(data[i].ferie);
};
// Build the chart
Highcharts.chart('zdravko', {
chart: {
plotBackgroundColor: null,
plotBorderWidth: null,
plotShadow: false,
type: 'pie'
},
title: {
text: 'Ore segnate oggi'
},
tooltip: {
pointFormat: '<b>{point.name}</b>: {point.y:.1f} h.'
},
accessibility: {
point: {
valueSuffix: '%'
}
},
plotOptions: {
pie: {
allowPointSelect: true,
cursor: 'pointer',
dataLabels: {
enabled: false
},
showInLegend: true
}
},
series: [{
name: nomeTecnico[0],
colorByPoint: true,
data: [{
name: '',
y:0,
sliced: true,
selected: true
}, {
name: 'Odinario',
y: oreTecico[0]
}, {
name: 'Malatia',
y: oreMalatia[0]
}, {
name: 'Straordinario',
y: oreStraordinario[0]
}, {
name: 'Infortunio',
y: oreInfortunio[0]
}, {
name: 'Ferie',
y: oreFerie[0]
}]
}]
});
});
}
It shows only the last "data". I want to make fo each array one pie. If i have 100 arrays I want 100 pies.
UPDATE:
I added this :
data.forEach(function (el) {
var chartData = [el.data1, el.data2];
var chartContainer = document.createElement('div');
document.getElementById('zdravko').append(chartContainer);
Highcharts.chart(chartContainer, {
series: [{
type: 'pie',
data: chartData
}]
});
});
The chartData is array of undefined objects.
Is it possible to make for or foreach inside Highcharts?
You need to use the Highcharts.chart method in a loop, for example:
var data = [{
data1: 12,
data2: 25
}, {
data1: 67,
data2: 11
}];
data.forEach(function(el) {
var chartData = [el.data1, el.data2];
var chartContainer = document.createElement('div');
document.getElementById('container').append(chartContainer);
Highcharts.chart(chartContainer, {
series: [{
type: 'pie',
data: chartData
}]
});
});
Live demo: http://jsfiddle.net/BlackLabel/x95pbw7j/
API Reference: https://api.highcharts.com/class-reference/Highcharts#.chart
I tried the code like this with many small restructuration and modification but without success.
Here is the code:
$(function () {
$.get('data.csv', function(data) {
// split the data set into ohlc and volume
var ohlc = [],
volume = [],
dataLength = data.length,
// set the allowed units for data grouping
groupingUnits = [[
'week', // unit name
[1] // allowed multiples
], [
'month',
[1, 2, 3, 4, 6]
]],
i = 0;
for (i; i < dataLength; i += 1) {
ohlc.push([
data[i][0], // the date
data[i][1], // open
data[i][2], // high
data[i][3], // low
data[i][4] // close
]);
volume.push([
data[i][0], // the date
data[i][5] // the volume
]);
}
$('#chart').highcharts({
rangeSelector: {
selected: 1
},
title: {
text: 'AAPL Historical'
},
yAxis: [{
labels: {
align: 'right',
x: -3
},
title: {
text: 'OHLC'
},
height: '60%',
lineWidth: 2
}, {
labels: {
align: 'right',
x: -3
},
title: {
text: 'Volume'
},
top: '65%',
height: '35%',
offset: 0,
lineWidth: 2
}],
data: {
csv: data
},
series: [{
type: 'candlestick',
name: 'AAPL',
data: ohlc,
dataGrouping: {
units: groupingUnits
}
}, {
type: 'column',
name: 'Volume',
data: volume,
yAxis: 1,
dataGrouping: {
units: groupingUnits
}
}]
});
});
});
Here is data.csv:
Date,Open,High,Low,Close,Volume
2013-12-20,9371.08,9413.09,9352.98,9400.18,161686900
2013-12-19,9279.68,9351.9,9257.24,9335.74,98276500
2013-12-18,9145.35,9190.73,9122.05,9181.75,82342700
2013-12-17,9142.75,9161.8,9085.12,9085.12,72207500
2013-12-16,9004.62,9187.78,8997.75,9163.56,99105600
2013-12-13,9016.78,9046.63,8990.58,9006.46,67761700
2013-12-12,9032.67,9060.54,8984.28,9017,75120200
2013-12-11,9093.26,9153.14,9065.51,9077.11,64845800
2013-12-10,9180.29,9223.73,9091.97,9114.44,74363400
Can you help me to figure out the problem or purpose new approch please ?
What is my goal ?
Is to be able to load a CSV file inside the chart instead of using JSON file.
Why ?
Because modifing CSV file is more easier for me using PHP than JSON, and it's for performance too.
Thank's
When you do data.length, you are getting length of the csv file string. What you need to do is split the data with the newline delimiter.
// sample from data
var data = `Date,Open,High,Low,Close,Volume
2013-12-20,9371.08,9413.09,9352.98,9400.18,161686900
2013-12-19,9279.68,9351.9,9257.24,9335.74,98276500`;
// split by \n (new line)
data = data.split('\n'); // now data is an array of rows
var finalObj = [];
// iterate over the rows
data.map(function(row){
var obj = {};
// row is a string separated by ','
row = row.split(','); // now row is an array
obj['date'] = row[0];
obj['open'] = row[1];
obj['high'] = row[2];
obj['low'] = row[3];
obj['close'] = row[4];
obj['volume'] = row[5];
finalObj.push(obj);
})
console.log(finalObj);
Output:
[
{
date:'Date',
open:'Open',
high:'High',
low:'Low',
close:'Close',
volume:'Volume'
},
{
date:'2013-12-20',
open:'9371.08',
high:'9413.09',
low:'9352.98',
close:'9400.18',
volume:'161686900'
},
{
date:'2013-12-19',
open:'9279.68',
high:'9351.9',
low:'9257.24',
close:'9335.74',
volume:'98276500'
}
]
Pie Chart looks like this:
Data is there in JSON:
EDIT
From console:
When i tried it with type of "bar" or "column" chart it works fine.
Still new to this and really appreciate your help, folks!
Django version: 1.10
Python version: 3.6
chartViewHigh.html
{% block main %}
<h1 align="center">Analysis</h1>
{% block content %}
<div id="container" style="width:50%; height:400px;"></div>
{% endblock %}
{% block extrajs %}
<script>
var endpoint = '/api/chart/data/';
var labels = [];
var defaultData = [];
var labels2 = [];
var defaultData2 = [];
$.ajax({
method: "GET",
url: endpoint,
/**
* #param data
* #param data.labels
* #param data.default
* #param data.default2
*/
success: function (data) {
labels = data.labels;
defaultData = data.default;
labels2 = data.labels2;
defaultData2 = data.default2;
setChart()
},
error: function (error_data) {
console.log("error");
console.log(error_data)
}
});
function setChart() {
$(document).ready(function() {
var chart = {
plotBackgroundColor: null,
plotBorderWidth: null,
plotShadow: false
};
var title = {
text: 'Total'
};
var tooltip = {
pointFormat: '{series.name}: <b>{point.percentage:.1f}%</b>'
};
var plotOptions = {
pie: {
allowPointSelect: true,
cursor: 'pointer',
dataLabels: {
enabled: true,
format: '<b>{point.name}%</b>: {point.percentage:.1f} %',
style: {
color: (Highcharts.theme && Highcharts.theme.contrastTextColor) || 'black'
}
}
}
};
var series= [{
type: 'pie',
name: 'Group share',
data: [
{ name: 'Board', y: defaultData },
{
name: 'Member',
y: defaultData2,
sliced: true,
selected: true
}
]
}];
var json = {};
json.chart = chart;
json.title = title;
json.tooltip = tooltip;
json.series = series;
json.plotOptions = plotOptions;
$('#container').highcharts(json);
});
views.py
class ChartData(APIView):
def get(self, request, format=None):
qs_count = Administrator.objects.filter(association=self.request.user.association).count()
qs_count2 = Member.objects.filter(association=self.request.user.association).count()
labels = ["Members"]
default_items = [qs_count2]
labels2 = ["Board"]
default_items2 = [qs_count]
data = {
"labels": labels,
"default": default_items,
"labels2": labels2,
"default2": default_items2
}
return Response(data)
The data array has incorrect format. y values must be numbers, you set them as an array.
Change series variable to:
var series= [{
type: 'pie',
name: 'Group share',
data: [{
name: 'Board',
y: defaultData[0] },
{
name: 'Member',
y: defaultData2[0],
sliced: true,
selected: true
}
]
}];
Or send those values as a single number instead of an array.
You look to be missing in your chart section a declaration of a pie chart.
var chart = {
plotBackgroundColor: null,
plotBorderWidth: null,
plotShadow: false,
type: 'pie' // this is missing.
};
Have not been able to run your code to test but fingers crossed this sorts it for you.
I was under the assumption the "Play with this data!" link was supposed the show up by default. Any ideas on why it may not appear? I am just working with a basic scatter plot.
Note that this code below is not standalone as is, it is just the excerpt that does the plotly work.
var xData = [];
var yData = [];
var h = results;
for(var k in h) {
var localdate = k;
var plotdate = moment(localdate).format('YYYY-MM-DD HH:mm:ss');
xData.push(plotdate);
if (currentPort === "t") {
yData.push(CtoF(h[k]));
} else {
yData.push(h[k]);
};
}
var plotdata = [
{
x: xData,
y: yData,
type: 'scatter',
mode: 'markers+lines',
line: {
'color': HELIUM_BLUE
},
marker: {
'symbol': 'circle',
'color': HELIUM_PINK,
'maxdisplayed': 50
}
}
];
var layout = {
title: currentData,
xaxis: {
'title': 'Date / Time'
},
yaxis: {
'title': title
}
};
Plotly.newPlot(plotHolder, plotdata, layout);
You would need to add {showLink: true} as the fourth argument (after layout). I guess the default value changed from true to false.
If you want to change the caption of the button, use {showLink: true, "linkText": "Play with this data"}
var xData = [1, 2, 3, 4, 5];
var yData = [10, 1, 25, 12, 9];
var plotdata = [
{
x: xData,
y: yData,
type: 'scatter',
mode: 'markers+lines',
}
];
var layout = {
title: 'Edit me',
xaxis: {
'title': 'x'
},
yaxis: {
'title': 'y'
}
};
Plotly.newPlot(plotHolder, plotdata, layout, {showLink: true, "linkText": "Play with this data"});
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<div id='plotHolder'>
</div>
So I wrote a python app to get stock data for highcharts. I get this in terminal
127.0.0.1 - - [28/Mar/2013 13:47:02] "GET / HTTP/1.1" 200 -
127.0.0.1 - - [28/Mar/2013 13:47:02] "GET /favicon.ico HTTP/1.1" 404 -
127.0.0.1 - - [28/Mar/2013 13:47:15] "GET /json/msft/?startdate=2010-01-01?enddate=2011-01-01 HTTP/1.1" 200 -
Here is my javascript code
<script type="text/javascript">
function stock() {
console.log('pass 1')
url = 'http://127.0.0.1:5000/json/' + document.getElementById("ticker").value + '/?startdate=' + document.getElementById("startdate").value + '?enddate=' + document.getElementById("enddate").value
$.getJSON(url, function(data) {
console.log('pass 2')
// split the data set into ohlc and volume
var ohlc = [],
volume = [],
dataLength = data.length;
for (i = 0; i < dataLength; i++) {
ohlc.push([
data[i]["date"], // the date
data[i]["open"], // open
data[i]["high"], // high
data[i]["low"], // low
data[i]["close"] // close
]);
volume.push([
data[i]["date"], // the date
data[i]["volume"] // the volume
])
}
// set the allowed units for data grouping
var groupingUnits = [[
'week', // unit name
[1] // allowed multiples
], [
'month',
[1, 2, 3, 4, 6]
]];
// create the chart
$('#container').highcharts('StockChart', {
rangeSelector: {
selected: 1
},
title: {
text: 'AAPL Historical'
},
yAxis: [{
title: {
text: 'OHLC'
},
height: 200,
lineWidth: 2
}, {
title: {
text: 'Volume'
},
top: 300,
height: 100,
offset: 0,
lineWidth: 2
}],
series: [{
type: 'candlestick',
name: 'AAPL',
data: ohlc,
dataGrouping: {
units: groupingUnits
}
}, {
type: 'column',
name: 'Volume',
data: volume,
yAxis: 1,
dataGrouping: {
units: groupingUnits
}
}]
});
});
}
</script>
and my python code for the json request
#app.route('/json/<ticker>/', methods = ['GET'])
def json_route(ticker):
startdate = request.args.get('startdate','')
enddate = request.args.get('enddate','')
check = datecheck(startdate,enddate)
if check != 'pass':
return check
check = datacheck(ticker,startdate,enddate)
if check != 'got data':
urldata(ticker)
print '-----------------data-----------------'
conn = engine.connect()
entries = conn.execute(select([Stocks]).where(Stocks.symbol == ticker ).where(Stocks.date.between(startdate,enddate))).fetchall()
stocklist = []
print entries
for x in entries:
stocklist.append({
'volume': float('%f' %x[7]),
'high': float('%.2f' %x[4]),
'open': float('%.2f' %x[3]),
'low': float('%.2f' %x[5]),
'close': float('%.2f' %x[6]),
'date': str(x[2]),
'stock': x[1],
})
conn.close()
return json.dumps(stocklist)
What am I doing wrong? I assumed getjson would be doable if its on the same domain localhost. only console.log('pass one') works and shows up in the console when inspecting the element. Pass two is never hit.
Judging by the error message "only supported for HTTP", it wants an HTTP URL. Have you tried:
url = 'http://127.0.0.1:5000/json/...