Elk doc update (#408)

* Update logstash's configuration to reflect changes in maxmind's geoip databases

* Update documentation on "ELK Stack"

* Add sample of filebeat configuration

* Update documentation for "ELK Stack" with FileBeat option

* Add some tips on configuration of kibana

* Remove outdated kibana-cowrie.conf

* Add link to XPack
This commit is contained in:
fe7ch
2017-01-13 16:56:35 +03:00
committed by Michel Oosterhof
parent df735dd286
commit cf16ff398e
4 changed files with 119 additions and 691 deletions

View File

@@ -7,35 +7,65 @@
* Working Cowrie installation
* Cowrie JSON log file (enable database json in cowrie.cfg)
* Java 8
## Installation
We'll examine simple installation, when we install ELK stack on the same machine that used for cowrie.
* Add Elastic's repository and key
```
wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
apt-get update
```
* Install logstash, elasticsearch and kibana
```
apt-get install logstash
apt-get install elasticsearch
````
* Install Kibana
This may be different depending on your operating system. Kibana will need additional components such as a web server
apt-get install elasticsearch logstash kibana
```
* Set them to autostart
```
update-rc.d elasticsearch defaults 95 10
update-rc.d kibana defaults 95 10
```
## ElasticSearch Configuration
TBD
## Kibana Configuration
* Make a folder for logs
```
mkdir /var/log/kibana
chown kibana:kibana /var/log/kibana
```
* Change the following parameters in /etc/kibana/kibana.yml to reflect your server setup:
```
"server.host" - set it to "localhost" if you use nginx for basic authentication or external interface if you use XPack (see below)
"server.name" - name of the server
"elasticsearch.url" - address of the elasticsearch
"elasticsearch.username", "elasticsearch.password" - needed only if you use XPack (see below)
"logging.dest" - set path to logs (/var/log/kibana/kibana.log)
```
## Logstash Configuration
* Download GeoIP data
```
wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
wget http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz
```
* Place these somewhere in your filesystem.
* Place these somewhere in your filesystem and make sure that "logstash" user can read it
* Configure logstash
@@ -65,3 +95,59 @@ http://<hostname>:9200/_search?q=cowrie&size=5
* If this gives output, your data is correctly loaded into ElasticSearch
* When you successfully configured logstash, remove "file" and "stdout" blocks from output section of logstash configuration.
## Distributed setup of sensors or multiple sensors on the same host
If you have multiple sensors, you will need to setup up FileBeat to feed logstash with logs from all sensors
On the logstash server:
* Change "input" section of the logstash to the following:
```
input {
beats {
port => 5044
}
}
```
On the sensor servers:
* Install filebeat
```
wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
apt-get update
apt-get install filebeat
```
* Enable autorun for it
```
update-rc.d filebeat defaults 95 10
```
* Configure filebeat
```
cp filebeat-cowrie.conf /etc/filebeat/filebeat.yml
```
* Check the following parameters
```
paths - path to cowrie's json logs
logstash - check ip of the logstash host
```
* Start filebeat
```
service filebeat start
```
## Tuning ELK stack
* Refer to elastic's documentation about proper configuration of the system for the best elasticsearch's performance
* You may avoid installing nginx for restricting access to the kibana by installing official elastic's plugin called "XPack" (https://www.elastic.co/products/x-pack)

View File

@@ -0,0 +1,22 @@
filebeat:
prospectors:
-
paths:
- /home/cowrie/cowrie/log/cowrie.json*
encoding: plain
input_type: log
document_type: cowrie
registry_file: /var/lib/filebeat/registry
output:
logstash:
hosts: ["10.10.0.11:5044"]
shipper:
logging:
to_syslog: false
to_files: true
files:
path: /var/log/filebeat/
name: mybeat
rotateeverybytes: 10485760 # = 10MB
keepfiles: 7
level: info

View File

@@ -1,675 +0,0 @@
{
"title": "Cowrie2ElasticSearch",
"services": {
"query": {
"list": {
"0": {
"query": "*",
"alias": "",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
}
},
"ids": [
0
]
},
"filter": {
"list": {
"0": {
"type": "terms",
"field": "_type",
"value": "cowrie",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "time",
"field": "@timestamp",
"from": "now-30d",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 1
}
},
"ids": [
0,
1
]
}
},
"rows": [
{
"title": "Graph",
"height": "250px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 3,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "sensor",
"exclude": [],
"missing": false,
"other": false,
"size": 5,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Sensors"
},
{
"error": false,
"span": 3,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "success",
"exclude": [],
"missing": true,
"other": true,
"size": 5,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Successes"
}
],
"notice": false
},
{
"title": "Histogram",
"height": "300px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "count",
"time_field": "timestamp",
"value_field": null,
"x-axis": true,
"y-axis": true,
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"queries": {
"mode": "all",
"ids": [
0
]
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"auto_int": false,
"resolution": 100,
"interval": "1d",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1y"
],
"lines": true,
"fill": 0,
"linewidth": 3,
"points": false,
"pointradius": 5,
"bars": false,
"stack": true,
"spyable": true,
"zoomlinks": true,
"options": true,
"legend": true,
"show_query": true,
"interactive": true,
"legend_counts": true,
"timezone": "browser",
"percentage": false,
"zerofill": true,
"derivative": false,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"title": "Histogram",
"scaleSeconds": false
}
],
"notice": false
},
{
"title": "Usernames",
"height": "300px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "username.raw",
"exclude": [],
"missing": false,
"other": false,
"size": 20,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "bar",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Usernames (top 20)"
},
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "username.raw",
"exclude": [],
"missing": false,
"other": false,
"size": 20,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Usernames (top 20)"
}
],
"notice": false
},
{
"title": "Passwords",
"height": "300px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "password.raw",
"exclude": [],
"missing": false,
"other": false,
"size": 20,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "bar",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Passwords (top 20)"
},
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "password.raw",
"exclude": [],
"missing": false,
"other": false,
"size": 20,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Passwords (top 20)"
}
],
"notice": false
},
{
"title": "Clients",
"height": "300px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "client.raw",
"exclude": [],
"missing": false,
"other": false,
"size": 20,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "bar",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "SSH clients (top 20)"
},
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "client.raw",
"exclude": [],
"missing": false,
"other": false,
"size": 20,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "SSH clients (top 20)"
}
],
"notice": false
},
{
"title": "Maps",
"height": "450px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 8,
"editable": true,
"type": "map",
"loadingEditor": false,
"map": "world",
"colors": [
"#A0E2E2",
"#265656"
],
"size": 100,
"exclude": [],
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"title": "Attack map (world)",
"field": "country_code2"
},
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "geoip.country_name.raw",
"exclude": [],
"missing": false,
"other": true,
"size": 13,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "count",
"valuefield": "",
"title": "Countries"
}
],
"notice": false
},
{
"title": "ASN",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "geoip.asn.raw",
"exclude": [],
"missing": false,
"other": true,
"size": 20,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "ASN"
}
],
"notice": false
},
{
"title": "Events",
"height": "650px",
"editable": true,
"collapse": true,
"collapsable": true,
"panels": [
{
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"_score",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [],
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"title": "Documents",
"all_fields": false,
"localTime": false,
"timeField": "@timestamp"
}
],
"notice": false
}
],
"editable": true,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "_all",
"warm_fields": false
},
"style": "dark",
"failover": false,
"panel_hints": true,
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"enable": true,
"now": true,
"filter_id": 1
}
],
"refresh": false
}

View File

@@ -33,16 +33,11 @@ filter {
geoip {
source => "src_ip"
target => "geoip"
database => "/opt/logstash/vendor/geoip/GeoLiteCity.dat"
database => "/opt/logstash/vendor/geoip/GeoLite2-City.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
geoip {
source => "src_ip"
database => "/opt/logstash/vendor/geoip/GeoIPASNum.dat"
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}