From cf16ff398e0655f0fe515c4b063a4b99d6293e73 Mon Sep 17 00:00:00 2001
From: fe7ch <fe7ch@users.noreply.github.com>
Date: Fri, 13 Jan 2017 16:56:35 +0300
Subject: [PATCH] Elk doc update (#408)

* Update logstash's configuration to reflect changes in maxmind's geoip databases

* Update documentation on "ELK Stack"

* Add sample of filebeat configuration

* Update documentation for "ELK Stack" with FileBeat option

* Add some tips on configuration of kibana

* Remove outdated kibana-cowrie.conf

* Add link to XPack
---
 doc/elk/README.md            | 106 +++++-
 doc/elk/filebeat-cowrie.conf |  22 ++
 doc/elk/kibana-cowrie.conf   | 675 -----------------------------------
 doc/elk/logstash-cowrie.conf |   7 +-
 4 files changed, 119 insertions(+), 691 deletions(-)
 create mode 100644 doc/elk/filebeat-cowrie.conf
 delete mode 100644 doc/elk/kibana-cowrie.conf

diff --git a/doc/elk/README.md b/doc/elk/README.md
index e930e23..6339005 100644
--- a/doc/elk/README.md
+++ b/doc/elk/README.md
@@ -7,35 +7,65 @@
 
 * Working Cowrie installation
 * Cowrie JSON log file (enable database json in cowrie.cfg)
+* Java 8
 
 ## Installation
 
+
+We'll examine simple installation, when we install ELK stack on the same machine that used for cowrie.
+
+* Add Elastic's repository and key
+```
+wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
+apt-get update
+```
+
 * Install logstash, elasticsearch and kibana
 
 ```
-apt-get install logstash
-apt-get install elasticsearch
-````
-
-* Install Kibana
-
-This may be different depending on your operating system. Kibana will need additional components such as a web server
+apt-get install elasticsearch logstash kibana
+```
 
+* Set them to autostart
+```
+update-rc.d elasticsearch defaults 95 10
+update-rc.d kibana defaults 95 10
+```
 
 ## ElasticSearch Configuration
 
 TBD
 
+## Kibana Configuration
+
+* Make a folder for logs
+
+```
+mkdir /var/log/kibana
+chown kibana:kibana /var/log/kibana
+```
+
+* Change the following parameters in /etc/kibana/kibana.yml to reflect your server setup:
+
+```
+"server.host"  - set it to "localhost" if you use nginx for basic authentication or external interface if you use XPack (see below)
+"server.name" - name of the server
+"elasticsearch.url" - address of the elasticsearch
+"elasticsearch.username", "elasticsearch.password" - needed only if you use XPack (see below)
+"logging.dest" - set path to logs (/var/log/kibana/kibana.log)
+```
+
 ## Logstash Configuration
 
 * Download GeoIP data
 
 ```
-wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
-wget http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
+wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
+wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz
 ```
 
-* Place these somewhere in your filesystem.
+* Place these somewhere in your filesystem and make sure that "logstash" user can read it
 
 * Configure logstash
 
@@ -65,3 +95,59 @@ http://<hostname>:9200/_search?q=cowrie&size=5
 
 * If this gives output, your data is correctly loaded into ElasticSearch
 
+* When you successfully configured logstash, remove "file" and "stdout" blocks from output section of logstash configuration.
+
+## Distributed setup of sensors or multiple sensors on the same host
+
+ If you have multiple sensors, you will need to setup up FileBeat to feed logstash with logs from all sensors
+ 
+ On the logstash server:
+ 
+ * Change "input" section of the logstash to the following:
+ 
+ ```
+ input {
+    beats {
+        port => 5044
+    }
+ }
+ ```
+ 
+ On the sensor servers:
+ 
+ * Install filebeat
+ ```
+ wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+ echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
+ apt-get update
+ apt-get install filebeat
+ ```
+ 
+ * Enable autorun for it
+ ```
+ update-rc.d filebeat defaults 95 10
+ ```
+
+ * Configure filebeat
+ 
+ ```
+ cp filebeat-cowrie.conf /etc/filebeat/filebeat.yml
+ ```
+
+ * Check the following parameters
+ ```
+ paths - path to cowrie's json logs
+ logstash - check ip of the logstash host
+ ```
+ 
+ * Start filebeat
+ 
+ ```
+ service filebeat start
+ ``` 
+
+## Tuning ELK stack
+
+* Refer to elastic's documentation about proper configuration of the system for the best elasticsearch's performance
+
+* You may avoid installing nginx for restricting access to the kibana by installing official elastic's plugin called "XPack" (https://www.elastic.co/products/x-pack) 
\ No newline at end of file
diff --git a/doc/elk/filebeat-cowrie.conf b/doc/elk/filebeat-cowrie.conf
new file mode 100644
index 0000000..592c183
--- /dev/null
+++ b/doc/elk/filebeat-cowrie.conf
@@ -0,0 +1,22 @@
+filebeat:
+  prospectors:
+    -
+      paths:
+        - /home/cowrie/cowrie/log/cowrie.json*
+      encoding: plain
+      input_type: log
+      document_type: cowrie
+  registry_file: /var/lib/filebeat/registry
+output:
+  logstash:
+    hosts: ["10.10.0.11:5044"]
+shipper:
+logging:
+  to_syslog: false
+  to_files: true
+  files:
+    path: /var/log/filebeat/
+    name: mybeat
+    rotateeverybytes: 10485760 # = 10MB
+    keepfiles: 7
+  level: info
\ No newline at end of file
diff --git a/doc/elk/kibana-cowrie.conf b/doc/elk/kibana-cowrie.conf
deleted file mode 100644
index d8d6017..0000000
--- a/doc/elk/kibana-cowrie.conf
+++ /dev/null
@@ -1,675 +0,0 @@
-{
-  "title": "Cowrie2ElasticSearch",
-  "services": {
-    "query": {
-      "list": {
-        "0": {
-          "query": "*",
-          "alias": "",
-          "color": "#7EB26D",
-          "id": 0,
-          "pin": false,
-          "type": "lucene",
-          "enable": true
-        }
-      },
-      "ids": [
-        0
-      ]
-    },
-    "filter": {
-      "list": {
-        "0": {
-          "type": "terms",
-          "field": "_type",
-          "value": "cowrie",
-          "mandate": "must",
-          "active": true,
-          "alias": "",
-          "id": 0
-        },
-        "1": {
-          "type": "time",
-          "field": "@timestamp",
-          "from": "now-30d",
-          "to": "now",
-          "mandate": "must",
-          "active": true,
-          "alias": "",
-          "id": 1
-        }
-      },
-      "ids": [
-        0,
-        1
-      ]
-    }
-  },
-  "rows": [
-    {
-      "title": "Graph",
-      "height": "250px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
-        {
-          "error": false,
-          "span": 3,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "sensor",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 5,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "table",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Sensors"
-        },
-        {
-          "error": false,
-          "span": 3,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "success",
-          "exclude": [],
-          "missing": true,
-          "other": true,
-          "size": 5,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "table",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Successes"
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "Histogram",
-      "height": "300px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
-        {
-          "span": 12,
-          "editable": true,
-          "type": "histogram",
-          "loadingEditor": false,
-          "mode": "count",
-          "time_field": "timestamp",
-          "value_field": null,
-          "x-axis": true,
-          "y-axis": true,
-          "scale": 1,
-          "y_format": "none",
-          "grid": {
-            "max": null,
-            "min": 0
-          },
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "annotate": {
-            "enable": false,
-            "query": "*",
-            "size": 20,
-            "field": "_type",
-            "sort": [
-              "_score",
-              "desc"
-            ]
-          },
-          "auto_int": false,
-          "resolution": 100,
-          "interval": "1d",
-          "intervals": [
-            "auto",
-            "1s",
-            "1m",
-            "5m",
-            "10m",
-            "30m",
-            "1h",
-            "3h",
-            "12h",
-            "1d",
-            "1w",
-            "1y"
-          ],
-          "lines": true,
-          "fill": 0,
-          "linewidth": 3,
-          "points": false,
-          "pointradius": 5,
-          "bars": false,
-          "stack": true,
-          "spyable": true,
-          "zoomlinks": true,
-          "options": true,
-          "legend": true,
-          "show_query": true,
-          "interactive": true,
-          "legend_counts": true,
-          "timezone": "browser",
-          "percentage": false,
-          "zerofill": true,
-          "derivative": false,
-          "tooltip": {
-            "value_type": "cumulative",
-            "query_as_alias": true
-          },
-          "title": "Histogram",
-          "scaleSeconds": false
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "Usernames",
-      "height": "300px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
-        {
-          "error": false,
-          "span": 6,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "username.raw",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 20,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "bar",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Usernames (top 20)"
-        },
-        {
-          "error": false,
-          "span": 6,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "username.raw",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 20,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "pie",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Usernames (top 20)"
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "Passwords",
-      "height": "300px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
-        {
-          "error": false,
-          "span": 6,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "password.raw",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 20,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "bar",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Passwords (top 20)"
-        },
-        {
-          "error": false,
-          "span": 6,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "password.raw",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 20,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "pie",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Passwords (top 20)"
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "Clients",
-      "height": "300px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
-        {
-          "error": false,
-          "span": 6,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "client.raw",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 20,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "bar",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "SSH clients (top 20)"
-        },
-        {
-          "error": false,
-          "span": 6,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "client.raw",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 20,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "pie",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "SSH clients (top 20)"
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "Maps",
-      "height": "450px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
-        {
-          "error": false,
-          "span": 8,
-          "editable": true,
-          "type": "map",
-          "loadingEditor": false,
-          "map": "world",
-          "colors": [
-            "#A0E2E2",
-            "#265656"
-          ],
-          "size": 100,
-          "exclude": [],
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "title": "Attack map (world)",
-          "field": "country_code2"
-        },
-        {
-          "error": false,
-          "span": 4,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "geoip.country_name.raw",
-          "exclude": [],
-          "missing": false,
-          "other": true,
-          "size": 13,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "table",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "count",
-          "valuefield": "",
-          "title": "Countries"
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "ASN",
-      "height": "150px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
-        {
-          "error": false,
-          "span": 4,
-          "editable": true,
-          "type": "terms",
-          "loadingEditor": false,
-          "field": "geoip.asn.raw",
-          "exclude": [],
-          "missing": false,
-          "other": true,
-          "size": 20,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "table",
-          "counter_pos": "above",
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "ASN"
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "Events",
-      "height": "650px",
-      "editable": true,
-      "collapse": true,
-      "collapsable": true,
-      "panels": [
-        {
-          "error": false,
-          "span": 12,
-          "editable": true,
-          "group": [
-            "default"
-          ],
-          "type": "table",
-          "size": 100,
-          "pages": 5,
-          "offset": 0,
-          "sort": [
-            "_score",
-            "desc"
-          ],
-          "style": {
-            "font-size": "9pt"
-          },
-          "overflow": "min-height",
-          "fields": [],
-          "highlight": [],
-          "sortable": true,
-          "header": true,
-          "paging": true,
-          "spyable": true,
-          "queries": {
-            "mode": "all",
-            "ids": [
-              0
-            ]
-          },
-          "field_list": true,
-          "status": "Stable",
-          "trimFactor": 300,
-          "normTimes": true,
-          "title": "Documents",
-          "all_fields": false,
-          "localTime": false,
-          "timeField": "@timestamp"
-        }
-      ],
-      "notice": false
-    }
-  ],
-  "editable": true,
-  "index": {
-    "interval": "day",
-    "pattern": "[logstash-]YYYY.MM.DD",
-    "default": "_all",
-    "warm_fields": false
-  },
-  "style": "dark",
-  "failover": false,
-  "panel_hints": true,
-  "loader": {
-    "save_gist": false,
-    "save_elasticsearch": true,
-    "save_local": true,
-    "save_default": true,
-    "save_temp": true,
-    "save_temp_ttl_enable": true,
-    "save_temp_ttl": "30d",
-    "load_gist": true,
-    "load_elasticsearch": true,
-    "load_elasticsearch_size": 20,
-    "load_local": true,
-    "hide": false
-  },
-  "pulldowns": [
-    {
-      "type": "query",
-      "collapse": false,
-      "notice": false,
-      "query": "*",
-      "pinned": true,
-      "history": [],
-      "remember": 10,
-      "enable": true
-    },
-    {
-      "type": "filtering",
-      "collapse": false,
-      "notice": true,
-      "enable": true
-    }
-  ],
-  "nav": [
-    {
-      "type": "timepicker",
-      "collapse": false,
-      "notice": false,
-      "status": "Stable",
-      "time_options": [
-        "5m",
-        "15m",
-        "1h",
-        "6h",
-        "12h",
-        "24h",
-        "2d",
-        "7d",
-        "30d"
-      ],
-      "refresh_intervals": [
-        "5s",
-        "10s",
-        "30s",
-        "1m",
-        "5m",
-        "15m",
-        "30m",
-        "1h",
-        "2h",
-        "1d"
-      ],
-      "timefield": "@timestamp",
-      "enable": true,
-      "now": true,
-      "filter_id": 1
-    }
-  ],
-  "refresh": false
-}
diff --git a/doc/elk/logstash-cowrie.conf b/doc/elk/logstash-cowrie.conf
index a5afc43..225eef0 100644
--- a/doc/elk/logstash-cowrie.conf
+++ b/doc/elk/logstash-cowrie.conf
@@ -33,16 +33,11 @@ filter {
             geoip {
                 source => "src_ip"
                 target => "geoip"
-                database => "/opt/logstash/vendor/geoip/GeoLiteCity.dat"
+                database => "/opt/logstash/vendor/geoip/GeoLite2-City.dat"
                 add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                 add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
             }
 
-            geoip {
-                source => "src_ip"
-                database => "/opt/logstash/vendor/geoip/GeoIPASNum.dat"
-            }
-
             mutate {
                 convert => [ "[geoip][coordinates]", "float" ]
             }