From 1685f3af79e5198e6393d479d68a6ea53cb49042 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 2 Jan 2017 14:54:59 +0400 Subject: [PATCH 01/22] ignore trial dirs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 67277cb..802e562 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ __pycache__/ *env/ twisted/plugins/dropin.cache .DS_Store +_trial_temp From 439ef8499f038dd11ce841f657762af53cc5e1d9 Mon Sep 17 00:00:00 2001 From: dwasserm Date: Wed, 4 Jan 2017 13:19:25 -0500 Subject: [PATCH 02/22] Fixed PIDFile path in cowrie.service template (#394) * Updated service template to use the virtual-env and proper PID file location * Fixed PIDFile path in cowrie.service template * Revert "Fixed PIDFile path in cowrie.service template" This reverts commit 022afb71345d49fb18d8b46d129275a988d22b74. * Revert "Revert "Fixed PIDFile path in cowrie.service template"" This reverts commit 5b01372176a72a851b07d4116387010221fd137b. Revert changes * Fixed PIDFile path in cowrie.service template --- doc/systemd/cowrie.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/systemd/cowrie.service b/doc/systemd/cowrie.service index f15fff9..ef88475 100644 --- a/doc/systemd/cowrie.service +++ b/doc/systemd/cowrie.service @@ -9,7 +9,7 @@ Wants=mysql.service Type=forking User=cowrie Group=cowrie -PIDFile=var/run/cowrie.pid +PIDFile=/var/run/cowrie.pid ExecStart=/home/cowrie/cowrie/start.sh cowrie-env ExecStop=/home/cowrie/cowrie/stop.sh ExecReload=/home/cowrie/cowrie/stop.sh && sleep 10 && /home/cowrie/cowrie/start.sh cowrie-env From 0254a78d53d6c6f8b1539e953e945dc491653597 Mon Sep 17 00:00:00 2001 From: dwasserm Date: Wed, 4 Jan 2017 14:26:53 -0500 Subject: [PATCH 03/22] Updated to correct PIDFile path for cowrie.service (#395) * Updated service template to use the virtual-env and proper PID file location * Fixed PIDFile path in cowrie.service template * Revert "Fixed PIDFile path in cowrie.service template" This reverts commit 022afb71345d49fb18d8b46d129275a988d22b74. * Revert "Revert "Fixed PIDFile path in cowrie.service template"" This reverts commit 5b01372176a72a851b07d4116387010221fd137b. Revert changes * Fixed PIDFile path in cowrie.service template * Updated to correct PIDFile path for cowrie.service --- doc/systemd/cowrie.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/systemd/cowrie.service b/doc/systemd/cowrie.service index ef88475..df35af3 100644 --- a/doc/systemd/cowrie.service +++ b/doc/systemd/cowrie.service @@ -9,7 +9,7 @@ Wants=mysql.service Type=forking User=cowrie Group=cowrie -PIDFile=/var/run/cowrie.pid +PIDFile=/home/cowrie/cowrie/var/run/cowrie.pid ExecStart=/home/cowrie/cowrie/start.sh cowrie-env ExecStop=/home/cowrie/cowrie/stop.sh ExecReload=/home/cowrie/cowrie/stop.sh && sleep 10 && /home/cowrie/cowrie/start.sh cowrie-env From cbb02c7f045aa686167f0839740f36e63c7f7fea Mon Sep 17 00:00:00 2001 From: Claud Xiao Date: Wed, 4 Jan 2017 22:33:23 -0800 Subject: [PATCH 04/22] Fixes #396 (#397) --- cowrie/core/honeypot.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index f153180..8c69036 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -45,9 +45,18 @@ class HoneyPotCommand(object): self.protocol.terminal.transport.session.id, re.sub('[^A-Za-z0-9]', '_', self.outfile)) perm = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - self.fs.mkfile(self.outfile, 0, 0, 0, stat.S_IFREG | perm) - with open(self.safeoutfile, 'a'): - self.fs.update_realfile(self.fs.getfile(self.outfile), self.safeoutfile) + try: + self.fs.mkfile(self.outfile, 0, 0, 0, stat.S_IFREG | perm) + except fs.FileNotFound: + # The outfile locates at a non-existing directory. + self.protocol.pp.outReceived('-bash: %s: No such file or directory\n' % self.outfile) + self.write = self.write_to_failed + self.outfile = None + self.safeoutfile = None + + else: + with open(self.safeoutfile, 'a'): + self.fs.update_realfile(self.fs.getfile(self.outfile), self.safeoutfile) def check_arguments(self, application, args): @@ -78,10 +87,16 @@ class HoneyPotCommand(object): self.fs.update_size(self.outfile, self.writtenBytes) + def write_to_failed(self, data): + """ + """ + pass + def start(self): """ """ - self.call() + if self.write != self.write_to_failed: + self.call() self.exit() From ed3ac80fa3195b956c0b171431572a769c6145ae Mon Sep 17 00:00:00 2001 From: lelonek1 Date: Thu, 5 Jan 2017 08:48:59 -0500 Subject: [PATCH 05/22] Fix #385 by implementing telnet_Command (#392) --- cowrie/telnet/transport.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cowrie/telnet/transport.py b/cowrie/telnet/transport.py index 7bffe76..a2f0a96 100644 --- a/cowrie/telnet/transport.py +++ b/cowrie/telnet/transport.py @@ -138,6 +138,10 @@ class HoneyPotTelnetAuthProtocol(AuthenticatingTelnetProtocol): return 'Discard' + def telnet_Command(self, command): + self.transport.protocol.dataReceived(command+'\r') + return "Command" + def _cbLogin(self, ial): """ Fired on a successful login From 37f178a9155e3f1265540cebe0b4a10b46415c45 Mon Sep 17 00:00:00 2001 From: Claud Xiao Date: Sun, 8 Jan 2017 09:29:20 -0800 Subject: [PATCH 06/22] Fixed issue #398 (#399) --- cowrie/core/protocol.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cowrie/core/protocol.py b/cowrie/core/protocol.py index 0b68434..5b32d5d 100644 --- a/cowrie/core/protocol.py +++ b/cowrie/core/protocol.py @@ -195,7 +195,8 @@ class HoneyPotBaseProtocol(insults.TerminalProtocol, TimeoutMixin): obj.set_input_data(pp.input_data) self.cmdstack.append(obj) obj.start() - self.pp.outConnectionLost() + if self.pp: + self.pp.outConnectionLost() def uptime(self): From 6380825960a85847849a109c7f57b111e0f9f304 Mon Sep 17 00:00:00 2001 From: funtimes-ninja Date: Tue, 10 Jan 2017 12:43:06 -0500 Subject: [PATCH 07/22] Update dshield.py (#401) fix for dshield.py --- cowrie/output/dshield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cowrie/output/dshield.py b/cowrie/output/dshield.py index ee29e4e..68143c9 100644 --- a/cowrie/output/dshield.py +++ b/cowrie/output/dshield.py @@ -87,7 +87,7 @@ class Output(cowrie.core.output.Output): auth_header = 'credentials={0} nonce={1} userid={2}'.format(digest, _nonceb64, self.userid) headers = {'X-ISC-Authorization': auth_header, 'Content-Type':'text/plain', - 'Content-Length': len(log_output)} + 'Content-Length': str(len(log_output))} #log.msg(headers) if self.debug: From df735dd286d3568dfb7a7b21b7560c7d70d54e49 Mon Sep 17 00:00:00 2001 From: funtimes-ninja Date: Wed, 11 Jan 2017 12:53:31 -0500 Subject: [PATCH 08/22] Update dshield.py (#404) --- cowrie/output/dshield.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cowrie/output/dshield.py b/cowrie/output/dshield.py index 68143c9..89dfcbc 100644 --- a/cowrie/output/dshield.py +++ b/cowrie/output/dshield.py @@ -86,8 +86,7 @@ class Output(cowrie.core.output.Output): base64.b64decode(self.auth_key), hashlib.sha256).digest()) auth_header = 'credentials={0} nonce={1} userid={2}'.format(digest, _nonceb64, self.userid) headers = {'X-ISC-Authorization': auth_header, - 'Content-Type':'text/plain', - 'Content-Length': str(len(log_output))} + 'Content-Type':'text/plain'} #log.msg(headers) if self.debug: From cf16ff398e0655f0fe515c4b063a4b99d6293e73 Mon Sep 17 00:00:00 2001 From: fe7ch Date: Fri, 13 Jan 2017 16:56:35 +0300 Subject: [PATCH 09/22] Elk doc update (#408) * Update logstash's configuration to reflect changes in maxmind's geoip databases * Update documentation on "ELK Stack" * Add sample of filebeat configuration * Update documentation for "ELK Stack" with FileBeat option * Add some tips on configuration of kibana * Remove outdated kibana-cowrie.conf * Add link to XPack --- doc/elk/README.md | 106 +++++- doc/elk/filebeat-cowrie.conf | 22 ++ doc/elk/kibana-cowrie.conf | 675 ----------------------------------- doc/elk/logstash-cowrie.conf | 7 +- 4 files changed, 119 insertions(+), 691 deletions(-) create mode 100644 doc/elk/filebeat-cowrie.conf delete mode 100644 doc/elk/kibana-cowrie.conf diff --git a/doc/elk/README.md b/doc/elk/README.md index e930e23..6339005 100644 --- a/doc/elk/README.md +++ b/doc/elk/README.md @@ -7,35 +7,65 @@ * Working Cowrie installation * Cowrie JSON log file (enable database json in cowrie.cfg) +* Java 8 ## Installation + +We'll examine simple installation, when we install ELK stack on the same machine that used for cowrie. + +* Add Elastic's repository and key +``` +wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - +echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list +apt-get update +``` + * Install logstash, elasticsearch and kibana ``` -apt-get install logstash -apt-get install elasticsearch -```` - -* Install Kibana - -This may be different depending on your operating system. Kibana will need additional components such as a web server +apt-get install elasticsearch logstash kibana +``` +* Set them to autostart +``` +update-rc.d elasticsearch defaults 95 10 +update-rc.d kibana defaults 95 10 +``` ## ElasticSearch Configuration TBD +## Kibana Configuration + +* Make a folder for logs + +``` +mkdir /var/log/kibana +chown kibana:kibana /var/log/kibana +``` + +* Change the following parameters in /etc/kibana/kibana.yml to reflect your server setup: + +``` +"server.host" - set it to "localhost" if you use nginx for basic authentication or external interface if you use XPack (see below) +"server.name" - name of the server +"elasticsearch.url" - address of the elasticsearch +"elasticsearch.username", "elasticsearch.password" - needed only if you use XPack (see below) +"logging.dest" - set path to logs (/var/log/kibana/kibana.log) +``` + ## Logstash Configuration * Download GeoIP data ``` -wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz -wget http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz +wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz +wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz ``` -* Place these somewhere in your filesystem. +* Place these somewhere in your filesystem and make sure that "logstash" user can read it * Configure logstash @@ -65,3 +95,59 @@ http://:9200/_search?q=cowrie&size=5 * If this gives output, your data is correctly loaded into ElasticSearch +* When you successfully configured logstash, remove "file" and "stdout" blocks from output section of logstash configuration. + +## Distributed setup of sensors or multiple sensors on the same host + + If you have multiple sensors, you will need to setup up FileBeat to feed logstash with logs from all sensors + + On the logstash server: + + * Change "input" section of the logstash to the following: + + ``` + input { + beats { + port => 5044 + } + } + ``` + + On the sensor servers: + + * Install filebeat + ``` + wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list + apt-get update + apt-get install filebeat + ``` + + * Enable autorun for it + ``` + update-rc.d filebeat defaults 95 10 + ``` + + * Configure filebeat + + ``` + cp filebeat-cowrie.conf /etc/filebeat/filebeat.yml + ``` + + * Check the following parameters + ``` + paths - path to cowrie's json logs + logstash - check ip of the logstash host + ``` + + * Start filebeat + + ``` + service filebeat start + ``` + +## Tuning ELK stack + +* Refer to elastic's documentation about proper configuration of the system for the best elasticsearch's performance + +* You may avoid installing nginx for restricting access to the kibana by installing official elastic's plugin called "XPack" (https://www.elastic.co/products/x-pack) \ No newline at end of file diff --git a/doc/elk/filebeat-cowrie.conf b/doc/elk/filebeat-cowrie.conf new file mode 100644 index 0000000..592c183 --- /dev/null +++ b/doc/elk/filebeat-cowrie.conf @@ -0,0 +1,22 @@ +filebeat: + prospectors: + - + paths: + - /home/cowrie/cowrie/log/cowrie.json* + encoding: plain + input_type: log + document_type: cowrie + registry_file: /var/lib/filebeat/registry +output: + logstash: + hosts: ["10.10.0.11:5044"] +shipper: +logging: + to_syslog: false + to_files: true + files: + path: /var/log/filebeat/ + name: mybeat + rotateeverybytes: 10485760 # = 10MB + keepfiles: 7 + level: info \ No newline at end of file diff --git a/doc/elk/kibana-cowrie.conf b/doc/elk/kibana-cowrie.conf deleted file mode 100644 index d8d6017..0000000 --- a/doc/elk/kibana-cowrie.conf +++ /dev/null @@ -1,675 +0,0 @@ -{ - "title": "Cowrie2ElasticSearch", - "services": { - "query": { - "list": { - "0": { - "query": "*", - "alias": "", - "color": "#7EB26D", - "id": 0, - "pin": false, - "type": "lucene", - "enable": true - } - }, - "ids": [ - 0 - ] - }, - "filter": { - "list": { - "0": { - "type": "terms", - "field": "_type", - "value": "cowrie", - "mandate": "must", - "active": true, - "alias": "", - "id": 0 - }, - "1": { - "type": "time", - "field": "@timestamp", - "from": "now-30d", - "to": "now", - "mandate": "must", - "active": true, - "alias": "", - "id": 1 - } - }, - "ids": [ - 0, - 1 - ] - } - }, - "rows": [ - { - "title": "Graph", - "height": "250px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 3, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "sensor", - "exclude": [], - "missing": false, - "other": false, - "size": 5, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "table", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "Sensors" - }, - { - "error": false, - "span": 3, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "success", - "exclude": [], - "missing": true, - "other": true, - "size": 5, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "table", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "Successes" - } - ], - "notice": false - }, - { - "title": "Histogram", - "height": "300px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "span": 12, - "editable": true, - "type": "histogram", - "loadingEditor": false, - "mode": "count", - "time_field": "timestamp", - "value_field": null, - "x-axis": true, - "y-axis": true, - "scale": 1, - "y_format": "none", - "grid": { - "max": null, - "min": 0 - }, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "annotate": { - "enable": false, - "query": "*", - "size": 20, - "field": "_type", - "sort": [ - "_score", - "desc" - ] - }, - "auto_int": false, - "resolution": 100, - "interval": "1d", - "intervals": [ - "auto", - "1s", - "1m", - "5m", - "10m", - "30m", - "1h", - "3h", - "12h", - "1d", - "1w", - "1y" - ], - "lines": true, - "fill": 0, - "linewidth": 3, - "points": false, - "pointradius": 5, - "bars": false, - "stack": true, - "spyable": true, - "zoomlinks": true, - "options": true, - "legend": true, - "show_query": true, - "interactive": true, - "legend_counts": true, - "timezone": "browser", - "percentage": false, - "zerofill": true, - "derivative": false, - "tooltip": { - "value_type": "cumulative", - "query_as_alias": true - }, - "title": "Histogram", - "scaleSeconds": false - } - ], - "notice": false - }, - { - "title": "Usernames", - "height": "300px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 6, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "username.raw", - "exclude": [], - "missing": false, - "other": false, - "size": 20, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "bar", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "Usernames (top 20)" - }, - { - "error": false, - "span": 6, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "username.raw", - "exclude": [], - "missing": false, - "other": false, - "size": 20, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "pie", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "Usernames (top 20)" - } - ], - "notice": false - }, - { - "title": "Passwords", - "height": "300px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 6, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "password.raw", - "exclude": [], - "missing": false, - "other": false, - "size": 20, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "bar", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "Passwords (top 20)" - }, - { - "error": false, - "span": 6, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "password.raw", - "exclude": [], - "missing": false, - "other": false, - "size": 20, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "pie", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "Passwords (top 20)" - } - ], - "notice": false - }, - { - "title": "Clients", - "height": "300px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 6, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "client.raw", - "exclude": [], - "missing": false, - "other": false, - "size": 20, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "bar", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "SSH clients (top 20)" - }, - { - "error": false, - "span": 6, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "client.raw", - "exclude": [], - "missing": false, - "other": false, - "size": 20, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "pie", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "SSH clients (top 20)" - } - ], - "notice": false - }, - { - "title": "Maps", - "height": "450px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 8, - "editable": true, - "type": "map", - "loadingEditor": false, - "map": "world", - "colors": [ - "#A0E2E2", - "#265656" - ], - "size": 100, - "exclude": [], - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "title": "Attack map (world)", - "field": "country_code2" - }, - { - "error": false, - "span": 4, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "geoip.country_name.raw", - "exclude": [], - "missing": false, - "other": true, - "size": 13, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "table", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "count", - "valuefield": "", - "title": "Countries" - } - ], - "notice": false - }, - { - "title": "ASN", - "height": "150px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 4, - "editable": true, - "type": "terms", - "loadingEditor": false, - "field": "geoip.asn.raw", - "exclude": [], - "missing": false, - "other": true, - "size": 20, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "table", - "counter_pos": "above", - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "tmode": "terms", - "tstat": "total", - "valuefield": "", - "title": "ASN" - } - ], - "notice": false - }, - { - "title": "Events", - "height": "650px", - "editable": true, - "collapse": true, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 12, - "editable": true, - "group": [ - "default" - ], - "type": "table", - "size": 100, - "pages": 5, - "offset": 0, - "sort": [ - "_score", - "desc" - ], - "style": { - "font-size": "9pt" - }, - "overflow": "min-height", - "fields": [], - "highlight": [], - "sortable": true, - "header": true, - "paging": true, - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "field_list": true, - "status": "Stable", - "trimFactor": 300, - "normTimes": true, - "title": "Documents", - "all_fields": false, - "localTime": false, - "timeField": "@timestamp" - } - ], - "notice": false - } - ], - "editable": true, - "index": { - "interval": "day", - "pattern": "[logstash-]YYYY.MM.DD", - "default": "_all", - "warm_fields": false - }, - "style": "dark", - "failover": false, - "panel_hints": true, - "loader": { - "save_gist": false, - "save_elasticsearch": true, - "save_local": true, - "save_default": true, - "save_temp": true, - "save_temp_ttl_enable": true, - "save_temp_ttl": "30d", - "load_gist": true, - "load_elasticsearch": true, - "load_elasticsearch_size": 20, - "load_local": true, - "hide": false - }, - "pulldowns": [ - { - "type": "query", - "collapse": false, - "notice": false, - "query": "*", - "pinned": true, - "history": [], - "remember": 10, - "enable": true - }, - { - "type": "filtering", - "collapse": false, - "notice": true, - "enable": true - } - ], - "nav": [ - { - "type": "timepicker", - "collapse": false, - "notice": false, - "status": "Stable", - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "timefield": "@timestamp", - "enable": true, - "now": true, - "filter_id": 1 - } - ], - "refresh": false -} diff --git a/doc/elk/logstash-cowrie.conf b/doc/elk/logstash-cowrie.conf index a5afc43..225eef0 100644 --- a/doc/elk/logstash-cowrie.conf +++ b/doc/elk/logstash-cowrie.conf @@ -33,16 +33,11 @@ filter { geoip { source => "src_ip" target => "geoip" - database => "/opt/logstash/vendor/geoip/GeoLiteCity.dat" + database => "/opt/logstash/vendor/geoip/GeoLite2-City.dat" add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] } - geoip { - source => "src_ip" - database => "/opt/logstash/vendor/geoip/GeoIPASNum.dat" - } - mutate { convert => [ "[geoip][coordinates]", "float" ] } From eb638750a170ad8fea23289fafb526a8465f1726 Mon Sep 17 00:00:00 2001 From: Claud Xiao Date: Fri, 13 Jan 2017 05:57:19 -0800 Subject: [PATCH 10/22] Added MongoDB output support (#407) --- cowrie.cfg.dist | 8 +++++++ cowrie/test/unittests.cfg | 8 +++++++ mongodb.py | 49 +++++++++++++++++++++++++++++++++++++++ requirements-output.txt | 3 +++ 4 files changed, 68 insertions(+) create mode 100644 mongodb.py diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index 144edcd..bc76c9b 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -385,6 +385,14 @@ logfile = log/cowrie.json #[output_sqlite] #db_file = cowrie.db +# MongoDB logging module +# +# MongoDB logging requires an extra Python module: pip install pymongo +# +#[output_mongodb] +#connection_string = mongodb://username:password@host:port/database +#database = dbname + # Splunk SDK output module - Legacy. Requires Splunk API installed # This sends logs directly to Splunk using the Python REST SDK diff --git a/cowrie/test/unittests.cfg b/cowrie/test/unittests.cfg index 8361fba..ae0409e 100644 --- a/cowrie/test/unittests.cfg +++ b/cowrie/test/unittests.cfg @@ -343,6 +343,14 @@ logfile = log/cowrie.json #[output_sqlite] #db_file = cowrie.db +# MongoDB logging module +# +# MongoDB logging requires an extra Python module: pip install pymongo +# +#[output_mongodb] +#connection_string = mongodb://username:password@host:port/database +#database = dbname + # Splunk SDK output module - EARLY RELEASE NOT RECOMMENDED # This sends logs directly to Splunk using the Python REST SDK diff --git a/mongodb.py b/mongodb.py new file mode 100644 index 0000000..cbb2f19 --- /dev/null +++ b/mongodb.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +import pymongo + +from twisted.python import log + +import cowrie.core.output + + +class Output(cowrie.core.output.Output): + """ + """ + + def __init__(self, cfg): + self.cfg = cfg + cowrie.core.output.Output.__init__(self, cfg) + + + def start(self): + """ + """ + db_addr = self.cfg.get('output_mongodb', 'connection_string') + db_name = self.cfg.get('output_mongodb', 'database') + + try: + self.mongo_client = pymongo.MongoClient(db_addr) + self.mongo_db = self.mongo_client[db_name] + self.coll = self.mongo_db['events'] + except Exception, e: + log.msg('output_mongodb: Error: %s' % str(e)) + + + def stop(self): + """ + """ + self.mongo_client.close() + + + def write(self, entry): + """ + """ + for i in list(entry.keys()): + # Remove twisted 15 legacy keys + if i.startswith('log_'): + del entry[i] + try: + self.coll.insert_one(entry) + except Exception,e: + log.msg('output_mongodb: MongoDB Error: %s' % str(e)) diff --git a/requirements-output.txt b/requirements-output.txt index 5016dcb..d70b363 100644 --- a/requirements-output.txt +++ b/requirements-output.txt @@ -10,6 +10,9 @@ pyes # mysql MySQL-python +# mongodb +pymongo + # rethinkdblog rethinkdb From 8c3286276aded15d9795f104ea963b9b587421d8 Mon Sep 17 00:00:00 2001 From: Claud Xiao Date: Sat, 14 Jan 2017 12:44:48 -0800 Subject: [PATCH 11/22] Move mongodb.py file to correct location. (#410) --- mongodb.py => cowrie/output/mongodb.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mongodb.py => cowrie/output/mongodb.py (100%) diff --git a/mongodb.py b/cowrie/output/mongodb.py similarity index 100% rename from mongodb.py rename to cowrie/output/mongodb.py From 2152a73cc6a8db487e1c51f88f1583dd08ee5212 Mon Sep 17 00:00:00 2001 From: Claud Xiao Date: Fri, 20 Jan 2017 22:28:50 -0800 Subject: [PATCH 12/22] Ignore parentheses in command. Fixed issue #361 (#416) --- cowrie/core/honeypot.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 8c69036..6778ffd 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -180,6 +180,14 @@ class HoneyPotShell(object): try: tok = self.lexer.get_token() # log.msg( "tok: %s" % (repr(tok)) ) + + # Ignore parentheses + tok_len = len(tok) + tok = tok.strip('(') + tok = tok.strip(')') + if len(tok) != tok_len and tok == '': + continue + if tok == self.lexer.eof: if len(tokens): self.cmdpending.append((tokens)) From c58056b01a2d312c56ddc6f404d963c2017aa536 Mon Sep 17 00:00:00 2001 From: lelonek1 Date: Sat, 21 Jan 2017 01:30:31 -0500 Subject: [PATCH 13/22] Refuse to enable SGA and LINEMODE during authentication (#415) Some telnet clients attempt to initiate negotiation about SGA and LINEMODE themselves, but only when a port isn't specified on the command line. If we allow them to enable SGA or LINEMODE, they change from sending a newline character when enter is pressed to sending a carriage return. Cowrie can't handle this properly at the login prompt, which prevents the client from being able to login. I left the old code commented out so it is easier re-enable once Cowrie/Twisted support the Telnet protocol better. Fixes #414 --- cowrie/telnet/transport.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cowrie/telnet/transport.py b/cowrie/telnet/transport.py index a2f0a96..bb92f26 100644 --- a/cowrie/telnet/transport.py +++ b/cowrie/telnet/transport.py @@ -180,15 +180,17 @@ class HoneyPotTelnetAuthProtocol(AuthenticatingTelnetProtocol): if opt == ECHO: return True elif opt == SGA: - return True + return False + #return True else: return False def enableRemote(self, opt): if opt == LINEMODE: - self.transport.requestNegotiation(LINEMODE, MODE + chr(TRAPSIG)) - return True + return False + #self.transport.requestNegotiation(LINEMODE, MODE + chr(TRAPSIG)) + #return True elif opt == NAWS: return True elif opt == SGA: From ce6b996de0b7c9207cf8bab388988699ceff2300 Mon Sep 17 00:00:00 2001 From: Claud Xiao Date: Sun, 22 Jan 2017 19:07:10 -0800 Subject: [PATCH 14/22] Fixed IndexError caused by intentionally constructed empty cmdstack (#418) Thanks! --- cowrie/core/honeypot.py | 5 +++-- cowrie/core/protocol.py | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 6778ffd..bf1d05b 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -112,8 +112,9 @@ class HoneyPotCommand(object): """ try: self.protocol.cmdstack.pop() - self.protocol.cmdstack[-1].resume() - except AttributeError: + if len(self.protocol.cmdstack): + self.protocol.cmdstack[-1].resume() + except (AttributeError, IndexError): # Cmdstack could be gone already (wget + disconnect) pass diff --git a/cowrie/core/protocol.py b/cowrie/core/protocol.py index 5b32d5d..ac88125 100644 --- a/cowrie/core/protocol.py +++ b/cowrie/core/protocol.py @@ -353,19 +353,22 @@ class HoneyPotInteractiveProtocol(HoneyPotBaseProtocol, recvline.HistoricRecvLin def handle_CTRL_C(self): """ """ - self.cmdstack[-1].handle_CTRL_C() + if len(self.cmdstack): + self.cmdstack[-1].handle_CTRL_C() def handle_CTRL_D(self): """ """ - self.cmdstack[-1].handle_CTRL_D() + if len(self.cmdstack): + self.cmdstack[-1].handle_CTRL_D() def handle_TAB(self): """ """ - self.cmdstack[-1].handle_TAB() + if len(self.cmdstack): + self.cmdstack[-1].handle_TAB() def handle_CTRL_K(self): From 0757e057ae2ffd29fa7baec8e4dd31015e28c92a Mon Sep 17 00:00:00 2001 From: fe7ch Date: Fri, 27 Jan 2017 09:40:55 +0300 Subject: [PATCH 15/22] Prevent cowrie from crashing on invalid host for wget (#429) Thanks for submitting this! --- cowrie/commands/wget.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cowrie/commands/wget.py b/cowrie/commands/wget.py index f09de95..c24cdb4 100644 --- a/cowrie/commands/wget.py +++ b/cowrie/commands/wget.py @@ -142,6 +142,9 @@ class command_wget(HoneyPotCommand): path = parsed.path or '/' if scheme != 'http' and scheme != 'https': raise NotImplementedError + if not host: + self.exit() + return None except: self.write('%s: Unsupported scheme.\n' % (url,)) self.exit() From 74916c1d72a326ce6e4e5d397c5f10fe6aa18157 Mon Sep 17 00:00:00 2001 From: fe7ch Date: Fri, 27 Jan 2017 09:42:07 +0300 Subject: [PATCH 16/22] Fix paths for egrep/fgrep commands (#426) * Support grep/egrep/fgrep without full path * Fixed path for egrep/fgrep * There is no grep/egrep/fgrep in /usr/bin in standard cowrie's fs.pickle. --- cowrie/commands/fs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cowrie/commands/fs.py b/cowrie/commands/fs.py index 50d0e66..69b3ebf 100644 --- a/cowrie/commands/fs.py +++ b/cowrie/commands/fs.py @@ -124,9 +124,8 @@ class command_grep(HoneyPotCommand): commands['/bin/grep'] = command_grep -commands['/usr/bin/grep'] = command_grep -commands['/usr/bin/egrep'] = command_grep -commands['/usr/bin/fgrep'] = command_grep +commands['/bin/egrep'] = command_grep +commands['/bin/fgrep'] = command_grep class command_tail(HoneyPotCommand): From 4711b1108a60622ea941c8171a796f31236c7cbc Mon Sep 17 00:00:00 2001 From: fe7ch Date: Fri, 27 Jan 2017 09:43:08 +0300 Subject: [PATCH 17/22] Remove entries from honeyfs/proc/mounts that does not have corresponding folders. (#423) --- honeyfs/proc/mounts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/honeyfs/proc/mounts b/honeyfs/proc/mounts index 28669c5..d5af897 100644 --- a/honeyfs/proc/mounts +++ b/honeyfs/proc/mounts @@ -5,25 +5,10 @@ udev /dev devtmpfs rw,relatime,size=10240k,nr_inodes=997843,mode=755 0 0 devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,nosuid,relatime,size=1613336k,mode=755 0 0 /dev/dm-0 / ext3 rw,relatime,errors=remount-ro,data=ordered 0 0 -securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,nosuid,nodev 0 0 tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0 -tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0 -cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd 0 0 -pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0 -cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0 -cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0 -cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0 -cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0 -cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0 -cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0 -cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=22,pgrp=1,timeout=300,minproto=5,maxproto=5,direct 0 0 -mqueue /dev/mqueue mqueue rw,relatime 0 0 -hugetlbfs /dev/hugepages hugetlbfs rw,relatime 0 0 -debugfs /sys/kernel/debug debugfs rw,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0 /dev/sda1 /boot ext2 rw,relatime 0 0 /dev/mapper/home /home ext3 rw,relatime,data=ordered 0 0 binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0 -tmpfs /run/user/1000 tmpfs rw,nosuid,nodev,relatime,size=806668k,mode=700,uid=1000,gid=1000 0 0 From fbf2dbaf3c1cf04b2dfd4124158adf42a22237bf Mon Sep 17 00:00:00 2001 From: TheHermit Date: Fri, 27 Jan 2017 07:16:11 +0000 Subject: [PATCH 18/22] Mongo Output (#413) * Extend mongo output to use more collections. Matching the style of the other output methods * Start to update session from other keys * Logging and endtime in to session --- cowrie/output/mongodb.py | 97 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/cowrie/output/mongodb.py b/cowrie/output/mongodb.py index cbb2f19..66d36f4 100644 --- a/cowrie/output/mongodb.py +++ b/cowrie/output/mongodb.py @@ -15,6 +15,19 @@ class Output(cowrie.core.output.Output): self.cfg = cfg cowrie.core.output.Output.__init__(self, cfg) + def insert_one(self, collection, event): + try: + object_id = collection.insert_one(event).inserted_id + return object_id + except Exception as e: + log.msg('mongo error - {0}'.format(e)) + + def update_one(self, collection, session, doc): + try: + object_id = collection.update({'session': session}, doc) + return object_id + except Exception as e: + log.msg('mongo error - {0}'.format(e)) def start(self): """ @@ -25,7 +38,17 @@ class Output(cowrie.core.output.Output): try: self.mongo_client = pymongo.MongoClient(db_addr) self.mongo_db = self.mongo_client[db_name] - self.coll = self.mongo_db['events'] + # Define Collections. + self.col_sensors = self.mongo_db['sensors'] + self.col_sessions = self.mongo_db['sessions'] + self.col_auth = self.mongo_db['auth'] + self.col_input = self.mongo_db['input'] + self.col_downloads = self.mongo_db['downloads'] + self.col_input = self.mongo_db['input'] + self.col_clients = self.mongo_db['clients'] + self.col_ttylog = self.mongo_db['ttylog'] + self.col_keyfingerprints = self.mongo_db['keyfingerprints'] + self.col_event = self.mongo_db['event'] except Exception, e: log.msg('output_mongodb: Error: %s' % str(e)) @@ -43,7 +66,71 @@ class Output(cowrie.core.output.Output): # Remove twisted 15 legacy keys if i.startswith('log_'): del entry[i] - try: - self.coll.insert_one(entry) - except Exception,e: - log.msg('output_mongodb: MongoDB Error: %s' % str(e)) + + eventid = entry["eventid"] + + if eventid == 'cowrie.session.connect': + # Check if sensor exists, else add it. + doc = self.col_sensors.find_one({'sensor': self.sensor}) + if doc: + sensorid = doc['sensor'] + else: + sensorid = self.insert_one(self.col_sensors, entry) + + # Prep extra elements just to make django happy later on + entry['starttime'] = entry['timestamp'] + entry['endtime'] = None + entry['sshversion'] = None + entry['termsize'] = None + log.msg('Session Created') + self.insert_one(self.col_sessions, entry) + + elif eventid in ['cowrie.login.success', 'cowrie.login.failed']: + self.insert_one(self.col_auth, entry) + + elif eventid in ['cowrie.command.success', 'cowrie.command.failed']: + self.insert_one(self.col_input, entry) + + elif eventid == 'cowrie.session.file_download': + # ToDo add a config section and offer to store the file in the db - useful for central logging + # we will add an option to set max size, if its 16mb or less we can store as normal, + # If over 16 either fail or we just use gridfs both are simple enough. + self.insert_one(self.col_downloads, entry) + + elif eventid == 'cowrie.client.version': + doc = self.col_sessions.find_one({'session': entry['session']}) + if doc: + doc['sshversion'] = entry['version'] + self.update_one(self.col_sessions, entry['session'], doc) + else: + pass + + elif eventid == 'cowrie.client.size': + doc = self.col_sessions.find_one({'session': entry['session']}) + if doc: + doc['termsize'] = '{0}x{1}'.format(entry['width'], entry['height']) + self.update_one(self.col_sessions, entry['session'], doc) + else: + pass + + elif eventid == 'cowrie.session.closed': + doc = self.col_sessions.find_one({'session': entry['session']}) + if doc: + doc['endtime'] = entry['timestamp'] + self.update_one(self.col_sessions, entry['session'], doc) + else: + pass + + elif eventid == 'cowrie.log.closed': + # ToDo Compress to opimise the space and if your sending to remote db + with open(entry["ttylog"]) as ttylog: + entry['ttylogpath'] = entry['ttylog'] + entry['ttylog'] = ttylog.read().encode('hex') + self.insert_one(self.col_ttylog, entry) + + elif eventid == 'cowrie.client.fingerprint': + self.insert_one(self.col_keyfingerprints, entry) + + # Catch any other event types + else: + self.insert_one(self.col_event, entry) From 8307b86e4da82d0443558ebc85a64544a5c81468 Mon Sep 17 00:00:00 2001 From: fe7ch Date: Sat, 28 Jan 2017 10:55:14 +0300 Subject: [PATCH 19/22] Remove empty tftp files, double logging fix (#430) * Remove empty tftp files, double logging fix * Remove duplicate of os.symlink() call, add transportID, sessionID to safeoutfile name * Remove empty file in case of exception --- cowrie/commands/tftp.py | 61 ++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/cowrie/commands/tftp.py b/cowrie/commands/tftp.py index dc1a39f..e4191e1 100644 --- a/cowrie/commands/tftp.py +++ b/cowrie/commands/tftp.py @@ -56,10 +56,12 @@ class command_tftp(HoneyPotCommand): self.download_path = cfg.get('honeypot', 'download_path') - self.safeoutfile = '%s/%s_%s' % \ - (self.download_path, - time.strftime('%Y%m%d%H%M%S'), - re.sub('[^A-Za-z0-9]', '_', self.file_to_get)) + tmp_fname = '%s_%s_%s_%s' % \ + (time.strftime('%Y%m%d%H%M%S'), + self.protocol.getProtoTransport().transportId, + self.protocol.terminal.transport.session.id, + re.sub('[^A-Za-z0-9]', '_', self.file_to_get)) + self.safeoutfile = os.path.join(self.download_path, tmp_fname) try: tclient.download(self.file_to_get, self.safeoutfile, progresshook) @@ -67,38 +69,41 @@ class command_tftp(HoneyPotCommand): self.fs.mkfile(self.file_to_get, 0, 0, tclient.context.metrics.bytes, 33188) self.fs.update_realfile(self.fs.getfile(self.file_to_get), self.safeoutfile) - shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest() - hash_path = '%s/%s' % (self.download_path, shasum) + if os.path.exists(self.safeoutfile): - # If we have content already, delete temp file - if not os.path.exists(hash_path): - os.rename(self.safeoutfile, hash_path) - else: - os.remove(self.safeoutfile) - log.msg("Not storing duplicate content " + shasum) + if os.path.getsize(self.safeoutfile) == 0: + os.remove(self.safeoutfile) + self.safeoutfile = None + return - log.msg(eventid='cowrie.session.file_download', - format='Downloaded tftpFile (%(url)s) with SHA-256 %(shasum)s to %(outfile)s', - url=self.file_to_get, - outfile=hash_path, - shasum=shasum) + with open(self.safeoutfile, 'rb') as f: + shasum = hashlib.sha256(f.read()).hexdigest() + hash_path = os.path.join(self.download_path, shasum) - # Link friendly name to hash - os.symlink(shasum, self.safeoutfile) + # If we have content already, delete temp file + if not os.path.exists(hash_path): + os.rename(self.safeoutfile, hash_path) + else: + os.remove(self.safeoutfile) + log.msg("Not storing duplicate content " + shasum) - # FIXME: is this necessary? - self.safeoutfile = hash_path + log.msg(eventid='cowrie.session.file_download', + format='Downloaded tftpFile (%(url)s) with SHA-256 %(shasum)s to %(outfile)s', + url=self.file_to_get, + outfile=hash_path, + shasum=shasum) - # Update the honeyfs to point to downloaded file - f = self.fs.getfile(self.file_to_get) - f[A_REALFILE] = hash_path + # Link friendly name to hash + os.symlink(shasum, self.safeoutfile) - log.msg(eventid='cowrie.session.file_download', - format='Downloaded tftpFile to %(outfile)s', - outfile=self.safeoutfile - ) + # Update the honeyfs to point to downloaded file + f = self.fs.getfile(self.file_to_get) + f[A_REALFILE] = hash_path except tftpy.TftpException, err: + if os.path.exists(self.safeoutfile): + if os.path.getsize(self.safeoutfile) == 0: + os.remove(self.safeoutfile) return except KeyboardInterrupt: From 7f003c2da33ff8e6cddd441d6f8940a07b604da9 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 30 Jan 2017 14:33:29 +0400 Subject: [PATCH 20/22] in README.md refer to requirements.txt --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fcc1da9..f4b98e4 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,9 @@ Additional functionality over standard kippo: Software required: * Python 2.7+, (Python 3 not yet supported due to Twisted dependencies) -* Zope Interface 3.6.0+ -* Twisted 12.0+ -* python-crypto -* python-cryptography -* python-pyasn1 -* python-gmpy2 (recommended) -* python-mysqldb (for MySQL output) -* python-OpenSSL +* python-virtualenv + +For Python dependencies, see requirements.txt ## Files of interest: From e2033c36f3ba94b4e493d9d1718c0279eba378c0 Mon Sep 17 00:00:00 2001 From: fe7ch Date: Tue, 31 Jan 2017 21:53:31 +0300 Subject: [PATCH 21/22] Append redirection support (#428) * Add support for '>>' redirection * Add redir files hashing * Delete only ">" or ">>" + file name from cmd args * Update stdin/redir messages to include SHA-256 hash of the file content * Small style fixes, log if we don't store duplicate * Bug fixes for wget command * Use os.path.join instead of string formatting * Use "with" for hashing a file to prevent handle leakage * Don't overwrite self.safeoutfile if it was already set in HoneyPotyCommand's init method * Don't overwrite self.safeoutfile with hash, else it will break stuff in insults.py * Revert "Delete only ">" or ">>" + file name from cmd args" This reverts commit f3f8b90cbe221da8ffba2670f4419da105ad8ac3. * Fix bugged check for presence of safeoutfile attribute. * Don't overwrite safeoutfile in curl * Don't store None objects * Include transportId and sessionId to all safeoutfiles to avoid collisions. --- cowrie/commands/curl.py | 16 ++++++++----- cowrie/commands/ftpget.py | 13 ++++++----- cowrie/commands/gcc.py | 11 +++++---- cowrie/commands/wget.py | 21 ++++++++++------- cowrie/core/honeypot.py | 47 +++++++++++++++++++++++---------------- cowrie/core/protocol.py | 3 +++ cowrie/insults/insults.py | 37 +++++++++++++++++++++++++++--- 7 files changed, 103 insertions(+), 45 deletions(-) diff --git a/cowrie/commands/curl.py b/cowrie/commands/curl.py index c227557..1608468 100644 --- a/cowrie/commands/curl.py +++ b/cowrie/commands/curl.py @@ -92,10 +92,14 @@ class command_curl(HoneyPotCommand): self.download_path = cfg.get('honeypot', 'download_path') - self.safeoutfile = '%s/%s_%s' % \ - (self.download_path, - time.strftime('%Y%m%d%H%M%S'), - re.sub('[^A-Za-z0-9]', '_', url)) + if not hasattr(self, 'safeoutfile'): + tmp_fname = '%s_%s_%s_%s' % \ + (time.strftime('%Y%m%d%H%M%S'), + self.protocol.getProtoTransport().transportId, + self.protocol.terminal.transport.session.id, + re.sub('[^A-Za-z0-9]', '_', url)) + self.safeoutfile = os.path.join(self.download_path, tmp_fname) + self.deferred = self.download(url, outfile, self.safeoutfile) if self.deferred: self.deferred.addCallback(self.success, outfile) @@ -311,7 +315,7 @@ Options: (H) means HTTP/HTTPS only, (F) means FTP only self.exit() shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest() - hashPath = '%s/%s' % (self.download_path, shasum) + hashPath = os.path.join(self.download_path, shasum) # If we have content already, delete temp file if not os.path.exists(hashPath): @@ -336,7 +340,7 @@ Options: (H) means HTTP/HTTPS only, (F) means FTP only os.symlink(shasum, self.safeoutfile) # FIXME: is this necessary? - self.safeoutfile = hashPath + # self.safeoutfile = hashPath # Update the honeyfs to point to downloaded file if outfile is not None: diff --git a/cowrie/commands/ftpget.py b/cowrie/commands/ftpget.py index 6135f06..97a0c87 100644 --- a/cowrie/commands/ftpget.py +++ b/cowrie/commands/ftpget.py @@ -93,10 +93,13 @@ Download a file via FTP cfg = self.protocol.cfg url = 'ftp://%s/%s' % (self.host, self.remote_path) self.download_path = cfg.get('honeypot', 'download_path') - self.safeoutfile = '%s/%s_%s' % \ - (self.download_path, - time.strftime('%Y%m%d%H%M%S'), - re.sub('[^A-Za-z0-9]', '_', url)) + + tmp_fname = '%s_%s_%s_%s' % \ + (time.strftime('%Y%m%d%H%M%S'), + self.protocol.getProtoTransport().transportId, + self.protocol.terminal.transport.session.id, + re.sub('[^A-Za-z0-9]', '_', url)) + self.safeoutfile = os.path.join(self.download_path, tmp_fname) result = self.ftp_download(self.safeoutfile) @@ -110,7 +113,7 @@ Download a file via FTP return shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest() - hash_path = '%s/%s' % (self.download_path, shasum) + hash_path = os.path.join(self.download_path, shasum) # If we have content already, delete temp file if not os.path.exists(hash_path): diff --git a/cowrie/commands/gcc.py b/cowrie/commands/gcc.py index 836f189..92ee7cc 100644 --- a/cowrie/commands/gcc.py +++ b/cowrie/commands/gcc.py @@ -4,6 +4,7 @@ import time import re import getopt import random +import os from twisted.internet import reactor @@ -164,10 +165,12 @@ gcc version %s (Debian %s-5)""" % (version, version_short, version_short, versio def generate_file(self, outfile): data = "" # TODO: make sure it is written to temp file, not downloads - safeoutfile = '%s/%s_%s' % \ - (self.protocol.cfg.get('honeypot', 'download_path'), - time.strftime('%Y%m%d%H%M%S'), - re.sub('[^A-Za-z0-9]', '_', outfile)) + tmp_fname = '%s_%s_%s_%s' % \ + (time.strftime('%Y%m%d%H%M%S'), + self.protocol.getProtoTransport().transportId, + self.protocol.terminal.transport.session.id, + re.sub('[^A-Za-z0-9]', '_', outfile)) + safeoutfile = os.path.join(self.protocol.cfg.get('honeypot', 'download_path'), tmp_fname) # Data contains random garbage from an actual file, so when # catting the file, you'll see some 'real' compiled data diff --git a/cowrie/commands/wget.py b/cowrie/commands/wget.py index c24cdb4..51d47de 100644 --- a/cowrie/commands/wget.py +++ b/cowrie/commands/wget.py @@ -121,10 +121,14 @@ class command_wget(HoneyPotCommand): self.download_path = cfg.get('honeypot', 'download_path') - self.safeoutfile = '%s/%s_%s' % \ - (self.download_path, - time.strftime('%Y%m%d%H%M%S'), - re.sub('[^A-Za-z0-9]', '_', url)) + if not hasattr(self, 'safeoutfile'): + tmp_fname = '%s_%s_%s_%s' % \ + (time.strftime('%Y%m%d%H%M%S'), + self.protocol.getProtoTransport().transportId, + self.protocol.terminal.transport.session.id, + re.sub('[^A-Za-z0-9]', '_', url)) + self.safeoutfile = os.path.join(self.download_path, tmp_fname) + self.deferred = self.download(url, outfile, self.safeoutfile) if self.deferred: self.deferred.addCallback(self.success, outfile) @@ -188,8 +192,9 @@ class command_wget(HoneyPotCommand): log.msg("there's no file " + self.safeoutfile) self.exit() - shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest() - hash_path = '%s/%s' % (self.download_path, shasum) + with open(self.safeoutfile, 'rb') as f: + shasum = hashlib.sha256(f.read()).hexdigest() + hash_path = os.path.join(self.download_path, shasum) # If we have content already, delete temp file if not os.path.exists(hash_path): @@ -211,10 +216,10 @@ class command_wget(HoneyPotCommand): shasum=shasum) # Link friendly name to hash - os.symlink( shasum, self.safeoutfile ) + os.symlink(shasum, self.safeoutfile) # FIXME: is this necessary? - self.safeoutfile = hash_path + # self.safeoutfile = hash_path # Update the honeyfs to point to downloaded file f = self.fs.getfile(outfile) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index bf1d05b..02ca9d2 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -32,31 +32,40 @@ class HoneyPotCommand(object): self.errorWrite = self.protocol.pp.errReceived # MS-DOS style redirect handling, inside the command # TODO: handle >>, 2>, etc - if '>' in self.args: + if '>' in self.args or '>>' in self.args: self.writtenBytes = 0 self.write = self.write_to_file - index = self.args.index(">") + if '>>' in self.args: + index = self.args.index('>>') + b_append = True + else: + index = self.args.index('>') + b_append = False self.outfile = self.fs.resolve_path(str(self.args[(index + 1)]), self.protocol.cwd) del self.args[index:] - self.safeoutfile = '%s/%s-%s-%s-redir_%s' % ( - self.protocol.cfg.get('honeypot', 'download_path'), - time.strftime('%Y%m%d-%H%M%S'), - self.protocol.getProtoTransport().transportId, - self.protocol.terminal.transport.session.id, - re.sub('[^A-Za-z0-9]', '_', self.outfile)) - perm = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - try: - self.fs.mkfile(self.outfile, 0, 0, 0, stat.S_IFREG | perm) - except fs.FileNotFound: - # The outfile locates at a non-existing directory. - self.protocol.pp.outReceived('-bash: %s: No such file or directory\n' % self.outfile) - self.write = self.write_to_failed - self.outfile = None - self.safeoutfile = None + p = self.fs.getfile(self.outfile) + if not p or not p[fs.A_REALFILE] or p[fs.A_REALFILE].startswith('honeyfs') or not b_append: + tmp_fname = '%s-%s-%s-redir_%s' % \ + (time.strftime('%Y%m%d-%H%M%S'), + self.protocol.getProtoTransport().transportId, + self.protocol.terminal.transport.session.id, + re.sub('[^A-Za-z0-9]', '_', self.outfile)) + self.safeoutfile = os.path.join(self.protocol.cfg.get('honeypot', 'download_path'), tmp_fname) + perm = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + try: + self.fs.mkfile(self.outfile, 0, 0, 0, stat.S_IFREG | perm) + except fs.FileNotFound: + # The outfile locates at a non-existing directory. + self.protocol.pp.outReceived('-bash: %s: No such file or directory\n' % self.outfile) + self.write = self.write_to_failed + self.outfile = None + self.safeoutfile = None + else: + with open(self.safeoutfile, 'a'): + self.fs.update_realfile(self.fs.getfile(self.outfile), self.safeoutfile) else: - with open(self.safeoutfile, 'a'): - self.fs.update_realfile(self.fs.getfile(self.outfile), self.safeoutfile) + self.safeoutfile = p[fs.A_REALFILE] def check_arguments(self, application, args): diff --git a/cowrie/core/protocol.py b/cowrie/core/protocol.py index ac88125..d3f70da 100644 --- a/cowrie/core/protocol.py +++ b/cowrie/core/protocol.py @@ -195,6 +195,9 @@ class HoneyPotBaseProtocol(insults.TerminalProtocol, TimeoutMixin): obj.set_input_data(pp.input_data) self.cmdstack.append(obj) obj.start() + if hasattr(obj, 'safeoutfile'): + if obj.safeoutfile: + self.terminal.redirFiles.add(obj.safeoutfile) if self.pp: self.pp.outConnectionLost() diff --git a/cowrie/insults/insults.py b/cowrie/insults/insults.py index 5bf431c..9b7ab86 100644 --- a/cowrie/insults/insults.py +++ b/cowrie/insults/insults.py @@ -31,6 +31,8 @@ class LoggingServerProtocol(insults.ServerProtocol): self.ttylogPath = cfg.get('honeypot', 'log_path') self.downloadPath = cfg.get('honeypot', 'download_path') + self.redirFiles = set() + try: self.bytesReceivedLimit = int(cfg.get('honeypot', 'download_limit_size')) @@ -134,14 +136,15 @@ class LoggingServerProtocol(insults.ServerProtocol): try: with open(self.stdinlogFile, 'rb') as f: shasum = hashlib.sha256(f.read()).hexdigest() - shasumfile = self.downloadPath + "/" + shasum - if (os.path.exists(shasumfile)): + shasumfile = os.path.join(self.downloadPath, shasum) + if os.path.exists(shasumfile): os.remove(self.stdinlogFile) + log.msg("Not storing duplicate content " + shasum) else: os.rename(self.stdinlogFile, shasumfile) os.symlink(shasum, self.stdinlogFile) log.msg(eventid='cowrie.session.file_download', - format='Saved stdin contents to %(outfile)s', + format='Saved stdin contents with SHA-256 %(shasum)s to %(outfile)s', url='stdin', outfile=shasumfile, shasum=shasum) @@ -150,6 +153,34 @@ class LoggingServerProtocol(insults.ServerProtocol): finally: self.stdinlogOpen = False + if self.redirFiles: + for rf in self.redirFiles: + try: + if not os.path.exists(rf): + continue + + if os.path.getsize(rf) == 0: + os.remove(rf) + continue + + with open(rf, 'rb') as f: + shasum = hashlib.sha256(f.read()).hexdigest() + shasumfile = os.path.join(self.downloadPath, shasum) + if os.path.exists(shasumfile): + os.remove(rf) + log.msg("Not storing duplicate content " + shasum) + else: + os.rename(rf, shasumfile) + os.symlink(shasum, rf) + log.msg(eventid='cowrie.session.file_download', + format='Saved redir contents with SHA-256 %(shasum)s to %(outfile)s', + url='redir', + outfile=shasumfile, + shasum=shasum) + except IOError: + pass + self.redirFiles.clear() + if self.ttylogOpen: # TODO: Add session duration to this entry log.msg(eventid='cowrie.log.closed', From 53ccf45c7c7c7c6171fa90e52f8479e9902a43aa Mon Sep 17 00:00:00 2001 From: fe7ch Date: Thu, 2 Feb 2017 13:11:15 +0300 Subject: [PATCH 22/22] Remove redundant [geoip][coordinates] field, since we already have [geoip][location] (added by geoip filter itself) (#440) --- doc/elk/README.md | 3 +-- doc/elk/logstash-cowrie.conf | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/doc/elk/README.md b/doc/elk/README.md index 6339005..6aae293 100644 --- a/doc/elk/README.md +++ b/doc/elk/README.md @@ -62,7 +62,6 @@ chown kibana:kibana /var/log/kibana ``` wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz -wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz ``` * Place these somewhere in your filesystem and make sure that "logstash" user can read it @@ -150,4 +149,4 @@ http://:9200/_search?q=cowrie&size=5 * Refer to elastic's documentation about proper configuration of the system for the best elasticsearch's performance -* You may avoid installing nginx for restricting access to the kibana by installing official elastic's plugin called "XPack" (https://www.elastic.co/products/x-pack) \ No newline at end of file +* You may avoid installing nginx for restricting access to the kibana by installing official elastic's plugin called "X-Pack" (https://www.elastic.co/products/x-pack) \ No newline at end of file diff --git a/doc/elk/logstash-cowrie.conf b/doc/elk/logstash-cowrie.conf index 225eef0..ecf5f4a 100644 --- a/doc/elk/logstash-cowrie.conf +++ b/doc/elk/logstash-cowrie.conf @@ -34,12 +34,6 @@ filter { source => "src_ip" target => "geoip" database => "/opt/logstash/vendor/geoip/GeoLite2-City.dat" - add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] - add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] - } - - mutate { - convert => [ "[geoip][coordinates]", "float" ] } } }