From f657af1a1a7997d92f0f006e89d683e075c40751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Mon, 8 Feb 2016 07:35:56 +0100 Subject: [PATCH 1/5] Add sqlite3 driver --- cowrie/output/sqlite.py | 151 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 cowrie/output/sqlite.py diff --git a/cowrie/output/sqlite.py b/cowrie/output/sqlite.py new file mode 100644 index 0000000..64b1ead --- /dev/null +++ b/cowrie/output/sqlite.py @@ -0,0 +1,151 @@ +import sqlite3 + +from twisted.internet import defer +from twisted.enterprise import adbapi +from twisted.python import log + +import cowrie.core.output +class Output(cowrie.core.output.Output): + """ + docstring here + """ + + def __init__(self, cfg): + self.cfg = cfg + cowrie.core.output.Output.__init__(self, cfg) + + def start(self): + """ + Start sqlite3 logging module using Twisted ConnectionPool. + """ + sqlite_file = self.cfg.get('output_sqlite', 'db_file') + try: + self.db = adbapi.ConnectionPool('sqlite3', + database = sqlite_file) + except sqlite3.OperationalError as e: + log.msg(e) + + self.db.start() + + def stop(self): + """ + Close connection to db + """ + self.db.close() + + def sqlerror(self, error): + """ + docstring here + """ + log.err( 'sqlite3 Error:', error.value ) + + def simpleQuery(self, sql, args): + """ + Just run a deferred sql query, only care about errors + """ + d = self.db.runQuery(sql, args) + d.addErrback(self.sqlerror) + + + @defer.inlineCallbacks + def write(self, entry): + """ + docstring here + """ + + if entry["eventid"] == 'cowrie.session.connect': + r = yield self.db.runQuery( + "SELECT `id` FROM `sensors` WHERE `ip` = %s", (self.sensor,)) + if r: + sensorid = r[0][0] + else: + yield self.db.runQuery( + 'INSERT INTO `sensors` (`ip`) VALUES (%s)', (self.sensor,)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + sensorid = int(r[0][0]) + self.simpleQuery( + "INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)" + + " VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)", + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + sensorid, entry["src_ip"])) + + elif entry["eventid"] == 'cowrie.login.success': + self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ + ', `username`, `password`, `timestamp`)' + \ + ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + (entry["session"], 1, entry['username'], entry['password'], + entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + + elif entry["eventid"] == 'cowrie.login.failed': + self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ + ', `username`, `password`, `timestamp`)' + \ + ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + (entry["session"], 0, entry['username'], entry['password'], + entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + + elif entry["eventid"] == 'cowrie.command.success': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `success`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + 1, entry["input"])) + + elif entry["eventid"] == 'cowrie.command.failed': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `success`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + 0, entry["input"])) + + elif entry["eventid"] == 'cowrie.session.file_download': + self.simpleQuery('INSERT INTO `downloads`' + \ + ' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s, %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + entry['url'], entry['outfile'], entry['shasum'])) + + elif entry["eventid"] == 'cowrie.session.file_download': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `realm`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + entry["realm"], entry["input"])) + + elif entry["eventid"] == 'cowrie.client.version': + r = yield self.db.runQuery( + 'SELECT `id` FROM `clients` WHERE `version` = %s', \ + (entry['version'],)) + if r: + id = int(r[0][0]) + else: + yield self.db.runQuery( + 'INSERT INTO `clients` (`version`) VALUES (%s)', \ + (entry['version'],)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + id = int(r[0][0]) + self.simpleQuery( + 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', + (id, entry["session"])) + + elif entry["eventid"] == 'cowrie.client.size': + self.simpleQuery( + 'UPDATE `sessions` SET `termsize` = %s WHERE `id` = %s', + ('%sx%s' % (entry['width'], entry['height']), + entry["session"])) + + elif entry["eventid"] == 'cowrie.session.closed': + self.simpleQuery( + 'UPDATE `sessions` SET `endtime` = STR_TO_DATE(%s, %s)' + \ + ' WHERE `id` = %s', (entry["timestamp"], + '%Y-%m-%dT%H:%i:%s.%fZ', entry["session"])) + + elif entry["eventid"] == 'cowrie.log.closed': + self.simpleQuery( + 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (%s, %s, %s)', + (entry["session"], entry["ttylog"], entry["size"])) + + elif entry["eventid"] == 'cowrie.client.fingerprint': + self.simpleQuery( + 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (%s, %s, %s)', + (entry["session"], entry["username"], entry["fingerprint"])) + From e7985a8c6290958702292bbe9791777207f5e0d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Thu, 18 Feb 2016 22:57:14 +0100 Subject: [PATCH 2/5] Add init sqlite3 script Index are not null and automatically incremented when they are declared INTEGER PRIMARY KEY. Thus, the integer size declaration has been removed --- doc/sql/sqlite3.sql | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/sql/sqlite3.sql diff --git a/doc/sql/sqlite3.sql b/doc/sql/sqlite3.sql new file mode 100644 index 0000000..0098311 --- /dev/null +++ b/doc/sql/sqlite3.sql @@ -0,0 +1,63 @@ +CREATE TABLE IF NOT EXISTS `auth` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `success` tinyint(1) NOT NULL, + `username` varchar(100) NOT NULL, + `password` varchar(100) NOT NULL, + `timestamp` datetime NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `clients` ( + `id` INTEGER PRIMARY KEY, + `version` varchar(50) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `input` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `timestamp` datetime NOT NULL, + `realm` varchar(50) default NULL, + `success` tinyint(1) default NULL, + `input` text NOT NULL +) ; +CREATE INDEX input_index ON input(session, timestamp, realm); + +CREATE TABLE IF NOT EXISTS `sensors` ( + `id` INTEGER PRIMARY KEY, + `ip` varchar(15) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `sessions` ( + `id` char(32) NOT NULL PRIMARY KEY, + `starttime` datetime NOT NULL, + `endtime` datetime default NULL, + `sensor` int(4) NOT NULL, + `ip` varchar(15) NOT NULL default '', + `termsize` varchar(7) default NULL, + `client` int(4) default NULL +) ; +CREATE INDEX sessions_index ON sessions(starttime, sensor); + +CREATE TABLE IF NOT EXISTS `ttylog` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `ttylog` varchar(100) NOT NULL, + `size` int(11) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `downloads` ( + `id` INTEGER PRIMARY KEY, + `session` CHAR( 32 ) NOT NULL, + `timestamp` datetime NOT NULL, + `url` text NOT NULL, + `outfile` text NOT NULL, + `shasum` varchar(64) default NULL +) ; +CREATE INDEX downloads_index ON downloads(session, timestamp); + +CREATE TABLE IF NOT EXISTS `keyfingerprints` ( + `id` INTEGER PRIMARY KEY, + `session` CHAR( 32 ) NOT NULL, + `username` varchar(100) NOT NULL, + `fingerprint` varchar(100) NOT NULL +) ; From d6646b76318acd62065cadc7148f2bedb2509d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Thu, 18 Feb 2016 22:59:58 +0100 Subject: [PATCH 3/5] Use placeholders and remove MySQL built-in functions usage --- cowrie/output/sqlite.py | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/cowrie/output/sqlite.py b/cowrie/output/sqlite.py index 64b1ead..9242645 100644 --- a/cowrie/output/sqlite.py +++ b/cowrie/output/sqlite.py @@ -17,11 +17,13 @@ class Output(cowrie.core.output.Output): def start(self): """ Start sqlite3 logging module using Twisted ConnectionPool. + Need to be started with check_same_thread=False. See + https://twistedmatrix.com/trac/ticket/3629. """ sqlite_file = self.cfg.get('output_sqlite', 'db_file') try: self.db = adbapi.ConnectionPool('sqlite3', - database = sqlite_file) + database = sqlite_file, check_same_thread=False) except sqlite3.OperationalError as e: log.msg(e) @@ -37,7 +39,8 @@ class Output(cowrie.core.output.Output): """ docstring here """ - log.err( 'sqlite3 Error:', error.value ) + log.err('sqlite error') + error.printTraceback() def simpleQuery(self, sql, args): """ @@ -55,97 +58,97 @@ class Output(cowrie.core.output.Output): if entry["eventid"] == 'cowrie.session.connect': r = yield self.db.runQuery( - "SELECT `id` FROM `sensors` WHERE `ip` = %s", (self.sensor,)) - if r: + "SELECT `id` FROM `sensors` WHERE `ip` = ?", (self.sensor,)) + if r and r[0][0]: sensorid = r[0][0] else: yield self.db.runQuery( - 'INSERT INTO `sensors` (`ip`) VALUES (%s)', (self.sensor,)) - r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + 'INSERT INTO `sensors` (`ip`) VALUES (?)', (self.sensor,)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()') sensorid = int(r[0][0]) self.simpleQuery( "INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)" - + " VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)", - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + + " VALUES (?, ?, ?, ?)", + (entry["session"], entry["timestamp"], sensorid, entry["src_ip"])) elif entry["eventid"] == 'cowrie.login.success': self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ ', `username`, `password`, `timestamp`)' + \ - ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + ' VALUES (?, ?, ?, ?, ?)', (entry["session"], 1, entry['username'], entry['password'], - entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + entry["timestamp"])) elif entry["eventid"] == 'cowrie.login.failed': self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ ', `username`, `password`, `timestamp`)' + \ - ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + ' VALUES (?, ?, ?, ?, ?)', (entry["session"], 0, entry['username'], entry['password'], - entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + entry["timestamp"])) elif entry["eventid"] == 'cowrie.command.success': + print str(entry) self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `success`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], 1, entry["input"])) elif entry["eventid"] == 'cowrie.command.failed': self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `success`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], 0, entry["input"])) elif entry["eventid"] == 'cowrie.session.file_download': self.simpleQuery('INSERT INTO `downloads`' + \ ' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s, %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?, ?)', + (entry["session"], entry["timestamp"], entry['url'], entry['outfile'], entry['shasum'])) elif entry["eventid"] == 'cowrie.session.file_download': self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `realm`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], entry["realm"], entry["input"])) elif entry["eventid"] == 'cowrie.client.version': r = yield self.db.runQuery( - 'SELECT `id` FROM `clients` WHERE `version` = %s', \ + 'SELECT `id` FROM `clients` WHERE `version` = ?', \ (entry['version'],)) - if r: + if r and r[0][0]: id = int(r[0][0]) else: yield self.db.runQuery( - 'INSERT INTO `clients` (`version`) VALUES (%s)', \ + 'INSERT INTO `clients` (`version`) VALUES (?)', \ (entry['version'],)) - r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()') id = int(r[0][0]) self.simpleQuery( - 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', + 'UPDATE `sessions` SET `client` = ? WHERE `id` = ?', (id, entry["session"])) elif entry["eventid"] == 'cowrie.client.size': self.simpleQuery( - 'UPDATE `sessions` SET `termsize` = %s WHERE `id` = %s', + 'UPDATE `sessions` SET `termsize` = ? WHERE `id` = ?', ('%sx%s' % (entry['width'], entry['height']), entry["session"])) elif entry["eventid"] == 'cowrie.session.closed': self.simpleQuery( - 'UPDATE `sessions` SET `endtime` = STR_TO_DATE(%s, %s)' + \ - ' WHERE `id` = %s', (entry["timestamp"], - '%Y-%m-%dT%H:%i:%s.%fZ', entry["session"])) + 'UPDATE `sessions` SET `endtime` = ?' + \ + ' WHERE `id` = ?', (entry["timestamp"], entry["session"])) elif entry["eventid"] == 'cowrie.log.closed': self.simpleQuery( - 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (%s, %s, %s)', + 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (?, ?, ?)', (entry["session"], entry["ttylog"], entry["size"])) elif entry["eventid"] == 'cowrie.client.fingerprint': self.simpleQuery( - 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (%s, %s, %s)', + 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (?, ?, ?)', (entry["session"], entry["username"], entry["fingerprint"])) From b5c91864f72497aa62f5acdd4fad00a3af53f198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Sun, 21 Feb 2016 22:40:28 +0100 Subject: [PATCH 4/5] Add documentation in cowrie.cfg.dist --- cowrie.cfg.dist | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index f3e1a20..05e5cd3 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -304,6 +304,13 @@ logfile = log/cowrie.json #port = 3306 +# SQLite3 logging module +# +# Logging to SQLite3 database +# +# [output_sqlite] +# db_file = cowrie.db + # Splunk SDK output module - EARLY RELEASE NOT RECOMMENDED # This sends logs directly to Splunk using the Python REST SDK # From b11730bcb2f26fc39ac2492c6898655a2254710d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Tue, 23 Feb 2016 20:17:04 +0100 Subject: [PATCH 5/5] Add some more documentation how to init database --- cowrie.cfg.dist | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index 05e5cd3..5db38cc 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -306,7 +306,9 @@ logfile = log/cowrie.json # SQLite3 logging module # -# Logging to SQLite3 database +# Logging to SQLite3 database. To init the database, use the script +# doc/sql/sqlite3.sql: +# sqlite3 < doc/sql/sqlite3.sql # # [output_sqlite] # db_file = cowrie.db