Merge pull request #122 from nirouviere/sqlite3-driver

Sqlite3 driver.
Thanks @nirouviere

cowrie.cfg.dist

@@ -304,6 +304,15 @@ logfile = log/cowrie.json
 #port = 3306
 
 
+# SQLite3 logging module
+#
+# Logging to SQLite3 database. To init the database, use the script
+# doc/sql/sqlite3.sql:
+#     sqlite3 <db_file> < doc/sql/sqlite3.sql
+#
+# [output_sqlite]
+# db_file = cowrie.db
+
 # Splunk SDK output module - EARLY RELEASE NOT RECOMMENDED
 # This sends logs directly to Splunk using the Python REST SDK
 #

cowrie/output/sqlite.py

@@ -0,0 +1,154 @@
+import sqlite3
+
+from twisted.internet import defer
+from twisted.enterprise import adbapi
+from twisted.python import log
+
+import cowrie.core.output
+class Output(cowrie.core.output.Output):
+    """
+    docstring here
+    """
+
+    def __init__(self, cfg):
+        self.cfg = cfg
+        cowrie.core.output.Output.__init__(self, cfg)
+
+    def start(self):
+        """
+        Start sqlite3 logging module using Twisted ConnectionPool.
+        Need to be started with check_same_thread=False. See
+        https://twistedmatrix.com/trac/ticket/3629.
+        """
+        sqlite_file = self.cfg.get('output_sqlite', 'db_file')
+        try:
+            self.db = adbapi.ConnectionPool('sqlite3',
+                    database = sqlite_file, check_same_thread=False)
+        except sqlite3.OperationalError as e:
+            log.msg(e)
+
+        self.db.start()
+        
+    def stop(self):
+        """
+        Close connection to db
+        """
+        self.db.close()
+
+    def sqlerror(self, error):
+        """
+        docstring here
+        """
+        log.err('sqlite error')
+        error.printTraceback()
+
+    def simpleQuery(self, sql, args):
+        """
+        Just run a deferred sql query, only care about errors
+        """
+        d = self.db.runQuery(sql, args)
+        d.addErrback(self.sqlerror)
+
+
+    @defer.inlineCallbacks
+    def write(self, entry):
+        """
+        docstring here
+        """
+
+        if entry["eventid"] == 'cowrie.session.connect':
+            r = yield self.db.runQuery(
+                "SELECT `id` FROM `sensors` WHERE `ip` = ?", (self.sensor,))
+            if r and r[0][0]:
+                sensorid = r[0][0]
+            else:
+                yield self.db.runQuery(
+                    'INSERT INTO `sensors` (`ip`) VALUES (?)', (self.sensor,))
+                r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()')
+                sensorid = int(r[0][0])
+            self.simpleQuery(
+                "INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)"
+                +  " VALUES (?, ?, ?, ?)",
+                (entry["session"], entry["timestamp"],
+                    sensorid, entry["src_ip"]))
+
+        elif entry["eventid"] == 'cowrie.login.success':
+            self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \
+                ', `username`, `password`, `timestamp`)' + \
+                ' VALUES (?, ?, ?, ?, ?)',
+                (entry["session"], 1, entry['username'], entry['password'],
+                entry["timestamp"]))
+
+        elif entry["eventid"] == 'cowrie.login.failed':
+            self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \
+                ', `username`, `password`, `timestamp`)' + \
+                ' VALUES (?, ?, ?, ?, ?)',
+                (entry["session"], 0, entry['username'], entry['password'],
+                entry["timestamp"]))
+
+        elif entry["eventid"] == 'cowrie.command.success':
+            print str(entry)
+            self.simpleQuery('INSERT INTO `input`' + \
+                ' (`session`, `timestamp`, `success`, `input`)' + \
+                ' VALUES (?, ?, ?, ?)',
+                (entry["session"], entry["timestamp"],
+                1, entry["input"]))
+
+        elif entry["eventid"] == 'cowrie.command.failed':
+            self.simpleQuery('INSERT INTO `input`' + \
+                ' (`session`, `timestamp`, `success`, `input`)' + \
+                ' VALUES (?, ?, ?, ?)',
+                (entry["session"], entry["timestamp"],
+                0, entry["input"]))
+
+        elif entry["eventid"] == 'cowrie.session.file_download':
+            self.simpleQuery('INSERT INTO `downloads`' + \
+                ' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \
+                ' VALUES (?, ?, ?, ?, ?)',
+                (entry["session"], entry["timestamp"],
+                entry['url'], entry['outfile'], entry['shasum']))
+
+        elif entry["eventid"] == 'cowrie.session.file_download':
+            self.simpleQuery('INSERT INTO `input`' + \
+                ' (`session`, `timestamp`, `realm`, `input`)' + \
+                ' VALUES (?, ?, ?, ?)',
+                (entry["session"], entry["timestamp"],
+                entry["realm"], entry["input"]))
+
+        elif entry["eventid"] == 'cowrie.client.version':
+            r = yield self.db.runQuery(
+                'SELECT `id` FROM `clients` WHERE `version` = ?', \
+                (entry['version'],))
+            if r and r[0][0]:
+                id = int(r[0][0])
+            else:
+                yield self.db.runQuery(
+                    'INSERT INTO `clients` (`version`) VALUES (?)', \
+                    (entry['version'],))
+                r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()')
+                id = int(r[0][0])
+            self.simpleQuery(
+                'UPDATE `sessions` SET `client` = ? WHERE `id` = ?',
+                (id, entry["session"]))
+
+        elif entry["eventid"] == 'cowrie.client.size':
+            self.simpleQuery(
+                'UPDATE `sessions` SET `termsize` = ? WHERE `id` = ?',
+                ('%sx%s' % (entry['width'], entry['height']),
+                    entry["session"]))
+
+        elif entry["eventid"] == 'cowrie.session.closed':
+            self.simpleQuery(
+                'UPDATE `sessions` SET `endtime` = ?' + \
+                ' WHERE `id` = ?', (entry["timestamp"], entry["session"]))
+
+        elif entry["eventid"] == 'cowrie.log.closed':
+            self.simpleQuery(
+                'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (?, ?, ?)',
+                (entry["session"], entry["ttylog"], entry["size"]))
+
+        elif entry["eventid"] == 'cowrie.client.fingerprint':
+            self.simpleQuery(
+                'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (?, ?, ?)',
+                (entry["session"], entry["username"], entry["fingerprint"]))
+

doc/sql/sqlite3.sql

@@ -0,0 +1,63 @@
+CREATE TABLE IF NOT EXISTS `auth` (
+  `id` INTEGER PRIMARY KEY,
+  `session` char(32) NOT NULL,
+  `success` tinyint(1) NOT NULL,
+  `username` varchar(100) NOT NULL,
+  `password` varchar(100) NOT NULL,
+  `timestamp` datetime NOT NULL
+) ;
+
+CREATE TABLE IF NOT EXISTS `clients` (
+  `id` INTEGER PRIMARY KEY,
+  `version` varchar(50) NOT NULL
+) ;
+
+CREATE TABLE IF NOT EXISTS `input` (
+  `id` INTEGER PRIMARY KEY,
+  `session` char(32) NOT NULL,
+  `timestamp` datetime NOT NULL,
+  `realm` varchar(50) default NULL,
+  `success` tinyint(1) default NULL,
+  `input` text NOT NULL
+) ;
+CREATE INDEX input_index ON input(session, timestamp, realm);
+
+CREATE TABLE IF NOT EXISTS `sensors` (
+  `id` INTEGER PRIMARY KEY,
+  `ip` varchar(15) NOT NULL
+) ;
+
+CREATE TABLE IF NOT EXISTS `sessions` (
+  `id` char(32) NOT NULL PRIMARY KEY,
+  `starttime` datetime NOT NULL,
+  `endtime` datetime default NULL,
+  `sensor` int(4) NOT NULL,
+  `ip` varchar(15) NOT NULL default '',
+  `termsize` varchar(7) default NULL,
+  `client` int(4) default NULL
+) ;
+CREATE INDEX sessions_index ON sessions(starttime, sensor);
+
+CREATE TABLE IF NOT EXISTS `ttylog` (
+  `id` INTEGER PRIMARY KEY,
+  `session` char(32) NOT NULL,
+  `ttylog` varchar(100) NOT NULL,
+  `size` int(11) NOT NULL
+) ;
+
+CREATE TABLE IF NOT EXISTS `downloads` (
+  `id` INTEGER PRIMARY KEY,
+  `session` CHAR( 32 ) NOT NULL,
+  `timestamp` datetime NOT NULL,
+  `url` text NOT NULL,
+  `outfile` text NOT NULL,
+  `shasum` varchar(64) default NULL
+) ;
+CREATE INDEX downloads_index ON downloads(session, timestamp);
+
+CREATE TABLE IF NOT EXISTS `keyfingerprints` (
+  `id` INTEGER PRIMARY KEY,
+  `session` CHAR( 32 ) NOT NULL,
+  `username` varchar(100) NOT NULL,
+  `fingerprint` varchar(100) NOT NULL
+) ;