This commit is contained in:
Michel Oosterhof
2016-03-01 12:07:02 +04:00
6 changed files with 616 additions and 27 deletions

View File

@@ -320,6 +320,15 @@ logfile = log/cowrie.json
#port = 3306
# SQLite3 logging module
#
# Logging to SQLite3 database. To init the database, use the script
# doc/sql/sqlite3.sql:
# sqlite3 <db_file> < doc/sql/sqlite3.sql
#
# [output_sqlite]
# db_file = cowrie.db
# Splunk SDK output module - EARLY RELEASE NOT RECOMMENDED
# This sends logs directly to Splunk using the Python REST SDK
#

View File

@@ -471,7 +471,10 @@ class command_yes(HoneyPotCommand):
def y(self):
"""
"""
self.write('y\n')
if len(self.args):
self.write(' '.join(self.args)+'\n')
else:
self.write('y\n')
self.scheduled = reactor.callLater(0.01, self.y)

View File

@@ -6,7 +6,6 @@ This module contains ...
"""
import os
import shlex
import re
import stat
import copy
@@ -17,6 +16,7 @@ from twisted.python import log, failure
from twisted.internet import error
from cowrie.core import fs
from cowrie.core import shlex
class HoneyPotCommand(object):
"""
@@ -124,22 +124,45 @@ class HoneyPotShell(object):
self.interactive = interactive
self.cmdpending = []
self.environ = protocol.environ
#self.lexer.debug = 1
self.showPrompt()
def lineReceived(self, line):
"""
"""
log.msg('CMD: %s' % (line,))
line = line[:500]
comment = re.compile('^\s*#')
for i in [x.strip() for x in re.split(';|&&|\n', line.strip())[:10]]:
if not len(i):
continue
if comment.match(i):
continue
self.cmdpending.append(i)
self.lexer = shlex.shlex(punctuation_chars=True);
self.lexer.push_source(line)
tokens = []
while True:
try:
tok = self.lexer.get_token()
#log.msg( "tok: %s" % (repr(tok)) )
# for now, execute all after &&
if tok == self.lexer.eof:
if len(tokens):
self.cmdpending.append((tokens))
tokens = []
break
if tok == ';' or tok == '&&' or tok == '||':
self.cmdpending.append((tokens))
tokens = []
if tok == ';':
continue
if tok == '&&':
continue
if tok == '||':
continue
tokens.append(tok)
except Exception as e:
self.protocol.terminal.write(
'bash: syntax error: unexpected end of file\n')
# Could run runCommand here, but i'll just clear the list instead
self.cmdpending = []
self.showPrompt()
return
if len(self.cmdpending):
self.runCommand()
else:
@@ -166,17 +189,8 @@ class HoneyPotShell(object):
self.protocol.terminal.transport.processEnded(ret)
return
line = self.cmdpending.pop(0)
try:
line = line.replace('>', ' > ').replace('|', ' | ').replace('<',' < ')
cmdAndArgs = shlex.split(line)
except:
self.protocol.terminal.write(
'bash: syntax error: unexpected end of file\n')
# Could run runCommand here, but i'll just clear the list instead
self.cmdpending = []
self.showPrompt()
return
cmdAndArgs = self.cmdpending.pop(0)
cmd2 = copy.copy(cmdAndArgs)
# Probably no reason to be this comprehensive for just PATH...
environ = copy.copy(self.environ)
@@ -204,14 +218,13 @@ class HoneyPotShell(object):
rargs.append(arg)
cmdclass = self.protocol.getCommand(cmd, environ['PATH'].split(':'))
if cmdclass:
log.msg(eventid='cowrie.command.success', input=line, format='Command found: %(input)s')
log.msg(eventid='cowrie.command.success', input=' '.join(cmd2), format='Command found: %(input)s')
self.protocol.call_command(cmdclass, *rargs)
else:
log.msg(eventid='cowrie.command.failed',
input=line, format='Command not found: %(input)s')
if len(line):
self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,))
runOrPrompt()
input=' '.join(cmd2), format='Command not found: %(input)s')
self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,))
runOrPrompt()
def resume(self):

347
cowrie/core/shlex.py Normal file
View File

@@ -0,0 +1,347 @@
# coding: UTF-8
"""A lexical analyzer class for simple shell-like syntaxes."""
# Module and documentation by Eric S. Raymond, 21 Dec 1998
# Input stacking and error message cleanup added by ESR, March 2000
# push_source() and pop_source() made explicit by ESR, January 2001.
# Posix compliance, split(), string arguments, and
# iterator interface by Gustavo Niemeyer, April 2003.
# changes to tokenize more like Posix shells by Vinay Sajip, January 2012.
import os
import re
import sys
from collections import deque
from io import StringIO, BytesIO
__all__ = ["shlex", "split", "quote"]
class shlex:
"A lexical analyzer class for simple shell-like syntaxes."
def __init__(self, instream=None, infile=None, posix=False,
punctuation_chars=False):
if isinstance(instream, str):
instream = StringIO(instream)
#instream = BytesIO(instream)
if instream is not None:
self.instream = instream
self.infile = infile
else:
self.instream = sys.stdin
self.infile = None
self.posix = posix
if posix:
self.eof = None
else:
self.eof = ''
self.commenters = '#'
self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
if self.posix:
self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
self.whitespace = ' \t\r\n'
self.whitespace_split = False
self.quotes = '\'"'
self.escape = '\\'
self.escapedquotes = '"'
self.state = ' '
self.pushback = deque()
self.lineno = 1
self.debug = 0
self.token = ''
self.filestack = deque()
self.source = None
if not punctuation_chars:
punctuation_chars = ''
elif punctuation_chars is True:
punctuation_chars = '();<>|&'
self.punctuation_chars = punctuation_chars
if punctuation_chars:
# _pushback_chars is a push back queue used by lookahead logic
self._pushback_chars = deque()
# these chars added because allowed in file names, args, wildcards
self.wordchars += '~-./*?='
#remove any punctuation chars from wordchars
self.wordchars = ''.join(c for c in self.wordchars if c not in
self.punctuation_chars)
for c in punctuation_chars:
if c in self.wordchars:
self.wordchars.remove(c)
if self.debug:
print('shlex: reading from %s, line %d' % (self.instream,
self.lineno))
def push_token(self, tok):
"Push a token onto the stack popped by the get_token method"
if self.debug >= 1:
print("shlex: pushing token " + repr(tok))
self.pushback.appendleft(tok)
def push_source(self, newstream, newfile=None):
"Push an input source onto the lexer's input source stack."
if isinstance(newstream, str):
#newstream = StringIO(newstream)
newstream = BytesIO(newstream)
self.filestack.appendleft((self.infile, self.instream, self.lineno))
self.infile = newfile
self.instream = newstream
self.lineno = 1
if self.debug:
if newfile is not None:
print('shlex: pushing to file %s' % (self.infile,))
else:
print('shlex: pushing to stream %s' % (self.instream,))
def pop_source(self):
"Pop the input source stack."
self.instream.close()
(self.infile, self.instream, self.lineno) = self.filestack.popleft()
if self.debug:
print('shlex: popping to %s, line %d' \
% (self.instream, self.lineno))
self.state = ' '
def get_token(self):
"Get a token from the input stream (or from stack if it's nonempty)"
if self.pushback:
tok = self.pushback.popleft()
if self.debug >= 1:
print("shlex: popping token " + repr(tok))
return tok
# No pushback. Get a token.
raw = self.read_token()
# Handle inclusions
if self.source is not None:
while raw == self.source:
spec = self.sourcehook(self.read_token())
if spec:
(newfile, newstream) = spec
self.push_source(newstream, newfile)
raw = self.get_token()
# Maybe we got EOF instead?
while raw == self.eof:
if not self.filestack:
return self.eof
else:
self.pop_source()
raw = self.get_token()
# Neither inclusion nor EOF
if self.debug >= 1:
if raw != self.eof:
print("shlex: token=" + repr(raw))
else:
print("shlex: token=EOF")
return raw
def read_token(self):
quoted = False
escapedstate = ' '
while True:
if self.punctuation_chars and self._pushback_chars:
nextchar = self._pushback_chars.pop()
else:
nextchar = self.instream.read(1)
if nextchar == '\n':
self.lineno += 1
if self.debug >= 3:
print("shlex: in state %r I see character: %r" % (self.state,
nextchar))
if self.state is None:
self.token = '' # past end of file
break
elif self.state == ' ':
if not nextchar:
self.state = None # end of file
break
elif nextchar in self.whitespace:
if self.debug >= 2:
print("shlex: I see whitespace in whitespace state")
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif nextchar in self.commenters:
self.instream.readline()
self.lineno += 1
elif self.posix and nextchar in self.escape:
escapedstate = 'a'
self.state = nextchar
elif nextchar in self.wordchars:
self.token = nextchar
self.state = 'a'
elif nextchar in self.punctuation_chars:
self.token = nextchar
self.state = 'c'
elif nextchar in self.quotes:
if not self.posix:
self.token = nextchar
self.state = nextchar
elif self.whitespace_split:
self.token = nextchar
self.state = 'a'
else:
self.token = nextchar
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif self.state in self.quotes:
quoted = True
if not nextchar: # end of file
if self.debug >= 2:
print("shlex: I see EOF in quotes state")
# XXX what error should be raised here?
raise ValueError("No closing quotation")
if nextchar == self.state:
if not self.posix:
self.token += nextchar
self.state = ' '
break
else:
self.state = 'a'
elif (self.posix and nextchar in self.escape and self.state
in self.escapedquotes):
escapedstate = self.state
self.state = nextchar
else:
self.token += nextchar
elif self.state in self.escape:
if not nextchar: # end of file
if self.debug >= 2:
print("shlex: I see EOF in escape state")
# XXX what error should be raised here?
raise ValueError("No escaped character")
# In posix shells, only the quote itself or the escape
# character may be escaped within quotes.
if (escapedstate in self.quotes and
nextchar != self.state and nextchar != escapedstate):
self.token += self.state
self.token += nextchar
self.state = escapedstate
elif self.state in ('a', 'c'):
if not nextchar:
self.state = None # end of file
break
elif nextchar in self.whitespace:
if self.debug >= 2:
print("shlex: I see whitespace in word state")
self.state = ' '
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif nextchar in self.commenters:
self.instream.readline()
self.lineno += 1
if self.posix:
self.state = ' '
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif self.posix and nextchar in self.quotes:
self.state = nextchar
elif self.posix and nextchar in self.escape:
escapedstate = 'a'
self.state = nextchar
elif self.state == 'c':
if nextchar in self.punctuation_chars:
self.token += nextchar
else:
if nextchar not in self.whitespace:
self._pushback_chars.append(nextchar)
self.state = ' '
break
elif (nextchar in self.wordchars or nextchar in self.quotes
or self.whitespace_split):
self.token += nextchar
else:
if self.punctuation_chars:
self._pushback_chars.append(nextchar)
else:
self.pushback.appendleft(nextchar)
if self.debug >= 2:
print("shlex: I see punctuation in word state")
self.state = ' '
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
result = self.token
self.token = ''
if self.posix and not quoted and result == '':
result = None
if self.debug > 1:
if result:
print("shlex: raw token=" + repr(result))
else:
print("shlex: raw token=EOF")
return result
def sourcehook(self, newfile):
"Hook called on a filename to be sourced."
if newfile[0] == '"':
newfile = newfile[1:-1]
# This implements cpp-like semantics for relative-path inclusion.
if isinstance(self.infile, str) and not os.path.isabs(newfile):
newfile = os.path.join(os.path.dirname(self.infile), newfile)
return (newfile, open(newfile, "r"))
def error_leader(self, infile=None, lineno=None):
"Emit a C-compiler-like, Emacs-friendly error-message leader."
if infile is None:
infile = self.infile
if lineno is None:
lineno = self.lineno
return "\"%s\", line %d: " % (infile, lineno)
def __iter__(self):
return self
# def __next__(self):
def next(self):
token = self.get_token()
if token == self.eof:
raise StopIteration
return token
def split(s, comments=False, posix=True):
lex = shlex(s, posix=posix)
lex.whitespace_split = True
if not comments:
lex.commenters = ''
return list(lex)
#_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search
# No ASCII in P2.x
_find_unsafe = re.compile(r'[^\w@%+=:,./-]' ).search
def quote(s):
"""Return a shell-escaped version of the string *s*."""
if not s:
return "''"
if _find_unsafe(s) is None:
return s
# use single quotes, and put single quotes into double quotes
# the string $'b is then quoted as '$'"'"'b'
return "'" + s.replace("'", "'\"'\"'") + "'"
if __name__ == '__main__':
if len(sys.argv) == 1:
lexer = shlex()
else:
file = sys.argv[1]
lexer = shlex(open(file), file)
while 1:
tt = lexer.get_token()
if tt:
print("Token: " + repr(tt))
else:
break

154
cowrie/output/sqlite.py Normal file
View File

@@ -0,0 +1,154 @@
import sqlite3
from twisted.internet import defer
from twisted.enterprise import adbapi
from twisted.python import log
import cowrie.core.output
class Output(cowrie.core.output.Output):
"""
docstring here
"""
def __init__(self, cfg):
self.cfg = cfg
cowrie.core.output.Output.__init__(self, cfg)
def start(self):
"""
Start sqlite3 logging module using Twisted ConnectionPool.
Need to be started with check_same_thread=False. See
https://twistedmatrix.com/trac/ticket/3629.
"""
sqlite_file = self.cfg.get('output_sqlite', 'db_file')
try:
self.db = adbapi.ConnectionPool('sqlite3',
database = sqlite_file, check_same_thread=False)
except sqlite3.OperationalError as e:
log.msg(e)
self.db.start()
def stop(self):
"""
Close connection to db
"""
self.db.close()
def sqlerror(self, error):
"""
docstring here
"""
log.err('sqlite error')
error.printTraceback()
def simpleQuery(self, sql, args):
"""
Just run a deferred sql query, only care about errors
"""
d = self.db.runQuery(sql, args)
d.addErrback(self.sqlerror)
@defer.inlineCallbacks
def write(self, entry):
"""
docstring here
"""
if entry["eventid"] == 'cowrie.session.connect':
r = yield self.db.runQuery(
"SELECT `id` FROM `sensors` WHERE `ip` = ?", (self.sensor,))
if r and r[0][0]:
sensorid = r[0][0]
else:
yield self.db.runQuery(
'INSERT INTO `sensors` (`ip`) VALUES (?)', (self.sensor,))
r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()')
sensorid = int(r[0][0])
self.simpleQuery(
"INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)"
+ " VALUES (?, ?, ?, ?)",
(entry["session"], entry["timestamp"],
sensorid, entry["src_ip"]))
elif entry["eventid"] == 'cowrie.login.success':
self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \
', `username`, `password`, `timestamp`)' + \
' VALUES (?, ?, ?, ?, ?)',
(entry["session"], 1, entry['username'], entry['password'],
entry["timestamp"]))
elif entry["eventid"] == 'cowrie.login.failed':
self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \
', `username`, `password`, `timestamp`)' + \
' VALUES (?, ?, ?, ?, ?)',
(entry["session"], 0, entry['username'], entry['password'],
entry["timestamp"]))
elif entry["eventid"] == 'cowrie.command.success':
print str(entry)
self.simpleQuery('INSERT INTO `input`' + \
' (`session`, `timestamp`, `success`, `input`)' + \
' VALUES (?, ?, ?, ?)',
(entry["session"], entry["timestamp"],
1, entry["input"]))
elif entry["eventid"] == 'cowrie.command.failed':
self.simpleQuery('INSERT INTO `input`' + \
' (`session`, `timestamp`, `success`, `input`)' + \
' VALUES (?, ?, ?, ?)',
(entry["session"], entry["timestamp"],
0, entry["input"]))
elif entry["eventid"] == 'cowrie.session.file_download':
self.simpleQuery('INSERT INTO `downloads`' + \
' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \
' VALUES (?, ?, ?, ?, ?)',
(entry["session"], entry["timestamp"],
entry['url'], entry['outfile'], entry['shasum']))
elif entry["eventid"] == 'cowrie.session.file_download':
self.simpleQuery('INSERT INTO `input`' + \
' (`session`, `timestamp`, `realm`, `input`)' + \
' VALUES (?, ?, ?, ?)',
(entry["session"], entry["timestamp"],
entry["realm"], entry["input"]))
elif entry["eventid"] == 'cowrie.client.version':
r = yield self.db.runQuery(
'SELECT `id` FROM `clients` WHERE `version` = ?', \
(entry['version'],))
if r and r[0][0]:
id = int(r[0][0])
else:
yield self.db.runQuery(
'INSERT INTO `clients` (`version`) VALUES (?)', \
(entry['version'],))
r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()')
id = int(r[0][0])
self.simpleQuery(
'UPDATE `sessions` SET `client` = ? WHERE `id` = ?',
(id, entry["session"]))
elif entry["eventid"] == 'cowrie.client.size':
self.simpleQuery(
'UPDATE `sessions` SET `termsize` = ? WHERE `id` = ?',
('%sx%s' % (entry['width'], entry['height']),
entry["session"]))
elif entry["eventid"] == 'cowrie.session.closed':
self.simpleQuery(
'UPDATE `sessions` SET `endtime` = ?' + \
' WHERE `id` = ?', (entry["timestamp"], entry["session"]))
elif entry["eventid"] == 'cowrie.log.closed':
self.simpleQuery(
'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (?, ?, ?)',
(entry["session"], entry["ttylog"], entry["size"]))
elif entry["eventid"] == 'cowrie.client.fingerprint':
self.simpleQuery(
'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (?, ?, ?)',
(entry["session"], entry["username"], entry["fingerprint"]))

63
doc/sql/sqlite3.sql Normal file
View File

@@ -0,0 +1,63 @@
CREATE TABLE IF NOT EXISTS `auth` (
`id` INTEGER PRIMARY KEY,
`session` char(32) NOT NULL,
`success` tinyint(1) NOT NULL,
`username` varchar(100) NOT NULL,
`password` varchar(100) NOT NULL,
`timestamp` datetime NOT NULL
) ;
CREATE TABLE IF NOT EXISTS `clients` (
`id` INTEGER PRIMARY KEY,
`version` varchar(50) NOT NULL
) ;
CREATE TABLE IF NOT EXISTS `input` (
`id` INTEGER PRIMARY KEY,
`session` char(32) NOT NULL,
`timestamp` datetime NOT NULL,
`realm` varchar(50) default NULL,
`success` tinyint(1) default NULL,
`input` text NOT NULL
) ;
CREATE INDEX input_index ON input(session, timestamp, realm);
CREATE TABLE IF NOT EXISTS `sensors` (
`id` INTEGER PRIMARY KEY,
`ip` varchar(15) NOT NULL
) ;
CREATE TABLE IF NOT EXISTS `sessions` (
`id` char(32) NOT NULL PRIMARY KEY,
`starttime` datetime NOT NULL,
`endtime` datetime default NULL,
`sensor` int(4) NOT NULL,
`ip` varchar(15) NOT NULL default '',
`termsize` varchar(7) default NULL,
`client` int(4) default NULL
) ;
CREATE INDEX sessions_index ON sessions(starttime, sensor);
CREATE TABLE IF NOT EXISTS `ttylog` (
`id` INTEGER PRIMARY KEY,
`session` char(32) NOT NULL,
`ttylog` varchar(100) NOT NULL,
`size` int(11) NOT NULL
) ;
CREATE TABLE IF NOT EXISTS `downloads` (
`id` INTEGER PRIMARY KEY,
`session` CHAR( 32 ) NOT NULL,
`timestamp` datetime NOT NULL,
`url` text NOT NULL,
`outfile` text NOT NULL,
`shasum` varchar(64) default NULL
) ;
CREATE INDEX downloads_index ON downloads(session, timestamp);
CREATE TABLE IF NOT EXISTS `keyfingerprints` (
`id` INTEGER PRIMARY KEY,
`session` CHAR( 32 ) NOT NULL,
`username` varchar(100) NOT NULL,
`fingerprint` varchar(100) NOT NULL
) ;