From ee27d792ce9a9a7ca2b7ca62fc927696fc559c04 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Fri, 16 Oct 2015 14:47:53 +0000 Subject: [PATCH 01/31] shlex update --- cowrie/core/honeypot.py | 3 +- cowrie/core/shlex.py | 344 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 cowrie/core/shlex.py diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 97906a0..af2e0bf 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -6,7 +6,6 @@ This module contains ... """ import os -import shlex import re import stat import copy @@ -17,6 +16,7 @@ from twisted.python import log, failure from twisted.internet import error from cowrie.core import fs +from cowrie.core import shlex class HoneyPotCommand(object): """ @@ -167,6 +167,7 @@ class HoneyPotShell(object): return line = self.cmdpending.pop(0) + cmdAndArgs = shlex.split(unicode(line)) try: line = line.replace('>', ' > ').replace('|', ' | ').replace('<',' < ') cmdAndArgs = shlex.split(line) diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py new file mode 100644 index 0000000..c7de1b5 --- /dev/null +++ b/cowrie/core/shlex.py @@ -0,0 +1,344 @@ +"""A lexical analyzer class for simple shell-like syntaxes.""" +# coding: UTF-8 + +# Module and documentation by Eric S. Raymond, 21 Dec 1998 +# Input stacking and error message cleanup added by ESR, March 2000 +# push_source() and pop_source() made explicit by ESR, January 2001. +# Posix compliance, split(), string arguments, and +# iterator interface by Gustavo Niemeyer, April 2003. +# changes to tokenize more like Posix shells by Vinay Sajip, January 2012. + +import os +import re +import sys +from collections import deque + +from io import StringIO + +__all__ = ["shlex", "split", "quote"] + +class shlex: + "A lexical analyzer class for simple shell-like syntaxes." + def __init__(self, instream=None, infile=None, posix=False, + punctuation_chars=False): + if isinstance(instream, str): + instream = StringIO(instream) + if instream is not None: + self.instream = instream + self.infile = infile + else: + self.instream = sys.stdin + self.infile = None + self.posix = posix + if posix: + self.eof = None + else: + self.eof = '' + self.commenters = '#' + self.wordchars = ('abcdfeghijklmnopqrstuvwxyz' + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_') + if self.posix: + self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ' + 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ') + self.whitespace = ' \t\r\n' + self.whitespace_split = False + self.quotes = '\'"' + self.escape = '\\' + self.escapedquotes = '"' + self.state = ' ' + self.pushback = deque() + self.lineno = 1 + self.debug = 0 + self.token = '' + self.filestack = deque() + self.source = None + if not punctuation_chars: + punctuation_chars = '' + elif punctuation_chars is True: + punctuation_chars = '();<>|&' + self.punctuation_chars = punctuation_chars + if punctuation_chars: + # _pushback_chars is a push back queue used by lookahead logic + self._pushback_chars = deque() + # these chars added because allowed in file names, args, wildcards + self.wordchars += '~-./*?=' + #remove any punctuation chars from wordchars + self.wordchars = ''.join(c for c in self.wordchars if c not in + self.punctuation_chars) + for c in punctuation_chars: + if c in self.wordchars: + self.wordchars.remove(c) + if self.debug: + print('shlex: reading from %s, line %d' % (self.instream, + self.lineno)) + + def push_token(self, tok): + "Push a token onto the stack popped by the get_token method" + if self.debug >= 1: + print("shlex: pushing token " + repr(tok)) + self.pushback.appendleft(tok) + + def push_source(self, newstream, newfile=None): + "Push an input source onto the lexer's input source stack." + if isinstance(newstream, str): + newstream = StringIO(newstream) + self.filestack.appendleft((self.infile, self.instream, self.lineno)) + self.infile = newfile + self.instream = newstream + self.lineno = 1 + if self.debug: + if newfile is not None: + print('shlex: pushing to file %s' % (self.infile,)) + else: + print('shlex: pushing to stream %s' % (self.instream,)) + + def pop_source(self): + "Pop the input source stack." + self.instream.close() + (self.infile, self.instream, self.lineno) = self.filestack.popleft() + if self.debug: + print('shlex: popping to %s, line %d' \ + % (self.instream, self.lineno)) + self.state = ' ' + + def get_token(self): + "Get a token from the input stream (or from stack if it's nonempty)" + if self.pushback: + tok = self.pushback.popleft() + if self.debug >= 1: + print("shlex: popping token " + repr(tok)) + return tok + # No pushback. Get a token. + raw = self.read_token() + # Handle inclusions + if self.source is not None: + while raw == self.source: + spec = self.sourcehook(self.read_token()) + if spec: + (newfile, newstream) = spec + self.push_source(newstream, newfile) + raw = self.get_token() + # Maybe we got EOF instead? + while raw == self.eof: + if not self.filestack: + return self.eof + else: + self.pop_source() + raw = self.get_token() + # Neither inclusion nor EOF + if self.debug >= 1: + if raw != self.eof: + print("shlex: token=" + repr(raw)) + else: + print("shlex: token=EOF") + return raw + + def read_token(self): + quoted = False + escapedstate = ' ' + while True: + if self.punctuation_chars and self._pushback_chars: + nextchar = self._pushback_chars.pop() + else: + nextchar = self.instream.read(1) + if nextchar == '\n': + self.lineno += 1 + if self.debug >= 3: + print("shlex: in state %r I see character: %r" % (self.state, + nextchar)) + if self.state is None: + self.token = '' # past end of file + break + elif self.state == ' ': + if not nextchar: + self.state = None # end of file + break + elif nextchar in self.whitespace: + if self.debug >= 2: + print("shlex: I see whitespace in whitespace state") + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar + elif nextchar in self.wordchars: + self.token = nextchar + self.state = 'a' + elif nextchar in self.punctuation_chars: + self.token = nextchar + self.state = 'c' + elif nextchar in self.quotes: + if not self.posix: + self.token = nextchar + self.state = nextchar + elif self.whitespace_split: + self.token = nextchar + self.state = 'a' + else: + self.token = nextchar + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif self.state in self.quotes: + quoted = True + if not nextchar: # end of file + if self.debug >= 2: + print("shlex: I see EOF in quotes state") + # XXX what error should be raised here? + raise ValueError("No closing quotation") + if nextchar == self.state: + if not self.posix: + self.token += nextchar + self.state = ' ' + break + else: + self.state = 'a' + elif (self.posix and nextchar in self.escape and self.state + in self.escapedquotes): + escapedstate = self.state + self.state = nextchar + else: + self.token += nextchar + elif self.state in self.escape: + if not nextchar: # end of file + if self.debug >= 2: + print("shlex: I see EOF in escape state") + # XXX what error should be raised here? + raise ValueError("No escaped character") + # In posix shells, only the quote itself or the escape + # character may be escaped within quotes. + if (escapedstate in self.quotes and + nextchar != self.state and nextchar != escapedstate): + self.token += self.state + self.token += nextchar + self.state = escapedstate + elif self.state in ('a', 'c'): + if not nextchar: + self.state = None # end of file + break + elif nextchar in self.whitespace: + if self.debug >= 2: + print("shlex: I see whitespace in word state") + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + if self.posix: + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif self.posix and nextchar in self.quotes: + self.state = nextchar + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar + elif self.state == 'c': + if nextchar in self.punctuation_chars: + self.token += nextchar + else: + if nextchar not in self.whitespace: + self._pushback_chars.append(nextchar) + self.state = ' ' + break + elif (nextchar in self.wordchars or nextchar in self.quotes + or self.whitespace_split): + self.token += nextchar + else: + if self.punctuation_chars: + self._pushback_chars.append(nextchar) + else: + self.pushback.appendleft(nextchar) + if self.debug >= 2: + print("shlex: I see punctuation in word state") + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + result = self.token + self.token = '' + if self.posix and not quoted and result == '': + result = None + if self.debug > 1: + if result: + print("shlex: raw token=" + repr(result)) + else: + print("shlex: raw token=EOF") + return result + + def sourcehook(self, newfile): + "Hook called on a filename to be sourced." + if newfile[0] == '"': + newfile = newfile[1:-1] + # This implements cpp-like semantics for relative-path inclusion. + if isinstance(self.infile, str) and not os.path.isabs(newfile): + newfile = os.path.join(os.path.dirname(self.infile), newfile) + return (newfile, open(newfile, "r")) + + def error_leader(self, infile=None, lineno=None): + "Emit a C-compiler-like, Emacs-friendly error-message leader." + if infile is None: + infile = self.infile + if lineno is None: + lineno = self.lineno + return "\"%s\", line %d: " % (infile, lineno) + + def __iter__(self): + return self + +# def __next__(self): + def next(self): + token = self.get_token() + if token == self.eof: + raise StopIteration + return token + +def split(s, comments=False, posix=True): + lex = shlex(s, posix=posix) + lex.whitespace_split = True + if not comments: + lex.commenters = '' + return list(lex) + + +#_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search +# No ASCII in P2.x +_find_unsafe = re.compile(r'[^\w@%+=:,./-]' ).search + +def quote(s): + """Return a shell-escaped version of the string *s*.""" + if not s: + return "''" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + + +if __name__ == '__main__': + if len(sys.argv) == 1: + lexer = shlex() + else: + file = sys.argv[1] + lexer = shlex(open(file), file) + while 1: + tt = lexer.get_token() + if tt: + print("Token: " + repr(tt)) + else: + break + From d87eafedf24554f2cb334f06cd1a6145a47d5841 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Sat, 17 Oct 2015 08:00:08 +0000 Subject: [PATCH 02/31] wip shlex --- cowrie/core/honeypot.py | 23 ++++++++++++++++------- cowrie/core/shlex.py | 8 +++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index af2e0bf..6eac8c7 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -124,22 +124,31 @@ class HoneyPotShell(object): self.interactive = interactive self.cmdpending = [] self.environ = protocol.environ + #self.lexer.debug = 1 self.showPrompt() - def lineReceived(self, line): """ """ log.msg('CMD: %s' % (line,)) - line = line[:500] - comment = re.compile('^\s*#') - for i in [x.strip() for x in re.split(';|&&|\n', line.strip())[:10]]: - if not len(i): + self.lexer = shlex.shlex(punctuation_chars=True); + self.lexer.push_source(line) + tokens = [] + while True: + tok = self.lexer.get_token() + log.msg( "tok: %s" % (repr(tok)) ) + # for now, execute all after && + if tok == ';' or tok == self.lexer.eof or tok == '&&': + self.cmdpending.append((tokens)) + tokens = [] + if tok == ';': continue - if comment.match(i): + if tok == '&&': continue - self.cmdpending.append(i) + if tok == self.lexer.eof: + break + tokens.append(tok) if len(self.cmdpending): self.runCommand() else: diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index c7de1b5..4e56ad5 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -13,7 +13,7 @@ import re import sys from collections import deque -from io import StringIO +from io import StringIO, BytesIO __all__ = ["shlex", "split", "quote"] @@ -22,7 +22,8 @@ class shlex: def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): if isinstance(instream, str): - instream = StringIO(instream) + #instream = StringIO(instream) + instream = BytesIO(instream) if instream is not None: self.instream = instream self.infile = infile @@ -81,7 +82,8 @@ class shlex: def push_source(self, newstream, newfile=None): "Push an input source onto the lexer's input source stack." if isinstance(newstream, str): - newstream = StringIO(newstream) + #newstream = StringIO(newstream) + newstream = BytesIO(newstream) self.filestack.appendleft((self.infile, self.instream, self.lineno)) self.infile = newfile self.instream = newstream From 3bf2480e75f62186a6c504675349ebfeb1be7ffd Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 14 Jan 2016 18:05:06 +0000 Subject: [PATCH 03/31] fix --- cowrie/core/shlex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index 4e56ad5..a68149f 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -1,6 +1,7 @@ -"""A lexical analyzer class for simple shell-like syntaxes.""" # coding: UTF-8 +"""A lexical analyzer class for simple shell-like syntaxes.""" + # Module and documentation by Eric S. Raymond, 21 Dec 1998 # Input stacking and error message cleanup added by ESR, March 2000 # push_source() and pop_source() made explicit by ESR, January 2001. From 5da73a0076a5b23484054d69f434d9029e2039dd Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 25 Jan 2016 10:54:15 +0000 Subject: [PATCH 04/31] works again --- cowrie/core/honeypot.py | 60 ++++++++++++++++++++--------------------- cowrie/core/shlex.py | 4 +-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 6eac8c7..ee75052 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -136,19 +136,31 @@ class HoneyPotShell(object): self.lexer.push_source(line) tokens = [] while True: - tok = self.lexer.get_token() - log.msg( "tok: %s" % (repr(tok)) ) - # for now, execute all after && - if tok == ';' or tok == self.lexer.eof or tok == '&&': - self.cmdpending.append((tokens)) - tokens = [] - if tok == ';': - continue - if tok == '&&': - continue - if tok == self.lexer.eof: - break - tokens.append(tok) + try: + tok = self.lexer.get_token() + #log.msg( "tok: %s" % (repr(tok)) ) + # for now, execute all after && + if tok == self.lexer.eof: + if len(tokens): + self.cmdpending.append((tokens)) + tokens = [] + break + if tok == ';' or tok == '&&': + self.cmdpending.append((tokens)) + tokens = [] + if tok == ';': + continue + if tok == '&&': + continue + tokens.append(tok) + except Exception as e: + self.protocol.terminal.write( + 'bash: syntax error: unexpected end of file\n') + # Could run runCommand here, but i'll just clear the list instead + self.cmdpending = [] + self.showPrompt() + return + if len(self.cmdpending): self.runCommand() else: @@ -175,18 +187,7 @@ class HoneyPotShell(object): self.protocol.terminal.transport.processEnded(ret) return - line = self.cmdpending.pop(0) - cmdAndArgs = shlex.split(unicode(line)) - try: - line = line.replace('>', ' > ').replace('|', ' | ').replace('<',' < ') - cmdAndArgs = shlex.split(line) - except: - self.protocol.terminal.write( - 'bash: syntax error: unexpected end of file\n') - # Could run runCommand here, but i'll just clear the list instead - self.cmdpending = [] - self.showPrompt() - return + cmdAndArgs = self.cmdpending.pop(0) # Probably no reason to be this comprehensive for just PATH... environ = copy.copy(self.environ) @@ -214,14 +215,13 @@ class HoneyPotShell(object): rargs.append(arg) cmdclass = self.protocol.getCommand(cmd, environ['PATH'].split(':')) if cmdclass: - log.msg(eventid='cowrie.command.success', input=line, format='Command found: %(input)s') + log.msg(eventid='cowrie.command.success', input=' '.join(cmdAndArgs), format='Command found: %(input)s') self.protocol.call_command(cmdclass, *rargs) else: log.msg(eventid='cowrie.command.failed', - input=line, format='Command not found: %(input)s') - if len(line): - self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) - runOrPrompt() + input=' '.join(cmdAndArgs), format='Command not found: %(input)s') + self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) + runOrPrompt() def resume(self): diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index a68149f..ea9634e 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -23,8 +23,8 @@ class shlex: def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): if isinstance(instream, str): - #instream = StringIO(instream) - instream = BytesIO(instream) + instream = StringIO(instream) + #instream = BytesIO(instream) if instream is not None: self.instream = instream self.infile = infile From d4c86389760d717f54cd7f39f4de208c1362c8ef Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 25 Jan 2016 11:00:21 +0000 Subject: [PATCH 05/31] 'yes' command prints args or 'y' --- cowrie/commands/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cowrie/commands/base.py b/cowrie/commands/base.py index 6efd308..356cd28 100644 --- a/cowrie/commands/base.py +++ b/cowrie/commands/base.py @@ -471,7 +471,10 @@ class command_yes(HoneyPotCommand): def y(self): """ """ - self.write('y\n') + if len(self.args): + self.write(' '.join(self.args)+'\n') + else: + self.write('y\n') self.scheduled = reactor.callLater(0.01, self.y) From 8b376b86c769de878db882d1bc472e77463c6311 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Fri, 16 Oct 2015 14:47:53 +0000 Subject: [PATCH 06/31] shlex update --- cowrie/core/honeypot.py | 3 +- cowrie/core/shlex.py | 344 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 cowrie/core/shlex.py diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 97906a0..af2e0bf 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -6,7 +6,6 @@ This module contains ... """ import os -import shlex import re import stat import copy @@ -17,6 +16,7 @@ from twisted.python import log, failure from twisted.internet import error from cowrie.core import fs +from cowrie.core import shlex class HoneyPotCommand(object): """ @@ -167,6 +167,7 @@ class HoneyPotShell(object): return line = self.cmdpending.pop(0) + cmdAndArgs = shlex.split(unicode(line)) try: line = line.replace('>', ' > ').replace('|', ' | ').replace('<',' < ') cmdAndArgs = shlex.split(line) diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py new file mode 100644 index 0000000..c7de1b5 --- /dev/null +++ b/cowrie/core/shlex.py @@ -0,0 +1,344 @@ +"""A lexical analyzer class for simple shell-like syntaxes.""" +# coding: UTF-8 + +# Module and documentation by Eric S. Raymond, 21 Dec 1998 +# Input stacking and error message cleanup added by ESR, March 2000 +# push_source() and pop_source() made explicit by ESR, January 2001. +# Posix compliance, split(), string arguments, and +# iterator interface by Gustavo Niemeyer, April 2003. +# changes to tokenize more like Posix shells by Vinay Sajip, January 2012. + +import os +import re +import sys +from collections import deque + +from io import StringIO + +__all__ = ["shlex", "split", "quote"] + +class shlex: + "A lexical analyzer class for simple shell-like syntaxes." + def __init__(self, instream=None, infile=None, posix=False, + punctuation_chars=False): + if isinstance(instream, str): + instream = StringIO(instream) + if instream is not None: + self.instream = instream + self.infile = infile + else: + self.instream = sys.stdin + self.infile = None + self.posix = posix + if posix: + self.eof = None + else: + self.eof = '' + self.commenters = '#' + self.wordchars = ('abcdfeghijklmnopqrstuvwxyz' + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_') + if self.posix: + self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ' + 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ') + self.whitespace = ' \t\r\n' + self.whitespace_split = False + self.quotes = '\'"' + self.escape = '\\' + self.escapedquotes = '"' + self.state = ' ' + self.pushback = deque() + self.lineno = 1 + self.debug = 0 + self.token = '' + self.filestack = deque() + self.source = None + if not punctuation_chars: + punctuation_chars = '' + elif punctuation_chars is True: + punctuation_chars = '();<>|&' + self.punctuation_chars = punctuation_chars + if punctuation_chars: + # _pushback_chars is a push back queue used by lookahead logic + self._pushback_chars = deque() + # these chars added because allowed in file names, args, wildcards + self.wordchars += '~-./*?=' + #remove any punctuation chars from wordchars + self.wordchars = ''.join(c for c in self.wordchars if c not in + self.punctuation_chars) + for c in punctuation_chars: + if c in self.wordchars: + self.wordchars.remove(c) + if self.debug: + print('shlex: reading from %s, line %d' % (self.instream, + self.lineno)) + + def push_token(self, tok): + "Push a token onto the stack popped by the get_token method" + if self.debug >= 1: + print("shlex: pushing token " + repr(tok)) + self.pushback.appendleft(tok) + + def push_source(self, newstream, newfile=None): + "Push an input source onto the lexer's input source stack." + if isinstance(newstream, str): + newstream = StringIO(newstream) + self.filestack.appendleft((self.infile, self.instream, self.lineno)) + self.infile = newfile + self.instream = newstream + self.lineno = 1 + if self.debug: + if newfile is not None: + print('shlex: pushing to file %s' % (self.infile,)) + else: + print('shlex: pushing to stream %s' % (self.instream,)) + + def pop_source(self): + "Pop the input source stack." + self.instream.close() + (self.infile, self.instream, self.lineno) = self.filestack.popleft() + if self.debug: + print('shlex: popping to %s, line %d' \ + % (self.instream, self.lineno)) + self.state = ' ' + + def get_token(self): + "Get a token from the input stream (or from stack if it's nonempty)" + if self.pushback: + tok = self.pushback.popleft() + if self.debug >= 1: + print("shlex: popping token " + repr(tok)) + return tok + # No pushback. Get a token. + raw = self.read_token() + # Handle inclusions + if self.source is not None: + while raw == self.source: + spec = self.sourcehook(self.read_token()) + if spec: + (newfile, newstream) = spec + self.push_source(newstream, newfile) + raw = self.get_token() + # Maybe we got EOF instead? + while raw == self.eof: + if not self.filestack: + return self.eof + else: + self.pop_source() + raw = self.get_token() + # Neither inclusion nor EOF + if self.debug >= 1: + if raw != self.eof: + print("shlex: token=" + repr(raw)) + else: + print("shlex: token=EOF") + return raw + + def read_token(self): + quoted = False + escapedstate = ' ' + while True: + if self.punctuation_chars and self._pushback_chars: + nextchar = self._pushback_chars.pop() + else: + nextchar = self.instream.read(1) + if nextchar == '\n': + self.lineno += 1 + if self.debug >= 3: + print("shlex: in state %r I see character: %r" % (self.state, + nextchar)) + if self.state is None: + self.token = '' # past end of file + break + elif self.state == ' ': + if not nextchar: + self.state = None # end of file + break + elif nextchar in self.whitespace: + if self.debug >= 2: + print("shlex: I see whitespace in whitespace state") + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar + elif nextchar in self.wordchars: + self.token = nextchar + self.state = 'a' + elif nextchar in self.punctuation_chars: + self.token = nextchar + self.state = 'c' + elif nextchar in self.quotes: + if not self.posix: + self.token = nextchar + self.state = nextchar + elif self.whitespace_split: + self.token = nextchar + self.state = 'a' + else: + self.token = nextchar + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif self.state in self.quotes: + quoted = True + if not nextchar: # end of file + if self.debug >= 2: + print("shlex: I see EOF in quotes state") + # XXX what error should be raised here? + raise ValueError("No closing quotation") + if nextchar == self.state: + if not self.posix: + self.token += nextchar + self.state = ' ' + break + else: + self.state = 'a' + elif (self.posix and nextchar in self.escape and self.state + in self.escapedquotes): + escapedstate = self.state + self.state = nextchar + else: + self.token += nextchar + elif self.state in self.escape: + if not nextchar: # end of file + if self.debug >= 2: + print("shlex: I see EOF in escape state") + # XXX what error should be raised here? + raise ValueError("No escaped character") + # In posix shells, only the quote itself or the escape + # character may be escaped within quotes. + if (escapedstate in self.quotes and + nextchar != self.state and nextchar != escapedstate): + self.token += self.state + self.token += nextchar + self.state = escapedstate + elif self.state in ('a', 'c'): + if not nextchar: + self.state = None # end of file + break + elif nextchar in self.whitespace: + if self.debug >= 2: + print("shlex: I see whitespace in word state") + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + if self.posix: + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif self.posix and nextchar in self.quotes: + self.state = nextchar + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar + elif self.state == 'c': + if nextchar in self.punctuation_chars: + self.token += nextchar + else: + if nextchar not in self.whitespace: + self._pushback_chars.append(nextchar) + self.state = ' ' + break + elif (nextchar in self.wordchars or nextchar in self.quotes + or self.whitespace_split): + self.token += nextchar + else: + if self.punctuation_chars: + self._pushback_chars.append(nextchar) + else: + self.pushback.appendleft(nextchar) + if self.debug >= 2: + print("shlex: I see punctuation in word state") + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + result = self.token + self.token = '' + if self.posix and not quoted and result == '': + result = None + if self.debug > 1: + if result: + print("shlex: raw token=" + repr(result)) + else: + print("shlex: raw token=EOF") + return result + + def sourcehook(self, newfile): + "Hook called on a filename to be sourced." + if newfile[0] == '"': + newfile = newfile[1:-1] + # This implements cpp-like semantics for relative-path inclusion. + if isinstance(self.infile, str) and not os.path.isabs(newfile): + newfile = os.path.join(os.path.dirname(self.infile), newfile) + return (newfile, open(newfile, "r")) + + def error_leader(self, infile=None, lineno=None): + "Emit a C-compiler-like, Emacs-friendly error-message leader." + if infile is None: + infile = self.infile + if lineno is None: + lineno = self.lineno + return "\"%s\", line %d: " % (infile, lineno) + + def __iter__(self): + return self + +# def __next__(self): + def next(self): + token = self.get_token() + if token == self.eof: + raise StopIteration + return token + +def split(s, comments=False, posix=True): + lex = shlex(s, posix=posix) + lex.whitespace_split = True + if not comments: + lex.commenters = '' + return list(lex) + + +#_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search +# No ASCII in P2.x +_find_unsafe = re.compile(r'[^\w@%+=:,./-]' ).search + +def quote(s): + """Return a shell-escaped version of the string *s*.""" + if not s: + return "''" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + + +if __name__ == '__main__': + if len(sys.argv) == 1: + lexer = shlex() + else: + file = sys.argv[1] + lexer = shlex(open(file), file) + while 1: + tt = lexer.get_token() + if tt: + print("Token: " + repr(tt)) + else: + break + From 09185e63cf0557a576a56a640fe1f62993f26b53 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Sat, 17 Oct 2015 08:00:08 +0000 Subject: [PATCH 07/31] wip shlex --- cowrie/core/honeypot.py | 23 ++++++++++++++++------- cowrie/core/shlex.py | 8 +++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index af2e0bf..6eac8c7 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -124,22 +124,31 @@ class HoneyPotShell(object): self.interactive = interactive self.cmdpending = [] self.environ = protocol.environ + #self.lexer.debug = 1 self.showPrompt() - def lineReceived(self, line): """ """ log.msg('CMD: %s' % (line,)) - line = line[:500] - comment = re.compile('^\s*#') - for i in [x.strip() for x in re.split(';|&&|\n', line.strip())[:10]]: - if not len(i): + self.lexer = shlex.shlex(punctuation_chars=True); + self.lexer.push_source(line) + tokens = [] + while True: + tok = self.lexer.get_token() + log.msg( "tok: %s" % (repr(tok)) ) + # for now, execute all after && + if tok == ';' or tok == self.lexer.eof or tok == '&&': + self.cmdpending.append((tokens)) + tokens = [] + if tok == ';': continue - if comment.match(i): + if tok == '&&': continue - self.cmdpending.append(i) + if tok == self.lexer.eof: + break + tokens.append(tok) if len(self.cmdpending): self.runCommand() else: diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index c7de1b5..4e56ad5 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -13,7 +13,7 @@ import re import sys from collections import deque -from io import StringIO +from io import StringIO, BytesIO __all__ = ["shlex", "split", "quote"] @@ -22,7 +22,8 @@ class shlex: def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): if isinstance(instream, str): - instream = StringIO(instream) + #instream = StringIO(instream) + instream = BytesIO(instream) if instream is not None: self.instream = instream self.infile = infile @@ -81,7 +82,8 @@ class shlex: def push_source(self, newstream, newfile=None): "Push an input source onto the lexer's input source stack." if isinstance(newstream, str): - newstream = StringIO(newstream) + #newstream = StringIO(newstream) + newstream = BytesIO(newstream) self.filestack.appendleft((self.infile, self.instream, self.lineno)) self.infile = newfile self.instream = newstream From 247c783908052e717665a5ebf9f6ce58ee50aec8 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 14 Jan 2016 18:05:06 +0000 Subject: [PATCH 08/31] fix --- cowrie/core/shlex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index 4e56ad5..a68149f 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -1,6 +1,7 @@ -"""A lexical analyzer class for simple shell-like syntaxes.""" # coding: UTF-8 +"""A lexical analyzer class for simple shell-like syntaxes.""" + # Module and documentation by Eric S. Raymond, 21 Dec 1998 # Input stacking and error message cleanup added by ESR, March 2000 # push_source() and pop_source() made explicit by ESR, January 2001. From 16dc08fbfd3ad65d7144d40085a41646dd8df815 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 25 Jan 2016 10:54:15 +0000 Subject: [PATCH 09/31] works again --- cowrie/core/honeypot.py | 60 ++++++++++++++++++++--------------------- cowrie/core/shlex.py | 4 +-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 6eac8c7..ee75052 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -136,19 +136,31 @@ class HoneyPotShell(object): self.lexer.push_source(line) tokens = [] while True: - tok = self.lexer.get_token() - log.msg( "tok: %s" % (repr(tok)) ) - # for now, execute all after && - if tok == ';' or tok == self.lexer.eof or tok == '&&': - self.cmdpending.append((tokens)) - tokens = [] - if tok == ';': - continue - if tok == '&&': - continue - if tok == self.lexer.eof: - break - tokens.append(tok) + try: + tok = self.lexer.get_token() + #log.msg( "tok: %s" % (repr(tok)) ) + # for now, execute all after && + if tok == self.lexer.eof: + if len(tokens): + self.cmdpending.append((tokens)) + tokens = [] + break + if tok == ';' or tok == '&&': + self.cmdpending.append((tokens)) + tokens = [] + if tok == ';': + continue + if tok == '&&': + continue + tokens.append(tok) + except Exception as e: + self.protocol.terminal.write( + 'bash: syntax error: unexpected end of file\n') + # Could run runCommand here, but i'll just clear the list instead + self.cmdpending = [] + self.showPrompt() + return + if len(self.cmdpending): self.runCommand() else: @@ -175,18 +187,7 @@ class HoneyPotShell(object): self.protocol.terminal.transport.processEnded(ret) return - line = self.cmdpending.pop(0) - cmdAndArgs = shlex.split(unicode(line)) - try: - line = line.replace('>', ' > ').replace('|', ' | ').replace('<',' < ') - cmdAndArgs = shlex.split(line) - except: - self.protocol.terminal.write( - 'bash: syntax error: unexpected end of file\n') - # Could run runCommand here, but i'll just clear the list instead - self.cmdpending = [] - self.showPrompt() - return + cmdAndArgs = self.cmdpending.pop(0) # Probably no reason to be this comprehensive for just PATH... environ = copy.copy(self.environ) @@ -214,14 +215,13 @@ class HoneyPotShell(object): rargs.append(arg) cmdclass = self.protocol.getCommand(cmd, environ['PATH'].split(':')) if cmdclass: - log.msg(eventid='cowrie.command.success', input=line, format='Command found: %(input)s') + log.msg(eventid='cowrie.command.success', input=' '.join(cmdAndArgs), format='Command found: %(input)s') self.protocol.call_command(cmdclass, *rargs) else: log.msg(eventid='cowrie.command.failed', - input=line, format='Command not found: %(input)s') - if len(line): - self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) - runOrPrompt() + input=' '.join(cmdAndArgs), format='Command not found: %(input)s') + self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) + runOrPrompt() def resume(self): diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index a68149f..ea9634e 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -23,8 +23,8 @@ class shlex: def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): if isinstance(instream, str): - #instream = StringIO(instream) - instream = BytesIO(instream) + instream = StringIO(instream) + #instream = BytesIO(instream) if instream is not None: self.instream = instream self.infile = infile From 5e71bea216f39a2b320c7c14434c408f524e7681 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 25 Jan 2016 11:00:21 +0000 Subject: [PATCH 10/31] 'yes' command prints args or 'y' --- cowrie/commands/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cowrie/commands/base.py b/cowrie/commands/base.py index 6efd308..356cd28 100644 --- a/cowrie/commands/base.py +++ b/cowrie/commands/base.py @@ -471,7 +471,10 @@ class command_yes(HoneyPotCommand): def y(self): """ """ - self.write('y\n') + if len(self.args): + self.write(' '.join(self.args)+'\n') + else: + self.write('y\n') self.scheduled = reactor.callLater(0.01, self.y) From 81f688be8b24a62b561bf81a84a5ea013c16bfb4 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 1 Feb 2016 10:03:34 +0400 Subject: [PATCH 11/31] don't store cookie & langcs/langsc --- cowrie/ssh/transport.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cowrie/ssh/transport.py b/cowrie/ssh/transport.py index 013cff6..3e8e710 100644 --- a/cowrie/ssh/transport.py +++ b/cowrie/ssh/transport.py @@ -234,13 +234,10 @@ class HoneyPotTransport(transport.SSHServerTransport, TimeoutMixin): def ssh_KEXINIT(self, packet): """ """ - cookie = packet[:16] - log.msg("EXPERIMENTAL KEXINIT cookie %s" % (cookie.encode('hex'),)) k = getNS(packet[16:], 10) strings, rest = k[:-1], k[-1] (kexAlgs, keyAlgs, encCS, encSC, macCS, macSC, compCS, compSC, langCS, langSC) = [s.split(',') for s in strings] - log.msg("EXPERIMENTAL KEXINIT langCS langSC %s %s" % (langCS,langSC,)) log.msg(eventid='cowrie.client.version', version=self.otherVersionString, kexAlgs=kexAlgs, keyAlgs=keyAlgs, encCS=encCS, macCS=macCS, compCS=compCS, format='Remote SSH version: %(version)s') From f657af1a1a7997d92f0f006e89d683e075c40751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Mon, 8 Feb 2016 07:35:56 +0100 Subject: [PATCH 12/31] Add sqlite3 driver --- cowrie/output/sqlite.py | 151 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 cowrie/output/sqlite.py diff --git a/cowrie/output/sqlite.py b/cowrie/output/sqlite.py new file mode 100644 index 0000000..64b1ead --- /dev/null +++ b/cowrie/output/sqlite.py @@ -0,0 +1,151 @@ +import sqlite3 + +from twisted.internet import defer +from twisted.enterprise import adbapi +from twisted.python import log + +import cowrie.core.output +class Output(cowrie.core.output.Output): + """ + docstring here + """ + + def __init__(self, cfg): + self.cfg = cfg + cowrie.core.output.Output.__init__(self, cfg) + + def start(self): + """ + Start sqlite3 logging module using Twisted ConnectionPool. + """ + sqlite_file = self.cfg.get('output_sqlite', 'db_file') + try: + self.db = adbapi.ConnectionPool('sqlite3', + database = sqlite_file) + except sqlite3.OperationalError as e: + log.msg(e) + + self.db.start() + + def stop(self): + """ + Close connection to db + """ + self.db.close() + + def sqlerror(self, error): + """ + docstring here + """ + log.err( 'sqlite3 Error:', error.value ) + + def simpleQuery(self, sql, args): + """ + Just run a deferred sql query, only care about errors + """ + d = self.db.runQuery(sql, args) + d.addErrback(self.sqlerror) + + + @defer.inlineCallbacks + def write(self, entry): + """ + docstring here + """ + + if entry["eventid"] == 'cowrie.session.connect': + r = yield self.db.runQuery( + "SELECT `id` FROM `sensors` WHERE `ip` = %s", (self.sensor,)) + if r: + sensorid = r[0][0] + else: + yield self.db.runQuery( + 'INSERT INTO `sensors` (`ip`) VALUES (%s)', (self.sensor,)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + sensorid = int(r[0][0]) + self.simpleQuery( + "INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)" + + " VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)", + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + sensorid, entry["src_ip"])) + + elif entry["eventid"] == 'cowrie.login.success': + self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ + ', `username`, `password`, `timestamp`)' + \ + ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + (entry["session"], 1, entry['username'], entry['password'], + entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + + elif entry["eventid"] == 'cowrie.login.failed': + self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ + ', `username`, `password`, `timestamp`)' + \ + ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + (entry["session"], 0, entry['username'], entry['password'], + entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + + elif entry["eventid"] == 'cowrie.command.success': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `success`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + 1, entry["input"])) + + elif entry["eventid"] == 'cowrie.command.failed': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `success`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + 0, entry["input"])) + + elif entry["eventid"] == 'cowrie.session.file_download': + self.simpleQuery('INSERT INTO `downloads`' + \ + ' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s, %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + entry['url'], entry['outfile'], entry['shasum'])) + + elif entry["eventid"] == 'cowrie.session.file_download': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `realm`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + entry["realm"], entry["input"])) + + elif entry["eventid"] == 'cowrie.client.version': + r = yield self.db.runQuery( + 'SELECT `id` FROM `clients` WHERE `version` = %s', \ + (entry['version'],)) + if r: + id = int(r[0][0]) + else: + yield self.db.runQuery( + 'INSERT INTO `clients` (`version`) VALUES (%s)', \ + (entry['version'],)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + id = int(r[0][0]) + self.simpleQuery( + 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', + (id, entry["session"])) + + elif entry["eventid"] == 'cowrie.client.size': + self.simpleQuery( + 'UPDATE `sessions` SET `termsize` = %s WHERE `id` = %s', + ('%sx%s' % (entry['width'], entry['height']), + entry["session"])) + + elif entry["eventid"] == 'cowrie.session.closed': + self.simpleQuery( + 'UPDATE `sessions` SET `endtime` = STR_TO_DATE(%s, %s)' + \ + ' WHERE `id` = %s', (entry["timestamp"], + '%Y-%m-%dT%H:%i:%s.%fZ', entry["session"])) + + elif entry["eventid"] == 'cowrie.log.closed': + self.simpleQuery( + 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (%s, %s, %s)', + (entry["session"], entry["ttylog"], entry["size"])) + + elif entry["eventid"] == 'cowrie.client.fingerprint': + self.simpleQuery( + 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (%s, %s, %s)', + (entry["session"], entry["username"], entry["fingerprint"])) + From e7985a8c6290958702292bbe9791777207f5e0d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Thu, 18 Feb 2016 22:57:14 +0100 Subject: [PATCH 13/31] Add init sqlite3 script Index are not null and automatically incremented when they are declared INTEGER PRIMARY KEY. Thus, the integer size declaration has been removed --- doc/sql/sqlite3.sql | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/sql/sqlite3.sql diff --git a/doc/sql/sqlite3.sql b/doc/sql/sqlite3.sql new file mode 100644 index 0000000..0098311 --- /dev/null +++ b/doc/sql/sqlite3.sql @@ -0,0 +1,63 @@ +CREATE TABLE IF NOT EXISTS `auth` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `success` tinyint(1) NOT NULL, + `username` varchar(100) NOT NULL, + `password` varchar(100) NOT NULL, + `timestamp` datetime NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `clients` ( + `id` INTEGER PRIMARY KEY, + `version` varchar(50) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `input` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `timestamp` datetime NOT NULL, + `realm` varchar(50) default NULL, + `success` tinyint(1) default NULL, + `input` text NOT NULL +) ; +CREATE INDEX input_index ON input(session, timestamp, realm); + +CREATE TABLE IF NOT EXISTS `sensors` ( + `id` INTEGER PRIMARY KEY, + `ip` varchar(15) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `sessions` ( + `id` char(32) NOT NULL PRIMARY KEY, + `starttime` datetime NOT NULL, + `endtime` datetime default NULL, + `sensor` int(4) NOT NULL, + `ip` varchar(15) NOT NULL default '', + `termsize` varchar(7) default NULL, + `client` int(4) default NULL +) ; +CREATE INDEX sessions_index ON sessions(starttime, sensor); + +CREATE TABLE IF NOT EXISTS `ttylog` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `ttylog` varchar(100) NOT NULL, + `size` int(11) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `downloads` ( + `id` INTEGER PRIMARY KEY, + `session` CHAR( 32 ) NOT NULL, + `timestamp` datetime NOT NULL, + `url` text NOT NULL, + `outfile` text NOT NULL, + `shasum` varchar(64) default NULL +) ; +CREATE INDEX downloads_index ON downloads(session, timestamp); + +CREATE TABLE IF NOT EXISTS `keyfingerprints` ( + `id` INTEGER PRIMARY KEY, + `session` CHAR( 32 ) NOT NULL, + `username` varchar(100) NOT NULL, + `fingerprint` varchar(100) NOT NULL +) ; From d6646b76318acd62065cadc7148f2bedb2509d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Thu, 18 Feb 2016 22:59:58 +0100 Subject: [PATCH 14/31] Use placeholders and remove MySQL built-in functions usage --- cowrie/output/sqlite.py | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/cowrie/output/sqlite.py b/cowrie/output/sqlite.py index 64b1ead..9242645 100644 --- a/cowrie/output/sqlite.py +++ b/cowrie/output/sqlite.py @@ -17,11 +17,13 @@ class Output(cowrie.core.output.Output): def start(self): """ Start sqlite3 logging module using Twisted ConnectionPool. + Need to be started with check_same_thread=False. See + https://twistedmatrix.com/trac/ticket/3629. """ sqlite_file = self.cfg.get('output_sqlite', 'db_file') try: self.db = adbapi.ConnectionPool('sqlite3', - database = sqlite_file) + database = sqlite_file, check_same_thread=False) except sqlite3.OperationalError as e: log.msg(e) @@ -37,7 +39,8 @@ class Output(cowrie.core.output.Output): """ docstring here """ - log.err( 'sqlite3 Error:', error.value ) + log.err('sqlite error') + error.printTraceback() def simpleQuery(self, sql, args): """ @@ -55,97 +58,97 @@ class Output(cowrie.core.output.Output): if entry["eventid"] == 'cowrie.session.connect': r = yield self.db.runQuery( - "SELECT `id` FROM `sensors` WHERE `ip` = %s", (self.sensor,)) - if r: + "SELECT `id` FROM `sensors` WHERE `ip` = ?", (self.sensor,)) + if r and r[0][0]: sensorid = r[0][0] else: yield self.db.runQuery( - 'INSERT INTO `sensors` (`ip`) VALUES (%s)', (self.sensor,)) - r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + 'INSERT INTO `sensors` (`ip`) VALUES (?)', (self.sensor,)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()') sensorid = int(r[0][0]) self.simpleQuery( "INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)" - + " VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)", - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + + " VALUES (?, ?, ?, ?)", + (entry["session"], entry["timestamp"], sensorid, entry["src_ip"])) elif entry["eventid"] == 'cowrie.login.success': self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ ', `username`, `password`, `timestamp`)' + \ - ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + ' VALUES (?, ?, ?, ?, ?)', (entry["session"], 1, entry['username'], entry['password'], - entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + entry["timestamp"])) elif entry["eventid"] == 'cowrie.login.failed': self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ ', `username`, `password`, `timestamp`)' + \ - ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + ' VALUES (?, ?, ?, ?, ?)', (entry["session"], 0, entry['username'], entry['password'], - entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + entry["timestamp"])) elif entry["eventid"] == 'cowrie.command.success': + print str(entry) self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `success`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], 1, entry["input"])) elif entry["eventid"] == 'cowrie.command.failed': self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `success`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], 0, entry["input"])) elif entry["eventid"] == 'cowrie.session.file_download': self.simpleQuery('INSERT INTO `downloads`' + \ ' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s, %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?, ?)', + (entry["session"], entry["timestamp"], entry['url'], entry['outfile'], entry['shasum'])) elif entry["eventid"] == 'cowrie.session.file_download': self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `realm`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], entry["realm"], entry["input"])) elif entry["eventid"] == 'cowrie.client.version': r = yield self.db.runQuery( - 'SELECT `id` FROM `clients` WHERE `version` = %s', \ + 'SELECT `id` FROM `clients` WHERE `version` = ?', \ (entry['version'],)) - if r: + if r and r[0][0]: id = int(r[0][0]) else: yield self.db.runQuery( - 'INSERT INTO `clients` (`version`) VALUES (%s)', \ + 'INSERT INTO `clients` (`version`) VALUES (?)', \ (entry['version'],)) - r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()') id = int(r[0][0]) self.simpleQuery( - 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', + 'UPDATE `sessions` SET `client` = ? WHERE `id` = ?', (id, entry["session"])) elif entry["eventid"] == 'cowrie.client.size': self.simpleQuery( - 'UPDATE `sessions` SET `termsize` = %s WHERE `id` = %s', + 'UPDATE `sessions` SET `termsize` = ? WHERE `id` = ?', ('%sx%s' % (entry['width'], entry['height']), entry["session"])) elif entry["eventid"] == 'cowrie.session.closed': self.simpleQuery( - 'UPDATE `sessions` SET `endtime` = STR_TO_DATE(%s, %s)' + \ - ' WHERE `id` = %s', (entry["timestamp"], - '%Y-%m-%dT%H:%i:%s.%fZ', entry["session"])) + 'UPDATE `sessions` SET `endtime` = ?' + \ + ' WHERE `id` = ?', (entry["timestamp"], entry["session"])) elif entry["eventid"] == 'cowrie.log.closed': self.simpleQuery( - 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (%s, %s, %s)', + 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (?, ?, ?)', (entry["session"], entry["ttylog"], entry["size"])) elif entry["eventid"] == 'cowrie.client.fingerprint': self.simpleQuery( - 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (%s, %s, %s)', + 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (?, ?, ?)', (entry["session"], entry["username"], entry["fingerprint"])) From b5c91864f72497aa62f5acdd4fad00a3af53f198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Sun, 21 Feb 2016 22:40:28 +0100 Subject: [PATCH 15/31] Add documentation in cowrie.cfg.dist --- cowrie.cfg.dist | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index f3e1a20..05e5cd3 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -304,6 +304,13 @@ logfile = log/cowrie.json #port = 3306 +# SQLite3 logging module +# +# Logging to SQLite3 database +# +# [output_sqlite] +# db_file = cowrie.db + # Splunk SDK output module - EARLY RELEASE NOT RECOMMENDED # This sends logs directly to Splunk using the Python REST SDK # From b11730bcb2f26fc39ac2492c6898655a2254710d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Tue, 23 Feb 2016 20:17:04 +0100 Subject: [PATCH 16/31] Add some more documentation how to init database --- cowrie.cfg.dist | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index 05e5cd3..5db38cc 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -306,7 +306,9 @@ logfile = log/cowrie.json # SQLite3 logging module # -# Logging to SQLite3 database +# Logging to SQLite3 database. To init the database, use the script +# doc/sql/sqlite3.sql: +# sqlite3 < doc/sql/sqlite3.sql # # [output_sqlite] # db_file = cowrie.db From cb3dfad87cc1f3420497c63153496cef96f0512e Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Sun, 28 Feb 2016 23:46:05 +0000 Subject: [PATCH 17/31] fix command logging --- cowrie/core/honeypot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index ee75052..f30d315 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -188,6 +188,7 @@ class HoneyPotShell(object): return cmdAndArgs = self.cmdpending.pop(0) + cmd2 = copy.copy(cmdAndArgs) # Probably no reason to be this comprehensive for just PATH... environ = copy.copy(self.environ) @@ -215,7 +216,7 @@ class HoneyPotShell(object): rargs.append(arg) cmdclass = self.protocol.getCommand(cmd, environ['PATH'].split(':')) if cmdclass: - log.msg(eventid='cowrie.command.success', input=' '.join(cmdAndArgs), format='Command found: %(input)s') + log.msg(eventid='cowrie.command.success', input=' '.join(cmd2), format='Command found: %(input)s') self.protocol.call_command(cmdclass, *rargs) else: log.msg(eventid='cowrie.command.failed', From bec3e93cc31daaf2cf495aaa4e8817c9914d98d9 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Sun, 28 Feb 2016 23:49:23 +0000 Subject: [PATCH 18/31] also fix logging for failed commands --- cowrie/core/honeypot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index f30d315..4b641de 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -220,7 +220,7 @@ class HoneyPotShell(object): self.protocol.call_command(cmdclass, *rargs) else: log.msg(eventid='cowrie.command.failed', - input=' '.join(cmdAndArgs), format='Command not found: %(input)s') + input=' '.join(cmd2), format='Command not found: %(input)s') self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) runOrPrompt() From ff1367c1b1a1e920d71617fdfc2b3d8e5365ef43 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Fri, 16 Oct 2015 14:47:53 +0000 Subject: [PATCH 19/31] shlex update --- cowrie/core/honeypot.py | 3 +- cowrie/core/shlex.py | 344 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 cowrie/core/shlex.py diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 97906a0..af2e0bf 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -6,7 +6,6 @@ This module contains ... """ import os -import shlex import re import stat import copy @@ -17,6 +16,7 @@ from twisted.python import log, failure from twisted.internet import error from cowrie.core import fs +from cowrie.core import shlex class HoneyPotCommand(object): """ @@ -167,6 +167,7 @@ class HoneyPotShell(object): return line = self.cmdpending.pop(0) + cmdAndArgs = shlex.split(unicode(line)) try: line = line.replace('>', ' > ').replace('|', ' | ').replace('<',' < ') cmdAndArgs = shlex.split(line) diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py new file mode 100644 index 0000000..c7de1b5 --- /dev/null +++ b/cowrie/core/shlex.py @@ -0,0 +1,344 @@ +"""A lexical analyzer class for simple shell-like syntaxes.""" +# coding: UTF-8 + +# Module and documentation by Eric S. Raymond, 21 Dec 1998 +# Input stacking and error message cleanup added by ESR, March 2000 +# push_source() and pop_source() made explicit by ESR, January 2001. +# Posix compliance, split(), string arguments, and +# iterator interface by Gustavo Niemeyer, April 2003. +# changes to tokenize more like Posix shells by Vinay Sajip, January 2012. + +import os +import re +import sys +from collections import deque + +from io import StringIO + +__all__ = ["shlex", "split", "quote"] + +class shlex: + "A lexical analyzer class for simple shell-like syntaxes." + def __init__(self, instream=None, infile=None, posix=False, + punctuation_chars=False): + if isinstance(instream, str): + instream = StringIO(instream) + if instream is not None: + self.instream = instream + self.infile = infile + else: + self.instream = sys.stdin + self.infile = None + self.posix = posix + if posix: + self.eof = None + else: + self.eof = '' + self.commenters = '#' + self.wordchars = ('abcdfeghijklmnopqrstuvwxyz' + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_') + if self.posix: + self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ' + 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ') + self.whitespace = ' \t\r\n' + self.whitespace_split = False + self.quotes = '\'"' + self.escape = '\\' + self.escapedquotes = '"' + self.state = ' ' + self.pushback = deque() + self.lineno = 1 + self.debug = 0 + self.token = '' + self.filestack = deque() + self.source = None + if not punctuation_chars: + punctuation_chars = '' + elif punctuation_chars is True: + punctuation_chars = '();<>|&' + self.punctuation_chars = punctuation_chars + if punctuation_chars: + # _pushback_chars is a push back queue used by lookahead logic + self._pushback_chars = deque() + # these chars added because allowed in file names, args, wildcards + self.wordchars += '~-./*?=' + #remove any punctuation chars from wordchars + self.wordchars = ''.join(c for c in self.wordchars if c not in + self.punctuation_chars) + for c in punctuation_chars: + if c in self.wordchars: + self.wordchars.remove(c) + if self.debug: + print('shlex: reading from %s, line %d' % (self.instream, + self.lineno)) + + def push_token(self, tok): + "Push a token onto the stack popped by the get_token method" + if self.debug >= 1: + print("shlex: pushing token " + repr(tok)) + self.pushback.appendleft(tok) + + def push_source(self, newstream, newfile=None): + "Push an input source onto the lexer's input source stack." + if isinstance(newstream, str): + newstream = StringIO(newstream) + self.filestack.appendleft((self.infile, self.instream, self.lineno)) + self.infile = newfile + self.instream = newstream + self.lineno = 1 + if self.debug: + if newfile is not None: + print('shlex: pushing to file %s' % (self.infile,)) + else: + print('shlex: pushing to stream %s' % (self.instream,)) + + def pop_source(self): + "Pop the input source stack." + self.instream.close() + (self.infile, self.instream, self.lineno) = self.filestack.popleft() + if self.debug: + print('shlex: popping to %s, line %d' \ + % (self.instream, self.lineno)) + self.state = ' ' + + def get_token(self): + "Get a token from the input stream (or from stack if it's nonempty)" + if self.pushback: + tok = self.pushback.popleft() + if self.debug >= 1: + print("shlex: popping token " + repr(tok)) + return tok + # No pushback. Get a token. + raw = self.read_token() + # Handle inclusions + if self.source is not None: + while raw == self.source: + spec = self.sourcehook(self.read_token()) + if spec: + (newfile, newstream) = spec + self.push_source(newstream, newfile) + raw = self.get_token() + # Maybe we got EOF instead? + while raw == self.eof: + if not self.filestack: + return self.eof + else: + self.pop_source() + raw = self.get_token() + # Neither inclusion nor EOF + if self.debug >= 1: + if raw != self.eof: + print("shlex: token=" + repr(raw)) + else: + print("shlex: token=EOF") + return raw + + def read_token(self): + quoted = False + escapedstate = ' ' + while True: + if self.punctuation_chars and self._pushback_chars: + nextchar = self._pushback_chars.pop() + else: + nextchar = self.instream.read(1) + if nextchar == '\n': + self.lineno += 1 + if self.debug >= 3: + print("shlex: in state %r I see character: %r" % (self.state, + nextchar)) + if self.state is None: + self.token = '' # past end of file + break + elif self.state == ' ': + if not nextchar: + self.state = None # end of file + break + elif nextchar in self.whitespace: + if self.debug >= 2: + print("shlex: I see whitespace in whitespace state") + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar + elif nextchar in self.wordchars: + self.token = nextchar + self.state = 'a' + elif nextchar in self.punctuation_chars: + self.token = nextchar + self.state = 'c' + elif nextchar in self.quotes: + if not self.posix: + self.token = nextchar + self.state = nextchar + elif self.whitespace_split: + self.token = nextchar + self.state = 'a' + else: + self.token = nextchar + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif self.state in self.quotes: + quoted = True + if not nextchar: # end of file + if self.debug >= 2: + print("shlex: I see EOF in quotes state") + # XXX what error should be raised here? + raise ValueError("No closing quotation") + if nextchar == self.state: + if not self.posix: + self.token += nextchar + self.state = ' ' + break + else: + self.state = 'a' + elif (self.posix and nextchar in self.escape and self.state + in self.escapedquotes): + escapedstate = self.state + self.state = nextchar + else: + self.token += nextchar + elif self.state in self.escape: + if not nextchar: # end of file + if self.debug >= 2: + print("shlex: I see EOF in escape state") + # XXX what error should be raised here? + raise ValueError("No escaped character") + # In posix shells, only the quote itself or the escape + # character may be escaped within quotes. + if (escapedstate in self.quotes and + nextchar != self.state and nextchar != escapedstate): + self.token += self.state + self.token += nextchar + self.state = escapedstate + elif self.state in ('a', 'c'): + if not nextchar: + self.state = None # end of file + break + elif nextchar in self.whitespace: + if self.debug >= 2: + print("shlex: I see whitespace in word state") + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif nextchar in self.commenters: + self.instream.readline() + self.lineno += 1 + if self.posix: + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif self.posix and nextchar in self.quotes: + self.state = nextchar + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar + elif self.state == 'c': + if nextchar in self.punctuation_chars: + self.token += nextchar + else: + if nextchar not in self.whitespace: + self._pushback_chars.append(nextchar) + self.state = ' ' + break + elif (nextchar in self.wordchars or nextchar in self.quotes + or self.whitespace_split): + self.token += nextchar + else: + if self.punctuation_chars: + self._pushback_chars.append(nextchar) + else: + self.pushback.appendleft(nextchar) + if self.debug >= 2: + print("shlex: I see punctuation in word state") + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + result = self.token + self.token = '' + if self.posix and not quoted and result == '': + result = None + if self.debug > 1: + if result: + print("shlex: raw token=" + repr(result)) + else: + print("shlex: raw token=EOF") + return result + + def sourcehook(self, newfile): + "Hook called on a filename to be sourced." + if newfile[0] == '"': + newfile = newfile[1:-1] + # This implements cpp-like semantics for relative-path inclusion. + if isinstance(self.infile, str) and not os.path.isabs(newfile): + newfile = os.path.join(os.path.dirname(self.infile), newfile) + return (newfile, open(newfile, "r")) + + def error_leader(self, infile=None, lineno=None): + "Emit a C-compiler-like, Emacs-friendly error-message leader." + if infile is None: + infile = self.infile + if lineno is None: + lineno = self.lineno + return "\"%s\", line %d: " % (infile, lineno) + + def __iter__(self): + return self + +# def __next__(self): + def next(self): + token = self.get_token() + if token == self.eof: + raise StopIteration + return token + +def split(s, comments=False, posix=True): + lex = shlex(s, posix=posix) + lex.whitespace_split = True + if not comments: + lex.commenters = '' + return list(lex) + + +#_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search +# No ASCII in P2.x +_find_unsafe = re.compile(r'[^\w@%+=:,./-]' ).search + +def quote(s): + """Return a shell-escaped version of the string *s*.""" + if not s: + return "''" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + + +if __name__ == '__main__': + if len(sys.argv) == 1: + lexer = shlex() + else: + file = sys.argv[1] + lexer = shlex(open(file), file) + while 1: + tt = lexer.get_token() + if tt: + print("Token: " + repr(tt)) + else: + break + From 1c81547cf75f16287f9816f4dbaa9760a4d1c288 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Sat, 17 Oct 2015 08:00:08 +0000 Subject: [PATCH 20/31] wip shlex --- cowrie/core/honeypot.py | 23 ++++++++++++++++------- cowrie/core/shlex.py | 8 +++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index af2e0bf..6eac8c7 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -124,22 +124,31 @@ class HoneyPotShell(object): self.interactive = interactive self.cmdpending = [] self.environ = protocol.environ + #self.lexer.debug = 1 self.showPrompt() - def lineReceived(self, line): """ """ log.msg('CMD: %s' % (line,)) - line = line[:500] - comment = re.compile('^\s*#') - for i in [x.strip() for x in re.split(';|&&|\n', line.strip())[:10]]: - if not len(i): + self.lexer = shlex.shlex(punctuation_chars=True); + self.lexer.push_source(line) + tokens = [] + while True: + tok = self.lexer.get_token() + log.msg( "tok: %s" % (repr(tok)) ) + # for now, execute all after && + if tok == ';' or tok == self.lexer.eof or tok == '&&': + self.cmdpending.append((tokens)) + tokens = [] + if tok == ';': continue - if comment.match(i): + if tok == '&&': continue - self.cmdpending.append(i) + if tok == self.lexer.eof: + break + tokens.append(tok) if len(self.cmdpending): self.runCommand() else: diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index c7de1b5..4e56ad5 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -13,7 +13,7 @@ import re import sys from collections import deque -from io import StringIO +from io import StringIO, BytesIO __all__ = ["shlex", "split", "quote"] @@ -22,7 +22,8 @@ class shlex: def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): if isinstance(instream, str): - instream = StringIO(instream) + #instream = StringIO(instream) + instream = BytesIO(instream) if instream is not None: self.instream = instream self.infile = infile @@ -81,7 +82,8 @@ class shlex: def push_source(self, newstream, newfile=None): "Push an input source onto the lexer's input source stack." if isinstance(newstream, str): - newstream = StringIO(newstream) + #newstream = StringIO(newstream) + newstream = BytesIO(newstream) self.filestack.appendleft((self.infile, self.instream, self.lineno)) self.infile = newfile self.instream = newstream From 96ac11fc415c440fcb82da20d362bb0300ba7491 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 14 Jan 2016 18:05:06 +0000 Subject: [PATCH 21/31] fix --- cowrie/core/shlex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index 4e56ad5..a68149f 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -1,6 +1,7 @@ -"""A lexical analyzer class for simple shell-like syntaxes.""" # coding: UTF-8 +"""A lexical analyzer class for simple shell-like syntaxes.""" + # Module and documentation by Eric S. Raymond, 21 Dec 1998 # Input stacking and error message cleanup added by ESR, March 2000 # push_source() and pop_source() made explicit by ESR, January 2001. From 58ae2b0e851c7e42405dc229103e8cce96ff9c87 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 25 Jan 2016 10:54:15 +0000 Subject: [PATCH 22/31] works again --- cowrie/core/honeypot.py | 60 ++++++++++++++++++++--------------------- cowrie/core/shlex.py | 4 +-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 6eac8c7..ee75052 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -136,19 +136,31 @@ class HoneyPotShell(object): self.lexer.push_source(line) tokens = [] while True: - tok = self.lexer.get_token() - log.msg( "tok: %s" % (repr(tok)) ) - # for now, execute all after && - if tok == ';' or tok == self.lexer.eof or tok == '&&': - self.cmdpending.append((tokens)) - tokens = [] - if tok == ';': - continue - if tok == '&&': - continue - if tok == self.lexer.eof: - break - tokens.append(tok) + try: + tok = self.lexer.get_token() + #log.msg( "tok: %s" % (repr(tok)) ) + # for now, execute all after && + if tok == self.lexer.eof: + if len(tokens): + self.cmdpending.append((tokens)) + tokens = [] + break + if tok == ';' or tok == '&&': + self.cmdpending.append((tokens)) + tokens = [] + if tok == ';': + continue + if tok == '&&': + continue + tokens.append(tok) + except Exception as e: + self.protocol.terminal.write( + 'bash: syntax error: unexpected end of file\n') + # Could run runCommand here, but i'll just clear the list instead + self.cmdpending = [] + self.showPrompt() + return + if len(self.cmdpending): self.runCommand() else: @@ -175,18 +187,7 @@ class HoneyPotShell(object): self.protocol.terminal.transport.processEnded(ret) return - line = self.cmdpending.pop(0) - cmdAndArgs = shlex.split(unicode(line)) - try: - line = line.replace('>', ' > ').replace('|', ' | ').replace('<',' < ') - cmdAndArgs = shlex.split(line) - except: - self.protocol.terminal.write( - 'bash: syntax error: unexpected end of file\n') - # Could run runCommand here, but i'll just clear the list instead - self.cmdpending = [] - self.showPrompt() - return + cmdAndArgs = self.cmdpending.pop(0) # Probably no reason to be this comprehensive for just PATH... environ = copy.copy(self.environ) @@ -214,14 +215,13 @@ class HoneyPotShell(object): rargs.append(arg) cmdclass = self.protocol.getCommand(cmd, environ['PATH'].split(':')) if cmdclass: - log.msg(eventid='cowrie.command.success', input=line, format='Command found: %(input)s') + log.msg(eventid='cowrie.command.success', input=' '.join(cmdAndArgs), format='Command found: %(input)s') self.protocol.call_command(cmdclass, *rargs) else: log.msg(eventid='cowrie.command.failed', - input=line, format='Command not found: %(input)s') - if len(line): - self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) - runOrPrompt() + input=' '.join(cmdAndArgs), format='Command not found: %(input)s') + self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) + runOrPrompt() def resume(self): diff --git a/cowrie/core/shlex.py b/cowrie/core/shlex.py index a68149f..ea9634e 100644 --- a/cowrie/core/shlex.py +++ b/cowrie/core/shlex.py @@ -23,8 +23,8 @@ class shlex: def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): if isinstance(instream, str): - #instream = StringIO(instream) - instream = BytesIO(instream) + instream = StringIO(instream) + #instream = BytesIO(instream) if instream is not None: self.instream = instream self.infile = infile From 6cd3b8c95fdf1cb741f2b1f9d6a5defd8b01da98 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 25 Jan 2016 11:00:21 +0000 Subject: [PATCH 23/31] 'yes' command prints args or 'y' --- cowrie/commands/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cowrie/commands/base.py b/cowrie/commands/base.py index 6efd308..356cd28 100644 --- a/cowrie/commands/base.py +++ b/cowrie/commands/base.py @@ -471,7 +471,10 @@ class command_yes(HoneyPotCommand): def y(self): """ """ - self.write('y\n') + if len(self.args): + self.write(' '.join(self.args)+'\n') + else: + self.write('y\n') self.scheduled = reactor.callLater(0.01, self.y) From 875486e1a0107cf4631d58ffc1168ee0d587c736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Mon, 8 Feb 2016 07:35:56 +0100 Subject: [PATCH 24/31] Add sqlite3 driver --- cowrie/output/sqlite.py | 151 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 cowrie/output/sqlite.py diff --git a/cowrie/output/sqlite.py b/cowrie/output/sqlite.py new file mode 100644 index 0000000..64b1ead --- /dev/null +++ b/cowrie/output/sqlite.py @@ -0,0 +1,151 @@ +import sqlite3 + +from twisted.internet import defer +from twisted.enterprise import adbapi +from twisted.python import log + +import cowrie.core.output +class Output(cowrie.core.output.Output): + """ + docstring here + """ + + def __init__(self, cfg): + self.cfg = cfg + cowrie.core.output.Output.__init__(self, cfg) + + def start(self): + """ + Start sqlite3 logging module using Twisted ConnectionPool. + """ + sqlite_file = self.cfg.get('output_sqlite', 'db_file') + try: + self.db = adbapi.ConnectionPool('sqlite3', + database = sqlite_file) + except sqlite3.OperationalError as e: + log.msg(e) + + self.db.start() + + def stop(self): + """ + Close connection to db + """ + self.db.close() + + def sqlerror(self, error): + """ + docstring here + """ + log.err( 'sqlite3 Error:', error.value ) + + def simpleQuery(self, sql, args): + """ + Just run a deferred sql query, only care about errors + """ + d = self.db.runQuery(sql, args) + d.addErrback(self.sqlerror) + + + @defer.inlineCallbacks + def write(self, entry): + """ + docstring here + """ + + if entry["eventid"] == 'cowrie.session.connect': + r = yield self.db.runQuery( + "SELECT `id` FROM `sensors` WHERE `ip` = %s", (self.sensor,)) + if r: + sensorid = r[0][0] + else: + yield self.db.runQuery( + 'INSERT INTO `sensors` (`ip`) VALUES (%s)', (self.sensor,)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + sensorid = int(r[0][0]) + self.simpleQuery( + "INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)" + + " VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)", + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + sensorid, entry["src_ip"])) + + elif entry["eventid"] == 'cowrie.login.success': + self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ + ', `username`, `password`, `timestamp`)' + \ + ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + (entry["session"], 1, entry['username'], entry['password'], + entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + + elif entry["eventid"] == 'cowrie.login.failed': + self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ + ', `username`, `password`, `timestamp`)' + \ + ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + (entry["session"], 0, entry['username'], entry['password'], + entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + + elif entry["eventid"] == 'cowrie.command.success': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `success`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + 1, entry["input"])) + + elif entry["eventid"] == 'cowrie.command.failed': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `success`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + 0, entry["input"])) + + elif entry["eventid"] == 'cowrie.session.file_download': + self.simpleQuery('INSERT INTO `downloads`' + \ + ' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s, %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + entry['url'], entry['outfile'], entry['shasum'])) + + elif entry["eventid"] == 'cowrie.session.file_download': + self.simpleQuery('INSERT INTO `input`' + \ + ' (`session`, `timestamp`, `realm`, `input`)' + \ + ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', + (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + entry["realm"], entry["input"])) + + elif entry["eventid"] == 'cowrie.client.version': + r = yield self.db.runQuery( + 'SELECT `id` FROM `clients` WHERE `version` = %s', \ + (entry['version'],)) + if r: + id = int(r[0][0]) + else: + yield self.db.runQuery( + 'INSERT INTO `clients` (`version`) VALUES (%s)', \ + (entry['version'],)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + id = int(r[0][0]) + self.simpleQuery( + 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', + (id, entry["session"])) + + elif entry["eventid"] == 'cowrie.client.size': + self.simpleQuery( + 'UPDATE `sessions` SET `termsize` = %s WHERE `id` = %s', + ('%sx%s' % (entry['width'], entry['height']), + entry["session"])) + + elif entry["eventid"] == 'cowrie.session.closed': + self.simpleQuery( + 'UPDATE `sessions` SET `endtime` = STR_TO_DATE(%s, %s)' + \ + ' WHERE `id` = %s', (entry["timestamp"], + '%Y-%m-%dT%H:%i:%s.%fZ', entry["session"])) + + elif entry["eventid"] == 'cowrie.log.closed': + self.simpleQuery( + 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (%s, %s, %s)', + (entry["session"], entry["ttylog"], entry["size"])) + + elif entry["eventid"] == 'cowrie.client.fingerprint': + self.simpleQuery( + 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (%s, %s, %s)', + (entry["session"], entry["username"], entry["fingerprint"])) + From 92d35462c370ecf0b8ea97fc23ccc32b4c2b4568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Thu, 18 Feb 2016 22:57:14 +0100 Subject: [PATCH 25/31] Add init sqlite3 script Index are not null and automatically incremented when they are declared INTEGER PRIMARY KEY. Thus, the integer size declaration has been removed --- doc/sql/sqlite3.sql | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/sql/sqlite3.sql diff --git a/doc/sql/sqlite3.sql b/doc/sql/sqlite3.sql new file mode 100644 index 0000000..0098311 --- /dev/null +++ b/doc/sql/sqlite3.sql @@ -0,0 +1,63 @@ +CREATE TABLE IF NOT EXISTS `auth` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `success` tinyint(1) NOT NULL, + `username` varchar(100) NOT NULL, + `password` varchar(100) NOT NULL, + `timestamp` datetime NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `clients` ( + `id` INTEGER PRIMARY KEY, + `version` varchar(50) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `input` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `timestamp` datetime NOT NULL, + `realm` varchar(50) default NULL, + `success` tinyint(1) default NULL, + `input` text NOT NULL +) ; +CREATE INDEX input_index ON input(session, timestamp, realm); + +CREATE TABLE IF NOT EXISTS `sensors` ( + `id` INTEGER PRIMARY KEY, + `ip` varchar(15) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `sessions` ( + `id` char(32) NOT NULL PRIMARY KEY, + `starttime` datetime NOT NULL, + `endtime` datetime default NULL, + `sensor` int(4) NOT NULL, + `ip` varchar(15) NOT NULL default '', + `termsize` varchar(7) default NULL, + `client` int(4) default NULL +) ; +CREATE INDEX sessions_index ON sessions(starttime, sensor); + +CREATE TABLE IF NOT EXISTS `ttylog` ( + `id` INTEGER PRIMARY KEY, + `session` char(32) NOT NULL, + `ttylog` varchar(100) NOT NULL, + `size` int(11) NOT NULL +) ; + +CREATE TABLE IF NOT EXISTS `downloads` ( + `id` INTEGER PRIMARY KEY, + `session` CHAR( 32 ) NOT NULL, + `timestamp` datetime NOT NULL, + `url` text NOT NULL, + `outfile` text NOT NULL, + `shasum` varchar(64) default NULL +) ; +CREATE INDEX downloads_index ON downloads(session, timestamp); + +CREATE TABLE IF NOT EXISTS `keyfingerprints` ( + `id` INTEGER PRIMARY KEY, + `session` CHAR( 32 ) NOT NULL, + `username` varchar(100) NOT NULL, + `fingerprint` varchar(100) NOT NULL +) ; From e0229241b8ebfbecd496aa2e5e340d3c49a2e50d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Thu, 18 Feb 2016 22:59:58 +0100 Subject: [PATCH 26/31] Use placeholders and remove MySQL built-in functions usage --- cowrie/output/sqlite.py | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/cowrie/output/sqlite.py b/cowrie/output/sqlite.py index 64b1ead..9242645 100644 --- a/cowrie/output/sqlite.py +++ b/cowrie/output/sqlite.py @@ -17,11 +17,13 @@ class Output(cowrie.core.output.Output): def start(self): """ Start sqlite3 logging module using Twisted ConnectionPool. + Need to be started with check_same_thread=False. See + https://twistedmatrix.com/trac/ticket/3629. """ sqlite_file = self.cfg.get('output_sqlite', 'db_file') try: self.db = adbapi.ConnectionPool('sqlite3', - database = sqlite_file) + database = sqlite_file, check_same_thread=False) except sqlite3.OperationalError as e: log.msg(e) @@ -37,7 +39,8 @@ class Output(cowrie.core.output.Output): """ docstring here """ - log.err( 'sqlite3 Error:', error.value ) + log.err('sqlite error') + error.printTraceback() def simpleQuery(self, sql, args): """ @@ -55,97 +58,97 @@ class Output(cowrie.core.output.Output): if entry["eventid"] == 'cowrie.session.connect': r = yield self.db.runQuery( - "SELECT `id` FROM `sensors` WHERE `ip` = %s", (self.sensor,)) - if r: + "SELECT `id` FROM `sensors` WHERE `ip` = ?", (self.sensor,)) + if r and r[0][0]: sensorid = r[0][0] else: yield self.db.runQuery( - 'INSERT INTO `sensors` (`ip`) VALUES (%s)', (self.sensor,)) - r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + 'INSERT INTO `sensors` (`ip`) VALUES (?)', (self.sensor,)) + r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()') sensorid = int(r[0][0]) self.simpleQuery( "INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)" - + " VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)", - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + + " VALUES (?, ?, ?, ?)", + (entry["session"], entry["timestamp"], sensorid, entry["src_ip"])) elif entry["eventid"] == 'cowrie.login.success': self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ ', `username`, `password`, `timestamp`)' + \ - ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + ' VALUES (?, ?, ?, ?, ?)', (entry["session"], 1, entry['username'], entry['password'], - entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + entry["timestamp"])) elif entry["eventid"] == 'cowrie.login.failed': self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \ ', `username`, `password`, `timestamp`)' + \ - ' VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s))', + ' VALUES (?, ?, ?, ?, ?)', (entry["session"], 0, entry['username'], entry['password'], - entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ')) + entry["timestamp"])) elif entry["eventid"] == 'cowrie.command.success': + print str(entry) self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `success`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], 1, entry["input"])) elif entry["eventid"] == 'cowrie.command.failed': self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `success`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], 0, entry["input"])) elif entry["eventid"] == 'cowrie.session.file_download': self.simpleQuery('INSERT INTO `downloads`' + \ ' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s, %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?, ?)', + (entry["session"], entry["timestamp"], entry['url'], entry['outfile'], entry['shasum'])) elif entry["eventid"] == 'cowrie.session.file_download': self.simpleQuery('INSERT INTO `input`' + \ ' (`session`, `timestamp`, `realm`, `input`)' + \ - ' VALUES (%s, STR_TO_DATE(%s, %s), %s , %s)', - (entry["session"], entry["timestamp"], '%Y-%m-%dT%H:%i:%s.%fZ', + ' VALUES (?, ?, ?, ?)', + (entry["session"], entry["timestamp"], entry["realm"], entry["input"])) elif entry["eventid"] == 'cowrie.client.version': r = yield self.db.runQuery( - 'SELECT `id` FROM `clients` WHERE `version` = %s', \ + 'SELECT `id` FROM `clients` WHERE `version` = ?', \ (entry['version'],)) - if r: + if r and r[0][0]: id = int(r[0][0]) else: yield self.db.runQuery( - 'INSERT INTO `clients` (`version`) VALUES (%s)', \ + 'INSERT INTO `clients` (`version`) VALUES (?)', \ (entry['version'],)) - r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') + r = yield self.db.runQuery('SELECT LAST_INSERT_ROWID()') id = int(r[0][0]) self.simpleQuery( - 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', + 'UPDATE `sessions` SET `client` = ? WHERE `id` = ?', (id, entry["session"])) elif entry["eventid"] == 'cowrie.client.size': self.simpleQuery( - 'UPDATE `sessions` SET `termsize` = %s WHERE `id` = %s', + 'UPDATE `sessions` SET `termsize` = ? WHERE `id` = ?', ('%sx%s' % (entry['width'], entry['height']), entry["session"])) elif entry["eventid"] == 'cowrie.session.closed': self.simpleQuery( - 'UPDATE `sessions` SET `endtime` = STR_TO_DATE(%s, %s)' + \ - ' WHERE `id` = %s', (entry["timestamp"], - '%Y-%m-%dT%H:%i:%s.%fZ', entry["session"])) + 'UPDATE `sessions` SET `endtime` = ?' + \ + ' WHERE `id` = ?', (entry["timestamp"], entry["session"])) elif entry["eventid"] == 'cowrie.log.closed': self.simpleQuery( - 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (%s, %s, %s)', + 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) VALUES (?, ?, ?)', (entry["session"], entry["ttylog"], entry["size"])) elif entry["eventid"] == 'cowrie.client.fingerprint': self.simpleQuery( - 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (%s, %s, %s)', + 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) VALUES (?, ?, ?)', (entry["session"], entry["username"], entry["fingerprint"])) From fec5306d3b89aa5ff5f08108464a87d9f16f30f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Sun, 21 Feb 2016 22:40:28 +0100 Subject: [PATCH 27/31] Add documentation in cowrie.cfg.dist --- cowrie.cfg.dist | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index 8657640..9e40e98 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -320,6 +320,13 @@ logfile = log/cowrie.json #port = 3306 +# SQLite3 logging module +# +# Logging to SQLite3 database +# +# [output_sqlite] +# db_file = cowrie.db + # Splunk SDK output module - EARLY RELEASE NOT RECOMMENDED # This sends logs directly to Splunk using the Python REST SDK # From f02c56743668d24ad0625a844c36e0a1a5f94343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20Rouvi=C3=A8re?= Date: Tue, 23 Feb 2016 20:17:04 +0100 Subject: [PATCH 28/31] Add some more documentation how to init database --- cowrie.cfg.dist | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index 9e40e98..63c2ab0 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -322,7 +322,9 @@ logfile = log/cowrie.json # SQLite3 logging module # -# Logging to SQLite3 database +# Logging to SQLite3 database. To init the database, use the script +# doc/sql/sqlite3.sql: +# sqlite3 < doc/sql/sqlite3.sql # # [output_sqlite] # db_file = cowrie.db From 976f678368d1fde791c7d65076d3b0085a8a2255 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Sun, 28 Feb 2016 23:46:05 +0000 Subject: [PATCH 29/31] fix command logging --- cowrie/core/honeypot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index ee75052..f30d315 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -188,6 +188,7 @@ class HoneyPotShell(object): return cmdAndArgs = self.cmdpending.pop(0) + cmd2 = copy.copy(cmdAndArgs) # Probably no reason to be this comprehensive for just PATH... environ = copy.copy(self.environ) @@ -215,7 +216,7 @@ class HoneyPotShell(object): rargs.append(arg) cmdclass = self.protocol.getCommand(cmd, environ['PATH'].split(':')) if cmdclass: - log.msg(eventid='cowrie.command.success', input=' '.join(cmdAndArgs), format='Command found: %(input)s') + log.msg(eventid='cowrie.command.success', input=' '.join(cmd2), format='Command found: %(input)s') self.protocol.call_command(cmdclass, *rargs) else: log.msg(eventid='cowrie.command.failed', From 0c16b3e9bae69f9e3d0798dcb6ccf0056742c2d6 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Sun, 28 Feb 2016 23:49:23 +0000 Subject: [PATCH 30/31] also fix logging for failed commands --- cowrie/core/honeypot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index f30d315..4b641de 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -220,7 +220,7 @@ class HoneyPotShell(object): self.protocol.call_command(cmdclass, *rargs) else: log.msg(eventid='cowrie.command.failed', - input=' '.join(cmdAndArgs), format='Command not found: %(input)s') + input=' '.join(cmd2), format='Command not found: %(input)s') self.protocol.terminal.write('bash: %s: command not found\n' % (cmd,)) runOrPrompt() From 7701da533d36c6989520202e3f81ecf1a9b6b0b8 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Mon, 29 Feb 2016 00:12:36 +0000 Subject: [PATCH 31/31] parse || also --- cowrie/core/honeypot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cowrie/core/honeypot.py b/cowrie/core/honeypot.py index 4b641de..ed20c0b 100644 --- a/cowrie/core/honeypot.py +++ b/cowrie/core/honeypot.py @@ -145,13 +145,15 @@ class HoneyPotShell(object): self.cmdpending.append((tokens)) tokens = [] break - if tok == ';' or tok == '&&': + if tok == ';' or tok == '&&' or tok == '||': self.cmdpending.append((tokens)) tokens = [] if tok == ';': continue if tok == '&&': continue + if tok == '||': + continue tokens.append(tok) except Exception as e: self.protocol.terminal.write(