From c2b8090315dee6953cf77acb599dfe0374661954 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Wed, 28 Sep 2016 22:51:34 +0400 Subject: [PATCH 1/9] add RSA key workaround --- INSTALL.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 13fc71d..57034ed 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -187,6 +187,13 @@ $ cd cowrie/data $ ssh-keygen -t dsa -b 1024 -f ssh_host_dsa_key ``` +* If there are issues creating the RSA keys, the following is a workaround: + +``` +$ cd cowrie/data +$ ssh-keygen -t rsa -b 2048 -f ssh_host_rsa_key +``` + * If you see `twistd: Unknown command: cowrie` there are two possibilities. If there's a python stack trace, it probably means there's a missing or broken dependency. If there's no stack trace, From 7dace024c73593e4da55f3a58493d6f0ec0e2058 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Wed, 28 Sep 2016 22:28:23 +0400 Subject: [PATCH 2/9] wip --- cowrie/output/hpfeeds.py | 315 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 cowrie/output/hpfeeds.py diff --git a/cowrie/output/hpfeeds.py b/cowrie/output/hpfeeds.py new file mode 100644 index 0000000..b5d639a --- /dev/null +++ b/cowrie/output/hpfeeds.py @@ -0,0 +1,315 @@ + +""" +Output plugin for HPFeeds +""" + +import os +import struct +import hashlib +import json +import socket +import uuid + +import cowrie.core.output + +""" +From https://github.com/threatstream/kippo/blob/master/kippo/dblog/hpfeeds.py +""" + +BUFSIZ = 16384 + +OP_ERROR = 0 +OP_INFO = 1 +OP_AUTH = 2 +OP_PUBLISH = 3 +OP_SUBSCRIBE = 4 + +MAXBUF = 1024**2 +SIZES = { + OP_ERROR: 5+MAXBUF, + OP_INFO: 5+256+20, + OP_AUTH: 5+256+20, + OP_PUBLISH: 5+MAXBUF, + OP_SUBSCRIBE: 5+256*2, +} + +COWRIECHAN = 'cowrie.sessions' + +class BadClient(Exception): + pass + + + +def strpack8(x): + """ + # packs a string with 1 byte length field + """ + if isinstance(x, str): x = x.encode('latin1') + return struct.pack('!B', len(x)) + x + + + +# unpacks a string with 1 byte length field +def strunpack8(x): + l = x[0] + return x[1:1+l], x[1+l:] + + + +def msghdr(op, data): + return struct.pack('!iB', 5+len(data), op) + data + + + +def msgpublish(ident, chan, data): + return msghdr(OP_PUBLISH, strpack8(ident) + strpack8(chan) + data) + + + +def msgsubscribe(ident, chan): + if isinstance(chan, str): chan = chan.encode('latin1') + return msghdr(OP_SUBSCRIBE, strpack8(ident) + chan) + + + +def msgauth(rand, ident, secret): + hash = hashlib.sha1(bytes(rand)+secret).digest() + return msghdr(OP_AUTH, strpack8(ident) + hash) + + + +class FeedUnpack(object): + def __init__(self): + self.buf = bytearray() + + + def __iter__(self): + return self + + + def next(self): + return self.unpack() + + + def feed(self, data): + self.buf.extend(data) + + + def unpack(self): + if len(self.buf) < 5: + raise StopIteration('No message.') + + ml, opcode = struct.unpack('!iB', buffer(self.buf,0,5)) + if ml > SIZES.get(opcode, MAXBUF): + raise BadClient('Not respecting MAXBUF.') + + if len(self.buf) < ml: + raise StopIteration('No message.') + + data = bytearray(buffer(self.buf, 5, ml-5)) + del self.buf[:ml] + return opcode, data + + + +class hpclient(object): + def __init__(self, server, port, ident, secret, debug): + print 'hpfeeds client init broker {0}:{1}, identifier {2}'.format(server, port, ident) + self.server, self.port = server, int(port) + self.ident, self.secret = ident.encode('latin1'), secret.encode('latin1') + self.debug = debug + self.unpacker = FeedUnpack() + self.state = 'INIT' + + self.connect() + self.sendfiles = [] + self.filehandle = None + + + def connect(self): + self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.s.settimeout(3) + try: self.s.connect((self.server, self.port)) + except: + print 'hpfeeds client could not connect to broker.' + self.s = None + else: + self.s.settimeout(None) + self.handle_established() + + + def send(self, data): + if not self.s: + self.connect() + + if not self.s: return + self.s.send(data) + + + def close(self): + self.s.close() + self.s = None + + + def handle_established(self): + if self.debug: print 'hpclient established' + while self.state != 'GOTINFO': + self.read() + + # Quickly try to see if there was an error message + self.s.settimeout(0.5) + self.read() + self.s.settimeout(None) + + + def read(self): + if not self.s: return + try: d = self.s.recv(BUFSIZ) + except socket.timeout: + return + + if not d: + if self.debug: log.msg('hpclient connection closed?') + self.close() + return + + self.unpacker.feed(d) + try: + for opcode, data in self.unpacker: + if self.debug: log.msg('hpclient msg opcode {0} data {1}'.format(opcode, data)) + if opcode == OP_INFO: + name, rand = strunpack8(data) + if self.debug: log.msg('hpclient server name {0} rand {1}'.format(name, rand)) + self.send(msgauth(rand, self.ident, self.secret)) + self.state = 'GOTINFO' + + elif opcode == OP_PUBLISH: + ident, data = strunpack8(data) + chan, data = strunpack8(data) + if self.debug: log.msg('publish to {0} by {1}: {2}'.format(chan, ident, data)) + + elif opcode == OP_ERROR: + log.err('errormessage from server: {0}'.format(data)) + else: + log.err('unknown opcode message: {0}'.format(opcode)) + except BadClient: + log.err('unpacker error, disconnecting.') + self.close() + + + def publish(self, channel, **kwargs): + try: + self.send(msgpublish(self.ident, channel, json.dumps(kwargs).encode('latin1'))) + except Exception as e: + log.err('connection to hpfriends lost: {0}'.format(e)) + log.err('connecting') + self.connect() + self.send(msgpublish(self.ident, channel, json.dumps(kwargs).encode('latin1'))) + + + def sendfile(self, filepath): + # does not read complete binary into memory, read and send chunks + if not self.filehandle: + self.sendfileheader(i.file) + self.sendfiledata() + else: self.sendfiles.append(filepath) + + + def sendfileheader(self, filepath): + self.filehandle = open(filepath, 'rb') + fsize = os.stat(filepath).st_size + headc = strpack8(self.ident) + strpack8(UNIQUECHAN) + headh = struct.pack('!iB', 5+len(headc)+fsize, OP_PUBLISH) + self.send(headh + headc) + + + def sendfiledata(self): + tmp = self.filehandle.read(BUFSIZ) + if not tmp: + if self.sendfiles: + fp = self.sendfiles.pop(0) + self.sendfileheader(fp) + else: + self.filehandle = None + self.handle_io_in(b'') + else: + self.send(tmp) + + + +class Output(cowrie.core.output.Output): + """ + Output plugin for HPFeeds + """ + + def __init__(self, cfg): + self.cfg = cfg + cowrie.core.output.Output.__init__(self, cfg) + + + def start(self, cfg): + """ + """ + server = cfg.get('database_hpfeeds', 'server') + port = cfg.get('database_hpfeeds', 'port') + ident = cfg.get('database_hpfeeds', 'identifier') + secret = cfg.get('database_hpfeeds', 'secret') + debug = cfg.get('database_hpfeeds', 'debug') + + self.client = hpclient(server, port, ident, secret, debug) + self.meta = {} + + + def stop(self): + """ + """ + pass + + + def write(self, entry): + """ + """ + session = entry["id"] + if entry["eventid"] == 'cowrie.session.connect': + startTime = entry["timestamp"] + self.meta[session] = {'session':session,'startTime':startTime, + 'endTime':'','peerIP': peerIP, 'peerPort': peerPort, + 'hostIP': hostIP, 'hostPort': hostPort, 'loggedin': None, + 'credentials':[], 'commands':[],"unknownCommands":[], + 'urls':[], 'version': None, 'ttylog': None } + + elif entry["eventid"] == 'cowrie.login.success': + u, p = args['username'], args['password'] + self.meta[session]['loggedin'] = (u,p) + + elif entry["eventid"] == 'cowrie.login.failed': + u, p = args['username'], args['password'] + self.meta[session]['credentials'].append((u,p)) + + elif entry["eventid"] == 'cowrie.command.success': + c = args['input'] + self.meta[session]['commands'].append(c) + + elif entry["eventid"] == 'cowrie.command.failed': + uc = args['input'] + self.meta[session]['unknownCommands'].append(uc) + + elif entry["eventid"] == 'cowrie.session.file_download': + url = args['url'] + self.meta[session]['urls'].append(url) + + elif entry["eventid"] == 'cowrie.client.version': + v = args['version'] + self.meta[session]['version'] = v + + elif entry["eventid"] == 'cowrie.log.closed': + # entry["ttylog"] + ttylog = self.ttylog(session) + if ttylog: + meta['ttylog'] = ttylog.encode('hex') + + elif entry["eventid"] == 'cowrie.session.closed': + log.msg('publishing metadata to hpfeeds') + meta = self.meta[session] + self.meta[session]['endTime']=entry["timestamp"] + self.client.publish(COWRIECHAN, **meta) From d547da41f6fd31b2036fb4f49e8e417e0915044b Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Wed, 28 Sep 2016 22:29:56 +0400 Subject: [PATCH 3/9] add output plugin --- cowrie.cfg.dist | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cowrie.cfg.dist b/cowrie.cfg.dist index 2374561..2c49450 100644 --- a/cowrie.cfg.dist +++ b/cowrie.cfg.dist @@ -403,7 +403,7 @@ logfile = log/cowrie.json #source = cowrie -# HPFeeds +# Legacy HPFeeds output # #[database_hpfeeds] #server = hpfeeds.mysite.org @@ -413,6 +413,16 @@ logfile = log/cowrie.json #debug=false +# HPFeeds +# +#[output_hpfeeds] +#server = hpfeeds.mysite.org +#port = 10000 +#identifier = abc123 +#secret = secret +#debug=false + + # VirusTotal output module # You must signup for an api key. # From 7f91be37c251acb24369f4cbbed0349cf62c4f53 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 29 Sep 2016 11:09:59 +0400 Subject: [PATCH 4/9] wip --- cowrie/output/hpfeeds.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cowrie/output/hpfeeds.py b/cowrie/output/hpfeeds.py index b5d639a..75be230 100644 --- a/cowrie/output/hpfeeds.py +++ b/cowrie/output/hpfeeds.py @@ -1,6 +1,7 @@ """ Output plugin for HPFeeds + """ import os @@ -10,6 +11,8 @@ import json import socket import uuid +from twisted.python import log + import cowrie.core.output """ @@ -99,7 +102,7 @@ class FeedUnpack(object): if len(self.buf) < 5: raise StopIteration('No message.') - ml, opcode = struct.unpack('!iB', buffer(self.buf,0,5)) + ml, opcode = struct.unpack('!iB', buffer(self.buf, 0, 5)) if ml > SIZES.get(opcode, MAXBUF): raise BadClient('Not respecting MAXBUF.') @@ -244,6 +247,7 @@ class Output(cowrie.core.output.Output): def __init__(self, cfg): self.cfg = cfg + log.msg("Early version of hpfeeds-output, untested!") cowrie.core.output.Output.__init__(self, cfg) @@ -272,19 +276,19 @@ class Output(cowrie.core.output.Output): session = entry["id"] if entry["eventid"] == 'cowrie.session.connect': startTime = entry["timestamp"] - self.meta[session] = {'session':session,'startTime':startTime, - 'endTime':'','peerIP': peerIP, 'peerPort': peerPort, + self.meta[session] = {'session':session, 'startTime':startTime, + 'endTime':'', 'peerIP': peerIP, 'peerPort': peerPort, 'hostIP': hostIP, 'hostPort': hostPort, 'loggedin': None, - 'credentials':[], 'commands':[],"unknownCommands":[], + 'credentials':[], 'commands':[], "unknownCommands":[], 'urls':[], 'version': None, 'ttylog': None } elif entry["eventid"] == 'cowrie.login.success': u, p = args['username'], args['password'] - self.meta[session]['loggedin'] = (u,p) + self.meta[session]['loggedin'] = (u, p) elif entry["eventid"] == 'cowrie.login.failed': u, p = args['username'], args['password'] - self.meta[session]['credentials'].append((u,p)) + self.meta[session]['credentials'].append((u, p)) elif entry["eventid"] == 'cowrie.command.success': c = args['input'] @@ -311,5 +315,5 @@ class Output(cowrie.core.output.Output): elif entry["eventid"] == 'cowrie.session.closed': log.msg('publishing metadata to hpfeeds') meta = self.meta[session] - self.meta[session]['endTime']=entry["timestamp"] + self.meta[session]['endTime'] = entry["timestamp"] self.client.publish(COWRIECHAN, **meta) From 3437a9ea8b88ebc89f482e26f6020795785da8d6 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 29 Sep 2016 12:02:55 +0400 Subject: [PATCH 5/9] wip --- cowrie/output/hpfeeds.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/cowrie/output/hpfeeds.py b/cowrie/output/hpfeeds.py index 75be230..d4b1b94 100644 --- a/cowrie/output/hpfeeds.py +++ b/cowrie/output/hpfeeds.py @@ -251,15 +251,14 @@ class Output(cowrie.core.output.Output): cowrie.core.output.Output.__init__(self, cfg) - def start(self, cfg): + def start(self): """ """ - server = cfg.get('database_hpfeeds', 'server') - port = cfg.get('database_hpfeeds', 'port') - ident = cfg.get('database_hpfeeds', 'identifier') - secret = cfg.get('database_hpfeeds', 'secret') - debug = cfg.get('database_hpfeeds', 'debug') - + server = self.cfg.get('output_hpfeeds', 'server') + port = self.cfg.get('output_hpfeeds', 'port') + ident = self.cfg.get('output_hpfeeds', 'identifier') + secret = self.cfg.get('output_hpfeeds', 'secret') + debug = self.cfg.get('output_hpfeeds', 'debug') self.client = hpclient(server, port, ident, secret, debug) self.meta = {} @@ -273,37 +272,37 @@ class Output(cowrie.core.output.Output): def write(self, entry): """ """ - session = entry["id"] + session = entry["session"] if entry["eventid"] == 'cowrie.session.connect': startTime = entry["timestamp"] self.meta[session] = {'session':session, 'startTime':startTime, - 'endTime':'', 'peerIP': peerIP, 'peerPort': peerPort, - 'hostIP': hostIP, 'hostPort': hostPort, 'loggedin': None, + 'endTime':'', 'peerIP': src_ip, 'peerPort': src_port, + 'hostIP': dst_ip, 'hostPort': dst_port, 'loggedin': None, 'credentials':[], 'commands':[], "unknownCommands":[], 'urls':[], 'version': None, 'ttylog': None } elif entry["eventid"] == 'cowrie.login.success': - u, p = args['username'], args['password'] + u, p = entry['username'], entry['password'] self.meta[session]['loggedin'] = (u, p) elif entry["eventid"] == 'cowrie.login.failed': - u, p = args['username'], args['password'] + u, p = entry['username'], entry['password'] self.meta[session]['credentials'].append((u, p)) elif entry["eventid"] == 'cowrie.command.success': - c = args['input'] + c = entry['input'] self.meta[session]['commands'].append(c) elif entry["eventid"] == 'cowrie.command.failed': - uc = args['input'] + uc = entry['input'] self.meta[session]['unknownCommands'].append(uc) elif entry["eventid"] == 'cowrie.session.file_download': - url = args['url'] + url = entry['url'] self.meta[session]['urls'].append(url) elif entry["eventid"] == 'cowrie.client.version': - v = args['version'] + v = entry['version'] self.meta[session]['version'] = v elif entry["eventid"] == 'cowrie.log.closed': From f522e298895530896717bb199408a1cad1030694 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 29 Sep 2016 12:24:29 +0400 Subject: [PATCH 6/9] wip --- cowrie/output/hpfeeds.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/cowrie/output/hpfeeds.py b/cowrie/output/hpfeeds.py index d4b1b94..35082ea 100644 --- a/cowrie/output/hpfeeds.py +++ b/cowrie/output/hpfeeds.py @@ -179,22 +179,26 @@ class hpclient(object): self.unpacker.feed(d) try: for opcode, data in self.unpacker: - if self.debug: log.msg('hpclient msg opcode {0} data {1}'.format(opcode, data)) + if self.debug: log.msg('hpclient msg opcode {0:x} data {1}'.format(opcode, + ''.join('{:02x}'.format(x) for x in data))) if opcode == OP_INFO: name, rand = strunpack8(data) - if self.debug: log.msg('hpclient server name {0} rand {1}'.format(name, rand)) + if self.debug: log.msg('hpclient server name {0} rand {1}'.format(name, + ''.join('{:02x}'.format(x) for x in rand))) self.send(msgauth(rand, self.ident, self.secret)) self.state = 'GOTINFO' elif opcode == OP_PUBLISH: ident, data = strunpack8(data) chan, data = strunpack8(data) - if self.debug: log.msg('publish to {0} by {1}: {2}'.format(chan, ident, data)) - + if self.debug: + log.msg('publish to {0} by {1}: {2}'.format(chan, ident, + ''.join('{:02x}'.format(x) for x in data))) elif opcode == OP_ERROR: - log.err('errormessage from server: {0}'.format(data)) + log.err('errormessage from server: {0}'.format( + ''.join('{:02x}'.format(x) for x in data))) else: - log.err('unknown opcode message: {0}'.format(opcode)) + log.err('unknown opcode message: {0:x}'.format(opcode)) except BadClient: log.err('unpacker error, disconnecting.') self.close() @@ -274,12 +278,13 @@ class Output(cowrie.core.output.Output): """ session = entry["session"] if entry["eventid"] == 'cowrie.session.connect': - startTime = entry["timestamp"] - self.meta[session] = {'session':session, 'startTime':startTime, - 'endTime':'', 'peerIP': src_ip, 'peerPort': src_port, - 'hostIP': dst_ip, 'hostPort': dst_port, 'loggedin': None, - 'credentials':[], 'commands':[], "unknownCommands":[], - 'urls':[], 'version': None, 'ttylog': None } + self.meta[session] = {'session':session, + 'startTime': entry["timestamp"], 'endTime':'', + 'peerIP': entry["src_ip"], 'peerPort': entry["src_port"], + 'hostIP': entry["dst_ip"], 'hostPort': entry["dst_port"], + 'loggedin': None, 'credentials':[], 'commands':[], + 'unknownCommands':[], 'urls':[], 'version': None, + 'ttylog': None } elif entry["eventid"] == 'cowrie.login.success': u, p = entry['username'], entry['password'] @@ -307,9 +312,8 @@ class Output(cowrie.core.output.Output): elif entry["eventid"] == 'cowrie.log.closed': # entry["ttylog"] - ttylog = self.ttylog(session) - if ttylog: - meta['ttylog'] = ttylog.encode('hex') + with open( entry["ttylog"]) as ttylog: + self.meta['ttylog'] = ttylog.read().encode('hex') elif entry["eventid"] == 'cowrie.session.closed': log.msg('publishing metadata to hpfeeds') From f5ba33ca7a0f2dae94f7afae634ac09d72f049d3 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 29 Sep 2016 15:08:17 +0400 Subject: [PATCH 7/9] improve display of wget --- cowrie/commands/wget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cowrie/commands/wget.py b/cowrie/commands/wget.py index d5c9c9e..ddcfa16 100644 --- a/cowrie/commands/wget.py +++ b/cowrie/commands/wget.py @@ -161,7 +161,7 @@ class command_wget(HoneyPotCommand): if scheme == 'https': contextFactory = ssl.ClientContextFactory() contextFactory.method = SSL.SSLv23_METHOD - reactor.connectSSL(host, port, factory, contextFactory) + self.connection = reactor.connectSSL(host, port, factory, contextFactory) else: # Can only be http, since we raised an error above for unknown schemes self.connection = reactor.connectTCP( host, port, factory, bindAddress=out_addr) @@ -311,7 +311,7 @@ class HTTPProgressDownloader(client.HTTPDownloader): if (time.time() - self.lastupdate) < 0.5: return client.HTTPDownloader.pagePart(self, data) if self.totallength: - percent = (self.currentlength/self.totallength)*100 + percent = int(self.currentlength/self.totallength*100) spercent = "{}%".format(percent) else: spercent = '%dK' % (self.currentlength/1000) From 06c8f57c4454b6de39b020bd044dbbaca72830cd Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 29 Sep 2016 15:31:34 +0400 Subject: [PATCH 8/9] respect outAddr for SSL --- cowrie/commands/wget.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cowrie/commands/wget.py b/cowrie/commands/wget.py index ddcfa16..fd8a61f 100644 --- a/cowrie/commands/wget.py +++ b/cowrie/commands/wget.py @@ -154,6 +154,7 @@ class command_wget(HoneyPotCommand): factory = HTTPProgressDownloader( self, fakeoutfile, url, outputfile, *args, **kwargs) + out_addr = None if self.protocol.cfg.has_option('honeypot', 'out_addr'): out_addr = (self.protocol.cfg.get('honeypot', 'out_addr'), 0) @@ -161,10 +162,13 @@ class command_wget(HoneyPotCommand): if scheme == 'https': contextFactory = ssl.ClientContextFactory() contextFactory.method = SSL.SSLv23_METHOD - self.connection = reactor.connectSSL(host, port, factory, contextFactory) - else: # Can only be http, since we raised an error above for unknown schemes + self.connection = reactor.connectSSL( + host, port, factory, contextFactory, bindAddress=out_addr) + elif scheme = 'http': self.connection = reactor.connectTCP( host, port, factory, bindAddress=out_addr) + else: + raise NotImplementedError return factory.deferred From 504c56b5b7e67f68b9cb93f845e259f1f829f152 Mon Sep 17 00:00:00 2001 From: Michel Oosterhof Date: Thu, 29 Sep 2016 15:40:29 +0400 Subject: [PATCH 9/9] typofix --- cowrie/commands/wget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cowrie/commands/wget.py b/cowrie/commands/wget.py index fd8a61f..05c7923 100644 --- a/cowrie/commands/wget.py +++ b/cowrie/commands/wget.py @@ -164,7 +164,7 @@ class command_wget(HoneyPotCommand): contextFactory.method = SSL.SSLv23_METHOD self.connection = reactor.connectSSL( host, port, factory, contextFactory, bindAddress=out_addr) - elif scheme = 'http': + elif scheme == 'http': self.connection = reactor.connectTCP( host, port, factory, bindAddress=out_addr) else: