remove all dblog

This commit is contained in:
Michel Oosterhof
2016-11-20 15:13:44 +04:00
parent b719908559
commit 9b418baff3
6 changed files with 0 additions and 658 deletions

View File

@@ -1,257 +0,0 @@
## From https://github.com/threatstream/kippo/blob/master/kippo/dblog/hpfeeds.py
from cowrie.core import dblog
from twisted.python import log
from datetime import datetime
import os
import struct
import hashlib
import json
import socket
import uuid
BUFSIZ = 16384
OP_ERROR = 0
OP_INFO = 1
OP_AUTH = 2
OP_PUBLISH = 3
OP_SUBSCRIBE = 4
MAXBUF = 1024**2
SIZES = {
OP_ERROR: 5+MAXBUF,
OP_INFO: 5+256+20,
OP_AUTH: 5+256+20,
OP_PUBLISH: 5+MAXBUF,
OP_SUBSCRIBE: 5+256*2,
}
COWRIECHAN = 'cowrie.sessions'
class BadClient(Exception):
pass
# packs a string with 1 byte length field
def strpack8(x):
if isinstance(x, str): x = x.encode('latin1')
return struct.pack('!B', len(x)) + x
# unpacks a string with 1 byte length field
def strunpack8(x):
l = x[0]
return x[1:1+l], x[1+l:]
def msghdr(op, data):
return struct.pack('!iB', 5+len(data), op) + data
def msgpublish(ident, chan, data):
return msghdr(OP_PUBLISH, strpack8(ident) + strpack8(chan) + data)
def msgsubscribe(ident, chan):
if isinstance(chan, str): chan = chan.encode('latin1')
return msghdr(OP_SUBSCRIBE, strpack8(ident) + chan)
def msgauth(rand, ident, secret):
hash = hashlib.sha1(bytes(rand)+secret).digest()
return msghdr(OP_AUTH, strpack8(ident) + hash)
class FeedUnpack(object):
def __init__(self):
self.buf = bytearray()
def __iter__(self):
return self
def next(self):
return self.unpack()
def feed(self, data):
self.buf.extend(data)
def unpack(self):
if len(self.buf) < 5:
raise StopIteration('No message.')
ml, opcode = struct.unpack('!iB', buffer(self.buf,0,5))
if ml > SIZES.get(opcode, MAXBUF):
raise BadClient('Not respecting MAXBUF.')
if len(self.buf) < ml:
raise StopIteration('No message.')
data = bytearray(buffer(self.buf, 5, ml-5))
del self.buf[:ml]
return opcode, data
class hpclient(object):
def __init__(self, server, port, ident, secret, debug):
print 'hpfeeds client init broker {0}:{1}, identifier {2}'.format(server, port, ident)
self.server, self.port = server, int(port)
self.ident, self.secret = ident.encode('latin1'), secret.encode('latin1')
self.debug = debug
self.unpacker = FeedUnpack()
self.state = 'INIT'
self.connect()
self.sendfiles = []
self.filehandle = None
def connect(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.settimeout(3)
try: self.s.connect((self.server, self.port))
except:
print 'hpfeeds client could not connect to broker.'
self.s = None
else:
self.s.settimeout(None)
self.handle_established()
def send(self, data):
if not self.s:
self.connect()
if not self.s: return
self.s.send(data)
def close(self):
self.s.close()
self.s = None
def handle_established(self):
if self.debug: print 'hpclient established'
while self.state != 'GOTINFO':
self.read()
#quickly try to see if there was an error message
self.s.settimeout(0.5)
self.read()
self.s.settimeout(None)
def read(self):
if not self.s: return
try: d = self.s.recv(BUFSIZ)
except socket.timeout:
return
if not d:
if self.debug: log.msg('hpclient connection closed?')
self.close()
return
self.unpacker.feed(d)
try:
for opcode, data in self.unpacker:
if self.debug: log.msg('hpclient msg opcode {0} data {1}'.format(opcode, data))
if opcode == OP_INFO:
name, rand = strunpack8(data)
if self.debug: log.msg('hpclient server name {0} rand {1}'.format(name, rand))
self.send(msgauth(rand, self.ident, self.secret))
self.state = 'GOTINFO'
elif opcode == OP_PUBLISH:
ident, data = strunpack8(data)
chan, data = strunpack8(data)
if self.debug: log.msg('publish to {0} by {1}: {2}'.format(chan, ident, data))
elif opcode == OP_ERROR:
log.err('errormessage from server: {0}'.format(data))
else:
log.err('unknown opcode message: {0}'.format(opcode))
except BadClient:
log.err('unpacker error, disconnecting.')
self.close()
def publish(self, channel, **kwargs):
try:
self.send(msgpublish(self.ident, channel, json.dumps(kwargs).encode('latin1')))
except Exception, e:
log.err('connection to hpfriends lost: {0}'.format(e))
log.err('connecting')
self.connect()
self.send(msgpublish(self.ident, channel, json.dumps(kwargs).encode('latin1')))
def sendfile(self, filepath):
# does not read complete binary into memory, read and send chunks
if not self.filehandle:
self.sendfileheader(i.file)
self.sendfiledata()
else: self.sendfiles.append(filepath)
def sendfileheader(self, filepath):
self.filehandle = open(filepath, 'rb')
fsize = os.stat(filepath).st_size
headc = strpack8(self.ident) + strpack8(UNIQUECHAN)
headh = struct.pack('!iB', 5+len(headc)+fsize, OP_PUBLISH)
self.send(headh + headc)
def sendfiledata(self):
tmp = self.filehandle.read(BUFSIZ)
if not tmp:
if self.sendfiles:
fp = self.sendfiles.pop(0)
self.sendfileheader(fp)
else:
self.filehandle = None
self.handle_io_in(b'')
else:
self.send(tmp)
class DBLogger(dblog.DBLogger):
def start(self, cfg):
print 'hpfeeds DBLogger start'
server = cfg.get('database_hpfeeds', 'server')
port = cfg.get('database_hpfeeds', 'port')
ident = cfg.get('database_hpfeeds', 'identifier')
secret = cfg.get('database_hpfeeds', 'secret')
debug = cfg.get('database_hpfeeds', 'debug')
self.client = hpclient(server, port, ident, secret, debug)
self.meta = {}
# We have to return an unique ID
def createSession(self, peerIP, peerPort, hostIP, hostPort):
session = uuid.uuid4().hex
startTime=datetime.now().isoformat()
self.meta[session] = {'session':session,'startTime':startTime,'endTime':'','peerIP': peerIP, 'peerPort': peerPort,
'hostIP': hostIP, 'hostPort': hostPort, 'loggedin': None,
'credentials':[], 'commands':[],"unknownCommands":[],'urls':[],'version': None, 'ttylog': None }
return session
def handleConnectionLost(self, session, args):
log.msg('publishing metadata to hpfeeds')
meta = self.meta[session]
self.meta[session]['endTime']=datetime.now().isoformat()
ttylog = self.ttylog(session)
if ttylog: meta['ttylog'] = ttylog.encode('hex')
self.client.publish(COWRIECHAN, **meta)
def handleLoginFailed(self, session, args):
u, p = args['username'], args['password']
self.meta[session]['credentials'].append((u,p))
def handleLoginSucceeded(self, session, args):
u, p = args['username'], args['password']
self.meta[session]['loggedin'] = (u,p)
def handleCommand(self, session, args):
c = args['input']
self.meta[session]['commands'].append(c)
def handleUnknownCommand(self, session, args):
uc = args['input']
self.meta[session]['unknownCommands'].append(uc)
def handleInput(self, session, args):
pass
def handleTerminalSize(self, session, args):
pass
def handleClientVersion(self, session, args):
v = args['version']
self.meta[session]['version'] = v
def handleFileDownload(self,session,args):
url = args['url']
self.meta[session]['urls'].append(url)
# vim: set sw=4 et:

View File

@@ -1,149 +0,0 @@
from cowrie.core import dblog
from twisted.enterprise import adbapi
from twisted.internet import defer
from twisted.python import log
import MySQLdb, uuid
class ReconnectingConnectionPool(adbapi.ConnectionPool):
"""Reconnecting adbapi connection pool for MySQL.
This class improves on the solution posted at
http://www.gelens.org/2008/09/12/reinitializing-twisted-connectionpool/
by checking exceptions by error code and only disconnecting the current
connection instead of all of them.
Also see:
http://twistedmatrix.com/pipermail/twisted-python/2009-July/020007.html
"""
def _runInteraction(self, interaction, *args, **kw):
try:
return adbapi.ConnectionPool._runInteraction(
self, interaction, *args, **kw)
except MySQLdb.OperationalError as e:
if e[0] not in (2003, 2006, 2013):
raise
log.msg("RCP: got error %s, retrying operation" %(e))
conn = self.connections.get(self.threadID())
self.disconnect(conn)
# try the interaction again
return adbapi.ConnectionPool._runInteraction(
self, interaction, *args, **kw)
class DBLogger(dblog.DBLogger):
def start(self, cfg):
if cfg.has_option('database_mysql', 'port'):
port = int(cfg.get('database_mysql', 'port'))
else:
port = 3306
self.db = ReconnectingConnectionPool('MySQLdb',
host = cfg.get('database_mysql', 'host'),
db = cfg.get('database_mysql', 'database'),
user = cfg.get('database_mysql', 'username'),
passwd = cfg.get('database_mysql', 'password'),
port = port,
cp_min = 1,
cp_max = 1)
def sqlerror(self, error):
log.msg( 'SQL Error:', error.value )
def simpleQuery(self, sql, args):
""" Just run a deferred sql query, only care about errors """
d = self.db.runQuery(sql, args)
d.addErrback(self.sqlerror)
def createSession(self, peerIP, peerPort, hostIP, hostPort):
sid = uuid.uuid4().hex
self.createSessionWhenever(sid, peerIP, hostIP)
return sid
# This is separate since we can't return with a value
@defer.inlineCallbacks
def createSessionWhenever(self, sid, peerIP, hostIP):
sensorname = self.getSensor() or hostIP
r = yield self.db.runQuery(
'SELECT `id` FROM `sensors` WHERE `ip` = %s', (sensorname,))
if r:
id = r[0][0]
else:
yield self.db.runQuery(
'INSERT INTO `sensors` (`ip`) VALUES (%s)', (sensorname,))
r = yield self.db.runQuery('SELECT LAST_INSERT_ID()')
id = int(r[0][0])
# now that we have a sensorID, continue creating the session
self.simpleQuery(
'INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)' + \
' VALUES (%s, FROM_UNIXTIME(%s), %s, %s)',
(sid, self.nowUnix(), id, peerIP))
def handleConnectionLost(self, session, args):
ttylog = self.ttylog(session)
if ttylog:
self.simpleQuery(
'INSERT INTO `ttylog` (`session`, `ttylog`) VALUES (%s, %s)',
(session, self.ttylog(session)))
self.simpleQuery(
'UPDATE `sessions` SET `endtime` = FROM_UNIXTIME(%s)' + \
' WHERE `id` = %s',
(self.nowUnix(), session))
def handleLoginFailed(self, session, args):
self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \
', `username`, `password`, `timestamp`)' + \
' VALUES (%s, %s, %s, %s, FROM_UNIXTIME(%s))',
(session, 0, args['username'], args['password'], self.nowUnix()))
def handleLoginSucceeded(self, session, args):
self.simpleQuery('INSERT INTO `auth` (`session`, `success`' + \
', `username`, `password`, `timestamp`)' + \
' VALUES (%s, %s, %s, %s, FROM_UNIXTIME(%s))',
(session, 1, args['username'], args['password'], self.nowUnix()))
def handleCommand(self, session, args):
self.simpleQuery('INSERT INTO `input`' + \
' (`session`, `timestamp`, `success`, `input`)' + \
' VALUES (%s, FROM_UNIXTIME(%s), %s, %s)',
(session, self.nowUnix(), 1, args['input']))
def handleUnknownCommand(self, session, args):
self.simpleQuery('INSERT INTO `input`' + \
' (`session`, `timestamp`, `success`, `input`)' + \
' VALUES (%s, FROM_UNIXTIME(%s), %s, %s)',
(session, self.nowUnix(), 0, args['input']))
def handleInput(self, session, args):
self.simpleQuery('INSERT INTO `input`' + \
' (`session`, `timestamp`, `realm`, `input`)' + \
' VALUES (%s, FROM_UNIXTIME(%s), %s, %s)',
(session, self.nowUnix(), args['realm'], args['input']))
def handleTerminalSize(self, session, args):
self.simpleQuery('UPDATE `sessions` SET `termsize` = %s' + \
' WHERE `id` = %s',
('%sx%s' % (args['width'], args['height']), session))
@defer.inlineCallbacks
def handleClientVersion(self, session, args):
r = yield self.db.runQuery(
'SELECT `id` FROM `clients` WHERE `version` = %s', \
(args['version'],))
if r:
id = int(r[0][0])
else:
yield self.db.runQuery(
'INSERT INTO `clients` (`version`) VALUES (%s)', \
(args['version'],))
r = yield self.db.runQuery('SELECT LAST_INSERT_ID()')
id = int(r[0][0])
self.simpleQuery(
'UPDATE `sessions` SET `client` = %s WHERE `id` = %s',
(id, session))
def handleFileDownload(self, session, args):
self.simpleQuery('INSERT INTO `downloads`' + \
' (`session`, `timestamp`, `url`, `outfile`, `shasum`)' + \
' VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s)',
(session, self.nowUnix(), args['url'], args['outfile'], args['shasum']))
# vim: set sw=4 et:

View File

@@ -1,56 +0,0 @@
#
# this module uses the dblog feature to create a "traditional" looking logfile
# ..so not exactly a dblog.
#
from cowrie.core import dblog
import time
import uuid
class DBLogger(dblog.DBLogger):
def start(self, cfg):
self.outfile = file(cfg.get('database_textlog', 'logfile'), 'a')
def write(self, session, msg):
self.outfile.write('%s [%s]: %s\r\n' % \
(session, time.strftime('%Y-%m-%d %H:%M:%S'), msg))
self.outfile.flush()
def createSession(self, peerIP, peerPort, hostIP, hostPort):
sid = uuid.uuid4().hex
sensorname = self.getSensor() or hostIP
self.write(sid, 'New connection: %s:%s' % (peerIP, peerPort))
return sid
def handleConnectionLost(self, session, args):
self.write(session, 'Connection lost')
def handleLoginFailed(self, session, args):
self.write(session, 'Login failed [%s/%s]' % \
(args['username'], args['password']))
def handleLoginSucceeded(self, session, args):
self.write(session, 'Login succeeded [%s/%s]' % \
(args['username'], args['password']))
def handleCommand(self, session, args):
self.write(session, 'Command [%s]' % (args['input'],))
def handleUnknownCommand(self, session, args):
self.write(session, 'Unknown command [%s]' % (args['input'],))
def handleInput(self, session, args):
self.write(session, 'Input [%s] @%s' % (args['input'], args['realm']))
def handleTerminalSize(self, session, args):
self.write(session, 'Terminal size: %sx%s' % \
(args['width'], args['height']))
def handleClientVersion(self, session, args):
self.write(session, 'Client version: [%s]' % (args['version'],))
def handleFileDownload(self, session, args):
self.write(session, 'File download: [%s] -> %s with SHA-256 %s' % \
(args['url'], args['outfile'], args['shasum']))
# vim: set sw=4 et:

View File

@@ -1,175 +0,0 @@
from twisted.words.xish import domish
from twisted.python import log
from wokkel.xmppim import AvailablePresence
from twisted.words.protocols.jabber.jid import JID
from wokkel import muc
import uuid
import json
class XMPPLoggerProtocol(muc.MUCClient):
def __init__(self, rooms, server, nick):
muc.MUCClient.__init__(self)
self.server = rooms.host
self.jrooms = rooms
self._roomOccupantMap = {}
log.msg(rooms.user)
log.msg(rooms.host)
self.nick = nick
self.last = {}
self.activity = None
def connectionInitialized(self):
"""The bot has connected to the xmpp server, now try to join the room.
"""
self.join(self.jrooms, self.nick);
def joinedRoom(self, room):
log.msg( 'Joined room %s' % room.name )
def connectionMade(self):
log.msg( 'Connected!' )
# send initial presence
self.send(AvailablePresence())
def connectionLost(self, reason):
log.msg( 'Disconnected!' )
def onMessage(self, msg):
pass
def receivedGroupChat(self, room, user, body):
pass
def receivedHistory(self, room, user, body, dely, frm=None):
pass
from twisted.application import service
from twisted.words.protocols.jabber import jid
from wokkel.client import XMPPClient
from cowrie.core import dblog
from twisted.words.xish import domish
class DBLogger(dblog.DBLogger):
def start(self, cfg):
from random import choice
import string
server = cfg.get('database_xmpp', 'server')
user = cfg.get('database_xmpp', 'user')
password = cfg.get('database_xmpp', 'password')
muc = cfg.get('database_xmpp', 'muc')
channels = {}
for i in ('createsession', 'connectionlost', 'loginfailed',
'loginsucceeded', 'command', 'clientversion'):
x = cfg.get('database_xmpp', 'signal_' + i)
if not x in channels:
channels[x] = []
channels[x].append(i)
resource = ''.join([choice(string.ascii_letters)
for i in range(8)])
jid = user + '/' + resource
application = service.Application('honeypot')
self.run(application, jid, password, JID(None,[muc,server,None]), channels)
def run(self, application, jidstr, password, muc, channels, anon=True):
self.xmppclient = XMPPClient(JID(jidstr), password)
if self.cfg.has_option('database_xmpp', 'debug') and \
self.cfg.getboolean('database_xmpp', 'debug') == True:
self.xmppclient.logTraffic = True # DEBUG HERE
(user, host, resource) = jid.parse(jidstr)
self.muc = XMPPLoggerProtocol(
muc, channels.keys(), user + '-' + resource)
self.muc.setHandlerParent(self.xmppclient)
self.xmppclient.setServiceParent(application)
self.signals = {}
for channel in channels:
for signal in channels[channel]:
self.signals[signal] = channel
self.anonymous = True
self.xmppclient.startService()
def broadcast(self, msgtype, msg):
if msgtype in self.signals:
self.report(msgtype, '%s@%s' %
(self.signals[msgtype], self.muc.server) , msg)
def report(self, msgtype, to, xmsg):
msg = {}
msg['type'] = msgtype
msg['message'] = xmsg
msgJson = json.dumps(msg,indent=5)
self.muc.groupChat(self.muc.jrooms, msgJson)
# We have to return an unique ID
def createSession(self, peerIP, peerPort, hostIP, hostPort):
session = uuid.uuid4().hex
ses = {}
ses['session'] = session
ses['remote_host'] = peerIP
ses['remote_port'] = str(peerPort)
if self.anonymous == True:
ses['local_host'] = '127.0.0.1'
else:
ses['local_host'] = hostIP
ses['local_port'] = str(hostPort)
self.broadcast('createsession', ses)
return session
def handleTTYLogOpened(self, session, args):
pass
def handleConnectionLost(self, session, args):
ses = {}
ses['session'] = session
self.broadcast('connectionlost', ses)
def handleLoginFailed(self, session, args):
ses = {}
ses['session'] = session
ses['username'] = args['username']
ses['password'] = args['password']
self.broadcast('loginfailed', ses)
def handleLoginSucceeded(self, session, args):
ses = {}
ses['session'] = session
ses['username'] = args['username']
ses['password'] = args['password']
self.broadcast('loginsucceeded', ses)
def handleCommand(self, session, args):
ses = {}
ses['session'] = session
ses['command'] = 'known'
ses['input'] = args['input']
self.broadcast('command', ses)
def handleUnknownCommand(self, session, args):
ses = {}
ses['session'] = session
ses['command'] = 'unknown'
ses['input'] = args['input']
self.broadcast('command', ses)
def handleInput(self, session, args):
ses = {}
ses['session'] = session
ses['realm'] = args['realm']
ses['input'] = args['input']
self.broadcast('input', ses)
def handleTerminalSize(self, session, args):
pass
def handleClientVersion(self, session, args):
ses = {}
ses['session'] = session
ses['version'] = args['version']
self.broadcast('clientversion', ses)
# vim: set sw=4 et:

View File

@@ -275,27 +275,6 @@ enabled = true
#reported_port = 23
# ============================================================================
# Database logging Specific Options
# ============================================================================
# XMPP Logging
# Log to an xmpp server.
#
#[database_xmpp]
#server = sensors.carnivore.it
#user = anonymous@sensors.carnivore.it
#password = anonymous
#muc = dionaea.sensors.carnivore.it
#signal_createsession = cowrie-events
#signal_connectionlost = cowrie-events
#signal_loginfailed = cowrie-events
#signal_loginsucceeded = cowrie-events
#signal_command = cowrie-events
#signal_clientversion = cowrie-events
#debug=true
# ============================================================================
# Output Plugins