Append redirection support (#428)

* Add support for '>>' redirection

* Add redir files hashing

* Delete only ">" or ">>" + file name from cmd args

* Update stdin/redir messages to include SHA-256 hash of the file content

* Small style fixes, log if we don't store duplicate

* Bug fixes for wget command

* Use os.path.join instead of string formatting
* Use "with" for hashing a file to prevent handle leakage
* Don't overwrite self.safeoutfile if it was already set in HoneyPotyCommand's init method
* Don't overwrite self.safeoutfile with hash, else it will break stuff in insults.py

* Revert "Delete only ">" or ">>" + file name from cmd args"

This reverts commit f3f8b90cbe221da8ffba2670f4419da105ad8ac3.

* Fix bugged check for presence of safeoutfile attribute.

* Don't overwrite safeoutfile in curl

* Don't store None objects

* Include transportId and sessionId to all safeoutfiles to avoid collisions.

cowrie/commands/curl.py

@@ -92,10 +92,14 @@ class command_curl(HoneyPotCommand):
 
         self.download_path = cfg.get('honeypot', 'download_path')
 
-        self.safeoutfile = '%s/%s_%s' % \
+        if not hasattr(self, 'safeoutfile'):
-            (self.download_path,
+            tmp_fname = '%s_%s_%s_%s' % \
-            time.strftime('%Y%m%d%H%M%S'),
+                        (time.strftime('%Y%m%d%H%M%S'),
-            re.sub('[^A-Za-z0-9]', '_', url))
+                         self.protocol.getProtoTransport().transportId,
+                         self.protocol.terminal.transport.session.id,
+                         re.sub('[^A-Za-z0-9]', '_', url))
+            self.safeoutfile = os.path.join(self.download_path, tmp_fname)
+
         self.deferred = self.download(url, outfile, self.safeoutfile)
         if self.deferred:
             self.deferred.addCallback(self.success, outfile)
@@ -311,7 +315,7 @@ Options: (H) means HTTP/HTTPS only, (F) means FTP only
             self.exit()
 
         shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest()
-        hashPath = '%s/%s' % (self.download_path, shasum)
+        hashPath = os.path.join(self.download_path, shasum)
 
         # If we have content already, delete temp file
         if not os.path.exists(hashPath):
@@ -336,7 +340,7 @@ Options: (H) means HTTP/HTTPS only, (F) means FTP only
         os.symlink(shasum, self.safeoutfile)
 
         # FIXME: is this necessary?
-        self.safeoutfile = hashPath
+        # self.safeoutfile = hashPath
 
         # Update the honeyfs to point to downloaded file
         if outfile is not None:

cowrie/commands/ftpget.py

@@ -93,10 +93,13 @@ Download a file via FTP
         cfg = self.protocol.cfg
         url = 'ftp://%s/%s' % (self.host, self.remote_path)
         self.download_path = cfg.get('honeypot', 'download_path')
-        self.safeoutfile = '%s/%s_%s' % \
+
-                           (self.download_path,
+        tmp_fname = '%s_%s_%s_%s' % \
-                            time.strftime('%Y%m%d%H%M%S'),
+                    (time.strftime('%Y%m%d%H%M%S'),
-                            re.sub('[^A-Za-z0-9]', '_', url))
+                     self.protocol.getProtoTransport().transportId,
+                     self.protocol.terminal.transport.session.id,
+                     re.sub('[^A-Za-z0-9]', '_', url))
+        self.safeoutfile = os.path.join(self.download_path, tmp_fname)
 
         result = self.ftp_download(self.safeoutfile)
 
@@ -110,7 +113,7 @@ Download a file via FTP
             return
 
         shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest()
-        hash_path = '%s/%s' % (self.download_path, shasum)
+        hash_path = os.path.join(self.download_path, shasum)
 
         # If we have content already, delete temp file
         if not os.path.exists(hash_path):

cowrie/commands/gcc.py

@@ -4,6 +4,7 @@ import time
 import re
 import getopt
 import random
+import os
 
 from twisted.internet import reactor
 
@@ -164,10 +165,12 @@ gcc version %s (Debian %s-5)""" % (version, version_short, version_short, versio
     def generate_file(self, outfile):
         data = ""
         # TODO: make sure it is written to temp file, not downloads
-        safeoutfile = '%s/%s_%s' % \
+        tmp_fname = '%s_%s_%s_%s' % \
-            (self.protocol.cfg.get('honeypot', 'download_path'),
+                    (time.strftime('%Y%m%d%H%M%S'),
-            time.strftime('%Y%m%d%H%M%S'),
+                     self.protocol.getProtoTransport().transportId,
-            re.sub('[^A-Za-z0-9]', '_', outfile))
+                     self.protocol.terminal.transport.session.id,
+                     re.sub('[^A-Za-z0-9]', '_', outfile))
+        safeoutfile = os.path.join(self.protocol.cfg.get('honeypot', 'download_path'), tmp_fname)
 
         # Data contains random garbage from an actual file, so when
         # catting the file, you'll see some 'real' compiled data

cowrie/commands/wget.py

@@ -121,10 +121,14 @@ class command_wget(HoneyPotCommand):
 
         self.download_path = cfg.get('honeypot', 'download_path')
 
-        self.safeoutfile = '%s/%s_%s' % \
+        if not hasattr(self, 'safeoutfile'):
-            (self.download_path,
+            tmp_fname = '%s_%s_%s_%s' % \
-            time.strftime('%Y%m%d%H%M%S'),
+                        (time.strftime('%Y%m%d%H%M%S'),
-            re.sub('[^A-Za-z0-9]', '_', url))
+                         self.protocol.getProtoTransport().transportId,
+                         self.protocol.terminal.transport.session.id,
+                         re.sub('[^A-Za-z0-9]', '_', url))
+            self.safeoutfile = os.path.join(self.download_path, tmp_fname)
+
         self.deferred = self.download(url, outfile, self.safeoutfile)
         if self.deferred:
             self.deferred.addCallback(self.success, outfile)
@@ -188,8 +192,9 @@ class command_wget(HoneyPotCommand):
             log.msg("there's no file " + self.safeoutfile)
             self.exit()
 
-        shasum = hashlib.sha256(open(self.safeoutfile, 'rb').read()).hexdigest()
+        with open(self.safeoutfile, 'rb') as f:
-        hash_path = '%s/%s' % (self.download_path, shasum)
+            shasum = hashlib.sha256(f.read()).hexdigest()
+        hash_path = os.path.join(self.download_path, shasum)
 
         # If we have content already, delete temp file
         if not os.path.exists(hash_path):
@@ -211,10 +216,10 @@ class command_wget(HoneyPotCommand):
                 shasum=shasum)
 
         # Link friendly name to hash
-        os.symlink( shasum, self.safeoutfile )
+        os.symlink(shasum, self.safeoutfile)
 
         # FIXME: is this necessary?
-        self.safeoutfile = hash_path
+        # self.safeoutfile = hash_path
 
         # Update the honeyfs to point to downloaded file
         f = self.fs.getfile(outfile)

cowrie/core/honeypot.py

@@ -32,31 +32,40 @@ class HoneyPotCommand(object):
         self.errorWrite = self.protocol.pp.errReceived
         # MS-DOS style redirect handling, inside the command
         # TODO: handle >>, 2>, etc
-        if '>' in self.args:
+        if '>' in self.args or '>>' in self.args:
             self.writtenBytes = 0
             self.write = self.write_to_file
-            index = self.args.index(">")
+            if '>>' in self.args:
+                index = self.args.index('>>')
+                b_append = True
+            else:
+                index = self.args.index('>')
+                b_append = False
             self.outfile = self.fs.resolve_path(str(self.args[(index + 1)]), self.protocol.cwd)
             del self.args[index:]
-            self.safeoutfile = '%s/%s-%s-%s-redir_%s' % (
+            p = self.fs.getfile(self.outfile)
-                self.protocol.cfg.get('honeypot', 'download_path'),
+            if not p or not p[fs.A_REALFILE] or p[fs.A_REALFILE].startswith('honeyfs') or not b_append:
-                time.strftime('%Y%m%d-%H%M%S'),
+                tmp_fname = '%s-%s-%s-redir_%s' % \
-                self.protocol.getProtoTransport().transportId,
+                            (time.strftime('%Y%m%d-%H%M%S'),
-                self.protocol.terminal.transport.session.id,
+                             self.protocol.getProtoTransport().transportId,
-                re.sub('[^A-Za-z0-9]', '_', self.outfile))
+                             self.protocol.terminal.transport.session.id,
-            perm = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+                             re.sub('[^A-Za-z0-9]', '_', self.outfile))
-            try:
+                self.safeoutfile = os.path.join(self.protocol.cfg.get('honeypot', 'download_path'), tmp_fname)
-                self.fs.mkfile(self.outfile, 0, 0, 0, stat.S_IFREG | perm)
+                perm = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
-            except fs.FileNotFound:
+                try:
-                # The outfile locates at a non-existing directory.
+                    self.fs.mkfile(self.outfile, 0, 0, 0, stat.S_IFREG | perm)
-                self.protocol.pp.outReceived('-bash: %s: No such file or directory\n' % self.outfile)
+                except fs.FileNotFound:
-                self.write = self.write_to_failed
+                    # The outfile locates at a non-existing directory.
-                self.outfile = None
+                    self.protocol.pp.outReceived('-bash: %s: No such file or directory\n' % self.outfile)
-                self.safeoutfile = None
+                    self.write = self.write_to_failed
+                    self.outfile = None
+                    self.safeoutfile = None
 
+                else:
+                    with open(self.safeoutfile, 'a'):
+                        self.fs.update_realfile(self.fs.getfile(self.outfile), self.safeoutfile)
             else:
-                with open(self.safeoutfile, 'a'):
+                self.safeoutfile = p[fs.A_REALFILE]
-                    self.fs.update_realfile(self.fs.getfile(self.outfile), self.safeoutfile)
 
 
     def check_arguments(self, application, args):

cowrie/core/protocol.py

@@ -195,6 +195,9 @@ class HoneyPotBaseProtocol(insults.TerminalProtocol, TimeoutMixin):
         obj.set_input_data(pp.input_data)
         self.cmdstack.append(obj)
         obj.start()
+        if hasattr(obj, 'safeoutfile'):
+            if obj.safeoutfile:
+                self.terminal.redirFiles.add(obj.safeoutfile)
         if self.pp:
             self.pp.outConnectionLost()
 

cowrie/insults/insults.py

@@ -31,6 +31,8 @@ class LoggingServerProtocol(insults.ServerProtocol):
         self.ttylogPath = cfg.get('honeypot', 'log_path')
         self.downloadPath = cfg.get('honeypot', 'download_path')
 
+        self.redirFiles = set()
+
         try:
             self.bytesReceivedLimit = int(cfg.get('honeypot',
                 'download_limit_size'))
@@ -134,14 +136,15 @@ class LoggingServerProtocol(insults.ServerProtocol):
             try:
                 with open(self.stdinlogFile, 'rb') as f:
                     shasum = hashlib.sha256(f.read()).hexdigest()
-                    shasumfile = self.downloadPath + "/" + shasum
+                    shasumfile = os.path.join(self.downloadPath, shasum)
-                    if (os.path.exists(shasumfile)):
+                    if os.path.exists(shasumfile):
                         os.remove(self.stdinlogFile)
+                        log.msg("Not storing duplicate content " + shasum)
                     else:
                         os.rename(self.stdinlogFile, shasumfile)
                     os.symlink(shasum, self.stdinlogFile)
                 log.msg(eventid='cowrie.session.file_download',
-                        format='Saved stdin contents to %(outfile)s',
+                        format='Saved stdin contents with SHA-256 %(shasum)s to %(outfile)s',
                         url='stdin',
                         outfile=shasumfile,
                         shasum=shasum)
@@ -150,6 +153,34 @@ class LoggingServerProtocol(insults.ServerProtocol):
             finally:
                 self.stdinlogOpen = False
 
+        if self.redirFiles:
+            for rf in self.redirFiles:
+                try:
+                    if not os.path.exists(rf):
+                        continue
+
+                    if os.path.getsize(rf) == 0:
+                        os.remove(rf)
+                        continue
+
+                    with open(rf, 'rb') as f:
+                        shasum = hashlib.sha256(f.read()).hexdigest()
+                        shasumfile = os.path.join(self.downloadPath, shasum)
+                        if os.path.exists(shasumfile):
+                            os.remove(rf)
+                            log.msg("Not storing duplicate content " + shasum)
+                        else:
+                            os.rename(rf, shasumfile)
+                        os.symlink(shasum, rf)
+                    log.msg(eventid='cowrie.session.file_download',
+                            format='Saved redir contents with SHA-256 %(shasum)s to %(outfile)s',
+                            url='redir',
+                            outfile=shasumfile,
+                            shasum=shasum)
+                except IOError:
+                    pass
+            self.redirFiles.clear()
+
         if self.ttylogOpen:
             # TODO: Add session duration to this entry
             log.msg(eventid='cowrie.log.closed',