From 75f60c0266bd37dcd1557e723e4dea0c033e00d3 Mon Sep 17 00:00:00 2001 From: epi052 Date: Sun, 12 Jan 2020 12:50:15 -0600 Subject: [PATCH] added interactive shell for improved user-experience --- luigid.service | 10 + recon-pipeline.py | 313 ++++++++++++++++++++++++++++++++ recon/config.py | 12 +- recon/masscan.py | 8 +- recon/nmap.py | 8 +- recon/web/aquatone.py | 4 +- recon/web/subdomain_takeover.py | 2 +- recon/web/webanalyze.py | 1 - recon/wrappers.py | 6 +- 9 files changed, 343 insertions(+), 21 deletions(-) create mode 100644 luigid.service create mode 100755 recon-pipeline.py diff --git a/luigid.service b/luigid.service new file mode 100644 index 0000000..9c53e0c --- /dev/null +++ b/luigid.service @@ -0,0 +1,10 @@ +[Unit] +Description=Spotify Luigi server +Documentation=https://luigi.readthedocs.io/en/stable/ +Requires=network.target remote-fs.target +After=network.target remote-fs.target +[Service] +Type=simple +ExecStart=/usr/local/bin/luigid --background --pidfile /var/run/luigid.pid --logdir /var/log/luigi +[Install] +WantedBy=multi-user.target diff --git a/recon-pipeline.py b/recon-pipeline.py new file mode 100755 index 0000000..10cbf8e --- /dev/null +++ b/recon-pipeline.py @@ -0,0 +1,313 @@ +import sys +import shlex +import pickle +import socket +import inspect +import pkgutil +import subprocess +from pathlib import Path +from collections import defaultdict + +import cmd2 +from cmd2.ansi import style + +import recon + + +def get_scans(): + """ Iterates over the recon package and its modules to find all of the *Scan classes. + + * A contract exists here that says any scans need to end with the word scan in order to be found by this function. + + Returns: + dict() containing mapping of {modulename: classname} for all potential recon-pipeline commands + """ + scans = defaultdict(list) + + # recursively walk packages; import each module in each package + for loader, module_name, is_pkg in pkgutil.walk_packages(recon.__path__, prefix="recon."): + _module = loader.find_module(module_name).load_module(module_name) + globals()[module_name] = _module + + # walk all modules, grabbing classes that we've written and add them to the classlist set + for name, obj in inspect.getmembers(sys.modules[__name__]): + if inspect.ismodule(obj) and not name.startswith("_"): + for subname, subobj in inspect.getmembers(obj): + if inspect.isclass(subobj) and subname.lower().endswith("scan"): + scans[subname].append(name) + + return scans + + +# tool definitions for the auto-installer +tools = { + "go": {"installed": False, "dependencies": None, "commands": ["apt-get install -y -q golang"]}, + "luigi": { + "installed": False, + "dependencies": ["pipenv", "luigi-service"], + "commands": ["pipenv install luigi"], + }, + "luigi-service": { + "installed": False, + "dependencies": None, + "commands": [ + f"cp {str(Path(__file__).parent / 'luigid.service')} /lib/systemd/system/luigid.service", + "systemctl start luigid.service", + "systemctl enable luigid.service", + ], + }, + "pipenv": { + "installed": False, + "dependencies": None, + "commands": ["apt-get install -y -q pipenv"], + }, + "masscan": { + "installed": False, + "dependencies": None, + "commands": [ + "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan", + "make -s -j -C /tmp/masscan", + "mv /tmp/bin/masscan /usr/bin/masscan", + "rm -rf /tmp/masscan", + ], + }, + "amass": {"installed": False, "dependencies": None, "commands": "apt-get install -y -q amass"}, + "aquatone": { + "installed": False, + "dependencies": None, + "commands": [ + "mkdir /tmp/aquatone", + "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip", + "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone", + "mv /tmp/aquatone/aquatone /usr/bin/aquatone", + "rm -rf /tmp/aquatone", + ], + }, + "corscanner": { + "installed": False, + "dependencies": None, + "commands": [ + "git clone https://github.com/chenjj/CORScanner.git /opt/CORScanner", + "pip install -r /opt/CORScanner/requirements.txt", + "pip install future", + ], + }, + "gobuster": { + "installed": False, + "dependencies": ["go"], + "commands": [ + "go get github.com/OJ/gobuster", + "(cd ~/go/src/github.com/OJ/gobuster && go build && go install)", + ], + "shell": True, + }, + "tko-subs": { + "installed": False, + "dependencies": ["go"], + "commands": [ + "go get github.com/anshumanbh/tko-subs", + "(cd ~/go/src/github.com/anshumanbh/tko-subs && go build && go install)", + ], + "shell": True, + }, + "subjack": { + "installed": False, + "dependencies": ["go"], + "commands": [ + "go get github.com/haccer/subjack", + "(cd ~/go/src/github.com/haccer/subjack && go build && go install)", + ], + "shell": True, + }, + "webanalyze": { + "installed": False, + "dependencies": ["go"], + "commands": [ + "go get github.com/rverton/webanalyze", + "(cd ~/go/src/github.com/rverton/webanalyze && go build && go install)", + ], + "shell": True, + }, + "recursive-gobuster": { + "installed": False, + "dependencies": None, + "commands": [ + "git clone https://github.com/epi052/recursive-gobuster.git /opt/recursive-gobuster", + "ln -s /opt/recursive-gobuster/recursive-gobuster.pyz /usr/local/bin", + ], + }, +} + +# options for ReconShell's 'scan' command +scan_parser = cmd2.Cmd2ArgumentParser() +scan_parser.add_argument("scantype", choices_function=get_scans) +scan_parser.add_argument( + "--target-file", + completer_method=cmd2.Cmd.path_complete, + help="file created by the user that defines the target's scope; list of ips/domains", +) +scan_parser.add_argument( + "--exempt-list", completer_method=cmd2.Cmd.path_complete, help="list of blacklisted ips/domains" +) +scan_parser.add_argument( + "--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster" +) +scan_parser.add_argument( + "--interface", + choices_function=lambda: [x[1] for x in socket.if_nameindex()], + help="which interface masscan should use", +) +scan_parser.add_argument( + "--recursive", action="store_true", help="whether or not to recursively gobust" +) +scan_parser.add_argument("--rate", help="rate at which masscan should scan") +scan_parser.add_argument( + "--top-ports", + help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)", +) +scan_parser.add_argument( + "--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)" +) +scan_parser.add_argument( + "--threads", help="number of threads for all of the threaded applications to use" +) +scan_parser.add_argument("--scan-timeout", help="scan timeout for aquatone") +scan_parser.add_argument("--proxy", help="proxy for gobuster if desired (ex. 127.0.0.1:8080)") +scan_parser.add_argument("--extensions", help="list of extensions for gobuster (ex. asp,html,aspx)") +scan_parser.add_argument( + "--local-scheduler", + action="store_true", + help="use the local scheduler instead of the central scheduler (luigid)", +) +scan_parser.add_argument( + "--verbose", + action="store_true", + help="shows debug messages from luigi, useful for troubleshooting", +) + +# options for ReconShell's 'install' command +install_parser = cmd2.Cmd2ArgumentParser() +install_parser.add_argument( + "tool", help="which tool to install", choices=list(tools.keys()) + ["all"] +) + + +class ReconShell(cmd2.Cmd): + prompt = "recon-pipeline> " + + @cmd2.with_argparser(scan_parser) + def do_scan(self, args): + """ Scan something. + + Possible scans include + AmassScan CORScannerScan GobusterScan SearchsploitScan + ThreadedNmapScan WebanalyzeScan AquatoneScan FullScan + MasscanScan SubjackScan TKOSubsScan + """ + scans = get_scans() + command = ["luigi", "--module", scans.get(args.scantype)[0]] + + command.extend(args.__statement__.arg_list) + + sentry = False + if args.verbose: + command.pop( + command.index("--verbose") + ) # verbose is not a luigi option, need to remove it + subprocess.run(command) + else: + # suppress luigi messages in favor of less verbose/cleaner output + proc = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + + while True: + output = proc.stderr.readline() + if not output and proc.poll() is not None: + break + if not output: + continue + + output = output.decode() + if "===== Luigi Execution Summary =====" in output: + self.poutput() + sentry = True + + # block below used for printing status messages + if sentry: + self.poutput(style(output.strip(), fg="bright_blue")) + elif output.startswith("INFO") and output.split()[-1] == "PENDING": + words = output.split() + self.poutput(style(f"[-] {words[5].split('_')[0]} queued", fg="bright_white")) + elif output.startswith("INFO") and "running" in output: + words = output.split() + # output looks similar to , pid=3938074) running MasscanScan( + # want to grab the index of the luigi task running + scantypeidx = words.index("running") + 1 + scantype = words[scantypeidx].split("(", 1)[0] + self.poutput(style(f"[*] {scantype} running...", fg="bright_yellow")) + elif output.startswith("INFO") and output.split()[-1] == "DONE": + words = output.split() + self.poutput( + style(f"[+] {words[5].split('_')[0]} complete!", fg="bright_green") + ) + self.async_alert( + style( + "If anything went wrong, rerun your command with --verbose to enable debug statements.", + fg="cyan", + dim=True, + ) + ) + + @cmd2.with_argparser(install_parser) + def do_install(self, args): + """ Install any/all of the libraries/tools necessary to make the recon-pipeline function. """ + global tools + + cachedir = Path.home() / ".cache/" + + cachedir.mkdir(parents=True, exist_ok=True) + + persistent_tool_dict = cachedir / ".tool-dict.pkl" + + if args.tool == "all": + for tool in tools.keys(): + self.do_install(tool) + return + + if persistent_tool_dict.exists(): + tools = pickle.loads(persistent_tool_dict.read_bytes()) + + if tools.get(args.tool).get("dependencies"): + for dependency in tools.get(args.tool).get("dependencies"): + if tools.get(dependency).get("installed"): + continue + + self.poutput( + style( + f"{args.tool} has an unmet dependency; installing {dependency}", + fg="blue", + bold=True, + ) + ) + self.do_install(dependency) + + if tools.get(args.tool).get("installed"): + return self.poutput(style(f"{args.tool} is already installed.", fg="yellow")) + else: + self.poutput(style(f"Installing {args.tool}...", fg="blue", bold=True)) + + for command in tools.get(args.tool).get("commands"): + if tools.get(args.tool).get("shell") and command.startswith( + "(" + ): # go installs use subshells (...) + subprocess.run(command, shell=True) + else: + subprocess.run(shlex.split(command)) + tools[args.tool]["installed"] = True + + self.poutput(style(f"{args.tool} installed!", fg="green", bold=True)) + pickle.dump(tools, persistent_tool_dict.open("wb")) + + +if __name__ == "__main__": + rs = ReconShell(persistent_history_file="~/.reconshell_history") + sys.exit(rs.cmdloop()) diff --git a/recon/config.py b/recon/config.py index fa5ad80..b2925ed 100644 --- a/recon/config.py +++ b/recon/config.py @@ -15,12 +15,12 @@ web_ports = {'80', '443', '8080', '8000', '8443'} tool_paths = { 'aquatone': '/opt/aquatone', - 'tko-subs': '/root/go/bin/tko-subs', - 'tko-subs-dir': '/root/go/src/github.com/anshumanbh/tko-subs', - 'subjack': '/root/go/bin/subjack', - 'subjack-fingerprints': '/root/go/src/github.com/haccer/subjack/fingerprints.json', + 'tko-subs': '~/go/bin/tko-subs', + 'tko-subs-dir': '~/go/src/github.com/anshumanbh/tko-subs', + 'subjack': '~/go/bin/subjack', + 'subjack-fingerprints': '~/go/src/github.com/haccer/subjack/fingerprints.json', 'CORScanner': '/opt/CORScanner/cors_scan.py', - 'gobuster': '/usr/local/go/bin/gobuster', + 'gobuster': '~/go/bin/gobuster', 'recursive-gobuster': '/usr/local/bin/recursive-gobuster.pyz', - 'webanalyze': '/root/go/bin/webanalyze' + 'webanalyze': '~/go/bin/webanalyze' } diff --git a/recon/masscan.py b/recon/masscan.py index 81a71f6..4a4ba6f 100644 --- a/recon/masscan.py +++ b/recon/masscan.py @@ -13,7 +13,7 @@ from recon.config import top_tcp_ports, top_udp_ports, defaults @inherits(TargetList, ParseAmassOutput) -class Masscan(luigi.Task): +class MasscanScan(luigi.Task): """ Run masscan against a target specified via the TargetList Task. Masscan commands are structured like the example below. When specified, --top_ports is processed and @@ -40,7 +40,7 @@ class Masscan(luigi.Task): ports = luigi.Parameter(default="") def __init__(self, *args, **kwargs): - super(Masscan, self).__init__(*args, **kwargs) + super(MasscanScan, self).__init__(*args, **kwargs) self.masscan_output = f"masscan.{self.target_file}.json" def output(self): @@ -107,7 +107,7 @@ class Masscan(luigi.Task): subprocess.run(command) -@inherits(Masscan) +@inherits(MasscanScan) class ParseMasscanOutput(luigi.Task): """ Read masscan JSON results and create a pickled dictionary of pertinent information for processing. @@ -134,7 +134,7 @@ class ParseMasscanOutput(luigi.Task): "interface": self.interface, "ports": self.ports, } - return Masscan(**args) + return MasscanScan(**args) def output(self): """ Returns the target output for this task. diff --git a/recon/nmap.py b/recon/nmap.py index 1645bad..6f9477e 100644 --- a/recon/nmap.py +++ b/recon/nmap.py @@ -12,7 +12,7 @@ from recon.masscan import ParseMasscanOutput @inherits(ParseMasscanOutput) -class ThreadedNmap(luigi.Task): +class ThreadedNmapScan(luigi.Task): """ Run nmap against specific targets and ports gained from the ParseMasscanOutput Task. nmap commands are structured like the example below. @@ -121,8 +121,8 @@ class ThreadedNmap(luigi.Task): executor.map(subprocess.run, commands) -@inherits(ThreadedNmap) -class Searchsploit(luigi.Task): +@inherits(ThreadedNmapScan) +class SearchsploitScan(luigi.Task): """ Run searchcploit against each nmap*.xml file in the TARGET-nmap-results directory and write results to disk. searchsploit commands are structured like the example below. @@ -160,7 +160,7 @@ class Searchsploit(luigi.Task): "interface": self.interface, "target_file": self.target_file, } - return ThreadedNmap(**args) + return ThreadedNmapScan(**args) def output(self): """ Returns the target output for this task. diff --git a/recon/web/aquatone.py b/recon/web/aquatone.py index 23d804c..7e78dc2 100644 --- a/recon/web/aquatone.py +++ b/recon/web/aquatone.py @@ -14,7 +14,7 @@ class AquatoneScan(luigi.Task): aquatone commands are structured like the example below. - aquatone --open -sT -sC -T 4 -sV -Pn -p 43,25,21,53,22 -oA htb-targets-nmap-results/nmap.10.10.10.155-tcp 10.10.10.155 + cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20 An example of the corresponding luigi command is shown below. @@ -66,7 +66,7 @@ class AquatoneScan(luigi.Task): def run(self): """ Defines the options/arguments sent to aquatone after processing. - /opt/aquatone -scan-timeout 900 -threads 20 + cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20 Returns: list: list of options/arguments, beginning with the name of the executable to run diff --git a/recon/web/subdomain_takeover.py b/recon/web/subdomain_takeover.py index 557592f..f8d8765 100644 --- a/recon/web/subdomain_takeover.py +++ b/recon/web/subdomain_takeover.py @@ -119,7 +119,7 @@ class SubjackScan(ExternalProgramTask): def output(self): """ Returns the target output for this task. - Naming convention for the output file is amass.TARGET_FILE.json. + Naming convention for the output file is subjack.TARGET_FILE.txt. Returns: luigi.local_target.LocalTarget diff --git a/recon/web/webanalyze.py b/recon/web/webanalyze.py index 5dbf9e2..b953ee6 100644 --- a/recon/web/webanalyze.py +++ b/recon/web/webanalyze.py @@ -104,7 +104,6 @@ class WebanalyzeScan(luigi.Task): for url_scheme in ("https://", "http://"): command = [tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}"] - commands.append(command) Path(self.output().path).mkdir(parents=True, exist_ok=True) diff --git a/recon/wrappers.py b/recon/wrappers.py index 05ca46f..92608ee 100644 --- a/recon/wrappers.py +++ b/recon/wrappers.py @@ -1,7 +1,7 @@ import luigi from luigi.util import inherits -from recon.nmap import Searchsploit +from recon.nmap import SearchsploitScan from recon.web.aquatone import AquatoneScan from recon.web.corscanner import CORScannerScan from recon.web.subdomain_takeover import TKOSubsScan, SubjackScan @@ -10,7 +10,7 @@ from recon.web.webanalyze import WebanalyzeScan @inherits( - Searchsploit, + SearchsploitScan, AquatoneScan, TKOSubsScan, SubjackScan, @@ -49,7 +49,7 @@ class FullScan(luigi.WrapperTask): del args["scan_timeout"] yield SubjackScan(**args) - yield Searchsploit(**args) + yield SearchsploitScan(**args) yield CORScannerScan(**args) yield WebanalyzeScan(**args)