From 5d05b8818db4ee5fadba74addb4bb017959d1741 Mon Sep 17 00:00:00 2001 From: epi052 Date: Sun, 20 Oct 2019 19:37:14 -0500 Subject: [PATCH] added GobusterScan and its related defaults; added it to FullScan; changed configs around a bit --- recon/config.py | 11 +++ recon/web/gobuster.py | 152 ++++++++++++++++++++++++++++++++++++++++++ recon/web/targets.py | 7 +- recon/wrappers.py | 16 ++++- 4 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 recon/web/gobuster.py diff --git a/recon/config.py b/recon/config.py index 4d3cec6..55bcb65 100644 --- a/recon/config.py +++ b/recon/config.py @@ -6,6 +6,15 @@ masscan_config = { 'rate': '1000', } +defaults = { + 'proxy': '', + 'threads': '10', + 'iface': 'tun0', + 'rate': '1000', + 'gobuster-extensions': "", + 'gobuster-wordlist': '/usr/share/seclists/Discovery/Web-Content/common.txt', +} + web_ports = {'80', '443', '8080', '8000', '8443'} tool_paths = { @@ -15,4 +24,6 @@ tool_paths = { 'subjack': '/root/go/bin/subjack', 'subjack-fingerprints': '/root/go/src/github.com/haccer/subjack/fingerprints.json', 'CORScanner': '/opt/CORScanner/cors_scan.py', + 'gobuster': '/usr/local/go/bin/gobuster', + 'recursive-gobuster': '/usr/local/bin/recursive-gobuster.pyz', } diff --git a/recon/web/gobuster.py b/recon/web/gobuster.py new file mode 100644 index 0000000..8cb8abf --- /dev/null +++ b/recon/web/gobuster.py @@ -0,0 +1,152 @@ +import os +import logging +import ipaddress +import subprocess +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor + +import luigi +from luigi.util import inherits + +from recon.config import tool_paths, defaults +from recon.web.targets import GatherWebTargets + + +@inherits(GatherWebTargets) +class GobusterScan(luigi.Task): + """ Use gobuster to scan perform forced browsing. + + gobuster commands are structured like the example below. + + gobuster dir -q -e -k -t 20 -u www.tesla.com -w /usr/share/seclists/Discovery/Web-Content/common.txt -p http://127.0.0.1:8080 -o gobuster.tesla.txt -x php,html + + An example of the corresponding luigi command is shown below. + + PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.gobuster GobusterScan --target-file tesla --top-ports 1000 \ + --interface eth0 --proxy http://127.0.0.1:8080 --extensions php,html \ + --wordlist /usr/share/seclists/Discovery/Web-Content/common.txt --threads 20 + + Install: + go get github.com/OJ/gobuster + git clone https://github.com/epi052/recursive-gobuster.git + + Args: + threads: number of threads for parallel gobuster command execution + wordlist: wordlist used for forced browsing + extensions: additional extensions to apply to each item in the wordlist + recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly) + exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task + top_ports: Scan top N most popular ports *--* Required by upstream Task + ports: specifies the port(s) to be scanned *--* Required by upstream Task + interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task + rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task + target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task + """ + + proxy = luigi.Parameter(default=defaults.get("proxy", "")) + threads = luigi.Parameter(default=defaults.get("threads", "")) + wordlist = luigi.Parameter(default=defaults.get("gobuster-wordlist", "")) + extensions = luigi.Parameter(default=defaults.get("gobuster-extensions", "")) + recursive = luigi.BoolParameter(default=False) + + def requires(self): + """ GobusterScan depends on GatherWebTargets to run. + + GatherWebTargets accepts exempt_list and expects rate, target_file, interface, + and either ports or top_ports as parameters + + Returns: + luigi.Task - GatherWebTargets + """ + args = { + "rate": self.rate, + "target_file": self.target_file, + "top_ports": self.top_ports, + "interface": self.interface, + "ports": self.ports, + "exempt_list": self.exempt_list, + } + return GatherWebTargets(**args) + + def output(self): + """ Returns the target output for this task. + + If recursion is disabled, the naming convention for the output file is gobuster.TARGET_FILE.txt + Otherwise the output file is recursive-gobuster_TARGET_FILE.log + + Results are stored in their own directory: gobuster-TARGET_FILE-results + + Returns: + luigi.local_target.LocalTarget + """ + return luigi.LocalTarget(f"gobuster-{self.target_file}-results") + + def run(self): + """ Defines the options/arguments sent to gobuster after processing. + + Returns: + list: list of options/arguments, beginning with the name of the executable to run + """ + try: + self.threads = abs(int(self.threads)) + except TypeError: + return logging.error("The value supplied to --threads must be a non-negative integer.") + + commands = list() + + with self.input().open() as f: + for target in f: + target = target.strip() + + try: + if isinstance(ipaddress.ip_address(target), ipaddress.IPv6Address): # ipv6 + target = f"[{target}]" + except ValueError: + # domain names raise ValueErrors, just assume we have a domain and keep on keepin on + pass + + for url_scheme in ("https://", "http://"): + if self.recursive: + command = [ + tool_paths.get("recursive-gobuster"), + "-w", + self.wordlist, + f"{url_scheme}{target}", + ] + else: + command = [ + tool_paths.get("gobuster"), + "dir", + "-q", + "-e", + "-k", + "-u", + f"{url_scheme}{target}", + "-w", + self.wordlist, + "-o", + Path(self.output().path).joinpath( + f"gobuster.{url_scheme.replace('//', '_').replace(':', '')}{target}.txt" + ), + ] + + if self.extensions: + command.extend(["-x", self.extensions]) + + if self.proxy: + command.extend(["-p", self.proxy]) + + commands.append(command) + + Path(self.output().path).mkdir(parents=True, exist_ok=True) + + if self.recursive: + # workaround for recursive gobuster not accepting output directory + cwd = Path().cwd() + os.chdir(self.output().path) + + with ThreadPoolExecutor(max_workers=self.threads) as executor: + executor.map(subprocess.run, commands) + + if self.recursive: + os.chdir(str(cwd)) diff --git a/recon/web/targets.py b/recon/web/targets.py index cdefc6e..087538c 100644 --- a/recon/web/targets.py +++ b/recon/web/targets.py @@ -62,8 +62,11 @@ class GatherWebTargets(luigi.Task): for target, protocol_dict in ip_dict.items(): for protocol, ports in protocol_dict.items(): - if ports.intersection(web_ports): # found a web port from masscan's results - targets.add(target) + for port in ports: + if port == "80": + targets.add(target) + elif port in web_ports: + targets.add(f"{target}:{port}") for amass_result in self.input().get("amass-output").values(): with amass_result.open() as f: diff --git a/recon/wrappers.py b/recon/wrappers.py index 7e4c2f6..1bb4374 100644 --- a/recon/wrappers.py +++ b/recon/wrappers.py @@ -5,9 +5,10 @@ from recon.nmap import Searchsploit from recon.web.aquatone import AquatoneScan from recon.web.corscanner import CORScannerScan from recon.web.subdomain_takeover import TKOSubsScan, SubjackScan +from recon.web.gobuster import GobusterScan -@inherits(Searchsploit, AquatoneScan, TKOSubsScan, SubjackScan, CORScannerScan) +@inherits(Searchsploit, AquatoneScan, TKOSubsScan, SubjackScan, CORScannerScan, GobusterScan) class FullScan(luigi.WrapperTask): """ Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time. """ @@ -21,8 +22,19 @@ class FullScan(luigi.WrapperTask): "ports": self.ports, "exempt_list": self.exempt_list, "threads": self.threads, - "scan_timeout": self.scan_timeout, + "proxy": self.proxy, + "wordlist": self.wordlist, + "extensions": self.extensions, + "recursive": self.recursive, } + + yield GobusterScan(**args) + + for gobuster_opt in ("proxy", "wordlist", "extensions", "recursive"): + del args[gobuster_opt] + + args.update({"scan_timeout": self.scan_timeout}) + yield AquatoneScan(**args) del args["scan_timeout"]