Add docs (#3)

* initial work on sphinx docs; much left to do

* first pass at docs complete; still has some warts

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/api/commands.rst

@@ -0,0 +1,25 @@
+Commands
+========
+
+``recon-pipeline`` provides two commands ``install`` and ``scan``.  All other commands are inherited
+from `cmd2 <https://github.com/python-cmd2/cmd2>`_.
+
+.. _install_command:
+
+install
+#######
+
+.. argparse::
+    :module: recon
+    :func: install_parser
+    :prog: install
+
+.. _scan_command:
+
+scan
+####
+
+.. argparse::
+    :module: recon
+    :func: scan_parser
+    :prog: install

docs/api/index.rst

@@ -0,0 +1,9 @@
+API Reference
+=============
+
+.. toctree::
+    :maxdepth: 1
+
+    scanners
+    parsers
+    commands

docs/api/parsers.rst

@@ -0,0 +1,17 @@
+Parsers
+=======
+
+Amass Parser
+############
+
+.. autoclass:: recon.amass.ParseAmassOutput
+
+Web Targets Parser
+##################
+
+.. autoclass:: recon.web.targets.GatherWebTargets
+
+Masscan Parser
+##############
+
+.. autoclass:: recon.masscan.ParseMasscanOutput

docs/api/scanners.rst

@@ -0,0 +1,62 @@
+Scanners
+================
+
+Amass Scanner
+#############
+
+.. autoclass:: recon.amass.AmassScan
+
+Aquatone Scanner
+################
+
+.. autoclass:: recon.web.aquatone.AquatoneScan
+
+CORS Scanner
+############
+
+.. autoclass:: recon.web.corscanner.CORScannerScan
+
+Full Scanner
+############
+
+.. autoclass:: recon.wrappers.FullScan
+
+Gobuster Scanner
+################
+
+.. autoclass:: recon.web.gobuster.GobusterScan
+
+Hackthebox Scanner
+##################
+
+.. autoclass:: recon.wrappers.HTBScan
+
+Masscan Scanner
+###############
+
+.. autoclass:: recon.masscan.MasscanScan
+
+Searchsploit Scanner
+####################
+
+.. autoclass:: recon.nmap.SearchsploitScan
+
+Subjack Scanner
+###############
+
+.. autoclass:: recon.web.subdomain_takeover.SubjackScan
+
+ThreadedNmap Scanner
+####################
+
+.. autoclass:: recon.nmap.ThreadedNmapScan
+
+TKOSubs Scanner
+###############
+
+.. autoclass:: recon.web.subdomain_takeover.TKOSubsScan
+
+Webanalyze Scanner
+##################
+
+.. autoclass:: recon.web.webanalyze.WebanalyzeScan

docs/conf.py

@@ -0,0 +1,66 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(".."))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "recon-pipeline"
+copyright = "2020, epi"
+author = "epi"
+
+# The full version, including alpha/beta/rc tags
+release = "0.7.3"
+rst_epilog = """
+.. |version| replace:: {versionnum}
+""".format(
+    versionnum=release
+)
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.coverage",
+    "sphinx.ext.napoleon",
+    "sphinxarg.ext",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]

docs/index.rst

@@ -0,0 +1,41 @@
+==============
+recon-pipeline
+==============
+
+``recon-pipeline`` was designed to chain together multiple security tools as part of a Flow-Based Programming paradigm.
+Each component is part of a network of "black box" processes.  These components exchange data between each other and
+can be reconnected in different ways to form different applications without any internal changes.
+
+Getting Started
+===============
+
+.. include:: overview/summary.rst
+
+.. toctree::
+    :maxdepth: 2
+    :hidden:
+
+    overview/index
+
+Changing the Code
+=================
+
+.. toctree::
+    :maxdepth: 1
+
+    modifications/index
+
+API Reference
+=============
+
+.. toctree::
+    :maxdepth: 2
+
+    api/index
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/modifications/index.rst

@@ -0,0 +1,7 @@
+Making Modifications
+====================
+
+.. toctree::
+    :maxdepth: 1
+
+    new_wrapper

docs/modifications/new_wrapper.rst

@@ -0,0 +1,59 @@
+Creating a New Wrapper Scan
+===========================
+
+If for whatever reason you want something other than FullScan, the process for defining a new scan is relatively simple.
+The ``HTBScan`` is a good example.
+
+1. Define your new class, inheriting from :class:`luigi.WrapperTask` and use the ``inherits`` decorator to include any scan you want to utilize
+
+.. code-block::
+
+    @inherits(SearchsploitScan, AquatoneScan, GobusterScan, WebanalyzeScan)
+    class HTBScan(luigi.WrapperTask):
+        ...
+
+2. Include all parameters needed by any of the scans passed to ``inherits``
+
+.. code-block::
+
+    def requires(self):
+        """ HTBScan is a wrapper, as such it requires any Tasks that it wraps. """
+        args = {
+            "results_dir": self.results_dir,
+            "rate": self.rate,
+            "target_file": self.target_file,
+            "top_ports": self.top_ports,
+            "interface": self.interface,
+            "ports": self.ports,
+            "exempt_list": self.exempt_list,
+            "threads": self.threads,
+            "proxy": self.proxy,
+            "wordlist": self.wordlist,
+            "extensions": self.extensions,
+            "recursive": self.recursive,
+        }
+        ...
+
+3. ``yield`` from each scan, keeping in mind that some of the parameters won't be universal (i.e. need to be removed/added)
+
+.. code-block::
+
+    def requires(self):
+        """ HTBScan is a wrapper, as such it requires any Tasks that it wraps. """
+        ...
+
+        yield GobusterScan(**args)
+
+        # remove options that are gobuster specific; if left dictionary unpacking to other scans throws an exception
+        for gobuster_opt in ("proxy", "wordlist", "extensions", "recursive"):
+            del args[gobuster_opt]
+
+        # add aquatone scan specific option
+        args.update({"scan_timeout": self.scan_timeout})
+
+        yield AquatoneScan(**args)
+
+        del args["scan_timeout"]
+
+        yield SearchsploitScan(**args)
+        yield WebanalyzeScan(**args)

docs/overview/index.rst

@@ -0,0 +1,13 @@
+Getting Started
+===============
+
+.. toctree::
+    :maxdepth: 1
+    :hidden:
+
+    installation
+    running_scans
+    scheduler
+    scope
+
+.. include:: summary.rst

docs/overview/installation.rst

@@ -0,0 +1,82 @@
+.. _install-ref-label:
+
+Installation Instructions
+=========================
+
+There are two primary phases for installation:
+
+* prior to `cmd2 <https://github.com/python-cmd2/cmd2>`_ being installed
+* everything else
+
+Manual Steps
+############
+
+First, the manual steps to get cmd2 installed in a virtual environment are as follows (and shown below)
+
+.. code-block::
+
+    apt install pipenv
+    git clone https://github.com/epi052/recon-pipeline.git
+    cd recon-pipeline
+    pipenv install cmd2
+
+
+.. raw:: html
+
+    <script id="asciicast-293306" src="https://asciinema.org/a/293306.js" async></script>
+
+Everything Else
+###############
+
+After manual installation of cmd2_ is complete, the recon-pipeline shell provides its own :ref:`install_command` command (seen below).
+A simple ``install all`` will handle all installation steps.  Installation has **only** been tested on **Kali 2019.4**.
+
+Individual tools may be installed by running ``install TOOLNAME`` where ``TOOLNAME`` is one of the known tools that make
+up the pipeline.
+
+The installer maintains a (naive) list of installed tools at ``~/.cache/.tool-dict.pkl``.  The installer in no way
+attempts to be a package manager.  It knows how to execute the steps necessary to install its tools.  Beyond that, it's
+like Jon Snow, it knows nothing.
+
+.. raw:: html
+
+    <script id="asciicast-294414" src="https://asciinema.org/a/294414.js" async></script>
+
+Alternative Distros
+###################
+
+If you're using a different distribution (i.e. not Kali), meeting the criteria below should be sufficient
+for the auto installer to function:
+
+- systemd-based system (luigid is installed as a systemd service)
+- derivative of debian (some tools are installed using apt)
+
+The alternatives would be to manually install each tool or to modify the distro-specific portions of the commands
+laid out in ``recon.__init__``.  For example, on Fedora, you could change the package manager from ``apt-get`` to
+``dnf`` and remove any ``apt-get`` specific options.
+
+Example from ``recon-pipeline/recon/__init__.py``
+
+.. code-block::
+
+    "pipenv": {
+        "installed": False,
+        "dependencies": None,
+        "commands": ["sudo apt-get install -y -q pipenv"],
+    },
+
+would become
+
+.. code-block::
+
+    "pipenv": {
+        "installed": False,
+        "dependencies": None,
+        "commands": ["sudo dnf install -y pipenv"],
+    },
+
+
+
+
+
+

docs/overview/running_scans.rst

@@ -0,0 +1,31 @@
+.. _scan-ref-label:
+
+Running Scans
+=============
+
+All scans are ran from within ``recon-pipeline``'s shell.  There are a number of individual scans, however to execute
+multiple scans at once, ``recon-pipeline`` includes wrappers around multiple commands.  As of version |version|, the
+following individual scans are available
+
+- :class:`recon.amass.AmassScan`
+- :class:`recon.web.aquatone.AquatoneScan`
+- :class:`recon.web.corscanner.CORScannerScan`
+- :class:`recon.web.gobuster.GobusterScan`
+- :class:`recon.masscan.MasscanScan`
+- :class:`recon.nmap.SearchsploitScan`
+- :class:`recon.web.subdomain_takeover.SubjackScan`
+- :class:`recon.nmap.ThreadedNmapScan`
+- :class:`recon.web.subdomain_takeover.TKOSubsScan`
+- :class:`recon.web.webanalyze.WebanalyzeScan`
+
+Additionally, two wrapper scans are made available as well.
+
+- :class:`recon.wrappers.FullScan` - runs the entire pipeline
+- :class:`recon.wrappers.HTBScan` - nicety for hackthebox players (myself included) that omits the scans in FullScan that don't make sense for HTB
+
+Example Scan
+============
+
+.. raw:: html
+
+    <script id="asciicast-293302" src="https://asciinema.org/a/293302.js" async></script>

docs/overview/scheduler.rst

@@ -0,0 +1,20 @@
+.. _scheduler-ref-label:
+
+Using a Scheduler
+=================
+
+The backbone of this pipeline is spotify's `luigi <https://github.com/spotify/luigi>`_ batch process management framework. Luigi uses the concept of a
+scheduler in order to manage task execution. Two types of scheduler are available, a local scheduler and a
+central scheduler. The local scheduler is useful for development and debugging while the central scheduler
+provides the following two benefits:
+
+- Make sure two instances of the same task are not running simultaneously
+- Provide visualization of everything that’s going on
+
+While in the ``recon-pipeline`` shell, running ``install luigi-service`` will copy the ``luigid.service``
+file provided in the repo to its appropriate systemd location and start/enable the service. The result is that the
+central scheduler is up and running easily.
+
+The other option is to add ``--local-scheduler`` to your :ref:`scan_command` command from within the ``recon-pipeline`` shell.
+
+

docs/overview/scope.rst

@@ -0,0 +1,27 @@
+.. _scope-ref-label:
+
+Defining Target Scope
+=====================
+
+The pipeline expects a file that describes the target's scope to be provided as an argument to the
+``--target-file`` option. The target file can consist of domains, ip addresses, and ip ranges, one per line.  Ip
+addresses and ip ranges can be mixed/matched, but domains cannot.
+
+.. code-block::
+
+    tesla.com
+    tesla.cn
+    teslamotors.com
+    ...
+
+Some bug bounty scopes have expressly verboten subdomains and/or top-level domains, for that there is the
+``--exempt-list`` option. The exempt list follows the same rules as the target file.
+
+.. code-block::
+
+    shop.eu.teslamotors.com
+    energysupport.tesla.com
+    feedback.tesla.com
+    ...
+
+

docs/overview/summary.rst

@@ -0,0 +1,8 @@
+There are an `accompanying set of blog posts <https://epi052.gitlab.io/notes-to-self/blog/2019-09-01-how-to-build-an-automated-recon-pipeline-with-python-and-luigi/>`_
+detailing the development process and underpinnings of the pipeline. Feel free to check them out if
+you're so inclined, but they're in no way required reading to use the tool.
+
+* :ref:`install-ref-label` - How to install ``recon-pipeline`` and associated dependencies
+* :ref:`scan-ref-label` - Example scan of **tesla.com** using ``recon-pipeline``
+* :ref:`scope-ref-label` - How to define the scope of your scans (list of targets and a blacklist)
+* :ref:`scheduler-ref-label` - The Luigi schedulers and which to choose

recon-pipeline.py

@@ -239,13 +239,13 @@ class ReconShell(cmd2.Cmd):
 
                     # go tools use subshells (cmd1 && cmd2 && cmd3 ...) during install, so need shell=True
                     proc = subprocess.Popen(
-                        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+                        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                     )
                 else:
 
                     # "normal" command, split up the string as usual and run it
                     proc = subprocess.Popen(
-                        shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+                        shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                     )
 
                 out, err = proc.communicate()

recon/__init__.py

@@ -26,7 +26,11 @@ tools = {
         ],
         "shell": True,
     },
-    "luigi": {"installed": False, "dependencies": ["pipenv"], "commands": ["pipenv install luigi"]},
+    "luigi": {
+        "installed": False,
+        "dependencies": ["pipenv"],
+        "commands": ["pipenv install luigi"],
+    },
     "pipenv": {
         "installed": False,
         "dependencies": None,
@@ -122,13 +126,15 @@ tools = {
 
 
 def get_scans():
-    """ Iterates over the recon package and its modules to find all of the *Scan classes.
+    """ Iterates over the recon package and its modules to find all of the \*Scan classes.
 
-    *** A contract exists here that says any scans need to end with the word scan in order to be found by this function.
+    **A contract exists here that says any scans need to end with the word scan in order to be found by this function.**
+
+    Example:
+        ``defaultdict(<class 'list'>, {'AmassScan': ['recon.amass'], 'MasscanScan': ['recon.masscan'], ... })``
 
     Returns:
-        dict() containing mapping of {classname: [modulename, ...]} for all potential recon-pipeline commands
+        dict containing mapping of ``classname -> [modulename, ...]`` for all potential recon-pipeline commands
-        ex:  defaultdict(<class 'list'>, {'AmassScan': ['recon.amass'], 'MasscanScan': ['recon.masscan'], ... })
     """
     scans = defaultdict(list)
 
@@ -168,7 +174,9 @@ scan_parser.add_argument(
     help="file created by the user that defines the target's scope; list of ips/domains",
 )
 scan_parser.add_argument(
-    "--exempt-list", completer_method=cmd2.Cmd.path_complete, help="list of blacklisted ips/domains"
+    "--exempt-list",
+    completer_method=cmd2.Cmd.path_complete,
+    help="list of blacklisted ips/domains",
 )
 scan_parser.add_argument(
     "--results-dir",
@@ -176,7 +184,7 @@ scan_parser.add_argument(
     help="directory in which to save scan results",
 )
 scan_parser.add_argument(
-    "--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster"
+    "--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster",
 )
 scan_parser.add_argument(
     "--interface",
@@ -192,7 +200,7 @@ scan_parser.add_argument(
     help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)",
 )
 scan_parser.add_argument(
-    "--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)"
+    "--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)",
 )
 scan_parser.add_argument(
     "--threads", help="number of threads for all of the threaded applications to use"

recon/web/gobuster.py

@@ -18,15 +18,20 @@ class GobusterScan(luigi.Task):
 
     gobuster commands are structured like the example below.
 
+    .. code-block::
+
         gobuster dir -q -e -k -t 20 -u www.tesla.com -w /usr/share/seclists/Discovery/Web-Content/common.txt -p http://127.0.0.1:8080 -o gobuster.tesla.txt -x php,html
 
     An example of the corresponding luigi command is shown below.
 
-    PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.gobuster GobusterScan --target-file tesla --top-ports 1000 \
+    Example:
-                            --interface eth0 --proxy http://127.0.0.1:8080 --extensions php,html \
+        .. code-block::
-                            --wordlist /usr/share/seclists/Discovery/Web-Content/common.txt --threads 20
+
+            PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.gobuster GobusterScan --target-file tesla --top-ports 1000 --interface eth0 --proxy http://127.0.0.1:8080 --extensions php,html --wordlist /usr/share/seclists/Discovery/Web-Content/common.txt --threads 20
 
     Install:
+        .. code-block::
+
             go get github.com/OJ/gobuster
             git clone https://github.com/epi052/recursive-gobuster.git
 
@@ -35,6 +40,7 @@ class GobusterScan(luigi.Task):
         wordlist: wordlist used for forced browsing
         extensions: additional extensions to apply to each item in the wordlist
         recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly)
+        proxy: protocol://ip:port proxy specification for gobuster
         exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
         top_ports: Scan top N most popular ports *--* Required by upstream Task
         ports: specifies the port(s) to be scanned *--* Required by upstream Task

recon/wrappers.py

@@ -19,7 +19,22 @@ from recon.web.webanalyze import WebanalyzeScan
     WebanalyzeScan,
 )
 class FullScan(luigi.WrapperTask):
-    """ Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time. """
+    """ Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time.
+
+    Args:
+        threads: number of threads for parallel gobuster command execution
+        wordlist: wordlist used for forced browsing
+        extensions: additional extensions to apply to each item in the wordlist
+        recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly)
+        proxy: protocol://ip:port proxy specification for gobuster
+        exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
+        top_ports: Scan top N most popular ports *--* Required by upstream Task
+        ports: specifies the port(s) to be scanned *--* Required by upstream Task
+        interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task
+        rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task
+        target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task
+        results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task
+    """
 
     def requires(self):
         """ FullScan is a wrapper, as such it requires any Tasks that it wraps. """
@@ -63,7 +78,22 @@ class FullScan(luigi.WrapperTask):
 
 @inherits(SearchsploitScan, AquatoneScan, GobusterScan, WebanalyzeScan)
 class HTBScan(luigi.WrapperTask):
-    """ Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time. """
+    """ Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time.
+
+    Args:
+        threads: number of threads for parallel gobuster command execution
+        wordlist: wordlist used for forced browsing
+        extensions: additional extensions to apply to each item in the wordlist
+        recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly)
+        proxy: protocol://ip:port proxy specification for gobuster
+        exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
+        top_ports: Scan top N most popular ports *--* Required by upstream Task
+        ports: specifies the port(s) to be scanned *--* Required by upstream Task
+        interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task
+        rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task
+        target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task
+        results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task
+    """
 
     def requires(self):
         """ HTBScan is a wrapper, as such it requires any Tasks that it wraps. """