minor polish / comments

.gitignore

@@ -102,3 +102,8 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+.idea
+.flake8
+.pre-commit-config.yaml
+pyproject.toml

recon-pipeline.py

@@ -16,11 +16,11 @@ os.environ["PYTHONPATH"] = f"{os.environ.get('PYTHONPATH')}:{str(Path(__file__).
 os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
 
 # third party imports
-import cmd2
+import cmd2  # noqa: E402
-from cmd2.ansi import style
+from cmd2.ansi import style  # noqa: E402
 
 # project's module imports
-from recon import get_scans, tools, scan_parser, install_parser
+from recon import get_scans, tools, scan_parser, install_parser  # noqa: F401,E402
 
 # select loop, handles async stdout/stderr processing of subprocesses
 selector = selectors.DefaultSelector()
@@ -150,15 +150,16 @@ class ReconShell(cmd2.Cmd):
             )
         )
 
-        # get_scans() returns mapping of {modulename: classname} in the recon module
+        # get_scans() returns mapping of {classname: [modulename, ...]} in the recon module
         # each classname corresponds to a potential recon-pipeline command, i.e. AmassScan, CORScannerScan ...
         scans = get_scans()
 
         # command is a list that will end up looking something like what's below
         # luigi --module recon.web.webanalyze WebanalyzeScan --target-file tesla --top-ports 1000 --interface eth0
         command = ["luigi", "--module", scans.get(args.scantype)[0]]
+
         command.extend(args.__statement__.arg_list)
-        self.async_alert(" ".join(command))
+
         if args.verbose:
             # verbose is not a luigi option, need to remove it
             command.pop(command.index("--verbose"))
@@ -209,11 +210,7 @@ class ReconShell(cmd2.Cmd):
                     continue
 
                 self.async_alert(
-                    style(
+                    style(f"[!] {args.tool} has an unmet dependency; installing {dependency}", fg="yellow", bold=True,)
-                        f"[!] {args.tool} has an unmet dependency; installing {dependency}",
-                        fg="yellow",
-                        bold=True,
-                    )
                 )
 
                 # install the dependency before continuing with installation
@@ -238,18 +235,12 @@ class ReconShell(cmd2.Cmd):
                 if tools.get(args.tool).get("shell"):
 
                     # go tools use subshells (cmd1 && cmd2 && cmd3 ...) during install, so need shell=True
-                    proc = subprocess.Popen(
+                    proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-                    )
                 else:
 
                     # "normal" command, split up the string as usual and run it
-                    proc = subprocess.Popen(
+                    proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                        shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE
-                    )
 
-                # register stderr for more visually pleasing output on failed commands
-                # selector.register(proc.stderr, selectors.EVENT_READ, self._install_error_reporter)
                 out, err = proc.communicate()
 
                 if err:
@@ -281,7 +272,5 @@ class ReconShell(cmd2.Cmd):
 
 
 if __name__ == "__main__":
-    rs = ReconShell(
+    rs = ReconShell(persistent_history_file="~/.reconshell_history", persistent_history_length=10000)
-        persistent_history_file="~/.reconshell_history", persistent_history_length=10000
-    )
     sys.exit(rs.cmdloop())

recon/__init__.py

@@ -1,7 +1,9 @@
+# flake8: noqa E231
 import sys
 import socket
 import inspect
 import pkgutil
+import importlib
 from pathlib import Path
 from collections import defaultdict
 
@@ -25,11 +27,7 @@ tools = {
         "shell": True,
     },
     "luigi": {"installed": False, "dependencies": ["pipenv"], "commands": ["pipenv install luigi"]},
-    "pipenv": {
+    "pipenv": {"installed": False, "dependencies": None, "commands": ["apt-get install -y -q pipenv"],},
-        "installed": False,
-        "dependencies": None,
-        "commands": ["apt-get install -y -q pipenv"],
-    },
     "masscan": {
         "installed": False,
         "dependencies": None,
@@ -40,11 +38,7 @@ tools = {
             "rm -rf /tmp/masscan",
         ],
     },
-    "amass": {
+    "amass": {"installed": False, "dependencies": None, "commands": ["apt-get install -y -q amass"],},
-        "installed": False,
-        "dependencies": None,
-        "commands": ["apt-get install -y -q amass"],
-    },
     "aquatone": {
         "installed": False,
         "dependencies": None,
@@ -88,10 +82,7 @@ tools = {
     "subjack": {
         "installed": False,
         "dependencies": ["go"],
-        "commands": [
+        "commands": ["go get github.com/haccer/subjack", "(cd ~/go/src/github.com/haccer/subjack && go install)",],
-            "go get github.com/haccer/subjack",
-            "(cd ~/go/src/github.com/haccer/subjack && go install)",
-        ],
         "shell": True,
     },
     "webanalyze": {
@@ -122,20 +113,26 @@ def get_scans():
     *** A contract exists here that says any scans need to end with the word scan in order to be found by this function.
 
     Returns:
-        dict() containing mapping of {modulename: classname} for all potential recon-pipeline commands
+        dict() containing mapping of {classname: [modulename, ...]} for all potential recon-pipeline commands
+        ex:  defaultdict(<class 'list'>, {'AmassScan': ['recon.amass'], 'MasscanScan': ['recon.masscan'], ... })
     """
     scans = defaultdict(list)
 
     # recursively walk packages; import each module in each package
-    for loader, module_name, is_pkg in pkgutil.walk_packages(recon.__path__, prefix="recon."):
+    # walk_packages yields ModuleInfo objects for all modules recursively on path
-        _module = loader.find_module(module_name).load_module(module_name)
+    # prefix is a string to output on the front of every module name on output.
-        globals()[module_name] = _module
+    for loader, module_name, is_pkg in pkgutil.walk_packages(path=recon.__path__, prefix="recon."):
+        importlib.import_module(module_name)
 
-    # walk all modules, grabbing classes that we've written and add them to the classlist set
+    # walk all modules, grabbing classes that we've written and add them to the classlist defaultdict
+    # getmembers returns all members of an object in a list of tuples (name, value)
     for name, obj in inspect.getmembers(sys.modules[__name__]):
         if inspect.ismodule(obj) and not name.startswith("_"):
+            # we're only interested in modules that don't begin with _ i.e. magic methods __len__ etc...
+
             for subname, subobj in inspect.getmembers(obj):
                 if inspect.isclass(subobj) and subname.lower().endswith("scan"):
+                    # now we only care about classes that end in [Ss]can
                     scans[subname].append(name)
 
     return scans
@@ -143,9 +140,7 @@ def get_scans():
 
 # options for ReconShell's 'install' command
 install_parser = cmd2.Cmd2ArgumentParser()
-install_parser.add_argument(
+install_parser.add_argument("tool", help="which tool to install", choices=list(tools.keys()) + ["all"])
-    "tool", help="which tool to install", choices=list(tools.keys()) + ["all"]
-)
 
 
 # options for ReconShell's 'scan' command
@@ -160,9 +155,7 @@ scan_parser.add_argument(
     "--exempt-list", completer_method=cmd2.Cmd.path_complete, help="list of blacklisted ips/domains"
 )
 scan_parser.add_argument(
-    "--results-dir",
+    "--results-dir", completer_method=cmd2.Cmd.path_complete, help="directory in which to save scan results",
-    completer_method=cmd2.Cmd.path_complete,
-    help="directory in which to save scan results",
 )
 scan_parser.add_argument(
     "--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster"
@@ -172,30 +165,19 @@ scan_parser.add_argument(
     choices_function=lambda: [x[1] for x in socket.if_nameindex()],
     help="which interface masscan should use",
 )
-scan_parser.add_argument(
+scan_parser.add_argument("--recursive", action="store_true", help="whether or not to recursively gobust")
-    "--recursive", action="store_true", help="whether or not to recursively gobust"
-)
 scan_parser.add_argument("--rate", help="rate at which masscan should scan")
 scan_parser.add_argument(
-    "--top-ports",
+    "--top-ports", help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)",
-    help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)",
-)
-scan_parser.add_argument(
-    "--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)"
-)
-scan_parser.add_argument(
-    "--threads", help="number of threads for all of the threaded applications to use"
 )
+scan_parser.add_argument("--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)")
+scan_parser.add_argument("--threads", help="number of threads for all of the threaded applications to use")
 scan_parser.add_argument("--scan-timeout", help="scan timeout for aquatone")
 scan_parser.add_argument("--proxy", help="proxy for gobuster if desired (ex. 127.0.0.1:8080)")
 scan_parser.add_argument("--extensions", help="list of extensions for gobuster (ex. asp,html,aspx)")
 scan_parser.add_argument(
-    "--local-scheduler",
+    "--local-scheduler", action="store_true", help="use the local scheduler instead of the central scheduler (luigid)",
-    action="store_true",
-    help="use the local scheduler instead of the central scheduler (luigid)",
 )
 scan_parser.add_argument(
-    "--verbose",
+    "--verbose", action="store_true", help="shows debug messages from luigi, useful for troubleshooting",
-    action="store_true",
-    help="shows debug messages from luigi, useful for troubleshooting",
 )