removed dependency on tool-dict.pkl (#93)

* removed tool_dict dependency

* updated tests

* updated go version

* added defaults for failing iteration during tool installation

* Update pythonapp.yml

* updated docs

.github/workflows/pythonapp.yml

@@ -1,6 +1,6 @@
 name: recon-pipeline build
 
-on: [push, pull_request]
+on: [push]
 
 jobs:
   lint:
@@ -24,9 +24,9 @@ jobs:
         # stop the build if there are Python syntax errors or undefined names
         pipenv run flake8 . --count
     - name: Check code formatting with black
-      uses: lgeiger/black-action@master
-      with:
-        args: ". --check"
+      run: |
+        pipenv install "black==19.10b0"
+        pipenv run black -l 120 --check . 
 
   test-shell:
 

.pre-commit-config.yaml

@@ -16,8 +16,9 @@ repos:
 -   repo: local
     hooks:
     - id: tests
+      pass_filenames: false
       name: run tests
-      entry: pytest
+      entry: python
       language: system
       types: [python]
-      args: ['tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models']
+      args: ['-m', 'pytest', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models']

README.md

@@ -86,7 +86,8 @@ After installing the python dependencies, the `recon-pipeline` shell provides it
 Individual tools may be installed by running `tools install TOOLNAME` where `TOOLNAME` is one of the known tools that make
 up the pipeline.
 
-The installer maintains a (naive) list of installed tools at `~/.local/recon-pipeline/tools/.tool-dict.pkl`.  The installer in no way attempts to be a package manager.  It knows how to execute the steps necessary to install and remove its tools.  Beyond that, it's like Jon Snow, **it knows nothing**.
+The installer does not maintain state.  In order to determine whether a tool is installed or not, it checks the `path` variable defined in the tool's .yaml file.  The installer in no way attempts to be a package manager.  It knows how to execute the steps necessary to install and remove its tools.  Beyond that, it's
+like Jon Snow, **it knows nothing**.
 
 [![asciicast](https://asciinema.org/a/343745.svg)](https://asciinema.org/a/343745)
 

docs/overview/installation.rst

@@ -59,7 +59,7 @@ A simple ``tools install all`` will handle all installation steps.  Installation
 Individual tools may be installed by running ``tools install TOOLNAME`` where ``TOOLNAME`` is one of the known tools that make
 up the pipeline.
 
-The installer maintains a (naive) list of installed tools at ``~/.local/recon-pipeline/tools/.tool-dict.pkl``.  The installer in no way attempts to be a package manager.  It knows how to execute the steps necessary to install and remove its tools.  Beyond that, it's
+The installer does not maintain state.  In order to determine whether a tool is installed or not, it checks the `path` variable defined in the tool's .yaml file.  The installer in no way attempts to be a package manager.  It knows how to execute the steps necessary to install and remove its tools.  Beyond that, it's
 like Jon Snow, **it knows nothing**.
 
 Current tool status can be viewed using ``tools list``. Tools can also be uninstalled using the ``tools uninstall all`` command. It is also possible to individually uninstall them in the same manner as shown above.

pipeline/recon-pipeline.py

@@ -5,7 +5,6 @@ import sys
 import time
 import shlex
 import shutil
-import pickle
 import tempfile
 import textwrap
 import selectors
@@ -109,7 +108,7 @@ class SelectorThread(threading.Thread):
 
         # close any fds that were registered and still haven't been unregistered
         for key in selector.get_map():
-            selector.get_key(key).fileobj.close()
+            selector.get_key(key).fileobj.close()  # pragma: no cover
 
     def stopped(self):
         """ Helper to determine whether the SelectorThread's Event is set or not. """
@@ -117,7 +116,7 @@ class SelectorThread(threading.Thread):
 
     def run(self):
         """ Run thread that executes a select loop; handles async stdout/stderr processing of subprocesses. """
-        while not self.stopped():
+        while not self.stopped():  # pragma: no cover
             for k, mask in selector.select():
                 callback = k.data
                 callback(k.fileobj)
@@ -346,15 +345,6 @@ class ReconShell(cmd2.Cmd):
 
     def _get_dict(self):
         """Retrieves tool dict if available"""
-
-        # imported tools variable is in global scope, and we reassign over it later
-        global tools
-
-        persistent_tool_dict = self.tools_dir / ".tool-dict.pkl"
-
-        if persistent_tool_dict.exists():
-            tools = pickle.loads(persistent_tool_dict.read_bytes())
-
         return tools
 
     def _finalize_tool_action(self, tool: str, tool_dict: dict, return_values: List[int], action: ToolActions):
@@ -387,16 +377,9 @@ class ReconShell(cmd2.Cmd):
                 )
             )
 
-        # store any tool installs/failures (back) to disk
-        persistent_tool_dict = self.tools_dir / ".tool-dict.pkl"
-
-        pickle.dump(tool_dict, persistent_tool_dict.open("wb"))
-
     def tools_install(self, args):
         """ Install any/all of the libraries/tools necessary to make the recon-pipeline function. """
 
-        tools = self._get_dict()
-
         if args.tool == "all":
             # show all tools have been queued for installation
             [
@@ -424,16 +407,6 @@ class ReconShell(cmd2.Cmd):
                 # install the dependency before continuing with installation
                 self.do_tools(f"install {dependency}")
 
-        # this prevents a stale copy of tools when dependency installs alter the state
-        # ex.
-        #   amass (which depends on go) grabs copy of tools (go installed false)
-        #   amass calls install with go as the arg
-        #   go grabs a copy of tools
-        #   go is installed and state is saved (go installed true)
-        #   recursion goes back to amass call (go installed false due to stale tools data)
-        #   amass installs and re-saves go's state as installed=false
-        tools = self._get_dict()
-
         if tools.get(args.tool).get("installed"):
             return self.poutput(style(f"[!] {args.tool} is already installed.", fg="yellow"))
         else:
@@ -448,7 +421,7 @@ class ReconShell(cmd2.Cmd):
             if addl_env_vars is not None:
                 addl_env_vars.update(dict(os.environ))
 
-            for command in tools.get(args.tool).get("install_commands"):
+            for command in tools.get(args.tool, {}).get("install_commands", []):
                 # run all commands required to install the tool
 
                 # print each command being run
@@ -478,7 +451,6 @@ class ReconShell(cmd2.Cmd):
 
     def tools_uninstall(self, args):
         """ Uninstall any/all of the libraries/tools used by recon-pipeline"""
-        tools = self._get_dict()
 
         if args.tool == "all":
             # show all tools have been queued for installation
@@ -525,7 +497,7 @@ class ReconShell(cmd2.Cmd):
 
     def tools_list(self, args):
         """ List status of pipeline tools """
-        for key, value in self._get_dict().items():
+        for key, value in tools.items():
             status = [style(":Missing:", fg="bright_magenta"), style("Installed", fg="bright_green")]
             self.poutput(style(f"[{status[value.get('installed')]}] - {value.get('path') or key}"))
 

pipeline/recon/helpers.py

@@ -1,6 +1,4 @@
 import sys
-import pickle
-import typing
 import inspect
 import pkgutil
 import importlib
@@ -10,13 +8,12 @@ from cmd2.ansi import style
 
 from collections import defaultdict
 
-from ..recon.config import defaults
+from ..tools import tools
 
 
 def meets_requirements(requirements, exception):
     """ Determine if tools required to perform task are installed. """
-    tools = get_tool_state()
-
+    print(tools.items())
     for tool in requirements:
         if not tools.get(tool).get("installed"):
             if exception:
@@ -29,14 +26,6 @@ def meets_requirements(requirements, exception):
     return True
 
 
-def get_tool_state() -> typing.Union[dict, None]:
-    """ Load current tool state from disk. """
-    tools = Path(defaults.get("tools-dir")) / ".tool-dict.pkl"
-
-    if tools.exists():
-        return pickle.loads(tools.read_bytes())
-
-
 def get_scans():
     """ Iterates over the recon package and its modules to find all of the classes that end in [Ss]can.
 

pipeline/tools/amass.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
 path: &path !join_path [!get_default "{gopath}", "bin/amass"]

pipeline/tools/aquatone.yaml

@@ -1,4 +1,3 @@
-installed: false
 tools: &tools !get_default "{tools-dir}"
 path: &path !join_path [*tools, aquatone]
 

pipeline/tools/exploitdb.yaml

@@ -1,3 +1,2 @@
-installed: true
 tools: &tools !get_default "{tools-dir}"
 path: !join_path [*tools, exploitdb]

pipeline/tools/go.yaml

@@ -1,7 +1,6 @@
-installed: false
 bashrc: &bashrc !join_path [!get_default "{home}", .bashrc]
 path: &gotool !join_path [!get_default "{goroot}", go/bin/go]
-dlpath: &dlpath !join_empty ["https://dl.google.com/go/go1.14.6.linux-", !get_default "{arch}", ".tar.gz"]
+dlpath: &dlpath !join_empty ["https://dl.google.com/go/go1.14.7.linux-", !get_default "{arch}", ".tar.gz"]
 
 install_commands:
 - !join ["wget -q", *dlpath, "-O /tmp/go.tar.gz"]

pipeline/tools/gobuster.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [go, seclists]
 go: &gotool !get_tool_path "{go[path]}"
 path: &path !join_path [!get_default "{gopath}", bin/gobuster]

pipeline/tools/loader.py

@@ -1,3 +1,4 @@
+import uuid
 import yaml
 from pathlib import Path
 
@@ -59,3 +60,6 @@ def load_yaml(file):
 for file in definitions.iterdir():
     if file.name.endswith(".yaml") and file.name.replace(".yaml", "") not in tools:
         load_yaml(file)
+
+for tool_name, tool_definition in tools.items():
+    tool_definition["installed"] = Path(tool_definition.get("path", f"/{uuid.uuid4()}")).exists()

pipeline/tools/luigi-service.yaml

@@ -1,9 +1,9 @@
-installed: false
 project-dir: &proj !get_default "{project-dir}"
 service-file: &svcfile !join_path [*proj, luigid.service]
+path: &path /lib/systemd/system/luigid.service
 
 install_commands:
-- !join [sudo cp, *svcfile, /lib/systemd/system/luigid.service]
+- !join [sudo cp, *svcfile, *path]
 - !join [sudo cp, $(which luigid), /usr/local/bin]
 - sudo systemctl daemon-reload
 - sudo systemctl start luigid.service

pipeline/tools/masscan.yaml

@@ -1,4 +1,3 @@
-installed: false
 tools: &tools !get_default "{tools-dir}"
 path: &path !join_path [*tools, masscan]
 

pipeline/tools/recursive-gobuster.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [gobuster]
 tools: &tools !get_default "{tools-dir}"
 path: &path !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz]

pipeline/tools/searchsploit.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [exploitdb]
 home: &home !get_default "{home}"
 tools: &tools !get_default "{tools-dir}"

pipeline/tools/seclists.yaml

@@ -1,4 +1,3 @@
-installed: false
 tools: &tools !get_default "{tools-dir}"
 path: &path !join_path [*tools, seclists]
 

pipeline/tools/subjack.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
 path: &path !join_path [!get_default "{gopath}", bin/subjack]

pipeline/tools/tko-subs.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
 path: &path !join_path [!get_default "{gopath}", bin/tko-subs]

pipeline/tools/waybackurls.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
 path: &path !join_path [!get_default "{gopath}", bin/waybackurls]

pipeline/tools/webanalyze.yaml

@@ -1,4 +1,3 @@
-installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
 path: &path !join_path [!get_default "{gopath}", bin/webanalyze]

tests/test_recon/test_helpers.py

@@ -32,30 +32,18 @@ def test_get_scans():
 
 
 @pytest.mark.parametrize(
-    "requirements, exception",
-    [
-        (["amass"], True),
-        (["masscan"], True),
-        (
-            [
-                "amass",
-                "aquatone",
-                "masscan",
-                "tko-subs",
-                "recursive-gobuster",
-                "searchsploit",
-                "subjack",
-                "gobuster",
-                "webanalyze",
-                "waybackurls",
-            ],
-            False,
-        ),
-    ],
+    "requirements, exception, expected",
+    [(["amass"], True, None), (["amass"], False, False), (["aquatone"], False, True)],
 )
-def test_meets_requirements(requirements, exception):
-    with patch("pipeline.recon.helpers.get_tool_state"):
-        assert meets_requirements(requirements, exception)
+def test_meets_requirements(requirements, exception, expected):
+    mdict = {"amass": {"installed": False}, "aquatone": {"installed": True}}
+    with patch("pipeline.recon.helpers.tools", autospec=dict) as mtools:
+        mtools.get.return_value = mdict.get(requirements[0])
+        if exception:
+            with pytest.raises(RuntimeError):
+                meets_requirements(requirements, exception)
+        else:
+            assert meets_requirements(requirements, exception) is expected
 
 
 @pytest.mark.parametrize(

tests/test_shell/test_recon_pipeline_shell.py

@@ -481,13 +481,15 @@ class TestReconShell:
 
         subfile.touch()
         assert subfile.exists()
+        old_loop = recon_shell.ReconShell.cmdloop
 
-        with patch("cmd2.Cmd.cmdloop"), patch("sys.exit"), patch("cmd2.Cmd.select") as mocked_select:
-            mocked_select.return_value = test_input
+        recon_shell.ReconShell.cmdloop = MagicMock()
+        recon_shell.cmd2.Cmd.select = MagicMock(return_value=test_input)
+        with patch("sys.exit"):
             recon_shell.main(
                 name="__main__", old_tools_dir=tooldir, old_tools_dict=tooldict, old_searchsploit_rc=searchsploit_rc
             )
-
+        recon_shell.ReconShell.cmdloop = old_loop
         for file in [subfile, tooldir, tooldict, searchsploit_rc]:
             if test_input == "Yes":
                 assert not file.exists()
@@ -503,16 +505,16 @@ class TestReconShell:
         new_tmp = tmp_path / f"check_scan_directory_test-{user_input}-{answer}"
         new_tmp.mkdir()
 
-        with patch("cmd2.Cmd.select") as mocked_select:
-            mocked_select.return_value = answer
+        recon_shell.cmd2.Cmd.select = MagicMock(return_value=answer)
 
-            self.shell.check_scan_directory(str(new_tmp))
+        print(list(tmp_path.iterdir()), new_tmp)
+        self.shell.check_scan_directory(str(new_tmp))
 
-            assert new_tmp.exists() == exists
-            assert len(list(tmp_path.iterdir())) == numdirs
-
-            if answer == "Save":
-                assert (
-                    re.search(r"check_scan_directory_test-3-Save-[0-9]{6,8}-[0-9]+", str(list(tmp_path.iterdir())[0]))
-                    is not None
-                )
+        assert new_tmp.exists() == exists
+        print(list(tmp_path.iterdir()), new_tmp)
+        assert len(list(tmp_path.iterdir())) == numdirs
+        if answer == "Save":
+            assert (
+                re.search(r"check_scan_directory_test-3-Save-[0-9]{6,8}-[0-9]+", str(list(tmp_path.iterdir())[0]))
+                is not None
+            )

tests/test_tools_install/test_tools_install_command.py

@@ -55,7 +55,7 @@ class TestUnmockedToolsInstall:
         tool_dict.get(dependency)["path"] = dependency_path
         tool_dict.get(dependency).get("install_commands")[
             0
-        ] = f"wget -q https://dl.google.com/go/go1.14.6.linux-amd64.tar.gz -O {tmp_path}/go.tar.gz"
+        ] = f"wget -q https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz -O {tmp_path}/go.tar.gz"
         tool_dict.get(dependency).get("install_commands")[
             1
         ] = f"tar -C {self.shell.tools_dir} -xvf {tmp_path}/go.tar.gz"

tests/test_web/test_gobuster.py

@@ -1,3 +1,4 @@
+import os
 import shutil
 import tempfile
 from pathlib import Path
@@ -41,7 +42,8 @@ class TestGobusterScan:
             assert mocked_run.called
             assert self.scan.parse_results.called
 
-    def test_scan_recursive_run(self):
+    def test_scan_recursive_run(self, tmp_path):
+        os.chdir(tmp_path)
         with patch("concurrent.futures.ThreadPoolExecutor.map") as mocked_run:
             self.scan.parse_results = MagicMock()
             self.scan.db_mgr.get_all_web_targets = MagicMock()