Modifies reference structure for tool paths (#61)

docs/modifications/new_scanner.rst

@@ -53,19 +53,18 @@ Dynamically creating strings and filesystem paths are handled by the following t
 
 In order to get values out of ``pipeline.recon.config.py``, you'll need to use one of the yaml helpers listed below.
 
-- ``!get_parent`` - (niche) get parent directory of a given tool that is defined in the ``pipeline.recon.config.tool_paths`` dictionary
 - ``!get_default`` - get a value from the ``pipeline.recon.config.defaults`` dictionary
-- ``!get_tool_path`` - get a value from the ``pipeline.recon.config.tool_paths`` dictionary
+- ``!get_tool_path`` - get a path value from the ``pipeline.tools.tools`` dictionary
 
 Simple Example Tool Definition
 ******************************
 
-The example below needs go to be installed prior to being installed itself.  It then grabs the path to the ``go`` binary from ``pipeline.recon.config.tool_paths`` by using ``!get_tool_path``.  After that, it creates a command using ``!join`` that will look like ``/usr/local/go/bin/go get github.com/tomnomnom/waybackurls``.  This command will be run by the ``install waybackurls`` command (or ``install all``).
+The example below needs go to be installed prior to being installed itself.  It then grabs the path to the ``go`` binary from ``pipeline.tools.tools`` by using ``!get_tool_path``.  After that, it creates a command using ``!join`` that will look like ``/usr/local/go/bin/go get github.com/tomnomnom/waybackurls``.  This command will be run by the ``install waybackurls`` command (or ``install all``).
 
 .. code-block:: yaml
 
     dependencies: [go]
-    go: &gobin !get_tool_path "{go}"
+    go: &gobin !get_tool_path "{go[path]}"
 
     commands:
     - !join [*gobin, get github.com/tomnomnom/waybackurls]

pipeline/recon-pipeline.py

@@ -341,13 +341,11 @@ class ReconShell(cmd2.Cmd):
                 self.do_install(tool)
 
             return
-
         if persistent_tool_dict.exists():
             tools = pickle.loads(persistent_tool_dict.read_bytes())
 
         if tools.get(args.tool).get("dependencies"):
             # get all of the requested tools dependencies
-
             for dependency in tools.get(args.tool).get("dependencies"):
                 if tools.get(dependency).get("installed"):
                     # already installed, skip it

pipeline/recon/__init__.py

@@ -4,7 +4,7 @@ from .wrappers import FullScan, HTBScan
 from .amass import AmassScan, ParseAmassOutput
 from .masscan import MasscanScan, ParseMasscanOutput
 from .nmap import ThreadedNmapScan, SearchsploitScan
-from .config import tool_paths, top_udp_ports, top_tcp_ports, defaults, web_ports
+from .config import top_udp_ports, top_tcp_ports, defaults, web_ports
 from .parsers import (
     install_parser,
     scan_parser,

pipeline/recon/amass.py

@@ -7,8 +7,8 @@ from luigi.util import inherits
 from luigi.contrib.sqla import SQLAlchemyTarget
 
 import pipeline.models.db_manager
-from .config import tool_paths
 from .targets import TargetList
+from ..tools import tools
 from ..models.target_model import Target
 
 
@@ -94,7 +94,7 @@ class AmassScan(luigi.Task):
             return subprocess.run(f"touch {self.output().path}".split())
 
         command = [
-            f"{tool_paths.get('amass')}",
+            tools.get("amass").get("path"),
             "enum",
             "-active",
             "-ip",

pipeline/recon/config.py

@@ -16,25 +16,7 @@ defaults = {
 defaults["tools-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/tools"
 defaults["database-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/databases"
 defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt"
-
+defaults["project-dir"] = str(Path(__file__).parents[2])
-tool_paths = {
-    "aquatone": f"{defaults.get('tools-dir')}/aquatone",
-    "tko-subs": f"{Path.home()}/go/bin/tko-subs",
-    "tko-subs-dir": f"{Path.home()}/go/src/github.com/anshumanbh/tko-subs",
-    "subjack": f"{Path.home()}/go/bin/subjack",
-    "subjack-fingerprints": f"{Path.home()}/go/src/github.com/haccer/subjack/fingerprints.json",
-    "gobuster": f"{Path.home()}/go/bin/gobuster",
-    "recursive-gobuster": f"{defaults.get('tools-dir')}/recursive-gobuster/recursive-gobuster.pyz",
-    "webanalyze": f"{Path.home()}/go/bin/webanalyze",
-    "masscan": f"{defaults.get('tools-dir')}/masscan",
-    "amass": f"{defaults.get('tools-dir')}/amass",
-    "go": "/usr/local/go/bin/go",
-    "searchsploit": f"{defaults.get('tools-dir')}/exploitdb/searchsploit",
-    "luigid": str(Path(__file__).parents[2] / "luigid.service"),
-    "seclists": f"{defaults.get('tools-dir')}/seclists",
-    "exploitdb": f"{defaults.get('tools-dir')}/exploitdb",
-    "waybackurls": f"{Path.home()}/go/bin/waybackurls",
-}
 
 web_ports = {
     "80",

pipeline/recon/masscan.py

@@ -9,11 +9,12 @@ from luigi.contrib.sqla import SQLAlchemyTarget
 
 import pipeline.models.db_manager
 from .targets import TargetList
+from ..tools import tools
 from .amass import ParseAmassOutput
 from ..models.port_model import Port
 from ..models.ip_address_model import IPAddress
 
-from .config import top_tcp_ports, top_udp_ports, defaults, tool_paths, web_ports
+from .config import top_tcp_ports, top_udp_ports, defaults, web_ports
 
 
 @inherits(TargetList, ParseAmassOutput)
@@ -108,7 +109,7 @@ class MasscanScan(luigi.Task):
             )
 
         command = [
-            tool_paths.get("masscan"),
+            tools.get("masscan").get("path"),
             "-v",
             "--open",
             "--banners",

pipeline/recon/nmap.py

@@ -12,9 +12,10 @@ from luigi.contrib.sqla import SQLAlchemyTarget
 
 import pipeline.models.db_manager
 from .masscan import ParseMasscanOutput
-from .config import defaults, tool_paths
+from .config import defaults
 from .helpers import get_ip_address_version, is_ip_address
 
+from ..tools import tools
 from ..models.port_model import Port
 from ..models.nse_model import NSEResult
 from ..models.target_model import Target
@@ -281,7 +282,7 @@ class SearchsploitScan(luigi.Task):
         """ Grabs the xml files created by ThreadedNmap and runs searchsploit --nmap on each one, saving the output. """
         for entry in Path(self.input().get("localtarget").path).glob("nmap*.xml"):
             proc = subprocess.run(
-                [tool_paths.get("searchsploit"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE
+                [tools.get("searchsploit").get("path"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE
             )
             if proc.stdout:
                 # change  wall-searchsploit-results/nmap.10.10.10.157-tcp to 10.10.10.157

pipeline/recon/web/aquatone.py

@@ -9,7 +9,8 @@ from luigi.util import inherits
 from luigi.contrib.sqla import SQLAlchemyTarget
 
 from .targets import GatherWebTargets
-from ..config import tool_paths, defaults
+from ..config import defaults
+from ...tools import tools
 
 import pipeline.models.db_manager
 from ...models.port_model import Port
@@ -250,7 +251,7 @@ class AquatoneScan(luigi.Task):
         self.results_subfolder.mkdir(parents=True, exist_ok=True)
 
         command = [
-            tool_paths.get("aquatone"),
+            tools.get("aquatone").get("path"),
             "-scan-timeout",
             self.scan_timeout,
             "-threads",

pipeline/recon/web/gobuster.py

@@ -11,7 +11,8 @@ from luigi.contrib.sqla import SQLAlchemyTarget
 
 import pipeline.models.db_manager
 from .targets import GatherWebTargets
-from ..config import tool_paths, defaults
+from ..config import defaults
+from ...tools import tools
 from ...models.endpoint_model import Endpoint
 from ..helpers import get_ip_address_version, is_ip_address
 
@@ -139,10 +140,16 @@ class GobusterScan(luigi.Task):
 
             for url_scheme in ("https://", "http://"):
                 if self.recursive:
-                    command = [tool_paths.get("recursive-gobuster"), "-s", "-w", self.wordlist, f"{url_scheme}{target}"]
+                    command = [
+                        tools.get("recursive-gobuster").get("path"),
+                        "-s",
+                        "-w",
+                        self.wordlist,
+                        f"{url_scheme}{target}",
+                    ]
                 else:
                     command = [
-                        tool_paths.get("gobuster"),
+                        tools.get("gobuster").get("path"),
                         "dir",
                         "-q",
                         "-e",

pipeline/recon/web/subdomain_takeover.py

@@ -8,8 +8,9 @@ from luigi.util import inherits
 from luigi.contrib.sqla import SQLAlchemyTarget
 
 import pipeline.models.db_manager
+from ...tools import tools
 from .targets import GatherWebTargets
-from ..config import tool_paths, defaults
+from ..config import defaults
 
 
 @inherits(GatherWebTargets)
@@ -120,9 +121,9 @@ class TKOSubsScan(luigi.Task):
             return
 
         command = [
-            tool_paths.get("tko-subs"),
+            tools.get("tko-subs").get("path"),
             f"-domain={','.join(domains)}",
-            f"-data={tool_paths.get('tko-subs-dir')}/providers-data.csv",
+            f"-data={tools.get('tko-subs').get('git_dir')}/providers-data.csv",
             f"-output={self.output_file}",
         ]
 
@@ -261,7 +262,7 @@ class SubjackScan(luigi.Task):
                 f.write(f"{hostname}\n")
 
         command = [
-            tool_paths.get("subjack"),
+            tools.get("subjack").get("path"),
             "-w",
             str(subjack_input_file),
             "-t",
@@ -274,7 +275,7 @@ class SubjackScan(luigi.Task):
             "-v",
             "-ssl",
             "-c",
-            tool_paths.get("subjack-fingerprints"),
+            tools.get("subjack").get("fingerprints"),
         ]
 
         subprocess.run(command)

pipeline/recon/web/webanalyze.py

@@ -11,8 +11,9 @@ from luigi.util import inherits
 from luigi.contrib.sqla import SQLAlchemyTarget
 
 import pipeline.models.db_manager
+from ...tools import tools
 from .targets import GatherWebTargets
-from ..config import tool_paths, defaults
+from ..config import defaults
 from ...models.technology_model import Technology
 from ..helpers import get_ip_address_version, is_ip_address
 
@@ -153,7 +154,7 @@ class WebanalyzeScan(luigi.Task):
                 target = f"[{target}]"
 
             for url_scheme in ("https://", "http://"):
-                command = [tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}", "-output", "csv"]
+                command = [tools.get("webanalyze").get("path"), "-host", f"{url_scheme}{target}", "-output", "csv"]
                 commands.append(command)
 
         self.results_subfolder.mkdir(parents=True, exist_ok=True)
@@ -162,7 +163,7 @@ class WebanalyzeScan(luigi.Task):
         os.chdir(self.results_subfolder)
 
         if not Path("apps.json").exists():
-            subprocess.run(f"{tool_paths.get('webanalyze')} -update".split())
+            subprocess.run(f"{tools.get('webanalyze').get('path')} -update".split())
 
         with ThreadPoolExecutor(max_workers=self.threads) as executor:
             executor.map(self._wrapped_subprocess, commands)

pipeline/tools/amass.yaml

@@ -1,10 +1,11 @@
 installed: false
 dependencies: [go]
-go: &gotool !get_tool_path "{go}"
+tools: &tools !get_default "{tools-dir}"
-amass: &amass !get_tool_path "{amass}"
+go: &gotool !get_tool_path "{go[path]}"
+path: &amass !join_path [*tools, "amass"]
 
 commands:
-- !join [*gotool, get -u github.com/OWASP/Amass/v3/...]
+- !join [*gotool, get github.com/OWASP/Amass/v3/...]
 - !join [cp ~/go/bin/amass, *amass]
 
 shell: true

pipeline/tools/aquatone.yaml

@@ -1,6 +1,6 @@
 installed: false
-dependencies:
+tools: &tools !get_default "{tools-dir}"
-aquatone: &aqua !get_tool_path "{aquatone}"
+path: &aqua !join_path [*tools, aquatone]
 
 commands:
 - mkdir /tmp/aquatone
@@ -10,5 +10,3 @@ commands:
 - !join [mv /tmp/aquatone/aquatone, *aqua]
 - rm -rf /tmp/aquatone
 - bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'
-
-shell: true

pipeline/tools/exploitdb.yaml

@@ -0,0 +1,3 @@
+installed: true
+tools: &tools !get_default "{tools-dir}"
+path: !join_path [*tools, exploitdb]

pipeline/tools/go.yaml

@@ -1,12 +1,9 @@
 installed: false
-dependencies:
 home: &home !get_default "{home}"
-go: &gotool !get_tool_path "{go}"
+path: &gotool /usr/local/go/bin/go
 bashrc: &bashrc !join_path [*home, "/.bashrc;"]
 
 commands:
-- wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz
+- wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -O /tmp/go.tar.gz
 - sudo tar -C /usr/local -xvf /tmp/go.tar.gz
 - !join ["bash -c 'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >>", *bashrc, "fi'"]
-
-shell: true

pipeline/tools/gobuster.yaml

@@ -1,7 +1,8 @@
 installed: false
 dependencies: [go, seclists]
 home: &home !get_default "{home}"
-go: &gotool !get_tool_path "{go}"
+path: !join_path [*home, go/bin/gobuster]
+go: &gotool !get_tool_path "{go[path]}"
 go_home: &gohome !join_path [*home, "/go/src/github.com/OJ/gobuster &&"]
 
 commands:

pipeline/tools/loader.py

@@ -1,9 +1,10 @@
 import yaml
 from pathlib import Path
 
-from ..recon.config import tool_paths, defaults
+from ..recon.config import defaults
 
 definitions = Path(__file__).parent
+tools = {}
 
 
 def join(loader, node):
@@ -30,16 +31,10 @@ def get_default(loader, node):
     return py_str.format(**defaults)
 
 
-def get_parent(loader, node):
-    """ yaml tag handler to access tool parents """
-    py_str = loader.construct_python_str(node)
-    return Path(py_str.format(**tool_paths)).parent
-
-
 def get_tool_path(loader, node):
-    """ yaml tag handler to access tool_paths dict at load time """
+    """ yaml tag handler to access tools dict at load time """
     py_str = loader.construct_python_str(node)
-    return py_str.format(**tool_paths)
+    return py_str.format(**tools)
 
 
 yaml.add_constructor("!join", join)
@@ -47,12 +42,20 @@ yaml.add_constructor("!join_empty", join_empty)
 yaml.add_constructor("!join_path", join_path)
 yaml.add_constructor("!get_default", get_default)
 yaml.add_constructor("!get_tool_path", get_tool_path)
-yaml.add_constructor("!get_parent", get_parent)
 
 
-tools = {}
+def load_yaml(file):
-for file in definitions.iterdir():
+    try:
-    if file.name.endswith(".yaml"):
         config = yaml.full_load(file.read_text())
         tool_name = str(file.name.replace(".yaml", ""))
         tools[tool_name] = config
+    except KeyError as error:  # load dependencies first
+        dependency = error.args[0]
+        dependency_file = definitions / (dependency + ".yaml")
+        load_yaml(dependency_file)
+        load_yaml(file)
+
+
+for file in definitions.iterdir():
+    if file.name.endswith(".yaml") and file.name.replace(".yaml", "") not in tools:
+        load_yaml(file)

pipeline/tools/luigi-service.yaml

@@ -1,6 +1,6 @@
 installed: false
-dependencies:
+project-dir: &proj !get_default "{project-dir}"
-service-file: &svcfile !get_tool_path "{luigid}"
+service-file: &svcfile !join_path [*proj, luigid.service]
 
 commands:
 - !join [sudo cp, *svcfile, /lib/systemd/system/luigid.service]

pipeline/tools/masscan.yaml

@@ -1,6 +1,6 @@
 installed: false
-dependencies:
+tools: &tools !get_default "{tools-dir}"
-masscan: &masscan !get_tool_path "{masscan}"
+path: &masscan !join_path [*tools, masscan]
 
 commands:
 - git clone https://github.com/robertdavidgraham/masscan /tmp/masscan
@@ -8,5 +8,3 @@ commands:
 - !join [mv /tmp/masscan/bin/masscan, *masscan]
 - rm -rf /tmp/masscan
 - !join [sudo setcap CAP_NET_RAW+ep, *masscan]
-
-shell: true

pipeline/tools/recursive-gobuster.yaml

@@ -1,10 +1,10 @@
 installed: false
 dependencies: [go]
-recursive-parent: &recpar !get_parent "{recursive-gobuster}"
+tools: &tools !get_default "{tools-dir}"
+path: !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz]
+recursive-parent: &recpar !join_path [*tools, recursive-gobuster]
 
 commands:
 - !join ["bash -c 'if [ -d", *recpar, "]; then cd", *recpar,
         "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git",
         *recpar, ; fi']
-
-shell: true

pipeline/tools/searchsploit.yaml

@@ -1,9 +1,9 @@
 installed: false
-dependencies:
+dependencies: [exploitdb]
 home: &home !get_default "{home}"
-tools-dir: &tools !get_default "{tools-dir}"
+tools: &tools !get_default "{tools-dir}"
-exploitdb-file: &exploitdb !get_tool_path "{exploitdb}"
+exploitdb-file: &exploitdb !get_tool_path "{exploitdb[path]}"
-searchsploit-file: &searchsploit !get_tool_path "{searchsploit}"
+path: &searchsploit !join_path [*tools, exploitdb/searchsploit]
 searchsploit-rc: &ss_rc !join_path [*exploitdb, ".searchsploit_rc"]
 homesploit: &homesploit !join_path [*home, ".searchsploit_rc"]
 sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"]
@@ -15,5 +15,3 @@ commands:
         "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, ; fi']
 - !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, ; fi']
 - !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, ; fi']
-
-shell: true

pipeline/tools/seclists.yaml

@@ -1,10 +1,8 @@
 installed: false
-depencencies:
+tools: &tools !get_default "{tools-dir}"
-seclists-file: &secfile !get_tool_path "{seclists}"
+path: &secfile !join_path [*tools, seclists]
 
 commands:
   - !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists",
           *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
           else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi']
-
-shell: true

pipeline/tools/subjack.yaml

@@ -1,8 +1,10 @@
 installed: false
 dependencies: [go]
-go: &gotool !get_tool_path "{go}"
+go: &gotool !get_tool_path "{go[path]}"
 home: &home !get_default "{home}"
+path: !join_path [*home, go/bin/subjack]
 subjack_home: &subjhome !join_path [*home, "/go/src/github.com/haccer/subjack &&"]
+fingerprints: !join_path [*home, go/src/github.com/haccer/subjack/fingerprints.json]
 
 commands:
 - !join [*gotool, get github.com/haccer/subjack]

pipeline/tools/tko-subs.yaml

@@ -1,8 +1,10 @@
 installed: false
 dependencies: [go]
-go: &gotool !get_tool_path "{go}"
+go: &gotool !get_tool_path "{go[path]}"
 home: &home !get_default "{home}"
+path: !join_path [*home, go/bin/tko-subs]
 tko_home: &tkohome !join_path [*home, "go/src/github.com/anshumanbh/tko-subs &&"]
+git_dir: !join_path [*home, go/src/github.com/anshumanbh/tko-subs]
 
 commands:
 - !join [*gotool, get, github.com/anshumanbh/tko-subs]

pipeline/tools/waybackurls.yaml

@@ -1,9 +1,7 @@
 installed: false
-dependencies: ["go"]
+dependencies: [go]
-go: &gotool !get_tool_path "{go}"
+go: &gotool !get_tool_path "{go[path]}"
 path: !join_path [!get_default "{home}", go, bin, waybackurls]
 
 commands:
 - !join [*gotool, get, github.com/tomnomnom/waybackurls]
-
-shell: false

pipeline/tools/webanalyze.yaml

@@ -1,7 +1,8 @@
 installed: false
 dependencies: [go]
 home: &home !get_default "{home}"
-go: &gotool !get_tool_path "{go}"
+go: &gotool !get_tool_path "{go[path]}"
+path: !join_path [*home, go/bin/webanalyze]
 webanalyze_home: &webhome !join_path [*home, "/go/src/github.com/rverton/webanalyze &&"]
 
 commands:

tests/test_recon/test_config.py

@@ -2,12 +2,14 @@ from pathlib import Path
 
 import pytest
 
-from pipeline.recon import tool_paths, defaults, web_ports, top_tcp_ports, top_udp_ports
+from pipeline.recon import defaults, web_ports, top_tcp_ports, top_udp_ports
+from pipeline.tools import tools
 
 
 def test_tool_paths_absolute():
-    for path in tool_paths.values():
+    for tool in tools.values():
-        assert Path(path).is_absolute()
+        if tool.get("path"):
+            assert Path(tool["path"]).is_absolute()
 
 
 @pytest.mark.parametrize("test_input", ["database-dir", "tools-dir", "gobuster-wordlist"])

tests/test_recon/test_nmap.py

@@ -7,7 +7,8 @@ import luigi
 import pytest
 from luigi.contrib.sqla import SQLAlchemyTarget
 
-from pipeline.recon import ThreadedNmapScan, SearchsploitScan, ParseMasscanOutput, config
+from pipeline.recon import ThreadedNmapScan, SearchsploitScan, ParseMasscanOutput
+from pipeline.tools import tools
 
 nmap_results = Path(__file__).parent.parent / "data" / "recon-results" / "nmap-results"
 
@@ -101,7 +102,7 @@ class TestSearchsploitScan:
 
         assert len(self.scan.db_mgr.get_all_searchsploit_results()) == 0
 
-        if not Path(config.tool_paths.get("searchsploit")).exists():
+        if not Path(tools.get("searchsploit").get("path")).exists():
             pytest.skip("exploit-db's searchsploit tool not installed")
 
         self.scan.run()

tests/test_shell/test_recon_pipeline_shell.py

@@ -12,7 +12,8 @@ import pytest
 from pipeline.models.port_model import Port
 from pipeline.models.target_model import Target
 from pipeline.models.db_manager import DBManager
-from pipeline.recon.config import defaults, tool_paths
+from pipeline.tools import tools
+from pipeline.recon.config import defaults
 from pipeline.models.ip_address_model import IPAddress
 
 recon_shell = importlib.import_module("pipeline.recon-pipeline")
@@ -294,143 +295,21 @@ class TestReconShell:
 
     # ("all", "commands failed and may have not installed properly", 1)
     # after tools moved to DB, update this test
-    @pytest.mark.parametrize("test_input, expected, return_code", [("all", "is already installed", 0)])
+    @pytest.mark.parametrize(
+        "test_input, expected, return_code", [("all", "is already installed", 0), ("amass", "dependency", 1)]
+    )
     def test_do_install(self, test_input, expected, return_code, capsys, tmp_path):
         process_mock = MagicMock()
         attrs = {"communicate.return_value": (b"output", b"error"), "returncode": return_code}
         process_mock.configure_mock(**attrs)
 
-        tool_dict = {
-            "luigi-service": {
-                "installed": False,
-                "dependencies": None,
-                "commands": [
-                    f"sudo cp {str(Path(__file__).parents[2] / 'luigid.service')} /lib/systemd/system/luigid.service",
-                    f"sudo cp $(which luigid) /usr/local/bin",
-                    "sudo systemctl daemon-reload",
-                    "sudo systemctl start luigid.service",
-                    "sudo systemctl enable luigid.service",
-                ],
-                "shell": True,
-            },
-            "seclists": {
-                "installed": False,
-                "dependencies": None,
-                "shell": True,
-                "commands": [
-                    f"bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists {defaults.get('tools-dir')}/seclists; elif [[ -d {defaults.get('tools-dir')}/seclists ]] ; then cd {defaults.get('tools-dir')}/seclists && git fetch --all && git pull; else git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists; fi'"
-                ],
-            },
-            "searchsploit": {
-                "installed": False,
-                "dependencies": None,
-                "shell": True,
-                "commands": [
-                    f"bash -c 'if [[ -d /usr/share/exploitdb ]]; then ln -s /usr/share/exploitdb {defaults.get('tools-dir')}/exploitdb && sudo ln -s $(which searchsploit) {defaults.get('tools-dir')}/exploitdb/searchsploit; elif [[ -d {Path(tool_paths.get('searchsploit')).parent} ]]; then cd {Path(tool_paths.get('searchsploit')).parent} && git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git {defaults.get('tools-dir')}/exploitdb; fi'",
-                    f"bash -c 'if [[ -f {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc ]]; then cp -n {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc {Path.home().expanduser().resolve()}; fi'",
-                    f"bash -c 'if [[ -f {Path.home().resolve()}/.searchsploit_rc ]]; then sed -i 's#/opt#{defaults.get('tools-dir')}#g' {Path.home().resolve()}/.searchsploit_rc; fi'",
-                ],
-            },
-            "masscan": {
-                "installed": False,
-                "dependencies": None,
-                "commands": [
-                    "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan",
-                    "make -s -j -C /tmp/masscan",
-                    f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}",
-                    "rm -rf /tmp/masscan",
-                    f"sudo setcap CAP_NET_RAW+ep {tool_paths.get('masscan')}",
-                ],
-            },
-            "amass": {
-                "installed": False,
-                "dependencies": ["go"],
-                "commands": [
-                    f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...",
-                    f"cp ~/go/bin/amass {tool_paths.get('amass')}",
-                ],
-                "shell": True,
-                "environ": {"GO111MODULE": "on"},
-            },
-            "aquatone": {
-                "installed": False,
-                "dependencies": None,
-                "shell": True,
-                "commands": [
-                    "mkdir /tmp/aquatone",
-                    "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip",
-                    "bash -c 'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'",
-                    "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone",
-                    f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}",
-                    "rm -rf /tmp/aquatone",
-                    "bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'",
-                ],
-            },
-            "gobuster": {
-                "installed": False,
-                "dependencies": ["go", "seclists"],
-                "commands": [
-                    f"{tool_paths.get('go')} get github.com/OJ/gobuster",
-                    f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
-                ],
-                "shell": True,
-            },
-            "tko-subs": {
-                "installed": False,
-                "dependencies": ["go"],
-                "commands": [
-                    f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs",
-                    f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
-                ],
-                "shell": True,
-            },
-            "subjack": {
-                "installed": False,
-                "dependencies": ["go"],
-                "commands": [
-                    f"{tool_paths.get('go')} get github.com/haccer/subjack",
-                    f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)",
-                ],
-                "shell": True,
-            },
-            "webanalyze": {
-                "installed": False,
-                "dependencies": ["go"],
-                "commands": [
-                    f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...",
-                    f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
-                ],
-                "shell": True,
-            },
-            "recursive-gobuster": {
-                "installed": False,
-                "dependencies": ["gobuster", "seclists"],
-                "shell": True,
-                "commands": [
-                    f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'"
-                ],
-            },
-            "go": {
-                "installed": False,
-                "dependencies": None,
-                "commands": [
-                    "wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz",
-                    "sudo tar -C /usr/local -xvf /tmp/go.tar.gz",
-                    f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'',
-                ],
-            },
-            "waybackurls": {
-                "installed": True,
-                "depencencies": ["go"],
-                "commands": ["/usr/local/go/bin/go get github.com/tomnomnom/waybackurls"],
-                "shell": True,
-            },
-        }
-
         tooldir = tmp_path / ".local" / "recon-pipeline" / "tools"
         tooldir.mkdir(parents=True, exist_ok=True)
 
-        pickle.dump(tool_dict, (tooldir / ".tool-dict.pkl").open("wb"))
+        tools["waybackurls"]["installed"] = True
+        tools["amass"]["shell"] = False
+
+        pickle.dump(tools, (tooldir / ".tool-dict.pkl").open("wb"))
 
         with patch("subprocess.Popen", autospec=True) as mocked_popen:
             mocked_popen.return_value = process_mock

tests/test_web/test_webanalyze.py

@@ -4,7 +4,7 @@ import tempfile
 from pathlib import Path
 from unittest.mock import patch, MagicMock
 
-from pipeline.recon.config import tool_paths
+from pipeline.tools import tools
 from pipeline.recon.web import WebanalyzeScan, GatherWebTargets
 
 webanalyze_results = Path(__file__).parent.parent / "data" / "recon-results" / "webanalyze-results"
@@ -65,7 +65,7 @@ class TestWebanalyzeScan:
             self.scan.results_subfolder.mkdir()
             os.chdir(self.scan.results_subfolder)
             assert len([x for x in self.scan.results_subfolder.iterdir()]) == 0
-            cmd = [tool_paths.get("webanalyze"), "-host", "https://google.com", "-output", "csv"]
+            cmd = [tools.get("webanalyze").get("path"), "-host", "https://google.com", "-output", "csv"]
             self.scan._wrapped_subprocess(cmd)
             assert len([x for x in self.scan.results_subfolder.iterdir()]) == 1
             assert next(self.scan.results_subfolder.iterdir()).name == "webanalyze-https_google.com.csv"