pivot to yaml structure for tool definitions (#33)

pipeline/recon-pipeline.py

@@ -59,7 +59,6 @@ from .models.searchsploit_model import SearchsploitResult  # noqa: F401,E402
 
 from .recon import (  # noqa: F401,E402
     get_scans,
-    tools,
     scan_parser,
     install_parser,
     status_parser,
@@ -77,6 +76,8 @@ from .recon import (  # noqa: F401,E402
     port_results_parser,
 )
 
+from .tools import tools  # noqa: F401,E402
+
 # select loop, handles async stdout/stderr processing of subprocesses
 selector = selectors.DefaultSelector()
 

pipeline/recon/__init__.py

@@ -1,6 +1,5 @@
 from .helpers import get_scans
 from .targets import TargetList
-from .tool_definitions import tools
 from .wrappers import FullScan, HTBScan
 from .amass import AmassScan, ParseAmassOutput
 from .masscan import MasscanScan, ParseMasscanOutput

pipeline/recon/config.py

@@ -12,6 +12,7 @@ defaults = {
     "aquatone-scan-timeout": "900",
     "tools-dir": f"{Path.home()}/.local/recon-pipeline/tools",
     "database-dir": f"{Path.home()}/.local/recon-pipeline/databases",
+    "home": Path.home().expanduser().resolve(),
 }
 
 defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt"
@@ -30,6 +31,9 @@ tool_paths = {
     "amass": f"{defaults.get('tools-dir')}/amass",
     "go": "/usr/local/go/bin/go",
     "searchsploit": f"{defaults.get('tools-dir')}/exploitdb/searchsploit",
+    "luigid": str(Path(__file__).parents[2] / "luigid.service"),
+    "seclists": f"{defaults.get('tools-dir')}/seclists",
+    "exploitdb": f"{defaults.get('tools-dir')}/exploitdb",
 }
 
 web_ports = {

pipeline/recon/parsers.py

@@ -4,7 +4,7 @@ import cmd2
 
 from .config import defaults
 from .helpers import get_scans
-from .tool_definitions import tools
+from ..tools import tools
 
 # options for ReconShell's 'install' command
 install_parser = cmd2.Cmd2ArgumentParser()

pipeline/recon/tool_definitions.py

@@ -1,125 +0,0 @@
-from pathlib import Path
-
-from .config import tool_paths, defaults
-
-# tool definitions for recon-pipeline's auto-installer
-tools = {
-    "luigi-service": {
-        "installed": False,
-        "dependencies": None,
-        "commands": [
-            f"sudo cp {str(Path(__file__).parents[2] / 'luigid.service')} /lib/systemd/system/luigid.service",
-            f"sudo cp $(which luigid) /usr/local/bin",
-            "sudo systemctl daemon-reload",
-            "sudo systemctl start luigid.service",
-            "sudo systemctl enable luigid.service",
-        ],
-        "shell": True,
-    },
-    "seclists": {
-        "installed": False,
-        "dependencies": None,
-        "shell": True,
-        "commands": [
-            f"bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists {defaults.get('tools-dir')}/seclists; elif [[ -d {defaults.get('tools-dir')}/seclists ]] ; then cd {defaults.get('tools-dir')}/seclists && git fetch --all && git pull; else git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists; fi'"
-        ],
-    },
-    "searchsploit": {
-        "installed": False,
-        "dependencies": None,
-        "shell": True,
-        "commands": [
-            f"bash -c 'if [[ -d /usr/share/exploitdb ]]; then ln -s /usr/share/exploitdb {defaults.get('tools-dir')}/exploitdb && sudo ln -s $(which searchsploit) {defaults.get('tools-dir')}/exploitdb/searchsploit; elif [[ -d {Path(tool_paths.get('searchsploit')).parent} ]]; then cd {Path(tool_paths.get('searchsploit')).parent} && git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git {defaults.get('tools-dir')}/exploitdb; fi'",
-            f"bash -c 'if [[ -f {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc ]]; then cp -n {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc {Path.home().expanduser().resolve()}; fi'",
-            f"bash -c 'if [[ -f {Path.home().resolve()}/.searchsploit_rc ]]; then sed -i 's#/opt#{defaults.get('tools-dir')}#g' {Path.home().resolve()}/.searchsploit_rc; fi'",
-        ],
-    },
-    "masscan": {
-        "installed": False,
-        "dependencies": None,
-        "commands": [
-            "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan",
-            "make -s -j -C /tmp/masscan",
-            f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}",
-            "rm -rf /tmp/masscan",
-            f"sudo setcap CAP_NET_RAW+ep {tool_paths.get('masscan')}",
-        ],
-    },
-    "amass": {
-        "installed": False,
-        "dependencies": ["go"],
-        "commands": [
-            f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...",
-            f"cp ~/go/bin/amass {tool_paths.get('amass')}",
-        ],
-        "shell": True,
-        "environ": {"GO111MODULE": "on"},
-    },
-    "aquatone": {
-        "installed": False,
-        "dependencies": None,
-        "shell": True,
-        "commands": [
-            "mkdir /tmp/aquatone",
-            "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip",
-            "bash -c 'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'",
-            "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone",
-            f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}",
-            "rm -rf /tmp/aquatone",
-            "bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'",
-        ],
-    },
-    "gobuster": {
-        "installed": False,
-        "dependencies": ["go", "seclists"],
-        "commands": [
-            f"{tool_paths.get('go')} get github.com/OJ/gobuster",
-            f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
-        ],
-        "shell": True,
-    },
-    "tko-subs": {
-        "installed": False,
-        "dependencies": ["go"],
-        "commands": [
-            f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs",
-            f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
-        ],
-        "shell": True,
-    },
-    "subjack": {
-        "installed": False,
-        "dependencies": ["go"],
-        "commands": [
-            f"{tool_paths.get('go')} get github.com/haccer/subjack",
-            f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)",
-        ],
-        "shell": True,
-    },
-    "webanalyze": {
-        "installed": False,
-        "dependencies": ["go"],
-        "commands": [
-            f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...",
-            f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
-        ],
-        "shell": True,
-    },
-    "recursive-gobuster": {
-        "installed": False,
-        "dependencies": ["gobuster", "seclists"],
-        "shell": True,
-        "commands": [
-            f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'"
-        ],
-    },
-    "go": {
-        "installed": False,
-        "dependencies": None,
-        "commands": [
-            "wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz",
-            "sudo tar -C /usr/local -xvf /tmp/go.tar.gz",
-            f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'',
-        ],
-    },
-}

pipeline/tools/__init__.py

@@ -0,0 +1 @@
+from .loader import tools  # noqa: F401,E402

pipeline/tools/amass.yaml

@@ -0,0 +1,11 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+amass: &amass !get_tool_path "{amass}"
+
+commands:
+- !join [*gotool, "get -u github.com/OWASP/Amass/v3/..."]
+- !join ["cp ~/go/bin/amass", *amass]
+
+shell: true
+environ: {"GO111MODULE": "on"}

pipeline/tools/aquatone.yaml

@@ -0,0 +1,14 @@
+installed: false
+dependencies:
+aquatone: &aqua !get_tool_path "{aquatone}"
+
+commands:
+- mkdir /tmp/aquatone
+- wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip
+- !join [bash, -c, "'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'"]
+- unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone
+- !join [mv, /tmp/aquatone/aquatone, *aqua]
+- rm -rf /tmp/aquatone
+- !join [bash, -c, "'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'"]
+
+shell: true

pipeline/tools/go.yaml

@@ -0,0 +1,10 @@
+installed: false
+dependencies:
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz
+- sudo tar -C /usr/local -xvf /tmp/go.tar.gz
+- !join [bash, -c, "'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >> ~/.bashrc; fi'"]
+
+shell: true

pipeline/tools/gobuster.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go", "seclists"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/OJ/gobuster]
+- !join [(cd, ~/go/src/github.com/OJ/gobuster, "&&", *gotool, "build &&", *gotool, install)]
+
+shell: true

pipeline/tools/loader.py

@@ -0,0 +1,58 @@
+import yaml
+from pathlib import Path
+
+from ..recon.config import tool_paths, defaults
+
+definitions = Path(__file__).parent
+
+
+def join(loader, node):
+    """ yaml tag handler to join a sequence of items into a space-separated string at load time """
+    seq = loader.construct_sequence(node)
+    return " ".join([str(val) for val in seq])
+
+
+def join_empty(loader, node):
+    """ yaml tag handler to join a sequence of items into a single string with no separations """
+    seq = loader.construct_sequence(node)
+    return "".join([str(val) for val in seq])
+
+
+def join_path(loader, node):
+    """ yaml tag handler to join a sequence of items into a filesystem path at load time """
+    seq = loader.construct_sequence(node)
+    return "/".join([str(i) for i in seq])
+
+
+def get_default(loader, node):
+    """ yaml tag handler to access defaults dict at load time """
+    py_str = loader.construct_python_str(node)
+    return py_str.format(**defaults)
+
+
+def get_parent(loader, node):
+    """ yaml tag handler to access tool parents """
+    py_str = loader.construct_python_str(node)
+    return Path(py_str.format(**tool_paths)).parent
+
+
+def get_tool_path(loader, node):
+    """ yaml tag handler to access tool_paths dict at load time """
+    py_str = loader.construct_python_str(node)
+    return py_str.format(**tool_paths)
+
+
+yaml.add_constructor("!join", join)
+yaml.add_constructor("!join_empty", join_empty)
+yaml.add_constructor("!join_path", join_path)
+yaml.add_constructor("!get_default", get_default)
+yaml.add_constructor("!get_tool_path", get_tool_path)
+yaml.add_constructor("!get_parent", get_parent)
+
+
+tools = {}
+for file in definitions.iterdir():
+    if file.name.endswith(".yaml"):
+        config = yaml.full_load(file.read_text())
+        tool_name = str(file.name.replace(".yaml", ""))
+        tools[tool_name] = config

pipeline/tools/luigi-service.yaml

@@ -0,0 +1,12 @@
+installed: false
+dependencies:
+service-file: &svcfile !get_tool_path "{luigid}"
+
+commands:
+- !join [sudo, cp, *svcfile, /lib/systemd/system/luigid.service]
+- !join [sudo, cp, *svcfile, $(which luigid), /usr/local/bin]
+- sudo systemctl daemon-reload
+- sudo systemctl start luigid.service
+- sudo systemctl enable luigid.service
+
+shell: true

pipeline/tools/masscan.yaml

@@ -0,0 +1,12 @@
+installed: false
+dependencies:
+masscan: &masscan !get_tool_path "{masscan}"
+
+commands:
+- git clone https://github.com/robertdavidgraham/masscan /tmp/masscan
+- make -s -j -C /tmp/masscan
+- !join [mv, /tmp/masscan/bin/masscan, *masscan]
+- rm -rf /tmp/masscan
+- !join [sudo, setcap, CAP_NET_RAW+ep, *masscan]
+
+shell: true

pipeline/tools/recursive-gobuster.yaml

@@ -0,0 +1,10 @@
+installed: false
+dependencies: ["go"]
+recursive-parent: &recpar !get_parent "{recursive-gobuster}"
+
+commands:
+- !join [bash, -c, "'if [ -d", *recpar, "]; then cd", *recpar,
+        "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git",
+        *recpar, "; fi'"]
+
+shell: true

pipeline/tools/searchsploit.yaml

@@ -0,0 +1,19 @@
+installed: false
+dependencies:
+home: &home !get_default "{home}"
+tools-dir: &tools !get_default "{tools-dir}"
+exploitdb-file: &exploitdb !get_tool_path "{exploitdb}"
+searchsploit-file: &searchsploit !get_tool_path "{searchsploit}"
+searchsploit-rc: &ss_rc !join_path [*exploitdb, ".searchsploit_rc"]
+homesploit: &homesploit !join_path [*home, ".searchsploit_rc"]
+sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"]
+
+commands:
+- !join [bash, -c, "'if [ -d /usr/share/exploitdb ]; then ln -fs /usr/share/exploitdb",
+        *exploitdb, "&& sudo ln -fs $(which searchsploit)", *searchsploit,
+        "; elif [ -d", *exploitdb, "]; then cd", *exploitdb,
+        "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, "; fi'"]
+- !join [bash, -c, "'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, "; fi'"]
+- !join [bash, -c, "'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, "; fi'"]
+
+shell: true

pipeline/tools/seclists.yaml

@@ -0,0 +1,10 @@
+installed: false
+depencencies:
+seclists-file: &secfile !get_tool_path "{seclists}"
+
+commands:
+  - !join [bash, -c, "'if [[ -d /usr/share/seclists ]];", "then ln -s /usr/share/seclists",
+          *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
+          "else git clone https://github.com/danielmiessler/SecLists.git", *secfile, "; fi'"]
+
+shell: true

pipeline/tools/subjack.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/haccer/subjack]
+- !join ["(cd ~/go/src/github.com/haccer/subjack &&", *gotool, "install)"]
+
+shell: true

pipeline/tools/tko-subs.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/haccer/subjack]
+- !join ["(cd ~/go/src/github.com/haccer/subjack &&", *gotool, "install)"]
+
+shell: true

pipeline/tools/webanalyze.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/rverton/webanalyze/...]
+- !join ["(cd ~/go/src/github.com/rverton/webanalyze &&", *gotool, "build &&", *gotool, "install)"]
+
+shell: true

tests/test_recon/test_parsers.py

@@ -2,7 +2,7 @@
 import pytest
 
 from pipeline.recon.parsers import *  # noqa: F403
-from pipeline.recon.tool_definitions import tools
+from pipeline.tools import tools
 
 
 @pytest.mark.parametrize("test_input", list(tools.keys()) + ["all"])