pivot to yaml structure for tool definitions (#33)

pipeline/tools/__init__.py

@@ -0,0 +1 @@
+from .loader import tools  # noqa: F401,E402

pipeline/tools/amass.yaml

@@ -0,0 +1,11 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+amass: &amass !get_tool_path "{amass}"
+
+commands:
+- !join [*gotool, "get -u github.com/OWASP/Amass/v3/..."]
+- !join ["cp ~/go/bin/amass", *amass]
+
+shell: true
+environ: {"GO111MODULE": "on"}

pipeline/tools/aquatone.yaml

@@ -0,0 +1,14 @@
+installed: false
+dependencies:
+aquatone: &aqua !get_tool_path "{aquatone}"
+
+commands:
+- mkdir /tmp/aquatone
+- wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip
+- !join [bash, -c, "'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'"]
+- unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone
+- !join [mv, /tmp/aquatone/aquatone, *aqua]
+- rm -rf /tmp/aquatone
+- !join [bash, -c, "'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'"]
+
+shell: true

pipeline/tools/go.yaml

@@ -0,0 +1,10 @@
+installed: false
+dependencies:
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz
+- sudo tar -C /usr/local -xvf /tmp/go.tar.gz
+- !join [bash, -c, "'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >> ~/.bashrc; fi'"]
+
+shell: true

pipeline/tools/gobuster.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go", "seclists"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/OJ/gobuster]
+- !join [(cd, ~/go/src/github.com/OJ/gobuster, "&&", *gotool, "build &&", *gotool, install)]
+
+shell: true

pipeline/tools/loader.py

@@ -0,0 +1,58 @@
+import yaml
+from pathlib import Path
+
+from ..recon.config import tool_paths, defaults
+
+definitions = Path(__file__).parent
+
+
+def join(loader, node):
+    """ yaml tag handler to join a sequence of items into a space-separated string at load time """
+    seq = loader.construct_sequence(node)
+    return " ".join([str(val) for val in seq])
+
+
+def join_empty(loader, node):
+    """ yaml tag handler to join a sequence of items into a single string with no separations """
+    seq = loader.construct_sequence(node)
+    return "".join([str(val) for val in seq])
+
+
+def join_path(loader, node):
+    """ yaml tag handler to join a sequence of items into a filesystem path at load time """
+    seq = loader.construct_sequence(node)
+    return "/".join([str(i) for i in seq])
+
+
+def get_default(loader, node):
+    """ yaml tag handler to access defaults dict at load time """
+    py_str = loader.construct_python_str(node)
+    return py_str.format(**defaults)
+
+
+def get_parent(loader, node):
+    """ yaml tag handler to access tool parents """
+    py_str = loader.construct_python_str(node)
+    return Path(py_str.format(**tool_paths)).parent
+
+
+def get_tool_path(loader, node):
+    """ yaml tag handler to access tool_paths dict at load time """
+    py_str = loader.construct_python_str(node)
+    return py_str.format(**tool_paths)
+
+
+yaml.add_constructor("!join", join)
+yaml.add_constructor("!join_empty", join_empty)
+yaml.add_constructor("!join_path", join_path)
+yaml.add_constructor("!get_default", get_default)
+yaml.add_constructor("!get_tool_path", get_tool_path)
+yaml.add_constructor("!get_parent", get_parent)
+
+
+tools = {}
+for file in definitions.iterdir():
+    if file.name.endswith(".yaml"):
+        config = yaml.full_load(file.read_text())
+        tool_name = str(file.name.replace(".yaml", ""))
+        tools[tool_name] = config

pipeline/tools/luigi-service.yaml

@@ -0,0 +1,12 @@
+installed: false
+dependencies:
+service-file: &svcfile !get_tool_path "{luigid}"
+
+commands:
+- !join [sudo, cp, *svcfile, /lib/systemd/system/luigid.service]
+- !join [sudo, cp, *svcfile, $(which luigid), /usr/local/bin]
+- sudo systemctl daemon-reload
+- sudo systemctl start luigid.service
+- sudo systemctl enable luigid.service
+
+shell: true

pipeline/tools/masscan.yaml

@@ -0,0 +1,12 @@
+installed: false
+dependencies:
+masscan: &masscan !get_tool_path "{masscan}"
+
+commands:
+- git clone https://github.com/robertdavidgraham/masscan /tmp/masscan
+- make -s -j -C /tmp/masscan
+- !join [mv, /tmp/masscan/bin/masscan, *masscan]
+- rm -rf /tmp/masscan
+- !join [sudo, setcap, CAP_NET_RAW+ep, *masscan]
+
+shell: true

pipeline/tools/recursive-gobuster.yaml

@@ -0,0 +1,10 @@
+installed: false
+dependencies: ["go"]
+recursive-parent: &recpar !get_parent "{recursive-gobuster}"
+
+commands:
+- !join [bash, -c, "'if [ -d", *recpar, "]; then cd", *recpar,
+        "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git",
+        *recpar, "; fi'"]
+
+shell: true

pipeline/tools/searchsploit.yaml

@@ -0,0 +1,19 @@
+installed: false
+dependencies:
+home: &home !get_default "{home}"
+tools-dir: &tools !get_default "{tools-dir}"
+exploitdb-file: &exploitdb !get_tool_path "{exploitdb}"
+searchsploit-file: &searchsploit !get_tool_path "{searchsploit}"
+searchsploit-rc: &ss_rc !join_path [*exploitdb, ".searchsploit_rc"]
+homesploit: &homesploit !join_path [*home, ".searchsploit_rc"]
+sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"]
+
+commands:
+- !join [bash, -c, "'if [ -d /usr/share/exploitdb ]; then ln -fs /usr/share/exploitdb",
+        *exploitdb, "&& sudo ln -fs $(which searchsploit)", *searchsploit,
+        "; elif [ -d", *exploitdb, "]; then cd", *exploitdb,
+        "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, "; fi'"]
+- !join [bash, -c, "'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, "; fi'"]
+- !join [bash, -c, "'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, "; fi'"]
+
+shell: true

pipeline/tools/seclists.yaml

@@ -0,0 +1,10 @@
+installed: false
+depencencies:
+seclists-file: &secfile !get_tool_path "{seclists}"
+
+commands:
+  - !join [bash, -c, "'if [[ -d /usr/share/seclists ]];", "then ln -s /usr/share/seclists",
+          *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
+          "else git clone https://github.com/danielmiessler/SecLists.git", *secfile, "; fi'"]
+
+shell: true

pipeline/tools/subjack.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/haccer/subjack]
+- !join ["(cd ~/go/src/github.com/haccer/subjack &&", *gotool, "install)"]
+
+shell: true

pipeline/tools/tko-subs.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/haccer/subjack]
+- !join ["(cd ~/go/src/github.com/haccer/subjack &&", *gotool, "install)"]
+
+shell: true

pipeline/tools/webanalyze.yaml

@@ -0,0 +1,9 @@
+installed: false
+dependencies: ["go"]
+go: &gotool !get_tool_path "{go}"
+
+commands:
+- !join [*gotool, get, github.com/rverton/webanalyze/...]
+- !join ["(cd ~/go/src/github.com/rverton/webanalyze &&", *gotool, "build &&", *gotool, "install)"]
+
+shell: true