Modifies reference structure for tool paths (#61)

pipeline/tools/amass.yaml

@@ -1,10 +1,11 @@
 installed: false
 dependencies: [go]
-go: &gotool !get_tool_path "{go}"
-amass: &amass !get_tool_path "{amass}"
+tools: &tools !get_default "{tools-dir}"
+go: &gotool !get_tool_path "{go[path]}"
+path: &amass !join_path [*tools, "amass"]
 
 commands:
-- !join [*gotool, get -u github.com/OWASP/Amass/v3/...]
+- !join [*gotool, get github.com/OWASP/Amass/v3/...]
 - !join [cp ~/go/bin/amass, *amass]
 
 shell: true

pipeline/tools/aquatone.yaml

@@ -1,6 +1,6 @@
 installed: false
-dependencies:
-aquatone: &aqua !get_tool_path "{aquatone}"
+tools: &tools !get_default "{tools-dir}"
+path: &aqua !join_path [*tools, aquatone]
 
 commands:
 - mkdir /tmp/aquatone
@@ -10,5 +10,3 @@ commands:
 - !join [mv /tmp/aquatone/aquatone, *aqua]
 - rm -rf /tmp/aquatone
 - bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'
-
-shell: true

pipeline/tools/exploitdb.yaml

@@ -0,0 +1,3 @@
+installed: true
+tools: &tools !get_default "{tools-dir}"
+path: !join_path [*tools, exploitdb]

pipeline/tools/go.yaml

@@ -1,12 +1,9 @@
 installed: false
-dependencies:
 home: &home !get_default "{home}"
-go: &gotool !get_tool_path "{go}"
+path: &gotool /usr/local/go/bin/go
 bashrc: &bashrc !join_path [*home, "/.bashrc;"]
 
 commands:
-- wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz
+- wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -O /tmp/go.tar.gz
 - sudo tar -C /usr/local -xvf /tmp/go.tar.gz
 - !join ["bash -c 'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >>", *bashrc, "fi'"]
-
-shell: true

pipeline/tools/gobuster.yaml

@@ -1,7 +1,8 @@
 installed: false
 dependencies: [go, seclists]
 home: &home !get_default "{home}"
-go: &gotool !get_tool_path "{go}"
+path: !join_path [*home, go/bin/gobuster]
+go: &gotool !get_tool_path "{go[path]}"
 go_home: &gohome !join_path [*home, "/go/src/github.com/OJ/gobuster &&"]
 
 commands:

pipeline/tools/loader.py

@@ -1,9 +1,10 @@
 import yaml
 from pathlib import Path
 
-from ..recon.config import tool_paths, defaults
+from ..recon.config import defaults
 
 definitions = Path(__file__).parent
+tools = {}
 
 
 def join(loader, node):
@@ -30,16 +31,10 @@ def get_default(loader, node):
     return py_str.format(**defaults)
 
 
-def get_parent(loader, node):
-    """ yaml tag handler to access tool parents """
-    py_str = loader.construct_python_str(node)
-    return Path(py_str.format(**tool_paths)).parent
-
-
 def get_tool_path(loader, node):
-    """ yaml tag handler to access tool_paths dict at load time """
+    """ yaml tag handler to access tools dict at load time """
     py_str = loader.construct_python_str(node)
-    return py_str.format(**tool_paths)
+    return py_str.format(**tools)
 
 
 yaml.add_constructor("!join", join)
@@ -47,12 +42,20 @@ yaml.add_constructor("!join_empty", join_empty)
 yaml.add_constructor("!join_path", join_path)
 yaml.add_constructor("!get_default", get_default)
 yaml.add_constructor("!get_tool_path", get_tool_path)
-yaml.add_constructor("!get_parent", get_parent)
 
 
-tools = {}
-for file in definitions.iterdir():
-    if file.name.endswith(".yaml"):
+def load_yaml(file):
+    try:
         config = yaml.full_load(file.read_text())
         tool_name = str(file.name.replace(".yaml", ""))
         tools[tool_name] = config
+    except KeyError as error:  # load dependencies first
+        dependency = error.args[0]
+        dependency_file = definitions / (dependency + ".yaml")
+        load_yaml(dependency_file)
+        load_yaml(file)
+
+
+for file in definitions.iterdir():
+    if file.name.endswith(".yaml") and file.name.replace(".yaml", "") not in tools:
+        load_yaml(file)

pipeline/tools/luigi-service.yaml

@@ -1,6 +1,6 @@
 installed: false
-dependencies:
-service-file: &svcfile !get_tool_path "{luigid}"
+project-dir: &proj !get_default "{project-dir}"
+service-file: &svcfile !join_path [*proj, luigid.service]
 
 commands:
 - !join [sudo cp, *svcfile, /lib/systemd/system/luigid.service]

pipeline/tools/masscan.yaml

@@ -1,6 +1,6 @@
 installed: false
-dependencies:
-masscan: &masscan !get_tool_path "{masscan}"
+tools: &tools !get_default "{tools-dir}"
+path: &masscan !join_path [*tools, masscan]
 
 commands:
 - git clone https://github.com/robertdavidgraham/masscan /tmp/masscan
@@ -8,5 +8,3 @@ commands:
 - !join [mv /tmp/masscan/bin/masscan, *masscan]
 - rm -rf /tmp/masscan
 - !join [sudo setcap CAP_NET_RAW+ep, *masscan]
-
-shell: true

pipeline/tools/recursive-gobuster.yaml

@@ -1,10 +1,10 @@
 installed: false
 dependencies: [go]
-recursive-parent: &recpar !get_parent "{recursive-gobuster}"
+tools: &tools !get_default "{tools-dir}"
+path: !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz]
+recursive-parent: &recpar !join_path [*tools, recursive-gobuster]
 
 commands:
 - !join ["bash -c 'if [ -d", *recpar, "]; then cd", *recpar,
         "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git",
-        *recpar, ; fi']
-
-shell: true
+        *recpar, ; fi']

pipeline/tools/searchsploit.yaml

@@ -1,9 +1,9 @@
 installed: false
-dependencies:
+dependencies: [exploitdb]
 home: &home !get_default "{home}"
-tools-dir: &tools !get_default "{tools-dir}"
-exploitdb-file: &exploitdb !get_tool_path "{exploitdb}"
-searchsploit-file: &searchsploit !get_tool_path "{searchsploit}"
+tools: &tools !get_default "{tools-dir}"
+exploitdb-file: &exploitdb !get_tool_path "{exploitdb[path]}"
+path: &searchsploit !join_path [*tools, exploitdb/searchsploit]
 searchsploit-rc: &ss_rc !join_path [*exploitdb, ".searchsploit_rc"]
 homesploit: &homesploit !join_path [*home, ".searchsploit_rc"]
 sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"]
@@ -15,5 +15,3 @@ commands:
         "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, ; fi']
 - !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, ; fi']
 - !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, ; fi']
-
-shell: true

pipeline/tools/seclists.yaml

@@ -1,10 +1,8 @@
 installed: false
-depencencies:
-seclists-file: &secfile !get_tool_path "{seclists}"
+tools: &tools !get_default "{tools-dir}"
+path: &secfile !join_path [*tools, seclists]
 
 commands:
   - !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists",
           *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
-          else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi']
-
-shell: true
+          else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi']

pipeline/tools/subjack.yaml

@@ -1,8 +1,10 @@
 installed: false
 dependencies: [go]
-go: &gotool !get_tool_path "{go}"
+go: &gotool !get_tool_path "{go[path]}"
 home: &home !get_default "{home}"
+path: !join_path [*home, go/bin/subjack]
 subjack_home: &subjhome !join_path [*home, "/go/src/github.com/haccer/subjack &&"]
+fingerprints: !join_path [*home, go/src/github.com/haccer/subjack/fingerprints.json]
 
 commands:
 - !join [*gotool, get github.com/haccer/subjack]

pipeline/tools/tko-subs.yaml

@@ -1,11 +1,13 @@
 installed: false
 dependencies: [go]
-go: &gotool !get_tool_path "{go}"
+go: &gotool !get_tool_path "{go[path]}"
 home: &home !get_default "{home}"
+path: !join_path [*home, go/bin/tko-subs]
 tko_home: &tkohome !join_path [*home, "go/src/github.com/anshumanbh/tko-subs &&"]
+git_dir: !join_path [*home, go/src/github.com/anshumanbh/tko-subs]
 
 commands:
 - !join [*gotool, get, github.com/anshumanbh/tko-subs]
 - !join [(cd, *tkohome, *gotool, "build &&", *gotool, "install)"]
 
-shell: true
+shell: true

pipeline/tools/waybackurls.yaml

@@ -1,9 +1,7 @@
 installed: false
-dependencies: ["go"]
-go: &gotool !get_tool_path "{go}"
+dependencies: [go]
+go: &gotool !get_tool_path "{go[path]}"
 path: !join_path [!get_default "{home}", go, bin, waybackurls]
 
 commands:
-- !join [*gotool, get, github.com/tomnomnom/waybackurls]
-
-shell: false
+- !join [*gotool, get, github.com/tomnomnom/waybackurls]

pipeline/tools/webanalyze.yaml

@@ -1,11 +1,12 @@
 installed: false
 dependencies: [go]
 home: &home !get_default "{home}"
-go: &gotool !get_tool_path "{go}"
+go: &gotool !get_tool_path "{go[path]}"
+path: !join_path [*home, go/bin/webanalyze]
 webanalyze_home: &webhome !join_path [*home, "/go/src/github.com/rverton/webanalyze &&"]
 
 commands:
 - !join [*gotool, get github.com/rverton/webanalyze/...]
 - !join [(cd, *webhome, *gotool, "build &&", *gotool, install)]
 
-shell: true
+shell: true