From db1dbb8e2bb3ca0ffea9a1de9ccc8b65767cec63 Mon Sep 17 00:00:00 2001 From: Ryan Good Date: Sat, 25 Apr 2020 16:00:02 -0400 Subject: [PATCH] pivot to yaml structure for tool definitions (#33) --- pipeline/recon-pipeline.py | 3 +- pipeline/recon/__init__.py | 1 - pipeline/recon/config.py | 4 + pipeline/recon/parsers.py | 2 +- pipeline/recon/tool_definitions.py | 125 ------------------------- pipeline/tools/__init__.py | 1 + pipeline/tools/amass.yaml | 11 +++ pipeline/tools/aquatone.yaml | 14 +++ pipeline/tools/go.yaml | 10 ++ pipeline/tools/gobuster.yaml | 9 ++ pipeline/tools/loader.py | 58 ++++++++++++ pipeline/tools/luigi-service.yaml | 12 +++ pipeline/tools/masscan.yaml | 12 +++ pipeline/tools/recursive-gobuster.yaml | 10 ++ pipeline/tools/searchsploit.yaml | 19 ++++ pipeline/tools/seclists.yaml | 10 ++ pipeline/tools/subjack.yaml | 9 ++ pipeline/tools/tko-subs.yaml | 9 ++ pipeline/tools/webanalyze.yaml | 9 ++ tests/test_recon/test_parsers.py | 2 +- 20 files changed, 201 insertions(+), 129 deletions(-) delete mode 100644 pipeline/recon/tool_definitions.py create mode 100644 pipeline/tools/__init__.py create mode 100644 pipeline/tools/amass.yaml create mode 100644 pipeline/tools/aquatone.yaml create mode 100644 pipeline/tools/go.yaml create mode 100644 pipeline/tools/gobuster.yaml create mode 100644 pipeline/tools/loader.py create mode 100644 pipeline/tools/luigi-service.yaml create mode 100644 pipeline/tools/masscan.yaml create mode 100644 pipeline/tools/recursive-gobuster.yaml create mode 100644 pipeline/tools/searchsploit.yaml create mode 100644 pipeline/tools/seclists.yaml create mode 100644 pipeline/tools/subjack.yaml create mode 100644 pipeline/tools/tko-subs.yaml create mode 100644 pipeline/tools/webanalyze.yaml diff --git a/pipeline/recon-pipeline.py b/pipeline/recon-pipeline.py index fe89e5b..8a9a631 100755 --- a/pipeline/recon-pipeline.py +++ b/pipeline/recon-pipeline.py @@ -59,7 +59,6 @@ from .models.searchsploit_model import SearchsploitResult # noqa: F401,E402 from .recon import ( # noqa: F401,E402 get_scans, - tools, scan_parser, install_parser, status_parser, @@ -77,6 +76,8 @@ from .recon import ( # noqa: F401,E402 port_results_parser, ) +from .tools import tools # noqa: F401,E402 + # select loop, handles async stdout/stderr processing of subprocesses selector = selectors.DefaultSelector() diff --git a/pipeline/recon/__init__.py b/pipeline/recon/__init__.py index 020aac3..d302131 100644 --- a/pipeline/recon/__init__.py +++ b/pipeline/recon/__init__.py @@ -1,6 +1,5 @@ from .helpers import get_scans from .targets import TargetList -from .tool_definitions import tools from .wrappers import FullScan, HTBScan from .amass import AmassScan, ParseAmassOutput from .masscan import MasscanScan, ParseMasscanOutput diff --git a/pipeline/recon/config.py b/pipeline/recon/config.py index 0aff3fa..93351a7 100644 --- a/pipeline/recon/config.py +++ b/pipeline/recon/config.py @@ -12,6 +12,7 @@ defaults = { "aquatone-scan-timeout": "900", "tools-dir": f"{Path.home()}/.local/recon-pipeline/tools", "database-dir": f"{Path.home()}/.local/recon-pipeline/databases", + "home": Path.home().expanduser().resolve(), } defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt" @@ -30,6 +31,9 @@ tool_paths = { "amass": f"{defaults.get('tools-dir')}/amass", "go": "/usr/local/go/bin/go", "searchsploit": f"{defaults.get('tools-dir')}/exploitdb/searchsploit", + "luigid": str(Path(__file__).parents[2] / "luigid.service"), + "seclists": f"{defaults.get('tools-dir')}/seclists", + "exploitdb": f"{defaults.get('tools-dir')}/exploitdb", } web_ports = { diff --git a/pipeline/recon/parsers.py b/pipeline/recon/parsers.py index 31cbcd4..c3f3e3d 100644 --- a/pipeline/recon/parsers.py +++ b/pipeline/recon/parsers.py @@ -4,7 +4,7 @@ import cmd2 from .config import defaults from .helpers import get_scans -from .tool_definitions import tools +from ..tools import tools # options for ReconShell's 'install' command install_parser = cmd2.Cmd2ArgumentParser() diff --git a/pipeline/recon/tool_definitions.py b/pipeline/recon/tool_definitions.py deleted file mode 100644 index bebae3f..0000000 --- a/pipeline/recon/tool_definitions.py +++ /dev/null @@ -1,125 +0,0 @@ -from pathlib import Path - -from .config import tool_paths, defaults - -# tool definitions for recon-pipeline's auto-installer -tools = { - "luigi-service": { - "installed": False, - "dependencies": None, - "commands": [ - f"sudo cp {str(Path(__file__).parents[2] / 'luigid.service')} /lib/systemd/system/luigid.service", - f"sudo cp $(which luigid) /usr/local/bin", - "sudo systemctl daemon-reload", - "sudo systemctl start luigid.service", - "sudo systemctl enable luigid.service", - ], - "shell": True, - }, - "seclists": { - "installed": False, - "dependencies": None, - "shell": True, - "commands": [ - f"bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists {defaults.get('tools-dir')}/seclists; elif [[ -d {defaults.get('tools-dir')}/seclists ]] ; then cd {defaults.get('tools-dir')}/seclists && git fetch --all && git pull; else git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists; fi'" - ], - }, - "searchsploit": { - "installed": False, - "dependencies": None, - "shell": True, - "commands": [ - f"bash -c 'if [[ -d /usr/share/exploitdb ]]; then ln -s /usr/share/exploitdb {defaults.get('tools-dir')}/exploitdb && sudo ln -s $(which searchsploit) {defaults.get('tools-dir')}/exploitdb/searchsploit; elif [[ -d {Path(tool_paths.get('searchsploit')).parent} ]]; then cd {Path(tool_paths.get('searchsploit')).parent} && git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git {defaults.get('tools-dir')}/exploitdb; fi'", - f"bash -c 'if [[ -f {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc ]]; then cp -n {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc {Path.home().expanduser().resolve()}; fi'", - f"bash -c 'if [[ -f {Path.home().resolve()}/.searchsploit_rc ]]; then sed -i 's#/opt#{defaults.get('tools-dir')}#g' {Path.home().resolve()}/.searchsploit_rc; fi'", - ], - }, - "masscan": { - "installed": False, - "dependencies": None, - "commands": [ - "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan", - "make -s -j -C /tmp/masscan", - f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}", - "rm -rf /tmp/masscan", - f"sudo setcap CAP_NET_RAW+ep {tool_paths.get('masscan')}", - ], - }, - "amass": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...", - f"cp ~/go/bin/amass {tool_paths.get('amass')}", - ], - "shell": True, - "environ": {"GO111MODULE": "on"}, - }, - "aquatone": { - "installed": False, - "dependencies": None, - "shell": True, - "commands": [ - "mkdir /tmp/aquatone", - "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip", - "bash -c 'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'", - "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone", - f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}", - "rm -rf /tmp/aquatone", - "bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'", - ], - }, - "gobuster": { - "installed": False, - "dependencies": ["go", "seclists"], - "commands": [ - f"{tool_paths.get('go')} get github.com/OJ/gobuster", - f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "tko-subs": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs", - f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "subjack": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get github.com/haccer/subjack", - f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "webanalyze": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...", - f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "recursive-gobuster": { - "installed": False, - "dependencies": ["gobuster", "seclists"], - "shell": True, - "commands": [ - f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'" - ], - }, - "go": { - "installed": False, - "dependencies": None, - "commands": [ - "wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz", - "sudo tar -C /usr/local -xvf /tmp/go.tar.gz", - f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'', - ], - }, -} diff --git a/pipeline/tools/__init__.py b/pipeline/tools/__init__.py new file mode 100644 index 0000000..25bd475 --- /dev/null +++ b/pipeline/tools/__init__.py @@ -0,0 +1 @@ +from .loader import tools # noqa: F401,E402 diff --git a/pipeline/tools/amass.yaml b/pipeline/tools/amass.yaml new file mode 100644 index 0000000..62da1bd --- /dev/null +++ b/pipeline/tools/amass.yaml @@ -0,0 +1,11 @@ +installed: false +dependencies: ["go"] +go: &gotool !get_tool_path "{go}" +amass: &amass !get_tool_path "{amass}" + +commands: +- !join [*gotool, "get -u github.com/OWASP/Amass/v3/..."] +- !join ["cp ~/go/bin/amass", *amass] + +shell: true +environ: {"GO111MODULE": "on"} \ No newline at end of file diff --git a/pipeline/tools/aquatone.yaml b/pipeline/tools/aquatone.yaml new file mode 100644 index 0000000..970c4ed --- /dev/null +++ b/pipeline/tools/aquatone.yaml @@ -0,0 +1,14 @@ +installed: false +dependencies: +aquatone: &aqua !get_tool_path "{aquatone}" + +commands: +- mkdir /tmp/aquatone +- wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip +- !join [bash, -c, "'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'"] +- unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone +- !join [mv, /tmp/aquatone/aquatone, *aqua] +- rm -rf /tmp/aquatone +- !join [bash, -c, "'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'"] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/go.yaml b/pipeline/tools/go.yaml new file mode 100644 index 0000000..0b9869f --- /dev/null +++ b/pipeline/tools/go.yaml @@ -0,0 +1,10 @@ +installed: false +dependencies: +go: &gotool !get_tool_path "{go}" + +commands: +- wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz +- sudo tar -C /usr/local -xvf /tmp/go.tar.gz +- !join [bash, -c, "'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >> ~/.bashrc; fi'"] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/gobuster.yaml b/pipeline/tools/gobuster.yaml new file mode 100644 index 0000000..9ab1a68 --- /dev/null +++ b/pipeline/tools/gobuster.yaml @@ -0,0 +1,9 @@ +installed: false +dependencies: ["go", "seclists"] +go: &gotool !get_tool_path "{go}" + +commands: +- !join [*gotool, get, github.com/OJ/gobuster] +- !join [(cd, ~/go/src/github.com/OJ/gobuster, "&&", *gotool, "build &&", *gotool, install)] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/loader.py b/pipeline/tools/loader.py new file mode 100644 index 0000000..ce64eb6 --- /dev/null +++ b/pipeline/tools/loader.py @@ -0,0 +1,58 @@ +import yaml +from pathlib import Path + +from ..recon.config import tool_paths, defaults + +definitions = Path(__file__).parent + + +def join(loader, node): + """ yaml tag handler to join a sequence of items into a space-separated string at load time """ + seq = loader.construct_sequence(node) + return " ".join([str(val) for val in seq]) + + +def join_empty(loader, node): + """ yaml tag handler to join a sequence of items into a single string with no separations """ + seq = loader.construct_sequence(node) + return "".join([str(val) for val in seq]) + + +def join_path(loader, node): + """ yaml tag handler to join a sequence of items into a filesystem path at load time """ + seq = loader.construct_sequence(node) + return "/".join([str(i) for i in seq]) + + +def get_default(loader, node): + """ yaml tag handler to access defaults dict at load time """ + py_str = loader.construct_python_str(node) + return py_str.format(**defaults) + + +def get_parent(loader, node): + """ yaml tag handler to access tool parents """ + py_str = loader.construct_python_str(node) + return Path(py_str.format(**tool_paths)).parent + + +def get_tool_path(loader, node): + """ yaml tag handler to access tool_paths dict at load time """ + py_str = loader.construct_python_str(node) + return py_str.format(**tool_paths) + + +yaml.add_constructor("!join", join) +yaml.add_constructor("!join_empty", join_empty) +yaml.add_constructor("!join_path", join_path) +yaml.add_constructor("!get_default", get_default) +yaml.add_constructor("!get_tool_path", get_tool_path) +yaml.add_constructor("!get_parent", get_parent) + + +tools = {} +for file in definitions.iterdir(): + if file.name.endswith(".yaml"): + config = yaml.full_load(file.read_text()) + tool_name = str(file.name.replace(".yaml", "")) + tools[tool_name] = config diff --git a/pipeline/tools/luigi-service.yaml b/pipeline/tools/luigi-service.yaml new file mode 100644 index 0000000..9b9ca3d --- /dev/null +++ b/pipeline/tools/luigi-service.yaml @@ -0,0 +1,12 @@ +installed: false +dependencies: +service-file: &svcfile !get_tool_path "{luigid}" + +commands: +- !join [sudo, cp, *svcfile, /lib/systemd/system/luigid.service] +- !join [sudo, cp, *svcfile, $(which luigid), /usr/local/bin] +- sudo systemctl daemon-reload +- sudo systemctl start luigid.service +- sudo systemctl enable luigid.service + +shell: true \ No newline at end of file diff --git a/pipeline/tools/masscan.yaml b/pipeline/tools/masscan.yaml new file mode 100644 index 0000000..e9e5831 --- /dev/null +++ b/pipeline/tools/masscan.yaml @@ -0,0 +1,12 @@ +installed: false +dependencies: +masscan: &masscan !get_tool_path "{masscan}" + +commands: +- git clone https://github.com/robertdavidgraham/masscan /tmp/masscan +- make -s -j -C /tmp/masscan +- !join [mv, /tmp/masscan/bin/masscan, *masscan] +- rm -rf /tmp/masscan +- !join [sudo, setcap, CAP_NET_RAW+ep, *masscan] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/recursive-gobuster.yaml b/pipeline/tools/recursive-gobuster.yaml new file mode 100644 index 0000000..572fea5 --- /dev/null +++ b/pipeline/tools/recursive-gobuster.yaml @@ -0,0 +1,10 @@ +installed: false +dependencies: ["go"] +recursive-parent: &recpar !get_parent "{recursive-gobuster}" + +commands: +- !join [bash, -c, "'if [ -d", *recpar, "]; then cd", *recpar, + "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git", + *recpar, "; fi'"] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/searchsploit.yaml b/pipeline/tools/searchsploit.yaml new file mode 100644 index 0000000..38b82ab --- /dev/null +++ b/pipeline/tools/searchsploit.yaml @@ -0,0 +1,19 @@ +installed: false +dependencies: +home: &home !get_default "{home}" +tools-dir: &tools !get_default "{tools-dir}" +exploitdb-file: &exploitdb !get_tool_path "{exploitdb}" +searchsploit-file: &searchsploit !get_tool_path "{searchsploit}" +searchsploit-rc: &ss_rc !join_path [*exploitdb, ".searchsploit_rc"] +homesploit: &homesploit !join_path [*home, ".searchsploit_rc"] +sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"] + +commands: +- !join [bash, -c, "'if [ -d /usr/share/exploitdb ]; then ln -fs /usr/share/exploitdb", + *exploitdb, "&& sudo ln -fs $(which searchsploit)", *searchsploit, + "; elif [ -d", *exploitdb, "]; then cd", *exploitdb, + "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, "; fi'"] +- !join [bash, -c, "'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, "; fi'"] +- !join [bash, -c, "'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, "; fi'"] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/seclists.yaml b/pipeline/tools/seclists.yaml new file mode 100644 index 0000000..48aa08c --- /dev/null +++ b/pipeline/tools/seclists.yaml @@ -0,0 +1,10 @@ +installed: false +depencencies: +seclists-file: &secfile !get_tool_path "{seclists}" + +commands: + - !join [bash, -c, "'if [[ -d /usr/share/seclists ]];", "then ln -s /usr/share/seclists", + *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;", + "else git clone https://github.com/danielmiessler/SecLists.git", *secfile, "; fi'"] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/subjack.yaml b/pipeline/tools/subjack.yaml new file mode 100644 index 0000000..6e8f45f --- /dev/null +++ b/pipeline/tools/subjack.yaml @@ -0,0 +1,9 @@ +installed: false +dependencies: ["go"] +go: &gotool !get_tool_path "{go}" + +commands: +- !join [*gotool, get, github.com/haccer/subjack] +- !join ["(cd ~/go/src/github.com/haccer/subjack &&", *gotool, "install)"] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/tko-subs.yaml b/pipeline/tools/tko-subs.yaml new file mode 100644 index 0000000..6e8f45f --- /dev/null +++ b/pipeline/tools/tko-subs.yaml @@ -0,0 +1,9 @@ +installed: false +dependencies: ["go"] +go: &gotool !get_tool_path "{go}" + +commands: +- !join [*gotool, get, github.com/haccer/subjack] +- !join ["(cd ~/go/src/github.com/haccer/subjack &&", *gotool, "install)"] + +shell: true \ No newline at end of file diff --git a/pipeline/tools/webanalyze.yaml b/pipeline/tools/webanalyze.yaml new file mode 100644 index 0000000..8ff0012 --- /dev/null +++ b/pipeline/tools/webanalyze.yaml @@ -0,0 +1,9 @@ +installed: false +dependencies: ["go"] +go: &gotool !get_tool_path "{go}" + +commands: +- !join [*gotool, get, github.com/rverton/webanalyze/...] +- !join ["(cd ~/go/src/github.com/rverton/webanalyze &&", *gotool, "build &&", *gotool, "install)"] + +shell: true diff --git a/tests/test_recon/test_parsers.py b/tests/test_recon/test_parsers.py index b45ffe5..f8891e9 100644 --- a/tests/test_recon/test_parsers.py +++ b/tests/test_recon/test_parsers.py @@ -2,7 +2,7 @@ import pytest from pipeline.recon.parsers import * # noqa: F403 -from pipeline.recon.tool_definitions import tools +from pipeline.tools import tools @pytest.mark.parametrize("test_input", list(tools.keys()) + ["all"])