diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5925db8..48ba76b 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -12,9 +12,9 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: -1. -2. -3. +1. +2. +3. **Expected behavior** A clear and concise description of what you expected to happen. diff --git a/docs/modifications/new_scanner.rst b/docs/modifications/new_scanner.rst index 2736ebf..79cdcd5 100644 --- a/docs/modifications/new_scanner.rst +++ b/docs/modifications/new_scanner.rst @@ -53,19 +53,18 @@ Dynamically creating strings and filesystem paths are handled by the following t In order to get values out of ``pipeline.recon.config.py``, you'll need to use one of the yaml helpers listed below. -- ``!get_parent`` - (niche) get parent directory of a given tool that is defined in the ``pipeline.recon.config.tool_paths`` dictionary - ``!get_default`` - get a value from the ``pipeline.recon.config.defaults`` dictionary -- ``!get_tool_path`` - get a value from the ``pipeline.recon.config.tool_paths`` dictionary +- ``!get_tool_path`` - get a path value from the ``pipeline.tools.tools`` dictionary Simple Example Tool Definition ****************************** -The example below needs go to be installed prior to being installed itself. It then grabs the path to the ``go`` binary from ``pipeline.recon.config.tool_paths`` by using ``!get_tool_path``. After that, it creates a command using ``!join`` that will look like ``/usr/local/go/bin/go get github.com/tomnomnom/waybackurls``. This command will be run by the ``install waybackurls`` command (or ``install all``). +The example below needs go to be installed prior to being installed itself. It then grabs the path to the ``go`` binary from ``pipeline.tools.tools`` by using ``!get_tool_path``. After that, it creates a command using ``!join`` that will look like ``/usr/local/go/bin/go get github.com/tomnomnom/waybackurls``. This command will be run by the ``install waybackurls`` command (or ``install all``). .. code-block:: yaml dependencies: [go] - go: &gobin !get_tool_path "{go}" + go: &gobin !get_tool_path "{go[path]}" commands: - !join [*gobin, get github.com/tomnomnom/waybackurls] diff --git a/pipeline/recon-pipeline.py b/pipeline/recon-pipeline.py index 0de90a7..48b2f4a 100755 --- a/pipeline/recon-pipeline.py +++ b/pipeline/recon-pipeline.py @@ -341,13 +341,11 @@ class ReconShell(cmd2.Cmd): self.do_install(tool) return - if persistent_tool_dict.exists(): tools = pickle.loads(persistent_tool_dict.read_bytes()) if tools.get(args.tool).get("dependencies"): # get all of the requested tools dependencies - for dependency in tools.get(args.tool).get("dependencies"): if tools.get(dependency).get("installed"): # already installed, skip it diff --git a/pipeline/recon/__init__.py b/pipeline/recon/__init__.py index d302131..8da6d0f 100644 --- a/pipeline/recon/__init__.py +++ b/pipeline/recon/__init__.py @@ -4,7 +4,7 @@ from .wrappers import FullScan, HTBScan from .amass import AmassScan, ParseAmassOutput from .masscan import MasscanScan, ParseMasscanOutput from .nmap import ThreadedNmapScan, SearchsploitScan -from .config import tool_paths, top_udp_ports, top_tcp_ports, defaults, web_ports +from .config import top_udp_ports, top_tcp_ports, defaults, web_ports from .parsers import ( install_parser, scan_parser, diff --git a/pipeline/recon/amass.py b/pipeline/recon/amass.py index 06d514a..77b0f8f 100644 --- a/pipeline/recon/amass.py +++ b/pipeline/recon/amass.py @@ -7,8 +7,8 @@ from luigi.util import inherits from luigi.contrib.sqla import SQLAlchemyTarget import pipeline.models.db_manager -from .config import tool_paths from .targets import TargetList +from ..tools import tools from ..models.target_model import Target @@ -94,7 +94,7 @@ class AmassScan(luigi.Task): return subprocess.run(f"touch {self.output().path}".split()) command = [ - f"{tool_paths.get('amass')}", + tools.get("amass").get("path"), "enum", "-active", "-ip", diff --git a/pipeline/recon/config.py b/pipeline/recon/config.py index a8b8543..ae9b999 100644 --- a/pipeline/recon/config.py +++ b/pipeline/recon/config.py @@ -16,25 +16,7 @@ defaults = { defaults["tools-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/tools" defaults["database-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/databases" defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt" - -tool_paths = { - "aquatone": f"{defaults.get('tools-dir')}/aquatone", - "tko-subs": f"{Path.home()}/go/bin/tko-subs", - "tko-subs-dir": f"{Path.home()}/go/src/github.com/anshumanbh/tko-subs", - "subjack": f"{Path.home()}/go/bin/subjack", - "subjack-fingerprints": f"{Path.home()}/go/src/github.com/haccer/subjack/fingerprints.json", - "gobuster": f"{Path.home()}/go/bin/gobuster", - "recursive-gobuster": f"{defaults.get('tools-dir')}/recursive-gobuster/recursive-gobuster.pyz", - "webanalyze": f"{Path.home()}/go/bin/webanalyze", - "masscan": f"{defaults.get('tools-dir')}/masscan", - "amass": f"{defaults.get('tools-dir')}/amass", - "go": "/usr/local/go/bin/go", - "searchsploit": f"{defaults.get('tools-dir')}/exploitdb/searchsploit", - "luigid": str(Path(__file__).parents[2] / "luigid.service"), - "seclists": f"{defaults.get('tools-dir')}/seclists", - "exploitdb": f"{defaults.get('tools-dir')}/exploitdb", - "waybackurls": f"{Path.home()}/go/bin/waybackurls", -} +defaults["project-dir"] = str(Path(__file__).parents[2]) web_ports = { "80", diff --git a/pipeline/recon/masscan.py b/pipeline/recon/masscan.py index 9729e76..ad9abb8 100644 --- a/pipeline/recon/masscan.py +++ b/pipeline/recon/masscan.py @@ -9,11 +9,12 @@ from luigi.contrib.sqla import SQLAlchemyTarget import pipeline.models.db_manager from .targets import TargetList +from ..tools import tools from .amass import ParseAmassOutput from ..models.port_model import Port from ..models.ip_address_model import IPAddress -from .config import top_tcp_ports, top_udp_ports, defaults, tool_paths, web_ports +from .config import top_tcp_ports, top_udp_ports, defaults, web_ports @inherits(TargetList, ParseAmassOutput) @@ -108,7 +109,7 @@ class MasscanScan(luigi.Task): ) command = [ - tool_paths.get("masscan"), + tools.get("masscan").get("path"), "-v", "--open", "--banners", diff --git a/pipeline/recon/nmap.py b/pipeline/recon/nmap.py index 747f81c..2169390 100644 --- a/pipeline/recon/nmap.py +++ b/pipeline/recon/nmap.py @@ -12,9 +12,10 @@ from luigi.contrib.sqla import SQLAlchemyTarget import pipeline.models.db_manager from .masscan import ParseMasscanOutput -from .config import defaults, tool_paths +from .config import defaults from .helpers import get_ip_address_version, is_ip_address +from ..tools import tools from ..models.port_model import Port from ..models.nse_model import NSEResult from ..models.target_model import Target @@ -281,7 +282,7 @@ class SearchsploitScan(luigi.Task): """ Grabs the xml files created by ThreadedNmap and runs searchsploit --nmap on each one, saving the output. """ for entry in Path(self.input().get("localtarget").path).glob("nmap*.xml"): proc = subprocess.run( - [tool_paths.get("searchsploit"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE + [tools.get("searchsploit").get("path"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE ) if proc.stdout: # change wall-searchsploit-results/nmap.10.10.10.157-tcp to 10.10.10.157 diff --git a/pipeline/recon/web/aquatone.py b/pipeline/recon/web/aquatone.py index d39ffee..4464a82 100644 --- a/pipeline/recon/web/aquatone.py +++ b/pipeline/recon/web/aquatone.py @@ -9,7 +9,8 @@ from luigi.util import inherits from luigi.contrib.sqla import SQLAlchemyTarget from .targets import GatherWebTargets -from ..config import tool_paths, defaults +from ..config import defaults +from ...tools import tools import pipeline.models.db_manager from ...models.port_model import Port @@ -250,7 +251,7 @@ class AquatoneScan(luigi.Task): self.results_subfolder.mkdir(parents=True, exist_ok=True) command = [ - tool_paths.get("aquatone"), + tools.get("aquatone").get("path"), "-scan-timeout", self.scan_timeout, "-threads", diff --git a/pipeline/recon/web/gobuster.py b/pipeline/recon/web/gobuster.py index 14e8d8c..f906e9e 100644 --- a/pipeline/recon/web/gobuster.py +++ b/pipeline/recon/web/gobuster.py @@ -11,7 +11,8 @@ from luigi.contrib.sqla import SQLAlchemyTarget import pipeline.models.db_manager from .targets import GatherWebTargets -from ..config import tool_paths, defaults +from ..config import defaults +from ...tools import tools from ...models.endpoint_model import Endpoint from ..helpers import get_ip_address_version, is_ip_address @@ -139,10 +140,16 @@ class GobusterScan(luigi.Task): for url_scheme in ("https://", "http://"): if self.recursive: - command = [tool_paths.get("recursive-gobuster"), "-s", "-w", self.wordlist, f"{url_scheme}{target}"] + command = [ + tools.get("recursive-gobuster").get("path"), + "-s", + "-w", + self.wordlist, + f"{url_scheme}{target}", + ] else: command = [ - tool_paths.get("gobuster"), + tools.get("gobuster").get("path"), "dir", "-q", "-e", diff --git a/pipeline/recon/web/subdomain_takeover.py b/pipeline/recon/web/subdomain_takeover.py index a175461..99acfbc 100644 --- a/pipeline/recon/web/subdomain_takeover.py +++ b/pipeline/recon/web/subdomain_takeover.py @@ -8,8 +8,9 @@ from luigi.util import inherits from luigi.contrib.sqla import SQLAlchemyTarget import pipeline.models.db_manager +from ...tools import tools from .targets import GatherWebTargets -from ..config import tool_paths, defaults +from ..config import defaults @inherits(GatherWebTargets) @@ -120,9 +121,9 @@ class TKOSubsScan(luigi.Task): return command = [ - tool_paths.get("tko-subs"), + tools.get("tko-subs").get("path"), f"-domain={','.join(domains)}", - f"-data={tool_paths.get('tko-subs-dir')}/providers-data.csv", + f"-data={tools.get('tko-subs').get('git_dir')}/providers-data.csv", f"-output={self.output_file}", ] @@ -261,7 +262,7 @@ class SubjackScan(luigi.Task): f.write(f"{hostname}\n") command = [ - tool_paths.get("subjack"), + tools.get("subjack").get("path"), "-w", str(subjack_input_file), "-t", @@ -274,7 +275,7 @@ class SubjackScan(luigi.Task): "-v", "-ssl", "-c", - tool_paths.get("subjack-fingerprints"), + tools.get("subjack").get("fingerprints"), ] subprocess.run(command) diff --git a/pipeline/recon/web/webanalyze.py b/pipeline/recon/web/webanalyze.py index 164c00f..15608dd 100644 --- a/pipeline/recon/web/webanalyze.py +++ b/pipeline/recon/web/webanalyze.py @@ -11,8 +11,9 @@ from luigi.util import inherits from luigi.contrib.sqla import SQLAlchemyTarget import pipeline.models.db_manager +from ...tools import tools from .targets import GatherWebTargets -from ..config import tool_paths, defaults +from ..config import defaults from ...models.technology_model import Technology from ..helpers import get_ip_address_version, is_ip_address @@ -153,7 +154,7 @@ class WebanalyzeScan(luigi.Task): target = f"[{target}]" for url_scheme in ("https://", "http://"): - command = [tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}", "-output", "csv"] + command = [tools.get("webanalyze").get("path"), "-host", f"{url_scheme}{target}", "-output", "csv"] commands.append(command) self.results_subfolder.mkdir(parents=True, exist_ok=True) @@ -162,7 +163,7 @@ class WebanalyzeScan(luigi.Task): os.chdir(self.results_subfolder) if not Path("apps.json").exists(): - subprocess.run(f"{tool_paths.get('webanalyze')} -update".split()) + subprocess.run(f"{tools.get('webanalyze').get('path')} -update".split()) with ThreadPoolExecutor(max_workers=self.threads) as executor: executor.map(self._wrapped_subprocess, commands) diff --git a/pipeline/tools/amass.yaml b/pipeline/tools/amass.yaml index c5b91e5..529bd69 100644 --- a/pipeline/tools/amass.yaml +++ b/pipeline/tools/amass.yaml @@ -1,10 +1,11 @@ installed: false dependencies: [go] -go: &gotool !get_tool_path "{go}" -amass: &amass !get_tool_path "{amass}" +tools: &tools !get_default "{tools-dir}" +go: &gotool !get_tool_path "{go[path]}" +path: &amass !join_path [*tools, "amass"] commands: -- !join [*gotool, get -u github.com/OWASP/Amass/v3/...] +- !join [*gotool, get github.com/OWASP/Amass/v3/...] - !join [cp ~/go/bin/amass, *amass] shell: true diff --git a/pipeline/tools/aquatone.yaml b/pipeline/tools/aquatone.yaml index 513c4f4..1f0662f 100644 --- a/pipeline/tools/aquatone.yaml +++ b/pipeline/tools/aquatone.yaml @@ -1,6 +1,6 @@ installed: false -dependencies: -aquatone: &aqua !get_tool_path "{aquatone}" +tools: &tools !get_default "{tools-dir}" +path: &aqua !join_path [*tools, aquatone] commands: - mkdir /tmp/aquatone @@ -10,5 +10,3 @@ commands: - !join [mv /tmp/aquatone/aquatone, *aqua] - rm -rf /tmp/aquatone - bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi' - -shell: true \ No newline at end of file diff --git a/pipeline/tools/exploitdb.yaml b/pipeline/tools/exploitdb.yaml new file mode 100644 index 0000000..712505a --- /dev/null +++ b/pipeline/tools/exploitdb.yaml @@ -0,0 +1,3 @@ +installed: true +tools: &tools !get_default "{tools-dir}" +path: !join_path [*tools, exploitdb] \ No newline at end of file diff --git a/pipeline/tools/go.yaml b/pipeline/tools/go.yaml index e99c6bd..ed24452 100644 --- a/pipeline/tools/go.yaml +++ b/pipeline/tools/go.yaml @@ -1,12 +1,9 @@ installed: false -dependencies: home: &home !get_default "{home}" -go: &gotool !get_tool_path "{go}" +path: &gotool /usr/local/go/bin/go bashrc: &bashrc !join_path [*home, "/.bashrc;"] commands: -- wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz +- wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -O /tmp/go.tar.gz - sudo tar -C /usr/local -xvf /tmp/go.tar.gz - !join ["bash -c 'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >>", *bashrc, "fi'"] - -shell: true \ No newline at end of file diff --git a/pipeline/tools/gobuster.yaml b/pipeline/tools/gobuster.yaml index 5cef7f3..aa47a5d 100644 --- a/pipeline/tools/gobuster.yaml +++ b/pipeline/tools/gobuster.yaml @@ -1,7 +1,8 @@ installed: false dependencies: [go, seclists] home: &home !get_default "{home}" -go: &gotool !get_tool_path "{go}" +path: !join_path [*home, go/bin/gobuster] +go: &gotool !get_tool_path "{go[path]}" go_home: &gohome !join_path [*home, "/go/src/github.com/OJ/gobuster &&"] commands: diff --git a/pipeline/tools/loader.py b/pipeline/tools/loader.py index ce64eb6..01f608d 100644 --- a/pipeline/tools/loader.py +++ b/pipeline/tools/loader.py @@ -1,9 +1,10 @@ import yaml from pathlib import Path -from ..recon.config import tool_paths, defaults +from ..recon.config import defaults definitions = Path(__file__).parent +tools = {} def join(loader, node): @@ -30,16 +31,10 @@ def get_default(loader, node): return py_str.format(**defaults) -def get_parent(loader, node): - """ yaml tag handler to access tool parents """ - py_str = loader.construct_python_str(node) - return Path(py_str.format(**tool_paths)).parent - - def get_tool_path(loader, node): - """ yaml tag handler to access tool_paths dict at load time """ + """ yaml tag handler to access tools dict at load time """ py_str = loader.construct_python_str(node) - return py_str.format(**tool_paths) + return py_str.format(**tools) yaml.add_constructor("!join", join) @@ -47,12 +42,20 @@ yaml.add_constructor("!join_empty", join_empty) yaml.add_constructor("!join_path", join_path) yaml.add_constructor("!get_default", get_default) yaml.add_constructor("!get_tool_path", get_tool_path) -yaml.add_constructor("!get_parent", get_parent) -tools = {} -for file in definitions.iterdir(): - if file.name.endswith(".yaml"): +def load_yaml(file): + try: config = yaml.full_load(file.read_text()) tool_name = str(file.name.replace(".yaml", "")) tools[tool_name] = config + except KeyError as error: # load dependencies first + dependency = error.args[0] + dependency_file = definitions / (dependency + ".yaml") + load_yaml(dependency_file) + load_yaml(file) + + +for file in definitions.iterdir(): + if file.name.endswith(".yaml") and file.name.replace(".yaml", "") not in tools: + load_yaml(file) diff --git a/pipeline/tools/luigi-service.yaml b/pipeline/tools/luigi-service.yaml index d92cf66..d8676c3 100644 --- a/pipeline/tools/luigi-service.yaml +++ b/pipeline/tools/luigi-service.yaml @@ -1,6 +1,6 @@ installed: false -dependencies: -service-file: &svcfile !get_tool_path "{luigid}" +project-dir: &proj !get_default "{project-dir}" +service-file: &svcfile !join_path [*proj, luigid.service] commands: - !join [sudo cp, *svcfile, /lib/systemd/system/luigid.service] diff --git a/pipeline/tools/masscan.yaml b/pipeline/tools/masscan.yaml index 3c7e20a..8f0b349 100644 --- a/pipeline/tools/masscan.yaml +++ b/pipeline/tools/masscan.yaml @@ -1,6 +1,6 @@ installed: false -dependencies: -masscan: &masscan !get_tool_path "{masscan}" +tools: &tools !get_default "{tools-dir}" +path: &masscan !join_path [*tools, masscan] commands: - git clone https://github.com/robertdavidgraham/masscan /tmp/masscan @@ -8,5 +8,3 @@ commands: - !join [mv /tmp/masscan/bin/masscan, *masscan] - rm -rf /tmp/masscan - !join [sudo setcap CAP_NET_RAW+ep, *masscan] - -shell: true \ No newline at end of file diff --git a/pipeline/tools/recursive-gobuster.yaml b/pipeline/tools/recursive-gobuster.yaml index a011d84..24f4541 100644 --- a/pipeline/tools/recursive-gobuster.yaml +++ b/pipeline/tools/recursive-gobuster.yaml @@ -1,10 +1,10 @@ installed: false dependencies: [go] -recursive-parent: &recpar !get_parent "{recursive-gobuster}" +tools: &tools !get_default "{tools-dir}" +path: !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz] +recursive-parent: &recpar !join_path [*tools, recursive-gobuster] commands: - !join ["bash -c 'if [ -d", *recpar, "]; then cd", *recpar, "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git", - *recpar, ; fi'] - -shell: true \ No newline at end of file + *recpar, ; fi'] \ No newline at end of file diff --git a/pipeline/tools/searchsploit.yaml b/pipeline/tools/searchsploit.yaml index 0ccf79b..91f15b9 100644 --- a/pipeline/tools/searchsploit.yaml +++ b/pipeline/tools/searchsploit.yaml @@ -1,9 +1,9 @@ installed: false -dependencies: +dependencies: [exploitdb] home: &home !get_default "{home}" -tools-dir: &tools !get_default "{tools-dir}" -exploitdb-file: &exploitdb !get_tool_path "{exploitdb}" -searchsploit-file: &searchsploit !get_tool_path "{searchsploit}" +tools: &tools !get_default "{tools-dir}" +exploitdb-file: &exploitdb !get_tool_path "{exploitdb[path]}" +path: &searchsploit !join_path [*tools, exploitdb/searchsploit] searchsploit-rc: &ss_rc !join_path [*exploitdb, ".searchsploit_rc"] homesploit: &homesploit !join_path [*home, ".searchsploit_rc"] sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"] @@ -15,5 +15,3 @@ commands: "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, ; fi'] - !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, ; fi'] - !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, ; fi'] - -shell: true \ No newline at end of file diff --git a/pipeline/tools/seclists.yaml b/pipeline/tools/seclists.yaml index ddf5306..aaef043 100644 --- a/pipeline/tools/seclists.yaml +++ b/pipeline/tools/seclists.yaml @@ -1,10 +1,8 @@ installed: false -depencencies: -seclists-file: &secfile !get_tool_path "{seclists}" +tools: &tools !get_default "{tools-dir}" +path: &secfile !join_path [*tools, seclists] commands: - !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists", *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;", - else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi'] - -shell: true \ No newline at end of file + else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi'] \ No newline at end of file diff --git a/pipeline/tools/subjack.yaml b/pipeline/tools/subjack.yaml index 04d45ee..ac4f37d 100644 --- a/pipeline/tools/subjack.yaml +++ b/pipeline/tools/subjack.yaml @@ -1,8 +1,10 @@ installed: false dependencies: [go] -go: &gotool !get_tool_path "{go}" +go: &gotool !get_tool_path "{go[path]}" home: &home !get_default "{home}" +path: !join_path [*home, go/bin/subjack] subjack_home: &subjhome !join_path [*home, "/go/src/github.com/haccer/subjack &&"] +fingerprints: !join_path [*home, go/src/github.com/haccer/subjack/fingerprints.json] commands: - !join [*gotool, get github.com/haccer/subjack] diff --git a/pipeline/tools/tko-subs.yaml b/pipeline/tools/tko-subs.yaml index be2f4be..6a3a37d 100644 --- a/pipeline/tools/tko-subs.yaml +++ b/pipeline/tools/tko-subs.yaml @@ -1,11 +1,13 @@ installed: false dependencies: [go] -go: &gotool !get_tool_path "{go}" +go: &gotool !get_tool_path "{go[path]}" home: &home !get_default "{home}" +path: !join_path [*home, go/bin/tko-subs] tko_home: &tkohome !join_path [*home, "go/src/github.com/anshumanbh/tko-subs &&"] +git_dir: !join_path [*home, go/src/github.com/anshumanbh/tko-subs] commands: - !join [*gotool, get, github.com/anshumanbh/tko-subs] - !join [(cd, *tkohome, *gotool, "build &&", *gotool, "install)"] -shell: true +shell: true \ No newline at end of file diff --git a/pipeline/tools/waybackurls.yaml b/pipeline/tools/waybackurls.yaml index 7badb9f..c86e6c7 100644 --- a/pipeline/tools/waybackurls.yaml +++ b/pipeline/tools/waybackurls.yaml @@ -1,9 +1,7 @@ installed: false -dependencies: ["go"] -go: &gotool !get_tool_path "{go}" +dependencies: [go] +go: &gotool !get_tool_path "{go[path]}" path: !join_path [!get_default "{home}", go, bin, waybackurls] commands: -- !join [*gotool, get, github.com/tomnomnom/waybackurls] - -shell: false +- !join [*gotool, get, github.com/tomnomnom/waybackurls] \ No newline at end of file diff --git a/pipeline/tools/webanalyze.yaml b/pipeline/tools/webanalyze.yaml index 6da2fad..533ffc7 100644 --- a/pipeline/tools/webanalyze.yaml +++ b/pipeline/tools/webanalyze.yaml @@ -1,11 +1,12 @@ installed: false dependencies: [go] home: &home !get_default "{home}" -go: &gotool !get_tool_path "{go}" +go: &gotool !get_tool_path "{go[path]}" +path: !join_path [*home, go/bin/webanalyze] webanalyze_home: &webhome !join_path [*home, "/go/src/github.com/rverton/webanalyze &&"] commands: - !join [*gotool, get github.com/rverton/webanalyze/...] - !join [(cd, *webhome, *gotool, "build &&", *gotool, install)] -shell: true +shell: true \ No newline at end of file diff --git a/tests/test_recon/test_config.py b/tests/test_recon/test_config.py index 5de268d..2d6f927 100644 --- a/tests/test_recon/test_config.py +++ b/tests/test_recon/test_config.py @@ -2,12 +2,14 @@ from pathlib import Path import pytest -from pipeline.recon import tool_paths, defaults, web_ports, top_tcp_ports, top_udp_ports +from pipeline.recon import defaults, web_ports, top_tcp_ports, top_udp_ports +from pipeline.tools import tools def test_tool_paths_absolute(): - for path in tool_paths.values(): - assert Path(path).is_absolute() + for tool in tools.values(): + if tool.get("path"): + assert Path(tool["path"]).is_absolute() @pytest.mark.parametrize("test_input", ["database-dir", "tools-dir", "gobuster-wordlist"]) diff --git a/tests/test_recon/test_nmap.py b/tests/test_recon/test_nmap.py index 755cfc2..5aa1b0a 100644 --- a/tests/test_recon/test_nmap.py +++ b/tests/test_recon/test_nmap.py @@ -7,7 +7,8 @@ import luigi import pytest from luigi.contrib.sqla import SQLAlchemyTarget -from pipeline.recon import ThreadedNmapScan, SearchsploitScan, ParseMasscanOutput, config +from pipeline.recon import ThreadedNmapScan, SearchsploitScan, ParseMasscanOutput +from pipeline.tools import tools nmap_results = Path(__file__).parent.parent / "data" / "recon-results" / "nmap-results" @@ -101,7 +102,7 @@ class TestSearchsploitScan: assert len(self.scan.db_mgr.get_all_searchsploit_results()) == 0 - if not Path(config.tool_paths.get("searchsploit")).exists(): + if not Path(tools.get("searchsploit").get("path")).exists(): pytest.skip("exploit-db's searchsploit tool not installed") self.scan.run() diff --git a/tests/test_shell/test_recon_pipeline_shell.py b/tests/test_shell/test_recon_pipeline_shell.py index 6a8df6b..43740b8 100644 --- a/tests/test_shell/test_recon_pipeline_shell.py +++ b/tests/test_shell/test_recon_pipeline_shell.py @@ -12,7 +12,8 @@ import pytest from pipeline.models.port_model import Port from pipeline.models.target_model import Target from pipeline.models.db_manager import DBManager -from pipeline.recon.config import defaults, tool_paths +from pipeline.tools import tools +from pipeline.recon.config import defaults from pipeline.models.ip_address_model import IPAddress recon_shell = importlib.import_module("pipeline.recon-pipeline") @@ -294,143 +295,21 @@ class TestReconShell: # ("all", "commands failed and may have not installed properly", 1) # after tools moved to DB, update this test - @pytest.mark.parametrize("test_input, expected, return_code", [("all", "is already installed", 0)]) + @pytest.mark.parametrize( + "test_input, expected, return_code", [("all", "is already installed", 0), ("amass", "dependency", 1)] + ) def test_do_install(self, test_input, expected, return_code, capsys, tmp_path): process_mock = MagicMock() attrs = {"communicate.return_value": (b"output", b"error"), "returncode": return_code} process_mock.configure_mock(**attrs) - tool_dict = { - "luigi-service": { - "installed": False, - "dependencies": None, - "commands": [ - f"sudo cp {str(Path(__file__).parents[2] / 'luigid.service')} /lib/systemd/system/luigid.service", - f"sudo cp $(which luigid) /usr/local/bin", - "sudo systemctl daemon-reload", - "sudo systemctl start luigid.service", - "sudo systemctl enable luigid.service", - ], - "shell": True, - }, - "seclists": { - "installed": False, - "dependencies": None, - "shell": True, - "commands": [ - f"bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists {defaults.get('tools-dir')}/seclists; elif [[ -d {defaults.get('tools-dir')}/seclists ]] ; then cd {defaults.get('tools-dir')}/seclists && git fetch --all && git pull; else git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists; fi'" - ], - }, - "searchsploit": { - "installed": False, - "dependencies": None, - "shell": True, - "commands": [ - f"bash -c 'if [[ -d /usr/share/exploitdb ]]; then ln -s /usr/share/exploitdb {defaults.get('tools-dir')}/exploitdb && sudo ln -s $(which searchsploit) {defaults.get('tools-dir')}/exploitdb/searchsploit; elif [[ -d {Path(tool_paths.get('searchsploit')).parent} ]]; then cd {Path(tool_paths.get('searchsploit')).parent} && git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git {defaults.get('tools-dir')}/exploitdb; fi'", - f"bash -c 'if [[ -f {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc ]]; then cp -n {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc {Path.home().expanduser().resolve()}; fi'", - f"bash -c 'if [[ -f {Path.home().resolve()}/.searchsploit_rc ]]; then sed -i 's#/opt#{defaults.get('tools-dir')}#g' {Path.home().resolve()}/.searchsploit_rc; fi'", - ], - }, - "masscan": { - "installed": False, - "dependencies": None, - "commands": [ - "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan", - "make -s -j -C /tmp/masscan", - f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}", - "rm -rf /tmp/masscan", - f"sudo setcap CAP_NET_RAW+ep {tool_paths.get('masscan')}", - ], - }, - "amass": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...", - f"cp ~/go/bin/amass {tool_paths.get('amass')}", - ], - "shell": True, - "environ": {"GO111MODULE": "on"}, - }, - "aquatone": { - "installed": False, - "dependencies": None, - "shell": True, - "commands": [ - "mkdir /tmp/aquatone", - "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip", - "bash -c 'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'", - "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone", - f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}", - "rm -rf /tmp/aquatone", - "bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'", - ], - }, - "gobuster": { - "installed": False, - "dependencies": ["go", "seclists"], - "commands": [ - f"{tool_paths.get('go')} get github.com/OJ/gobuster", - f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "tko-subs": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs", - f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "subjack": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get github.com/haccer/subjack", - f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "webanalyze": { - "installed": False, - "dependencies": ["go"], - "commands": [ - f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...", - f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", - ], - "shell": True, - }, - "recursive-gobuster": { - "installed": False, - "dependencies": ["gobuster", "seclists"], - "shell": True, - "commands": [ - f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'" - ], - }, - "go": { - "installed": False, - "dependencies": None, - "commands": [ - "wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz", - "sudo tar -C /usr/local -xvf /tmp/go.tar.gz", - f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'', - ], - }, - "waybackurls": { - "installed": True, - "depencencies": ["go"], - "commands": ["/usr/local/go/bin/go get github.com/tomnomnom/waybackurls"], - "shell": True, - }, - } - tooldir = tmp_path / ".local" / "recon-pipeline" / "tools" tooldir.mkdir(parents=True, exist_ok=True) - pickle.dump(tool_dict, (tooldir / ".tool-dict.pkl").open("wb")) + tools["waybackurls"]["installed"] = True + tools["amass"]["shell"] = False + + pickle.dump(tools, (tooldir / ".tool-dict.pkl").open("wb")) with patch("subprocess.Popen", autospec=True) as mocked_popen: mocked_popen.return_value = process_mock diff --git a/tests/test_web/test_webanalyze.py b/tests/test_web/test_webanalyze.py index 1926b44..2861e54 100644 --- a/tests/test_web/test_webanalyze.py +++ b/tests/test_web/test_webanalyze.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path from unittest.mock import patch, MagicMock -from pipeline.recon.config import tool_paths +from pipeline.tools import tools from pipeline.recon.web import WebanalyzeScan, GatherWebTargets webanalyze_results = Path(__file__).parent.parent / "data" / "recon-results" / "webanalyze-results" @@ -65,7 +65,7 @@ class TestWebanalyzeScan: self.scan.results_subfolder.mkdir() os.chdir(self.scan.results_subfolder) assert len([x for x in self.scan.results_subfolder.iterdir()]) == 0 - cmd = [tool_paths.get("webanalyze"), "-host", "https://google.com", "-output", "csv"] + cmd = [tools.get("webanalyze").get("path"), "-host", "https://google.com", "-output", "csv"] self.scan._wrapped_subprocess(cmd) assert len([x for x in self.scan.results_subfolder.iterdir()]) == 1 assert next(self.scan.results_subfolder.iterdir()).name == "webanalyze-https_google.com.csv"