diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 7a90d26..3f65476 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -107,3 +107,23 @@ jobs: run: | pipenv install pytest cmd2 luigi sqlalchemy python-libnmap pipenv run python -m pytest tests/test_models + + test-unmocked-tool-installs: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up pipenv + run: | + python -m pip install --upgrade pip + pip install pipenv + pipenv install -d + - name: Test with pytest + run: | + pipenv install pytest cmd2 luigi sqlalchemy python-libnmap + pipenv run python -m pytest tests/test_tools_install diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6087de4..19277c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,8 +3,7 @@ repos: rev: stable hooks: - id: black - language_version: python3 - args: ['pipeline', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models'] + args: ['pipeline', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models', 'tests/test_tools_install'] - repo: https://gitlab.com/pycqa/flake8 rev: 3.7.9 hooks: diff --git a/Pipfile b/Pipfile index 84afe9d..b2d53d8 100644 --- a/Pipfile +++ b/Pipfile @@ -22,4 +22,4 @@ python-libnmap = "==0.7.0" pyyaml = "==5.3.1" [requires] -python_version = "3.7" +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 80499c9..74e9dee 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "3337a01cba6c46ec2ec9e0a0ea5678b5bd4c5e6f29c5a7bb5c2c50c503c4c1e4" + "sha256": "e67d6cfc23215132f5512382c7e1004761d6de45818252643c1b3f3d0e56af20" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "3" }, "sources": [ { @@ -147,10 +147,10 @@ }, "appdirs": { "hashes": [ - "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", - "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" ], - "version": "==1.4.3" + "version": "==1.4.4" }, "attrs": { "hashes": [ @@ -295,14 +295,6 @@ ], "version": "==1.2.0" }, - "importlib-metadata": { - "hashes": [ - "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", - "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e" - ], - "markers": "python_version < '3.8'", - "version": "==1.6.0" - }, "jinja2": { "hashes": [ "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", @@ -473,29 +465,29 @@ }, "regex": { "hashes": [ - "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", - "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", - "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", - "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", - "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", - "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", - "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", - "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", - "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", - "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", - "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", - "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", - "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", - "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", - "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", - "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", - "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", - "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", - "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", - "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", - "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" + "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349", + "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608", + "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf", + "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938", + "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998", + "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918", + "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945", + "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd", + "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d", + "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e", + "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74", + "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2", + "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8", + "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4", + "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451", + "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388", + "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc", + "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494", + "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1", + "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03", + "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8" ], - "version": "==2020.4.4" + "version": "==2020.5.7" }, "requests": { "hashes": [ @@ -633,10 +625,10 @@ }, "virtualenv": { "hashes": [ - "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675", - "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1" + "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74", + "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637" ], - "version": "==20.0.18" + "version": "==20.0.20" }, "wcwidth": { "hashes": [ @@ -644,13 +636,6 @@ "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1" ], "version": "==0.1.9" - }, - "zipp": { - "hashes": [ - "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", - "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96" - ], - "version": "==3.1.0" } } } diff --git a/pipeline/recon-pipeline.py b/pipeline/recon-pipeline.py index c5565b6..49fa296 100755 --- a/pipeline/recon-pipeline.py +++ b/pipeline/recon-pipeline.py @@ -124,6 +124,8 @@ class ReconShell(cmd2.Cmd): self.tools_dir.mkdir(parents=True, exist_ok=True) Path(defaults.get("database-dir")).mkdir(parents=True, exist_ok=True) + Path(defaults.get("gopath")).mkdir(parents=True, exist_ok=True) + Path(defaults.get("goroot")).mkdir(parents=True, exist_ok=True) # register hooks to handle selector loop start and cleanup self.register_preloop_hook(self._preloop_hook) diff --git a/pipeline/recon/config.py b/pipeline/recon/config.py index ae9b999..baa8555 100644 --- a/pipeline/recon/config.py +++ b/pipeline/recon/config.py @@ -13,10 +13,13 @@ defaults = { "home": Path.home(), } +defaults["project-dir"] = str(Path(__file__).parents[2]) defaults["tools-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/tools" defaults["database-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/databases" + +defaults["goroot"] = f"{defaults.get('tools-dir')}/pipeline-go" +defaults["gopath"] = f"{defaults.get('tools-dir')}/pipeline-go-workspace" defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt" -defaults["project-dir"] = str(Path(__file__).parents[2]) web_ports = { "80", diff --git a/pipeline/tools/amass.yaml b/pipeline/tools/amass.yaml index 529bd69..62c1219 100644 --- a/pipeline/tools/amass.yaml +++ b/pipeline/tools/amass.yaml @@ -1,12 +1,10 @@ installed: false dependencies: [go] -tools: &tools !get_default "{tools-dir}" go: &gotool !get_tool_path "{go[path]}" -path: &amass !join_path [*tools, "amass"] +path: !join_path [!get_default "{gopath}", bin/amass] +environ: {"GO111MODULE": "on", "GOPATH": !get_default "{gopath}"} commands: - !join [*gotool, get github.com/OWASP/Amass/v3/...] -- !join [cp ~/go/bin/amass, *amass] -shell: true -environ: {"GO111MODULE": "on"} \ No newline at end of file + diff --git a/pipeline/tools/go.yaml b/pipeline/tools/go.yaml index ed24452..e2335b7 100644 --- a/pipeline/tools/go.yaml +++ b/pipeline/tools/go.yaml @@ -1,9 +1,8 @@ installed: false -home: &home !get_default "{home}" -path: &gotool /usr/local/go/bin/go -bashrc: &bashrc !join_path [*home, "/.bashrc;"] +bashrc: &bashrc !join_path [!get_default "{home}", .bashrc] +path: &gotool !join_path [!get_default "{goroot}", go/bin/go] commands: - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -O /tmp/go.tar.gz -- sudo tar -C /usr/local -xvf /tmp/go.tar.gz -- !join ["bash -c 'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >>", *bashrc, "fi'"] +- !join [tar -C, !get_default "{goroot}", -xvf /tmp/go.tar.gz] +- !join ["bash -c 'if [ ! $(grep $(dirname", *gotool, ")", *bashrc, ") ]; then echo PATH=${PATH}:$(dirname", *gotool, ") >>", *bashrc, "; fi'"] diff --git a/pipeline/tools/gobuster.yaml b/pipeline/tools/gobuster.yaml index aa47a5d..122347d 100644 --- a/pipeline/tools/gobuster.yaml +++ b/pipeline/tools/gobuster.yaml @@ -1,12 +1,8 @@ installed: false dependencies: [go, seclists] -home: &home !get_default "{home}" -path: !join_path [*home, go/bin/gobuster] go: &gotool !get_tool_path "{go[path]}" -go_home: &gohome !join_path [*home, "/go/src/github.com/OJ/gobuster &&"] +path: !join_path [!get_default "{gopath}", bin/gobuster] +environ: {"GOPATH": !get_default "{gopath}"} commands: - !join [*gotool, get github.com/OJ/gobuster] -- !join [(cd, *gohome, *gotool, build &&, *gotool, install)] - -shell: true \ No newline at end of file diff --git a/pipeline/tools/recursive-gobuster.yaml b/pipeline/tools/recursive-gobuster.yaml index 24f4541..a4c7358 100644 --- a/pipeline/tools/recursive-gobuster.yaml +++ b/pipeline/tools/recursive-gobuster.yaml @@ -1,10 +1,10 @@ installed: false -dependencies: [go] +dependencies: [gobuster] tools: &tools !get_default "{tools-dir}" path: !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz] -recursive-parent: &recpar !join_path [*tools, recursive-gobuster] +recursive-parent: &parent !join_path [*tools, recursive-gobuster] commands: -- !join ["bash -c 'if [ -d", *recpar, "]; then cd", *recpar, +- !join ["bash -c 'if [ -d", *parent, "]; then cd", *parent, "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git", - *recpar, ; fi'] \ No newline at end of file + *parent, " ; fi'"] \ No newline at end of file diff --git a/pipeline/tools/searchsploit.yaml b/pipeline/tools/searchsploit.yaml index 91f15b9..5820341 100644 --- a/pipeline/tools/searchsploit.yaml +++ b/pipeline/tools/searchsploit.yaml @@ -10,8 +10,8 @@ sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"] commands: - !join ["bash -c 'if [ -d /usr/share/exploitdb ]; then ln -fs /usr/share/exploitdb", - *exploitdb, "&& sudo ln -fs $(which searchsploit)", *searchsploit, + *exploitdb, "&& ln -fs $(which searchsploit)", *searchsploit, "; elif [ -d", *exploitdb, "]; then cd", *exploitdb, - "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, ; fi'] -- !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, ; fi'] -- !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, ; fi'] + "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, "; fi'"] +- !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, "; fi'"] +- !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, "; fi'"] diff --git a/pipeline/tools/seclists.yaml b/pipeline/tools/seclists.yaml index aaef043..f32e2e0 100644 --- a/pipeline/tools/seclists.yaml +++ b/pipeline/tools/seclists.yaml @@ -3,6 +3,6 @@ tools: &tools !get_default "{tools-dir}" path: &secfile !join_path [*tools, seclists] commands: - - !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists", - *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;", - else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi'] \ No newline at end of file +- !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists", + *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;", + "else git clone https://github.com/danielmiessler/SecLists.git", *secfile, "; fi'"] \ No newline at end of file diff --git a/pipeline/tools/subjack.yaml b/pipeline/tools/subjack.yaml index ac4f37d..97158e9 100644 --- a/pipeline/tools/subjack.yaml +++ b/pipeline/tools/subjack.yaml @@ -1,13 +1,10 @@ installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" -home: &home !get_default "{home}" -path: !join_path [*home, go/bin/subjack] -subjack_home: &subjhome !join_path [*home, "/go/src/github.com/haccer/subjack &&"] -fingerprints: !join_path [*home, go/src/github.com/haccer/subjack/fingerprints.json] +path: !join_path [!get_default "{gopath}", bin/subjack] +environ: {"GOPATH": !get_default "{gopath}"} + +fingerprints: !join_path [!get_default "{gopath}", src/github.com/haccer/subjack/fingerprints.json] commands: - !join [*gotool, get github.com/haccer/subjack] -- !join [(cd, *subjhome, *gotool, install)] - -shell: true \ No newline at end of file diff --git a/pipeline/tools/tko-subs.yaml b/pipeline/tools/tko-subs.yaml index 6a3a37d..83eac46 100644 --- a/pipeline/tools/tko-subs.yaml +++ b/pipeline/tools/tko-subs.yaml @@ -1,13 +1,8 @@ installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" -home: &home !get_default "{home}" -path: !join_path [*home, go/bin/tko-subs] -tko_home: &tkohome !join_path [*home, "go/src/github.com/anshumanbh/tko-subs &&"] -git_dir: !join_path [*home, go/src/github.com/anshumanbh/tko-subs] +path: !join_path [!get_default "{gopath}", bin/tko-subs] +environ: {"GOPATH": !get_default "{gopath}"} commands: - !join [*gotool, get, github.com/anshumanbh/tko-subs] -- !join [(cd, *tkohome, *gotool, "build &&", *gotool, "install)"] - -shell: true \ No newline at end of file diff --git a/pipeline/tools/waybackurls.yaml b/pipeline/tools/waybackurls.yaml index c86e6c7..da9358b 100644 --- a/pipeline/tools/waybackurls.yaml +++ b/pipeline/tools/waybackurls.yaml @@ -1,7 +1,8 @@ installed: false dependencies: [go] go: &gotool !get_tool_path "{go[path]}" -path: !join_path [!get_default "{home}", go, bin, waybackurls] +path: !join_path [!get_default "{gopath}", bin/waybackurls] +environ: {"GOPATH": !get_default "{gopath}"} commands: - !join [*gotool, get, github.com/tomnomnom/waybackurls] \ No newline at end of file diff --git a/pipeline/tools/webanalyze.yaml b/pipeline/tools/webanalyze.yaml index 533ffc7..2361718 100644 --- a/pipeline/tools/webanalyze.yaml +++ b/pipeline/tools/webanalyze.yaml @@ -1,12 +1,8 @@ installed: false dependencies: [go] -home: &home !get_default "{home}" go: &gotool !get_tool_path "{go[path]}" -path: !join_path [*home, go/bin/webanalyze] -webanalyze_home: &webhome !join_path [*home, "/go/src/github.com/rverton/webanalyze &&"] +path: !join_path [!get_default "{gopath}", bin/webanalyze] +environ: {"GOPATH": !get_default "{gopath}"} commands: - !join [*gotool, get github.com/rverton/webanalyze/...] -- !join [(cd, *webhome, *gotool, "build &&", *gotool, install)] - -shell: true \ No newline at end of file diff --git a/tests/test_tools_install/__init__.py b/tests/test_tools_install/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_tools_install/test_tools_install_command.py b/tests/test_tools_install/test_tools_install_command.py new file mode 100644 index 0000000..8888ef7 --- /dev/null +++ b/tests/test_tools_install/test_tools_install_command.py @@ -0,0 +1,300 @@ +import pickle +import shutil +import tempfile +import importlib +import subprocess +from pathlib import Path + +import pytest + +from tests import utils + +recon_pipeline = importlib.import_module("pipeline.recon-pipeline") +tools = recon_pipeline.tools + + +class TestUnmockedToolsInstall: + def setup_method(self): + self.shell = recon_pipeline.ReconShell() + self.tmp_path = Path(tempfile.mkdtemp()) + self.shell.tools_dir = self.tmp_path / ".local" / "recon-pipeline" / "tools" + self.shell.tools_dir.mkdir(parents=True, exist_ok=True) + + def teardown_method(self): + def onerror(func, path, exc_info): + subprocess.run(f"sudo rm -rf {self.shell.tools_dir}".split()) + + shutil.rmtree(self.tmp_path, onerror=onerror) + + def perform_install(self, tools_dict, tool_name, exists=False): + pickle.dump(tools_dict, Path(self.shell.tools_dir / ".tool-dict.pkl").open("wb")) + + tool = Path(tools_dict.get(tool_name).get("path")) + + if exists is False: + assert tool.exists() is False + + utils.run_cmd(self.shell, f"install {tool_name}") + + assert tool.exists() is True + + def setup_go_test(self, tool_name, tool_dict): + # install go in tmp location + dependency = "go" + dependency_path = f"{self.shell.tools_dir}/go/bin/go" + + tool_dict.get(dependency)["path"] = dependency_path + tool_dict.get(dependency).get("commands")[1] = f"tar -C {self.shell.tools_dir} -xvf /tmp/go.tar.gz" + + # handle env for local go install + tmp_go_path = f"{self.shell.tools_dir}/mygo" + Path(tmp_go_path).mkdir(parents=True, exist_ok=True) + tool_dict.get(tool_name)["environ"]["GOPATH"] = tmp_go_path + + tool_path = f"{tool_dict.get(tool_name).get('environ').get('GOPATH')}/bin/{tool_name}" + tool_dict.get(tool_name)["path"] = tool_path + + return tool_dict + + def test_install_masscan(self): + tool = "masscan" + tools_copy = tools.copy() + + tool_path = f"{self.shell.tools_dir}/{tool}" + + tools_copy.get(tool)["path"] = tool_path + tools_copy.get(tool).get("commands")[2] = f"mv /tmp/masscan/bin/masscan {tool_path}" + tools_copy.get(tool).get("commands")[4] = f"sudo setcap CAP_NET_RAW+ep {tool_path}" + + self.perform_install(tools_copy, tool) + + def test_install_amass(self): + tool = "amass" + url = "github.com/OWASP/Amass/v3/..." + tools_copy = tools.copy() + + tools_copy.update(self.setup_go_test(tool, tools_copy)) + + tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}" + + self.perform_install(tools_copy, tool) + + def test_install_aquatone(self): + tool = "aquatone" + tools_copy = tools.copy() + + tool_path = f"{self.shell.tools_dir}/{tool}" + + tools_copy.get(tool)["path"] = tool_path + tools_copy.get(tool).get("commands")[4] = f"mv /tmp/aquatone/aquatone {tool_path}" + + self.perform_install(tools_copy, tool) + + def test_install_go(self): + tool = "go" + tools_copy = tools.copy() + + tool_path = f"{self.shell.tools_dir}/go/bin/go" + + tools_copy.get(tool)["path"] = tool_path + tools_copy.get(tool).get("commands")[1] = f"tar -C {self.shell.tools_dir} -xvf /tmp/go.tar.gz" + + self.perform_install(tools_copy, tool) + + def test_install_gobuster(self): + tool = "gobuster" + dependency = "go" + tools_copy = tools.copy() + + tools_copy.update(self.setup_go_test(tool, tools_copy)) + + tools_copy.get(tool)["dependencies"] = [dependency] + tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get(dependency).get('path')} get github.com/OJ/gobuster" + + self.perform_install(tools_copy, tool) + + def test_install_luigi_service(self): + luigi_service = Path("/lib/systemd/system/luigid.service") + + if luigi_service.exists(): + subprocess.run(f"sudo rm {luigi_service}".split()) + + tools_copy = tools.copy() + pickle.dump(tools_copy, Path(self.shell.tools_dir / ".tool-dict.pkl").open("wb")) + + proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) + + if proc.stdout.decode().strip() == "enabled": + subprocess.run("sudo systemctl disable luigid.service".split()) + + proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) + + if proc.stdout.decode().strip() == "active": + subprocess.run("sudo systemctl stop luigid.service".split()) + + if Path("/usr/local/bin/luigid").exists(): + subprocess.run("sudo rm /usr/local/bin/luigid".split()) + + assert ( + subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) + .stdout.decode() + .strip() + != "enabled" + ) + + assert ( + subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE).stdout.decode().strip() + != "active" + ) + + assert not Path("/usr/local/bin/luigid").exists() + + utils.run_cmd(self.shell, "install luigi-service") + + assert Path("/lib/systemd/system/luigid.service").exists() + + proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) + assert proc.stdout.decode().strip() == "enabled" + + proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) + assert proc.stdout.decode().strip() == "active" + + assert Path("/usr/local/bin/luigid").exists() + + @pytest.mark.parametrize("test_input", ["install", "update"]) + def test_install_recursive_gobuster(self, test_input): + tool = "recursive-gobuster" + tools_copy = tools.copy() + + parent = f"{self.shell.tools_dir}/{tool}" + tool_path = f"{parent}/recursive-gobuster.pyz" + + if test_input == "update": + subprocess.run(f"git clone https://github.com/epi052/recursive-gobuster.git {parent}".split()) + + tools_copy.get(tool)["path"] = tool_path + tools_copy.get(tool)["dependencies"] = None + + tools_copy.get(tool).get("commands")[0] = f"bash -c 'if [ -d {parent} ]; " + tools_copy.get(tool).get("commands")[0] += f"then cd {parent} && git fetch --all && git pull; " + tools_copy.get(tool).get("commands")[0] += "else git clone https://github.com/epi052/recursive-gobuster.git " + tools_copy.get(tool).get("commands")[0] += f"{parent} ; fi'" + + if test_input == "update": + self.perform_install(tools_copy, tool, exists=True) + else: + self.perform_install(tools_copy, tool) + + @pytest.mark.parametrize("test_input", ["install", "update"]) + def test_install_searchsploit(self, test_input): + tool = "searchsploit" + tools_copy = tools.copy() + + home_path = f"{self.shell.tools_dir}/home" + Path(home_path).mkdir(parents=True, exist_ok=True) + + copied_searchsploit_rc = f"{home_path}/.searchsploit_rc" + + dependency_path = f"{self.shell.tools_dir}/exploitdb" + searchsploit_rc_path = f"{dependency_path}/.searchsploit_rc" + tool_path = f"{dependency_path}/{tool}" + + sed_cmd = f"s#/opt#{self.shell.tools_dir}#g" + + if test_input == "update": + subprocess.run(f"git clone https://github.com/offensive-security/exploitdb.git {dependency_path}".split()) + + tools_copy.get(tool)["path"] = tool_path + + first_cmd = "bash -c 'if [ -d /usr/share/exploitdb ]; then ln -fs " + first_cmd += f"/usr/share/exploitdb {dependency_path} && ln -fs $(which searchsploit) {tool_path}" + first_cmd += f"; elif [ -d {dependency_path} ]; then cd {dependency_path} && git fetch --all && git pull; else " + first_cmd += f"git clone https://github.com/offensive-security/exploitdb.git {dependency_path}; fi'" + + tools_copy.get(tool).get("commands")[0] = first_cmd + + tools_copy.get(tool).get("commands")[1] = f"bash -c 'if [ -f {searchsploit_rc_path} ]; " + tools_copy.get(tool).get("commands")[1] += f"then cp -n {searchsploit_rc_path} {home_path} ; fi'" + + tools_copy.get(tool).get("commands")[2] = f"bash -c 'if [ -f {copied_searchsploit_rc} ]; " + tools_copy.get(tool).get("commands")[2] += f"then sed -i {sed_cmd} {copied_searchsploit_rc}; fi'" + + pickle.dump(tools_copy, Path(self.shell.tools_dir / ".tool-dict.pkl").open("wb")) + + if test_input == "install": + assert not Path(tool_path).exists() + assert not Path(copied_searchsploit_rc).exists() + assert not Path(dependency_path).exists() + + utils.run_cmd(self.shell, f"install {tool}") + + assert subprocess.run(f"grep {self.shell.tools_dir} {copied_searchsploit_rc}".split()).returncode == 0 + assert Path(copied_searchsploit_rc).exists() + assert Path(dependency_path).exists() + + @pytest.mark.parametrize("test_input", ["install", "update"]) + def test_install_seclists(self, test_input): + tool = "seclists" + tools_copy = tools.copy() + + tool_path = f"{self.shell.tools_dir}/seclists" + + tools_copy.get(tool)["path"] = tool_path + + if test_input == "update": + subprocess.run(f"git clone https://github.com/danielmiessler/SecLists.git {tool_path}".split()) + + first_cmd = f"bash -c 'if [ -d /usr/share/seclists ]; then ln -s /usr/share/seclists {tool_path}; elif " + first_cmd += f"[[ -d {tool_path} ]] ; then cd {tool_path} && git fetch --all && git pull; " + first_cmd += f"else git clone https://github.com/danielmiessler/SecLists.git {tool_path}; fi'" + + tools_copy.get(tool).get("commands")[0] = first_cmd + + if test_input == "update": + self.perform_install(tools_copy, tool, exists=True) + else: + self.perform_install(tools_copy, tool) + + def test_install_subjack(self): + tool = "subjack" + url = "github.com/haccer/subjack" + tools_copy = tools.copy() + + tools_copy.update(self.setup_go_test(tool, tools_copy)) + + tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}" + + self.perform_install(tools_copy, tool) + + def test_install_tkosubs(self): + tool = "tko-subs" + url = "github.com/anshumanbh/tko-subs" + tools_copy = tools.copy() + + tools_copy.update(self.setup_go_test(tool, tools_copy)) + + tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}" + + self.perform_install(tools_copy, tool) + + def test_install_waybackurls(self): + tool = "waybackurls" + url = "github.com/tomnomnom/waybackurls" + tools_copy = tools.copy() + + tools_copy.update(self.setup_go_test(tool, tools_copy)) + + tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}" + + self.perform_install(tools_copy, tool) + + def test_install_webanalyze(self): + tool = "webanalyze" + url = "github.com/rverton/webanalyze/..." + tools_copy = tools.copy() + + tools_copy.update(self.setup_go_test(tool, tools_copy)) + + tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}" + + self.perform_install(tools_copy, tool) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..b066399 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,39 @@ +import sys +from contextlib import redirect_stdout, redirect_stderr + +from cmd2.utils import StdSim + + +def normalize(block): + """ Normalize a block of text to perform comparison. + Strip newlines from the very beginning and very end Then split into separate lines and strip trailing whitespace + from each line. + """ + assert isinstance(block, str) + block = block.strip("\n") + return [line.rstrip() for line in block.splitlines()] + + +def run_cmd(app, cmd): + """ Clear out and err StdSim buffers, run the command, and return out and err """ + saved_sysout = sys.stdout + sys.stdout = app.stdout + + # This will be used to capture app.stdout and sys.stdout + copy_cmd_stdout = StdSim(app.stdout) + + # This will be used to capture sys.stderr + copy_stderr = StdSim(sys.stderr) + + try: + app.stdout = copy_cmd_stdout + with redirect_stdout(copy_cmd_stdout): + with redirect_stderr(copy_stderr): + app.onecmd_plus_hooks(cmd) + finally: + app.stdout = copy_cmd_stdout.inner_stream + sys.stdout = saved_sysout + + out = copy_cmd_stdout.getvalue() + err = copy_stderr.getvalue() + return normalize(out), normalize(err)