added tests for new ci stage; restructured go tool installs; closes #63; closes #46 (#64)

.github/workflows/pythonapp.yml

@@ -107,3 +107,23 @@ jobs:
         run: |
           pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
           pipenv run python -m pytest tests/test_models
+
+  test-unmocked-tool-installs:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up pipenv
+      run: |
+        python -m pip install --upgrade pip
+        pip install pipenv
+        pipenv install -d
+    - name: Test with pytest
+      run: |
+        pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
+        pipenv run python -m pytest tests/test_tools_install

.pre-commit-config.yaml

@@ -3,8 +3,7 @@ repos:
     rev: stable
     hooks:
     - id: black
-      language_version: python3
+      args: ['pipeline', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models', 'tests/test_tools_install']
-      args: ['pipeline', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models']
 -   repo: https://gitlab.com/pycqa/flake8
     rev: 3.7.9
     hooks:

Pipfile

@@ -22,4 +22,4 @@ python-libnmap = "==0.7.0"
 pyyaml = "==5.3.1"
 
 [requires]
-python_version = "3.7"
+python_version = "3"

Pipfile.lock

@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "3337a01cba6c46ec2ec9e0a0ea5678b5bd4c5e6f29c5a7bb5c2c50c503c4c1e4"
+            "sha256": "e67d6cfc23215132f5512382c7e1004761d6de45818252643c1b3f3d0e56af20"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.7"
+            "python_version": "3"
         },
         "sources": [
             {
@@ -147,10 +147,10 @@
         },
         "appdirs": {
             "hashes": [
-                "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92",
+                "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
-                "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"
+                "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
             ],
-            "version": "==1.4.3"
+            "version": "==1.4.4"
         },
         "attrs": {
             "hashes": [
@@ -295,14 +295,6 @@
             ],
             "version": "==1.2.0"
         },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f",
-                "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==1.6.0"
-        },
         "jinja2": {
             "hashes": [
                 "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0",
@@ -473,29 +465,29 @@
         },
         "regex": {
             "hashes": [
-                "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b",
+                "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349",
-                "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8",
+                "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608",
-                "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3",
+                "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf",
-                "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e",
+                "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938",
-                "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683",
+                "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998",
-                "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1",
+                "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918",
-                "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142",
+                "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945",
-                "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3",
+                "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd",
-                "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468",
+                "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d",
-                "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e",
+                "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e",
-                "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3",
+                "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74",
-                "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a",
+                "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2",
-                "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f",
+                "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8",
-                "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6",
+                "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4",
-                "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156",
+                "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451",
-                "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b",
+                "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388",
-                "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db",
+                "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc",
-                "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd",
+                "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494",
-                "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a",
+                "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1",
-                "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948",
+                "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03",
-                "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89"
+                "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8"
             ],
-            "version": "==2020.4.4"
+            "version": "==2020.5.7"
         },
         "requests": {
             "hashes": [
@@ -633,10 +625,10 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675",
+                "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74",
-                "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1"
+                "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637"
             ],
-            "version": "==20.0.18"
+            "version": "==20.0.20"
         },
         "wcwidth": {
             "hashes": [
@@ -644,13 +636,6 @@
                 "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1"
             ],
             "version": "==0.1.9"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
-                "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"
-            ],
-            "version": "==3.1.0"
         }
     }
 }

pipeline/recon-pipeline.py

@@ -124,6 +124,8 @@ class ReconShell(cmd2.Cmd):
 
         self.tools_dir.mkdir(parents=True, exist_ok=True)
         Path(defaults.get("database-dir")).mkdir(parents=True, exist_ok=True)
+        Path(defaults.get("gopath")).mkdir(parents=True, exist_ok=True)
+        Path(defaults.get("goroot")).mkdir(parents=True, exist_ok=True)
 
         # register hooks to handle selector loop start and cleanup
         self.register_preloop_hook(self._preloop_hook)

pipeline/recon/config.py

@@ -13,10 +13,13 @@ defaults = {
     "home": Path.home(),
 }
 
+defaults["project-dir"] = str(Path(__file__).parents[2])
 defaults["tools-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/tools"
 defaults["database-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/databases"
+
+defaults["goroot"] = f"{defaults.get('tools-dir')}/pipeline-go"
+defaults["gopath"] = f"{defaults.get('tools-dir')}/pipeline-go-workspace"
 defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt"
-defaults["project-dir"] = str(Path(__file__).parents[2])
 
 web_ports = {
     "80",

pipeline/tools/amass.yaml

@@ -1,12 +1,10 @@
 installed: false
 dependencies: [go]
-tools: &tools !get_default "{tools-dir}"
 go: &gotool !get_tool_path "{go[path]}"
-path: &amass !join_path [*tools, "amass"]
+path: !join_path [!get_default "{gopath}", bin/amass]
+environ: {"GO111MODULE": "on", "GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get github.com/OWASP/Amass/v3/...]
-- !join [cp ~/go/bin/amass, *amass]
 
-shell: true
+
-environ: {"GO111MODULE": "on"}

pipeline/tools/go.yaml

@@ -1,9 +1,8 @@
 installed: false
-home: &home !get_default "{home}"
+bashrc: &bashrc !join_path [!get_default "{home}", .bashrc]
-path: &gotool /usr/local/go/bin/go
+path: &gotool !join_path [!get_default "{goroot}", go/bin/go]
-bashrc: &bashrc !join_path [*home, "/.bashrc;"]
 
 commands:
 - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -O /tmp/go.tar.gz
-- sudo tar -C /usr/local -xvf /tmp/go.tar.gz
+- !join [tar -C, !get_default "{goroot}", -xvf /tmp/go.tar.gz]
-- !join ["bash -c 'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >>", *bashrc, "fi'"]
+- !join ["bash -c 'if [ ! $(grep $(dirname", *gotool, ")", *bashrc, ") ]; then echo PATH=${PATH}:$(dirname", *gotool, ") >>", *bashrc, "; fi'"]

pipeline/tools/gobuster.yaml

@@ -1,12 +1,8 @@
 installed: false
 dependencies: [go, seclists]
-home: &home !get_default "{home}"
-path: !join_path [*home, go/bin/gobuster]
 go: &gotool !get_tool_path "{go[path]}"
-go_home: &gohome !join_path [*home, "/go/src/github.com/OJ/gobuster &&"]
+path: !join_path [!get_default "{gopath}", bin/gobuster]
+environ: {"GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get github.com/OJ/gobuster]
-- !join [(cd, *gohome, *gotool, build &&, *gotool, install)]
-
-shell: true

pipeline/tools/recursive-gobuster.yaml

@@ -1,10 +1,10 @@
 installed: false
-dependencies: [go]
+dependencies: [gobuster]
 tools: &tools !get_default "{tools-dir}"
 path: !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz]
-recursive-parent: &recpar !join_path [*tools, recursive-gobuster]
+recursive-parent: &parent !join_path [*tools, recursive-gobuster]
 
 commands:
-- !join ["bash -c 'if [ -d", *recpar, "]; then cd", *recpar,
+- !join ["bash -c 'if [ -d", *parent, "]; then cd", *parent,
         "&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git",
-        *recpar, ; fi']
+        *parent, " ; fi'"]

pipeline/tools/searchsploit.yaml

@@ -10,8 +10,8 @@ sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"]
 
 commands:
 - !join ["bash -c 'if [ -d /usr/share/exploitdb ]; then ln -fs /usr/share/exploitdb",
-        *exploitdb, "&& sudo ln -fs $(which searchsploit)", *searchsploit,
+        *exploitdb, "&& ln -fs $(which searchsploit)", *searchsploit,
         "; elif [ -d", *exploitdb, "]; then cd", *exploitdb,
-        "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, ; fi']
+        "&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, "; fi'"]
-- !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, ; fi']
+- !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, "; fi'"]
-- !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, ; fi']
+- !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, "; fi'"]

pipeline/tools/seclists.yaml

@@ -3,6 +3,6 @@ tools: &tools !get_default "{tools-dir}"
 path: &secfile !join_path [*tools, seclists]
 
 commands:
-  - !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists",
+- !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists",
-          *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
+        *secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
-          else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi']
+        "else git clone https://github.com/danielmiessler/SecLists.git", *secfile, "; fi'"]

pipeline/tools/subjack.yaml

@@ -1,13 +1,10 @@
 installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
-home: &home !get_default "{home}"
+path: !join_path [!get_default "{gopath}", bin/subjack]
-path: !join_path [*home, go/bin/subjack]
+environ: {"GOPATH": !get_default "{gopath}"}
-subjack_home: &subjhome !join_path [*home, "/go/src/github.com/haccer/subjack &&"]
+
-fingerprints: !join_path [*home, go/src/github.com/haccer/subjack/fingerprints.json]
+fingerprints: !join_path [!get_default "{gopath}", src/github.com/haccer/subjack/fingerprints.json]
 
 commands:
 - !join [*gotool, get github.com/haccer/subjack]
-- !join [(cd, *subjhome, *gotool, install)]
-
-shell: true

pipeline/tools/tko-subs.yaml

@@ -1,13 +1,8 @@
 installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
-home: &home !get_default "{home}"
+path: !join_path [!get_default "{gopath}", bin/tko-subs]
-path: !join_path [*home, go/bin/tko-subs]
+environ: {"GOPATH": !get_default "{gopath}"}
-tko_home: &tkohome !join_path [*home, "go/src/github.com/anshumanbh/tko-subs &&"]
-git_dir: !join_path [*home, go/src/github.com/anshumanbh/tko-subs]
 
 commands:
 - !join [*gotool, get, github.com/anshumanbh/tko-subs]
-- !join [(cd, *tkohome, *gotool, "build &&", *gotool, "install)"]
-
-shell: true

pipeline/tools/waybackurls.yaml

@@ -1,7 +1,8 @@
 installed: false
 dependencies: [go]
 go: &gotool !get_tool_path "{go[path]}"
-path: !join_path [!get_default "{home}", go, bin, waybackurls]
+path: !join_path [!get_default "{gopath}", bin/waybackurls]
+environ: {"GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get, github.com/tomnomnom/waybackurls]

pipeline/tools/webanalyze.yaml

@@ -1,12 +1,8 @@
 installed: false
 dependencies: [go]
-home: &home !get_default "{home}"
 go: &gotool !get_tool_path "{go[path]}"
-path: !join_path [*home, go/bin/webanalyze]
+path: !join_path [!get_default "{gopath}", bin/webanalyze]
-webanalyze_home: &webhome !join_path [*home, "/go/src/github.com/rverton/webanalyze &&"]
+environ: {"GOPATH": !get_default "{gopath}"}
 
 commands:
 - !join [*gotool, get github.com/rverton/webanalyze/...]
-- !join [(cd, *webhome, *gotool, "build &&", *gotool, install)]
-
-shell: true

tests/test_tools_install/test_tools_install_command.py

@@ -0,0 +1,300 @@
+import pickle
+import shutil
+import tempfile
+import importlib
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from tests import utils
+
+recon_pipeline = importlib.import_module("pipeline.recon-pipeline")
+tools = recon_pipeline.tools
+
+
+class TestUnmockedToolsInstall:
+    def setup_method(self):
+        self.shell = recon_pipeline.ReconShell()
+        self.tmp_path = Path(tempfile.mkdtemp())
+        self.shell.tools_dir = self.tmp_path / ".local" / "recon-pipeline" / "tools"
+        self.shell.tools_dir.mkdir(parents=True, exist_ok=True)
+
+    def teardown_method(self):
+        def onerror(func, path, exc_info):
+            subprocess.run(f"sudo rm -rf {self.shell.tools_dir}".split())
+
+        shutil.rmtree(self.tmp_path, onerror=onerror)
+
+    def perform_install(self, tools_dict, tool_name, exists=False):
+        pickle.dump(tools_dict, Path(self.shell.tools_dir / ".tool-dict.pkl").open("wb"))
+
+        tool = Path(tools_dict.get(tool_name).get("path"))
+
+        if exists is False:
+            assert tool.exists() is False
+
+        utils.run_cmd(self.shell, f"install {tool_name}")
+
+        assert tool.exists() is True
+
+    def setup_go_test(self, tool_name, tool_dict):
+        # install go in tmp location
+        dependency = "go"
+        dependency_path = f"{self.shell.tools_dir}/go/bin/go"
+
+        tool_dict.get(dependency)["path"] = dependency_path
+        tool_dict.get(dependency).get("commands")[1] = f"tar -C {self.shell.tools_dir} -xvf /tmp/go.tar.gz"
+
+        # handle env for local go install
+        tmp_go_path = f"{self.shell.tools_dir}/mygo"
+        Path(tmp_go_path).mkdir(parents=True, exist_ok=True)
+        tool_dict.get(tool_name)["environ"]["GOPATH"] = tmp_go_path
+
+        tool_path = f"{tool_dict.get(tool_name).get('environ').get('GOPATH')}/bin/{tool_name}"
+        tool_dict.get(tool_name)["path"] = tool_path
+
+        return tool_dict
+
+    def test_install_masscan(self):
+        tool = "masscan"
+        tools_copy = tools.copy()
+
+        tool_path = f"{self.shell.tools_dir}/{tool}"
+
+        tools_copy.get(tool)["path"] = tool_path
+        tools_copy.get(tool).get("commands")[2] = f"mv /tmp/masscan/bin/masscan {tool_path}"
+        tools_copy.get(tool).get("commands")[4] = f"sudo setcap CAP_NET_RAW+ep {tool_path}"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_amass(self):
+        tool = "amass"
+        url = "github.com/OWASP/Amass/v3/..."
+        tools_copy = tools.copy()
+
+        tools_copy.update(self.setup_go_test(tool, tools_copy))
+
+        tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_aquatone(self):
+        tool = "aquatone"
+        tools_copy = tools.copy()
+
+        tool_path = f"{self.shell.tools_dir}/{tool}"
+
+        tools_copy.get(tool)["path"] = tool_path
+        tools_copy.get(tool).get("commands")[4] = f"mv /tmp/aquatone/aquatone {tool_path}"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_go(self):
+        tool = "go"
+        tools_copy = tools.copy()
+
+        tool_path = f"{self.shell.tools_dir}/go/bin/go"
+
+        tools_copy.get(tool)["path"] = tool_path
+        tools_copy.get(tool).get("commands")[1] = f"tar -C {self.shell.tools_dir} -xvf /tmp/go.tar.gz"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_gobuster(self):
+        tool = "gobuster"
+        dependency = "go"
+        tools_copy = tools.copy()
+
+        tools_copy.update(self.setup_go_test(tool, tools_copy))
+
+        tools_copy.get(tool)["dependencies"] = [dependency]
+        tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get(dependency).get('path')} get github.com/OJ/gobuster"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_luigi_service(self):
+        luigi_service = Path("/lib/systemd/system/luigid.service")
+
+        if luigi_service.exists():
+            subprocess.run(f"sudo rm {luigi_service}".split())
+
+        tools_copy = tools.copy()
+        pickle.dump(tools_copy, Path(self.shell.tools_dir / ".tool-dict.pkl").open("wb"))
+
+        proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE)
+
+        if proc.stdout.decode().strip() == "enabled":
+            subprocess.run("sudo systemctl disable luigid.service".split())
+
+        proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE)
+
+        if proc.stdout.decode().strip() == "active":
+            subprocess.run("sudo systemctl stop luigid.service".split())
+
+        if Path("/usr/local/bin/luigid").exists():
+            subprocess.run("sudo rm /usr/local/bin/luigid".split())
+
+        assert (
+            subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE)
+            .stdout.decode()
+            .strip()
+            != "enabled"
+        )
+
+        assert (
+            subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE).stdout.decode().strip()
+            != "active"
+        )
+
+        assert not Path("/usr/local/bin/luigid").exists()
+
+        utils.run_cmd(self.shell, "install luigi-service")
+
+        assert Path("/lib/systemd/system/luigid.service").exists()
+
+        proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE)
+        assert proc.stdout.decode().strip() == "enabled"
+
+        proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE)
+        assert proc.stdout.decode().strip() == "active"
+
+        assert Path("/usr/local/bin/luigid").exists()
+
+    @pytest.mark.parametrize("test_input", ["install", "update"])
+    def test_install_recursive_gobuster(self, test_input):
+        tool = "recursive-gobuster"
+        tools_copy = tools.copy()
+
+        parent = f"{self.shell.tools_dir}/{tool}"
+        tool_path = f"{parent}/recursive-gobuster.pyz"
+
+        if test_input == "update":
+            subprocess.run(f"git clone https://github.com/epi052/recursive-gobuster.git {parent}".split())
+
+        tools_copy.get(tool)["path"] = tool_path
+        tools_copy.get(tool)["dependencies"] = None
+
+        tools_copy.get(tool).get("commands")[0] = f"bash -c 'if [ -d {parent} ]; "
+        tools_copy.get(tool).get("commands")[0] += f"then cd {parent} && git fetch --all && git pull; "
+        tools_copy.get(tool).get("commands")[0] += "else git clone https://github.com/epi052/recursive-gobuster.git "
+        tools_copy.get(tool).get("commands")[0] += f"{parent} ; fi'"
+
+        if test_input == "update":
+            self.perform_install(tools_copy, tool, exists=True)
+        else:
+            self.perform_install(tools_copy, tool)
+
+    @pytest.mark.parametrize("test_input", ["install", "update"])
+    def test_install_searchsploit(self, test_input):
+        tool = "searchsploit"
+        tools_copy = tools.copy()
+
+        home_path = f"{self.shell.tools_dir}/home"
+        Path(home_path).mkdir(parents=True, exist_ok=True)
+
+        copied_searchsploit_rc = f"{home_path}/.searchsploit_rc"
+
+        dependency_path = f"{self.shell.tools_dir}/exploitdb"
+        searchsploit_rc_path = f"{dependency_path}/.searchsploit_rc"
+        tool_path = f"{dependency_path}/{tool}"
+
+        sed_cmd = f"s#/opt#{self.shell.tools_dir}#g"
+
+        if test_input == "update":
+            subprocess.run(f"git clone https://github.com/offensive-security/exploitdb.git {dependency_path}".split())
+
+        tools_copy.get(tool)["path"] = tool_path
+
+        first_cmd = "bash -c 'if [ -d /usr/share/exploitdb ]; then ln -fs "
+        first_cmd += f"/usr/share/exploitdb {dependency_path} && ln -fs $(which searchsploit) {tool_path}"
+        first_cmd += f"; elif [ -d {dependency_path} ]; then cd {dependency_path} && git fetch --all && git pull; else "
+        first_cmd += f"git clone https://github.com/offensive-security/exploitdb.git {dependency_path}; fi'"
+
+        tools_copy.get(tool).get("commands")[0] = first_cmd
+
+        tools_copy.get(tool).get("commands")[1] = f"bash -c 'if [ -f {searchsploit_rc_path} ]; "
+        tools_copy.get(tool).get("commands")[1] += f"then cp -n {searchsploit_rc_path} {home_path} ; fi'"
+
+        tools_copy.get(tool).get("commands")[2] = f"bash -c 'if [ -f {copied_searchsploit_rc} ]; "
+        tools_copy.get(tool).get("commands")[2] += f"then sed -i {sed_cmd} {copied_searchsploit_rc}; fi'"
+
+        pickle.dump(tools_copy, Path(self.shell.tools_dir / ".tool-dict.pkl").open("wb"))
+
+        if test_input == "install":
+            assert not Path(tool_path).exists()
+            assert not Path(copied_searchsploit_rc).exists()
+            assert not Path(dependency_path).exists()
+
+        utils.run_cmd(self.shell, f"install {tool}")
+
+        assert subprocess.run(f"grep {self.shell.tools_dir} {copied_searchsploit_rc}".split()).returncode == 0
+        assert Path(copied_searchsploit_rc).exists()
+        assert Path(dependency_path).exists()
+
+    @pytest.mark.parametrize("test_input", ["install", "update"])
+    def test_install_seclists(self, test_input):
+        tool = "seclists"
+        tools_copy = tools.copy()
+
+        tool_path = f"{self.shell.tools_dir}/seclists"
+
+        tools_copy.get(tool)["path"] = tool_path
+
+        if test_input == "update":
+            subprocess.run(f"git clone https://github.com/danielmiessler/SecLists.git {tool_path}".split())
+
+        first_cmd = f"bash -c 'if [ -d /usr/share/seclists ]; then ln -s /usr/share/seclists {tool_path}; elif "
+        first_cmd += f"[[ -d {tool_path} ]] ; then cd {tool_path} && git fetch --all && git pull; "
+        first_cmd += f"else git clone https://github.com/danielmiessler/SecLists.git {tool_path}; fi'"
+
+        tools_copy.get(tool).get("commands")[0] = first_cmd
+
+        if test_input == "update":
+            self.perform_install(tools_copy, tool, exists=True)
+        else:
+            self.perform_install(tools_copy, tool)
+
+    def test_install_subjack(self):
+        tool = "subjack"
+        url = "github.com/haccer/subjack"
+        tools_copy = tools.copy()
+
+        tools_copy.update(self.setup_go_test(tool, tools_copy))
+
+        tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_tkosubs(self):
+        tool = "tko-subs"
+        url = "github.com/anshumanbh/tko-subs"
+        tools_copy = tools.copy()
+
+        tools_copy.update(self.setup_go_test(tool, tools_copy))
+
+        tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_waybackurls(self):
+        tool = "waybackurls"
+        url = "github.com/tomnomnom/waybackurls"
+        tools_copy = tools.copy()
+
+        tools_copy.update(self.setup_go_test(tool, tools_copy))
+
+        tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}"
+
+        self.perform_install(tools_copy, tool)
+
+    def test_install_webanalyze(self):
+        tool = "webanalyze"
+        url = "github.com/rverton/webanalyze/..."
+        tools_copy = tools.copy()
+
+        tools_copy.update(self.setup_go_test(tool, tools_copy))
+
+        tools_copy.get(tool).get("commands")[0] = f"{tools_copy.get('go').get('path')} get {url}"
+
+        self.perform_install(tools_copy, tool)

tests/utils.py

@@ -0,0 +1,39 @@
+import sys
+from contextlib import redirect_stdout, redirect_stderr
+
+from cmd2.utils import StdSim
+
+
+def normalize(block):
+    """ Normalize a block of text to perform comparison.
+    Strip newlines from the very beginning and very end  Then split into separate lines and strip trailing whitespace
+    from each line.
+    """
+    assert isinstance(block, str)
+    block = block.strip("\n")
+    return [line.rstrip() for line in block.splitlines()]
+
+
+def run_cmd(app, cmd):
+    """ Clear out and err StdSim buffers, run the command, and return out and err """
+    saved_sysout = sys.stdout
+    sys.stdout = app.stdout
+
+    # This will be used to capture app.stdout and sys.stdout
+    copy_cmd_stdout = StdSim(app.stdout)
+
+    # This will be used to capture sys.stderr
+    copy_stderr = StdSim(sys.stderr)
+
+    try:
+        app.stdout = copy_cmd_stdout
+        with redirect_stdout(copy_cmd_stdout):
+            with redirect_stderr(copy_stderr):
+                app.onecmd_plus_hooks(cmd)
+    finally:
+        app.stdout = copy_cmd_stdout.inner_stream
+        sys.stdout = saved_sysout
+
+    out = copy_cmd_stdout.getvalue()
+    err = copy_stderr.getvalue()
+    return normalize(out), normalize(err)