added CI pipeline (#2)

* Create pythonapp.yml

* Update pythonapp.yml

* fixing up flake8/black

* Update pythonapp.yml

* testing addition of tests

* testing masscan install

* testing pipenv install

* test install command done?

* first set of tests complete

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 100
+select = C,E,F,W,B,B950
+ignore = E203, E501, W503
+max-complexity = 13

.github/workflows/pythonapp.yml

@@ -0,0 +1,51 @@
+name: recon-pipeline build
+
+on: [push]
+
+jobs:
+  lint:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up pipenv
+      run: |
+        python -m pip install --upgrade pip
+        pip install pipenv
+        pipenv install -d
+    - name: Lint with flake8
+      run: |
+        pipenv install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        pipenv run flake8 . --count
+    - name: Check code formatting with black
+      uses: lgeiger/black-action@master
+      with:
+        args: ". --check"
+
+  test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Golang
+      uses: actions/setup-go@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up pipenv
+      run: |
+        python -m pip install --upgrade pip
+        pip install pipenv
+        pipenv install -d
+    - name: Test with pytest
+      run: |
+        pipenv install pytest
+        pipenv run python -m pytest tests/

.gitignore

@@ -103,7 +103,4 @@ venv.bak/
 # mypy
 .mypy_cache/
 
-.idea
-.flake8
-.pre-commit-config.yaml
-pyproject.toml
+.idea

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.7
+      args: ['.']
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.2.3
+    hooks:
+    - id: flake8
+

Pipfile

@@ -6,8 +6,8 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-luigi = "*"
 cmd2 = "*"
+luigi = "*"
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -25,11 +25,11 @@
         },
         "cmd2": {
             "hashes": [
-                "sha256:208812035933cdb5c1c254cf266ffd7f560ca3a075569f3a39fc4e4a4427c2a0",
-                "sha256:8ad12ef3cc46d03073c545b6e80a3f84a5921f6653073a60e7d9a7ff3b352c9e"
+                "sha256:a07b165603e6cdf6730c95007160036f13b83415f9074dcb475e91f897ec324d",
+                "sha256:bdb4d7a56023c800c4428abf8e198e28d6a566f23fa172208763c30e298be4ee"
             ],
             "index": "pypi",
-            "version": "==0.9.23"
+            "version": "==0.9.24"
         },
         "colorama": {
             "hashes": [

README.md

@@ -1,9 +1,10 @@
 # Automated Reconnaissance Pipeline
 
-![version](https://img.shields.io/badge/version-0.7.3-informational?style=for-the-badge)
+![version](https://img.shields.io/github/v/release/epi052/recon-pipeline?style=for-the-badge)
+![Python application](https://img.shields.io/github/workflow/status/epi052/recon-pipeline/recon-pipeline%20build?style=for-the-badge)
 ![python](https://img.shields.io/badge/python-3.7-informational?style=for-the-badge)
-![luigi](https://img.shields.io/badge/luigi-2.8.11-yellowgreen?style=for-the-badge)
-![cmd2](https://img.shields.io/badge/cmd2-0.9.23-yellowgreen?style=for-the-badge)
+![luigi](https://img.shields.io/github/pipenv/locked/dependency-version/epi052/recon-pipeline/luigi?style=for-the-badge)
+![cmd2](https://img.shields.io/github/pipenv/locked/dependency-version/epi052/recon-pipeline/cmd2?style=for-the-badge)
 ![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=for-the-badge)
 
 There are an [accompanying set of blog posts](https://epi052.gitlab.io/notes-to-self/blog/2019-09-01-how-to-build-an-automated-recon-pipeline-with-python-and-luigi/) detailing the development process and underpinnings of the pipeline.  Feel free to check them out if you're so inclined, but they're in no way required reading to use the tool.

pyproject.toml

@@ -0,0 +1,4 @@
+[tool.black]
+line-length = 100
+include = '\.pyi?$'
+exclude = '.*config.*py$|\.git'

recon-pipeline.py

@@ -251,7 +251,7 @@ class ReconShell(cmd2.Cmd):
                 out, err = proc.communicate()
 
                 if err:
-                    self.async_alert(style(f"[!] {err.decode().strip()}", fg="bright_red"))
+                    self.poutput(style(f"[!] {err.decode().strip()}", fg="bright_red"))
 
                 retvals.append(proc.returncode)
 

recon/__init__.py

@@ -18,11 +18,11 @@ tools = {
         "installed": False,
         "dependencies": ["luigi"],
         "commands": [
-            f"cp {str(Path(__file__).parent.parent / 'luigid.service')} /lib/systemd/system/luigid.service",
-            f"cp $(which luigid) /usr/local/bin",
-            "systemctl daemon-reload",
-            "systemctl start luigid.service",
-            "systemctl enable luigid.service",
+            f"sudo cp {str(Path(__file__).parent.parent / 'luigid.service')} /lib/systemd/system/luigid.service",
+            f"sudo cp $(which luigid) /usr/local/bin",
+            "sudo systemctl daemon-reload",
+            "sudo systemctl start luigid.service",
+            "sudo systemctl enable luigid.service",
         ],
         "shell": True,
     },
@@ -30,7 +30,7 @@ tools = {
     "pipenv": {
         "installed": False,
         "dependencies": None,
-        "commands": ["apt-get install -y -q pipenv"],
+        "commands": ["sudo apt-get install -y -q pipenv"],
     },
     "masscan": {
         "installed": False,
@@ -38,14 +38,14 @@ tools = {
         "commands": [
             "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan",
             "make -s -j -C /tmp/masscan",
-            f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}",
+            f"sudo mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}",
             "rm -rf /tmp/masscan",
         ],
     },
     "amass": {
         "installed": False,
         "dependencies": None,
-        "commands": ["apt-get install -y -q amass"],
+        "commands": ["sudo apt-get install -y -q amass"],
     },
     "aquatone": {
         "installed": False,
@@ -55,7 +55,7 @@ tools = {
             "mkdir /tmp/aquatone",
             "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip",
             "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone",
-            f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}",
+            f"sudo mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}",
             "rm -rf /tmp/aquatone",
         ],
     },
@@ -64,7 +64,7 @@ tools = {
         "dependencies": None,
         "shell": True,
         "commands": [
-            f"bash -c 'if [[ -d {Path(tool_paths.get('CORScanner')).parent} ]] ; then cd {Path(tool_paths.get('CORScanner')).parent} && git pull; else git clone https://github.com/chenjj/CORScanner.git {Path(tool_paths.get('CORScanner')).parent}; fi'",
+            f"sudo bash -c 'if [[ -d {Path(tool_paths.get('CORScanner')).parent} ]] ; then cd {Path(tool_paths.get('CORScanner')).parent} && git pull; else git clone https://github.com/chenjj/CORScanner.git {Path(tool_paths.get('CORScanner')).parent}; fi'",
             f"pip install -q -r {Path(tool_paths.get('CORScanner')).parent / 'requirements.txt'}",
             "pip install -q future",
         ],
@@ -110,11 +110,14 @@ tools = {
         "dependencies": None,
         "shell": True,
         "commands": [
-            f"bash -c 'if [[ -d /opt/recursive-gobuster ]] ; then cd /opt/recursive-gobuster && git pull; else git clone https://github.com/epi052/recursive-gobuster.git /opt/recursive-gobuster; fi'",
-            f"ln -fs /opt/recursive-gobuster/recursive-gobuster.pyz {tool_paths.get('recursive-gobuster')}",
+            f"sudo bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'",
         ],
     },
-    "go": {"installed": False, "dependencies": None, "commands": ["apt-get install -y -q golang"]},
+    "go": {
+        "installed": False,
+        "dependencies": None,
+        "commands": ["sudo apt-get install -y -q golang"],
+    },
 }
 
 

tests/test_install_command.py

@@ -0,0 +1,231 @@
+import shutil
+import importlib
+import subprocess
+from pathlib import Path
+
+import utils
+from recon.config import tool_paths
+
+recon_pipeline = importlib.import_module("recon-pipeline")
+
+
+def test_install_masscan():
+    masscan = Path(tool_paths.get("masscan"))
+
+    utils.setup_install_test(masscan)
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install masscan")
+
+    assert masscan.exists() is True
+
+
+def test_install_amass():
+    utils.setup_install_test()
+
+    if not utils.is_kali():
+        return True
+
+    if shutil.which("amass") is not None:
+        subprocess.run("sudo apt remove amass -y".split())
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install amass")
+
+    assert shutil.which("amass") is not None
+
+
+def test_install_pipenv():
+    utils.setup_install_test()
+
+    if not utils.is_kali():
+        return True
+
+    if shutil.which("pipenv") is not None:
+        subprocess.run("sudo apt remove pipenv -y".split())
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install pipenv")
+
+    assert shutil.which("pipenv") is not None
+
+
+def test_install_luigi():
+    utils.setup_install_test()
+
+    if shutil.which("luigi") is not None:
+        subprocess.run("pipenv uninstall luigi".split())
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install luigi")
+
+    assert shutil.which("luigi") is not None
+
+
+def test_install_aquatone():
+    aquatone = Path(tool_paths.get("aquatone"))
+
+    utils.setup_install_test(aquatone)
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install aquatone")
+
+    assert aquatone.exists() is True
+
+
+def test_install_gobuster():
+    gobuster = Path(tool_paths.get("gobuster"))
+
+    utils.setup_install_test(gobuster)
+
+    assert shutil.which("go") is not None
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install gobuster")
+
+    assert gobuster.exists() is True
+
+
+def test_install_tkosubs():
+    tkosubs = Path(tool_paths.get("tko-subs"))
+
+    utils.setup_install_test(tkosubs)
+
+    assert shutil.which("go") is not None
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install tko-subs")
+
+    assert tkosubs.exists() is True
+
+
+def test_install_subjack():
+    subjack = Path(tool_paths.get("subjack"))
+
+    utils.setup_install_test(subjack)
+
+    assert shutil.which("go") is not None
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install subjack")
+
+    assert subjack.exists() is True
+
+
+def test_install_webanalyze():
+    webanalyze = Path(tool_paths.get("webanalyze"))
+
+    utils.setup_install_test(webanalyze)
+
+    assert shutil.which("go") is not None
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install webanalyze")
+
+    assert webanalyze.exists() is True
+
+
+def test_install_corscanner():
+    corscanner = Path(tool_paths.get("CORScanner"))
+
+    utils.setup_install_test(corscanner)
+
+    if corscanner.parent.exists():
+        shutil.rmtree(corscanner.parent)
+
+    rs = recon_pipeline.ReconShell()
+
+    script_out, script_err = utils.run_cmd(rs, "install corscanner")
+
+    assert corscanner.exists() is True
+
+
+def test_update_corscanner():
+    corscanner = Path(tool_paths.get("CORScanner"))
+
+    utils.setup_install_test()
+
+    if not corscanner.parent.exists():
+        subprocess.run(
+            f"sudo git clone https://github.com/chenjj/CORScanner.git {corscanner.parent}".split()
+        )
+
+    rs = recon_pipeline.ReconShell()
+
+    utils.run_cmd(rs, "install corscanner")
+
+    assert corscanner.exists() is True
+
+
+def test_install_recursive_gobuster():
+    recursive_gobuster = Path(tool_paths.get("recursive-gobuster"))
+
+    utils.setup_install_test(recursive_gobuster)
+
+    if recursive_gobuster.parent.exists():
+        shutil.rmtree(recursive_gobuster.parent)
+
+    rs = recon_pipeline.ReconShell()
+
+    utils.run_cmd(rs, "install recursive-gobuster")
+
+    assert recursive_gobuster.exists() is True
+
+
+def test_update_recursive_gobuster():
+    recursive_gobuster = Path(tool_paths.get("recursive-gobuster"))
+
+    utils.setup_install_test()
+
+    if not recursive_gobuster.parent.exists():
+        subprocess.run(
+            f"sudo git clone https://github.com/epi052/recursive-gobuster.git {recursive_gobuster.parent}".split()
+        )
+
+    rs = recon_pipeline.ReconShell()
+
+    utils.run_cmd(rs, "install recursive-gobuster")
+
+    assert recursive_gobuster.exists() is True
+
+
+def test_install_luigi_service():
+    luigi_service = Path("/lib/systemd/system/luigid.service")
+
+    utils.setup_install_test(luigi_service)
+
+    proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE)
+
+    if proc.stdout.decode().strip() == "enabled":
+        subprocess.run("systemctl disable luigid.service".split())
+
+    proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE)
+
+    if proc.stdout.decode().strip() == "active":
+        subprocess.run("systemctl stop luigid.service".split())
+
+    if Path("/usr/local/bin/luigid").exists():
+        Path("/usr/local/bin/luigid").unlink()
+
+    rs = recon_pipeline.ReconShell()
+
+    utils.run_cmd(rs, "install luigi-service")
+
+    assert Path("/lib/systemd/system/luigid.service").exists()
+
+    proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE)
+    assert proc.stdout.decode().strip() == "enabled"
+
+    proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE)
+    assert proc.stdout.decode().strip() == "active"
+
+    assert Path("/usr/local/bin/luigid").exists()

tests/utils.py

@@ -0,0 +1,67 @@
+import sys
+import subprocess
+from pathlib import Path
+from contextlib import redirect_stdout, redirect_stderr
+
+from cmd2.utils import StdSim
+
+
+def is_kali():
+    return any(
+        [
+            "kali" in x
+            for x in subprocess.run("cat /etc/lsb-release".split(), stdout=subprocess.PIPE)
+            .stdout.decode()
+            .split()
+        ]
+    )
+
+
+def normalize(block):
+    """ Normalize a block of text to perform comparison.
+    Strip newlines from the very beginning and very end  Then split into separate lines and strip trailing whitespace
+    from each line.
+    """
+    assert isinstance(block, str)
+    block = block.strip("\n")
+    return [line.rstrip() for line in block.splitlines()]
+
+
+def run_cmd(app, cmd):
+    """ Clear out and err StdSim buffers, run the command, and return out and err """
+    saved_sysout = sys.stdout
+    sys.stdout = app.stdout
+
+    # This will be used to capture app.stdout and sys.stdout
+    copy_cmd_stdout = StdSim(app.stdout)
+
+    # This will be used to capture sys.stderr
+    copy_stderr = StdSim(sys.stderr)
+
+    try:
+        app.stdout = copy_cmd_stdout
+        with redirect_stdout(copy_cmd_stdout):
+            with redirect_stderr(copy_stderr):
+                app.onecmd_plus_hooks(cmd)
+    finally:
+        app.stdout = copy_cmd_stdout.inner_stream
+        sys.stdout = saved_sysout
+
+    out = copy_cmd_stdout.getvalue()
+    err = copy_stderr.getvalue()
+    return normalize(out), normalize(err)
+
+
+def setup_install_test(tool=None):
+    tools = Path.home() / ".cache" / ".tool-dict.pkl"
+
+    try:
+        tools.unlink()
+    except FileNotFoundError:
+        pass
+
+    if tool is not None:
+        try:
+            tool.unlink()
+        except FileNotFoundError:
+            pass