diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 3d28e57..f64bb4c 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -47,7 +47,7 @@ jobs: pipenv install -d - name: Test with pytest run: | - pipenv install pytest + pipenv install pytest cmd2 luigi pipenv run python -m pytest tests/test_install test-recon: @@ -67,7 +67,7 @@ jobs: pipenv install -d - name: Test with pytest run: | - pipenv install pytest + pipenv install pytest cmd2 luigi pipenv run python -m pytest tests/test_recon test-web: @@ -87,5 +87,5 @@ jobs: pipenv install -d - name: Test with pytest run: | - pipenv install pytest + pipenv install pytest cmd2 luigi pipenv run python -m pytest tests/test_web \ No newline at end of file diff --git a/.gitignore b/.gitignore index 486f201..4dc5d66 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,6 @@ venv.bak/ # mypy .mypy_cache/ -.idea \ No newline at end of file +.idea +Pipfile +Pipfile.lock diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 3552eb0..0000000 --- a/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] - -[packages] -cmd2 = "*" -luigi = "*" - -[requires] -python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 64a7ff2..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,110 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "f6e3610c10920d9297afe9d974892f5e970a04cfb8c3f8164e874031c6274037" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.7" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "attrs": { - "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" - ], - "version": "==19.3.0" - }, - "cmd2": { - "hashes": [ - "sha256:a07b165603e6cdf6730c95007160036f13b83415f9074dcb475e91f897ec324d", - "sha256:bdb4d7a56023c800c4428abf8e198e28d6a566f23fa172208763c30e298be4ee" - ], - "index": "pypi", - "version": "==0.9.24" - }, - "colorama": { - "hashes": [ - "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", - "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" - ], - "version": "==0.4.3" - }, - "docutils": { - "hashes": [ - "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", - "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" - ], - "version": "==0.16" - }, - "lockfile": { - "hashes": [ - "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", - "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa" - ], - "version": "==0.12.2" - }, - "luigi": { - "hashes": [ - "sha256:c2b3dcecc565fe77920553434ed475fa21f562d4b76da6bd1a179a8b732fcc9e" - ], - "index": "pypi", - "version": "==2.8.11" - }, - "pyperclip": { - "hashes": [ - "sha256:979325468ccf682104d5dcaf753f869868100631301d3e72f47babdea5700d1c" - ], - "version": "==1.7.0" - }, - "python-daemon": { - "hashes": [ - "sha256:57c84f50a04d7825515e4dbf3a31c70cc44414394a71608dee6cfde469e81766", - "sha256:a0d5dc0b435a02c7e0b401e177a7c17c3f4c7b4e22e2d06271122c8fec5f8946" - ], - "version": "==2.2.4" - }, - "python-dateutil": { - "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" - ], - "version": "==2.8.1" - }, - "six": { - "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" - ], - "version": "==1.14.0" - }, - "tornado": { - "hashes": [ - "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", - "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", - "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", - "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", - "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", - "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", - "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" - ], - "version": "==5.1.1" - }, - "wcwidth": { - "hashes": [ - "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", - "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" - ], - "version": "==0.1.8" - } - }, - "develop": {} -} diff --git a/docs/conf.py b/docs/conf.py index 3ea41ec..17b5d77 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,12 +38,7 @@ master_doc = "index" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.coverage", - "sphinx.ext.napoleon", - "sphinxarg.ext", -] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.coverage", "sphinx.ext.napoleon", "sphinxarg.ext"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/recon-pipeline.py b/recon-pipeline.py index 8fe0b49..970442f 100755 --- a/recon-pipeline.py +++ b/recon-pipeline.py @@ -16,12 +16,16 @@ os.environ["PYTHONPATH"] = f"{os.environ.get('PYTHONPATH')}:{str(Path(__file__). # suppress "You should consider upgrading via the 'pip install --upgrade pip' command." warning os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" +# in case we need pipenv, add its default --user installed directory to the PATH +sys.path.append(str(Path.home() / ".local" / "bin")) + # third party imports import cmd2 # noqa: E402 from cmd2.ansi import style # noqa: E402 # project's module imports from recon import get_scans, tools, scan_parser, install_parser, status_parser # noqa: F401,E402 +from recon.config import defaults # noqa: F401,E402 # select loop, handles async stdout/stderr processing of subprocesses selector = selectors.DefaultSelector() @@ -58,6 +62,9 @@ class ReconShell(cmd2.Cmd): self.sentry = False self.prompt = "recon-pipeline> " self.selectorloop = None + self.continue_install = True + + Path(defaults.get("tools-dir")).mkdir(parents=True, exist_ok=True) # register hooks to handle selector loop start and cleanup self.register_preloop_hook(self._preloop_hook) @@ -215,7 +222,7 @@ class ReconShell(cmd2.Cmd): continue self.async_alert( - style(f"[!] {args.tool} has an unmet dependency; installing {dependency}", fg="yellow", bold=True,) + style(f"[!] {args.tool} has an unmet dependency; installing {dependency}", fg="yellow", bold=True) ) # install the dependency before continuing with installation @@ -224,13 +231,17 @@ class ReconShell(cmd2.Cmd): if tools.get(args.tool).get("installed"): return self.async_alert(style(f"[!] {args.tool} is already installed.", fg="yellow")) else: - # list of return values from commands run during each tool installation # used to determine whether the tool installed correctly or not retvals = list() self.async_alert(style(f"[*] Installing {args.tool}...", fg="bright_yellow")) + addl_env_vars = tools.get(args.tool).get("environ") + + if addl_env_vars is not None: + addl_env_vars.update(dict(os.environ)) + for command in tools.get(args.tool).get("commands"): # run all commands required to install the tool @@ -240,11 +251,15 @@ class ReconShell(cmd2.Cmd): if tools.get(args.tool).get("shell"): # go tools use subshells (cmd1 && cmd2 && cmd3 ...) during install, so need shell=True - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,) + proc = subprocess.Popen( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=addl_env_vars + ) else: # "normal" command, split up the string as usual and run it - proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE,) + proc = subprocess.Popen( + shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=addl_env_vars + ) out, err = proc.communicate() diff --git a/recon/__init__.py b/recon/__init__.py index 2f7c551..dd94bc4 100644 --- a/recon/__init__.py +++ b/recon/__init__.py @@ -10,7 +10,7 @@ from collections import defaultdict import cmd2 import recon -from recon.config import tool_paths +from recon.config import tool_paths, defaults # tool definitions for recon-pipeline's auto-installer tools = { @@ -26,19 +26,32 @@ tools = { ], "shell": True, }, - "luigi": {"installed": False, "dependencies": ["pipenv"], "commands": ["pipenv install luigi"],}, - "pipenv": {"installed": False, "dependencies": None, "commands": ["sudo apt-get install -y -q pipenv"],}, + "luigi": {"installed": False, "dependencies": None, "commands": ["pip install luigi"]}, + "seclists": { + "installed": False, + "dependencies": None, + "commands": [f"git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists"], + }, "masscan": { "installed": False, "dependencies": None, "commands": [ "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan", "make -s -j -C /tmp/masscan", - f"sudo mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}", + f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}", "rm -rf /tmp/masscan", ], }, - "amass": {"installed": False, "dependencies": None, "commands": ["sudo apt-get install -y -q amass"],}, + "amass": { + "installed": False, + "dependencies": ["go"], + "commands": [ + f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...", + f"cp ~/go/bin/amass {tool_paths.get('amass')}", + ], + "shell": True, + "environ": {"GO111MODULE": "on"}, + }, "aquatone": { "installed": False, "dependencies": None, @@ -47,7 +60,7 @@ tools = { "mkdir /tmp/aquatone", "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip", "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone", - f"sudo mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}", + f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}", "rm -rf /tmp/aquatone", ], }, @@ -56,17 +69,17 @@ tools = { "dependencies": None, "shell": True, "commands": [ - f"sudo bash -c 'if [[ -d {Path(tool_paths.get('CORScanner')).parent} ]] ; then cd {Path(tool_paths.get('CORScanner')).parent} && git pull; else git clone https://github.com/chenjj/CORScanner.git {Path(tool_paths.get('CORScanner')).parent}; fi'", - f"pip install -q -r {Path(tool_paths.get('CORScanner')).parent / 'requirements.txt'}", - "pip install -q future", + f"bash -c 'if [[ -d {Path(tool_paths.get('CORScanner')).parent} ]] ; then cd {Path(tool_paths.get('CORScanner')).parent} && git fetch --all && git pull; else git clone https://github.com/chenjj/CORScanner.git {Path(tool_paths.get('CORScanner')).parent}; fi'", + f"pip install -r {Path(tool_paths.get('CORScanner')).parent / 'requirements.txt'}", + "pip install future", ], }, "gobuster": { "installed": False, - "dependencies": ["go"], + "dependencies": ["go", "seclists"], "commands": [ - "go get github.com/OJ/gobuster", - "(cd ~/go/src/github.com/OJ/gobuster && go build && go install)", + f"{tool_paths.get('go')} get github.com/OJ/gobuster", + f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", ], "shell": True, }, @@ -74,35 +87,46 @@ tools = { "installed": False, "dependencies": ["go"], "commands": [ - "go get github.com/anshumanbh/tko-subs", - "(cd ~/go/src/github.com/anshumanbh/tko-subs && go build && go install)", + f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs", + f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", ], "shell": True, }, "subjack": { "installed": False, "dependencies": ["go"], - "commands": ["go get github.com/haccer/subjack", "(cd ~/go/src/github.com/haccer/subjack && go install)",], + "commands": [ + f"{tool_paths.get('go')} get github.com/haccer/subjack", + f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)", + ], "shell": True, }, "webanalyze": { "installed": False, "dependencies": ["go"], "commands": [ - "go get github.com/rverton/webanalyze/...", - "(cd ~/go/src/github.com/rverton/webanalyze && go build && go install)", + f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...", + f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)", ], "shell": True, }, "recursive-gobuster": { "installed": False, - "dependencies": None, + "dependencies": ["gobuster", "seclists"], "shell": True, "commands": [ - f"sudo bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'", + f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'" + ], + }, + "go": { + "installed": False, + "dependencies": None, + "commands": [ + "wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz", + "sudo tar -C /usr/local -xvf /tmp/go.tar.gz", + f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'', ], }, - "go": {"installed": False, "dependencies": None, "commands": ["sudo apt-get install -y -q golang"],}, } @@ -167,13 +191,13 @@ scan_parser.add_argument( help="file created by the user that defines the target's scope; list of ips/domains", ) scan_parser.add_argument( - "--exempt-list", completer_method=cmd2.Cmd.path_complete, help="list of blacklisted ips/domains", + "--exempt-list", completer_method=cmd2.Cmd.path_complete, help="list of blacklisted ips/domains" ) scan_parser.add_argument( - "--results-dir", completer_method=cmd2.Cmd.path_complete, help="directory in which to save scan results", + "--results-dir", completer_method=cmd2.Cmd.path_complete, help="directory in which to save scan results" ) scan_parser.add_argument( - "--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster", + "--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster" ) scan_parser.add_argument( "--interface", @@ -183,11 +207,9 @@ scan_parser.add_argument( scan_parser.add_argument("--recursive", action="store_true", help="whether or not to recursively gobust") scan_parser.add_argument("--rate", help="rate at which masscan should scan") scan_parser.add_argument( - "--top-ports", help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)", -) -scan_parser.add_argument( - "--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)", + "--top-ports", help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)" ) +scan_parser.add_argument("--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)") scan_parser.add_argument("--threads", help="number of threads for all of the threaded applications to use") scan_parser.add_argument("--scan-timeout", help="scan timeout for aquatone") scan_parser.add_argument("--proxy", help="proxy for gobuster if desired (ex. 127.0.0.1:8080)") @@ -198,8 +220,8 @@ scan_parser.add_argument( help="ppen a web browser to Luigi's central scheduler's visualization site (see how the sausage is made!)", ) scan_parser.add_argument( - "--local-scheduler", action="store_true", help="use the local scheduler instead of the central scheduler (luigid)", + "--local-scheduler", action="store_true", help="use the local scheduler instead of the central scheduler (luigid)" ) scan_parser.add_argument( - "--verbose", action="store_true", help="shows debug messages from luigi, useful for troubleshooting", + "--verbose", action="store_true", help="shows debug messages from luigi, useful for troubleshooting" ) diff --git a/recon/amass.py b/recon/amass.py index f900c60..77b26a3 100644 --- a/recon/amass.py +++ b/recon/amass.py @@ -6,6 +6,7 @@ import luigi from luigi.util import inherits from luigi.contrib.external_program import ExternalProgramTask +from recon.config import tool_paths from recon.targets import TargetList @@ -77,7 +78,7 @@ class AmassScan(ExternalProgramTask): return f"touch {self.output().path}".split() command = [ - "amass", + f"{tool_paths.get('amass')}", "enum", "-active", "-ip", @@ -117,11 +118,7 @@ class ParseAmassOutput(luigi.Task): luigi.ExternalTask - TargetList """ - args = { - "target_file": self.target_file, - "exempt_list": self.exempt_list, - "results_dir": self.results_dir, - } + args = {"target_file": self.target_file, "exempt_list": self.exempt_list, "results_dir": self.results_dir} return AmassScan(**args) def output(self): diff --git a/recon/config.py b/recon/config.py index fe040f5..0e49a09 100644 --- a/recon/config.py +++ b/recon/config.py @@ -1,30 +1,35 @@ # flake8: noqa E231 from pathlib import Path + defaults = { "proxy": "", "threads": "10", "masscan-rate": "1000", "masscan-iface": "tun0", + "tools-dir": f"{Path.home()}/.recon-tools", "results-dir": "recon-results", "aquatone-scan-timeout": "900", "gobuster-extensions": "", - "gobuster-wordlist": "/usr/share/seclists/Discovery/Web-Content/common.txt", } +defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt" + web_ports = {"80", "443", "8080", "8000", "8443"} tool_paths = { - "aquatone": "/usr/local/bin/aquatone", + "aquatone": f"{defaults.get('tools-dir')}/aquatone", "tko-subs": f"{Path.home()}/go/bin/tko-subs", "tko-subs-dir": f"{Path.home()}/go/src/github.com/anshumanbh/tko-subs", "subjack": f"{Path.home()}/go/bin/subjack", "subjack-fingerprints": f"{Path.home()}/go/src/github.com/haccer/subjack/fingerprints.json", - "CORScanner": "/opt/CORScanner/cors_scan.py", + "CORScanner": f"{defaults.get('tools-dir')}/CORScanner/cors_scan.py", "gobuster": f"{Path.home()}/go/bin/gobuster", - "recursive-gobuster": "/opt/recursive-gobuster/recursive-gobuster.pyz", + "recursive-gobuster": f"{defaults.get('tools-dir')}/recursive-gobuster/recursive-gobuster.pyz", "webanalyze": f"{Path.home()}/go/bin/webanalyze", - "masscan": "/usr/local/bin/masscan", + "masscan": f"{defaults.get('tools-dir')}/masscan", + "amass": f"{defaults.get('tools-dir')}/amass", + "go": "/usr/local/go/bin/go" } top_tcp_ports = [ diff --git a/recon/masscan.py b/recon/masscan.py index a431912..7f1c431 100644 --- a/recon/masscan.py +++ b/recon/masscan.py @@ -105,7 +105,7 @@ class MasscanScan(luigi.Task): if target_list.path.endswith("domains"): yield ParseAmassOutput( - target_file=self.target_file, exempt_list=self.exempt_list, results_dir=self.results_dir, + target_file=self.target_file, exempt_list=self.exempt_list, results_dir=self.results_dir ) command = [ diff --git a/recon/web/gobuster.py b/recon/web/gobuster.py index 9c819ea..77d5006 100644 --- a/recon/web/gobuster.py +++ b/recon/web/gobuster.py @@ -114,12 +114,7 @@ class GobusterScan(luigi.Task): for url_scheme in ("https://", "http://"): if self.recursive: - command = [ - tool_paths.get("recursive-gobuster"), - "-w", - self.wordlist, - f"{url_scheme}{target}", - ] + command = [tool_paths.get("recursive-gobuster"), "-w", self.wordlist, f"{url_scheme}{target}"] else: command = [ tool_paths.get("gobuster"), diff --git a/recon/web/targets.py b/recon/web/targets.py index 99b5dd3..013b930 100644 --- a/recon/web/targets.py +++ b/recon/web/targets.py @@ -43,7 +43,7 @@ class GatherWebTargets(luigi.Task): return { "masscan-output": ParseMasscanOutput(**args), "amass-output": ParseAmassOutput( - exempt_list=self.exempt_list, target_file=self.target_file, results_dir=self.results_dir, + exempt_list=self.exempt_list, target_file=self.target_file, results_dir=self.results_dir ), } diff --git a/recon/web/webanalyze.py b/recon/web/webanalyze.py index 59f2ab0..4d29684 100644 --- a/recon/web/webanalyze.py +++ b/recon/web/webanalyze.py @@ -110,11 +110,7 @@ class WebanalyzeScan(luigi.Task): pass for url_scheme in ("https://", "http://"): - command = [ - tool_paths.get("webanalyze"), - "-host", - f"{url_scheme}{target}", - ] + command = [tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}"] commands.append(command) Path(self.output().path).mkdir(parents=True, exist_ok=True) diff --git a/recon/wrappers.py b/recon/wrappers.py index 14e72a9..6297124 100644 --- a/recon/wrappers.py +++ b/recon/wrappers.py @@ -9,9 +9,7 @@ from recon.web.gobuster import GobusterScan from recon.web.webanalyze import WebanalyzeScan -@inherits( - SearchsploitScan, AquatoneScan, TKOSubsScan, SubjackScan, CORScannerScan, GobusterScan, WebanalyzeScan, -) +@inherits(SearchsploitScan, AquatoneScan, TKOSubsScan, SubjackScan, CORScannerScan, GobusterScan, WebanalyzeScan) class FullScan(luigi.WrapperTask): """ Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time. diff --git a/tests/test_install/test_install_command.py b/tests/test_install/test_install_command.py index 61a467e..b7a5db6 100644 --- a/tests/test_install/test_install_command.py +++ b/tests/test_install/test_install_command.py @@ -3,12 +3,22 @@ import importlib import subprocess from pathlib import Path -from ..utils import setup_install_test, run_cmd, is_kali -from recon.config import tool_paths +from recon.config import tool_paths, defaults +from ..utils import setup_install_test, run_cmd recon_pipeline = importlib.import_module("recon-pipeline") +def test_install_go(): + go = Path(tool_paths.get("go")) + + rs = recon_pipeline.ReconShell() + + run_cmd(rs, "install go") + + assert go.exists() + + def test_install_masscan(): masscan = Path(tool_paths.get("masscan")) @@ -16,51 +26,37 @@ def test_install_masscan(): rs = recon_pipeline.ReconShell() + assert Path(defaults.get("tools-dir")).exists() + run_cmd(rs, "install masscan") - assert masscan.exists() is True + assert masscan.exists() -def test_install_amass(): - setup_install_test() - - if not is_kali(): - return True - - if shutil.which("amass") is not None: - subprocess.run("sudo apt remove amass -y".split()) - - rs = recon_pipeline.ReconShell() - - run_cmd(rs, "install amass") - - assert shutil.which("amass") is not None - - -def test_install_pipenv(): - setup_install_test() - - if not is_kali(): - return True - - if shutil.which("pipenv") is not None: - subprocess.run("sudo apt remove pipenv -y".split()) - - rs = recon_pipeline.ReconShell() - - run_cmd(rs, "install pipenv") - - assert shutil.which("pipenv") is not None +# def test_install_amass(): +# amass = Path(tool_paths.get("amass")) +# +# setup_install_test(amass) +# +# rs = recon_pipeline.ReconShell() +# +# assert Path(defaults.get("tools-dir")).exists() +# +# run_cmd(rs, "install amass") +# +# assert amass.exists() def test_install_luigi(): setup_install_test() if shutil.which("luigi") is not None: - subprocess.run("pipenv uninstall luigi".split()) + subprocess.run("pip uninstall luigi".split()) rs = recon_pipeline.ReconShell() + assert Path(defaults.get("tools-dir")).exists() + run_cmd(rs, "install luigi") assert shutil.which("luigi") is not None @@ -73,65 +69,75 @@ def test_install_aquatone(): rs = recon_pipeline.ReconShell() + assert Path(defaults.get("tools-dir")).exists() + run_cmd(rs, "install aquatone") - assert aquatone.exists() is True + assert aquatone.exists() -def test_install_gobuster(): - gobuster = Path(tool_paths.get("gobuster")) - - setup_install_test(gobuster) - - assert shutil.which("go") is not None - - rs = recon_pipeline.ReconShell() - - run_cmd(rs, "install gobuster") - - assert gobuster.exists() is True +# def test_install_gobuster(): +# gobuster = Path(tool_paths.get("gobuster")) +# +# setup_install_test(gobuster) +# +# assert Path(tool_paths.get("go")).exists() +# +# rs = recon_pipeline.ReconShell() +# +# assert Path(defaults.get("tools-dir")).exists() +# +# run_cmd(rs, "install gobuster") +# +# assert gobuster.exists() +# +# +# def test_install_tkosubs(): +# tkosubs = Path(tool_paths.get("tko-subs")) +# +# setup_install_test(tkosubs) +# +# assert Path(tool_paths.get("go")).exists() +# +# rs = recon_pipeline.ReconShell() +# +# assert Path(defaults.get("tools-dir")).exists() +# +# run_cmd(rs, "install tko-subs") +# +# assert tkosubs.exists() -def test_install_tkosubs(): - tkosubs = Path(tool_paths.get("tko-subs")) - - setup_install_test(tkosubs) - - assert shutil.which("go") is not None - - rs = recon_pipeline.ReconShell() - - run_cmd(rs, "install tko-subs") - - assert tkosubs.exists() is True +# def test_install_subjack(): +# subjack = Path(tool_paths.get("subjack")) +# +# setup_install_test(subjack) +# +# assert Path(tool_paths.get("go")).exists() +# +# rs = recon_pipeline.ReconShell() +# +# assert Path(defaults.get("tools-dir")).exists() +# +# run_cmd(rs, "install subjack") +# +# assert subjack.exists() -def test_install_subjack(): - subjack = Path(tool_paths.get("subjack")) - - setup_install_test(subjack) - - assert shutil.which("go") is not None - - rs = recon_pipeline.ReconShell() - - run_cmd(rs, "install subjack") - - assert subjack.exists() is True - - -def test_install_webanalyze(): - webanalyze = Path(tool_paths.get("webanalyze")) - - setup_install_test(webanalyze) - - assert shutil.which("go") is not None - - rs = recon_pipeline.ReconShell() - - run_cmd(rs, "install webanalyze") - - assert webanalyze.exists() is True +# def test_install_webanalyze(): +# webanalyze = Path(tool_paths.get("webanalyze")) +# +# setup_install_test(webanalyze) +# +# assert Path(tool_paths.get("go")).exists() +# +# rs = recon_pipeline.ReconShell() +# +# assert Path(defaults.get("tools-dir")).exists() +# +# run_cmd(rs, "install webanalyze") +# +# assert webanalyze.exists() def test_install_corscanner(): @@ -144,9 +150,11 @@ def test_install_corscanner(): rs = recon_pipeline.ReconShell() + assert Path(defaults.get("tools-dir")).exists() + run_cmd(rs, "install corscanner") - assert corscanner.exists() is True + assert corscanner.exists() def test_update_corscanner(): @@ -155,13 +163,15 @@ def test_update_corscanner(): setup_install_test() if not corscanner.parent.exists(): - subprocess.run(f"sudo git clone https://github.com/chenjj/CORScanner.git {corscanner.parent}".split()) + subprocess.run(f"git clone https://github.com/chenjj/CORScanner.git {corscanner.parent}".split()) rs = recon_pipeline.ReconShell() + assert Path(defaults.get("tools-dir")).exists() + run_cmd(rs, "install corscanner") - assert corscanner.exists() is True + assert corscanner.exists() def test_install_recursive_gobuster(): @@ -174,9 +184,11 @@ def test_install_recursive_gobuster(): rs = recon_pipeline.ReconShell() + assert Path(defaults.get("tools-dir")).exists() + run_cmd(rs, "install recursive-gobuster") - assert recursive_gobuster.exists() is True + assert recursive_gobuster.exists() def test_update_recursive_gobuster(): @@ -186,14 +198,16 @@ def test_update_recursive_gobuster(): if not recursive_gobuster.parent.exists(): subprocess.run( - f"sudo git clone https://github.com/epi052/recursive-gobuster.git {recursive_gobuster.parent}".split() + f"git clone https://github.com/epi052/recursive-gobuster.git {recursive_gobuster.parent}".split() ) rs = recon_pipeline.ReconShell() + assert Path(defaults.get("tools-dir")).exists() + run_cmd(rs, "install recursive-gobuster") - assert recursive_gobuster.exists() is True + assert recursive_gobuster.exists() def test_install_luigi_service(): @@ -201,18 +215,18 @@ def test_install_luigi_service(): setup_install_test(luigi_service) - proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) + proc = subprocess.run("sudo systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) if proc.stdout.decode().strip() == "enabled": - subprocess.run("systemctl disable luigid.service".split()) + subprocess.run("sudo systemctl disable luigid.service".split()) - proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) + proc = subprocess.run("sudo systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) if proc.stdout.decode().strip() == "active": - subprocess.run("systemctl stop luigid.service".split()) + subprocess.run("sudo systemctl stop luigid.service".split()) if Path("/usr/local/bin/luigid").exists(): - Path("/usr/local/bin/luigid").unlink() + subprocess.run("sudo rm /usr/local/bin/luigid".split()) rs = recon_pipeline.ReconShell() @@ -220,10 +234,10 @@ def test_install_luigi_service(): assert Path("/lib/systemd/system/luigid.service").exists() - proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) + proc = subprocess.run("sudo systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) assert proc.stdout.decode().strip() == "enabled" - proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) + proc = subprocess.run("sudo systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) assert proc.stdout.decode().strip() == "active" assert Path("/usr/local/bin/luigid").exists() diff --git a/tests/test_recon/test_amass.py b/tests/test_recon/test_amass.py index 02e7be9..58dfc95 100644 --- a/tests/test_recon/test_amass.py +++ b/tests/test_recon/test_amass.py @@ -27,12 +27,7 @@ ips = [ "104.20.61.51", "104.20.60.51", ] -ip6s = [ - "2606:4700:10::6814:3c33", - "2606:4700:10::6814:3d33", - "2606:4700:10::6814:3d33", - "2606:4700:10::6814:3c33", -] +ip6s = ["2606:4700:10::6814:3c33", "2606:4700:10::6814:3d33", "2606:4700:10::6814:3d33", "2606:4700:10::6814:3c33"] subdomains = [ "blog.bitdiscovery.com", "bitdiscovery.com", diff --git a/tests/test_recon/test_config.py b/tests/test_recon/test_config.py index e81cbd2..f17dd23 100644 --- a/tests/test_recon/test_config.py +++ b/tests/test_recon/test_config.py @@ -1,5 +1,4 @@ from pathlib import Path -from ..utils import is_kali from recon.config import tool_paths, defaults, web_ports, top_tcp_ports, top_udp_ports @@ -9,11 +8,6 @@ def test_tool_paths_absolute(): assert Path(path).is_absolute() -def test_wordlist_exists(): - if is_kali(): - assert Path(defaults.get("gobuster-wordlist")).exists() - - def test_threads_numeric(): assert defaults.get("threads").isnumeric() diff --git a/tests/test_recon/test_targets.py b/tests/test_recon/test_targets.py index bcb5966..2cbd484 100644 --- a/tests/test_recon/test_targets.py +++ b/tests/test_recon/test_targets.py @@ -36,7 +36,7 @@ def test_results_dir_relative(tmp_path): targetfile = tmp_path / "test_targetlist" targetfile.write_text("stuff.com") - tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / ".." / tmp_path / "recon-results")),) + tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / ".." / tmp_path / "recon-results"))) out = tl.output() assert out.path == str((tmp_path / "recon-results" / "target-results" / "domains").resolve()) @@ -46,7 +46,7 @@ def test_results_dir_absolute(tmp_path): targetfile = tmp_path / "test_targetlist" targetfile.write_text("stuff.com") - tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / "recon-results").resolve()),) + tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / "recon-results").resolve())) out = tl.output() assert out.path == str((tmp_path / "recon-results" / "target-results" / "domains").resolve()) diff --git a/tests/utils.py b/tests/utils.py index 25e0507..92fc788 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -61,5 +61,5 @@ def setup_install_test(tool=None): if tool is not None: try: tool.unlink() - except FileNotFoundError: + except (FileNotFoundError, PermissionError): pass