aljaz/recon-pipeline
1
0
Fork 0
mirror of https://github.com/aljazceru/recon-pipeline.git synced 2025-12-20 07:44:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove OS assumption (#15)

Browse Source 
* removed kali specific installs - test 1

* removed kali specific installs - test 2

* removed kali specific installs - test 2

* removed kali specific installs - test 4

* removed kali specific installs - test 5

* removed kali specific installs - test 6

* removed kali specific installs - test 7

* removed kali specific installs - test 8

* removed kali specific installs - test 9

* removed kali specific installs - test 10

* removed kali specific installs - test 11

* removed kali specific installs - test 12

* removed kali specific installs - test 13

* removed kali specific installs - test 14

* all works locally, fixing up tests et al

* trying pipeline again; round 2

* looks good locally; testing pipeline again

* test 18?

* test 19

* 20

* 21
This commit is contained in:
epi052
2020-02-07 22:54:15 -06:00
committed by GitHub
parent 25da9574e3
commit b0534ceb27
19 changed files with 211 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/pythonapp.yml vendored
View File

@@ -47,7 +47,7 @@ jobs:
pipenv install -d pipenv install -d
- name: Test with pytest - name: Test with pytest
run: | run: |
pipenv install pytest pipenv install pytest cmd2 luigi
pipenv run python -m pytest tests/test_install pipenv run python -m pytest tests/test_install
test-recon: test-recon:
@@ -67,7 +67,7 @@ jobs:
pipenv install -d pipenv install -d
- name: Test with pytest - name: Test with pytest
run: | run: |
pipenv install pytest pipenv install pytest cmd2 luigi
pipenv run python -m pytest tests/test_recon pipenv run python -m pytest tests/test_recon
test-web: test-web:
@@ -87,5 +87,5 @@ jobs:
pipenv install -d pipenv install -d
- name: Test with pytest - name: Test with pytest
run: | run: |
pipenv install pytest pipenv install pytest cmd2 luigi
pipenv run python -m pytest tests/test_web pipenv run python -m pytest tests/test_web

4
.gitignore vendored
View File

@@ -103,4 +103,6 @@ venv.bak/
# mypy # mypy
.mypy_cache/ .mypy_cache/
.idea .idea
Pipfile
Pipfile.lock

13
Pipfile
View File

@@ -1,13 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
cmd2 = "*"
luigi = "*"
[requires]
python_version = "3.7"

110
Pipfile.lock generated
View File

@@ -1,110 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "f6e3610c10920d9297afe9d974892f5e970a04cfb8c3f8164e874031c6274037"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.7"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"attrs": {
"hashes": [
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
],
"version": "==19.3.0"
},
"cmd2": {
"hashes": [
"sha256:a07b165603e6cdf6730c95007160036f13b83415f9074dcb475e91f897ec324d",
"sha256:bdb4d7a56023c800c4428abf8e198e28d6a566f23fa172208763c30e298be4ee"
],
"index": "pypi",
"version": "==0.9.24"
},
"colorama": {
"hashes": [
"sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
"sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"
],
"version": "==0.4.3"
},
"docutils": {
"hashes": [
"sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
"sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
],
"version": "==0.16"
},
"lockfile": {
"hashes": [
"sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799",
"sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa"
],
"version": "==0.12.2"
},
"luigi": {
"hashes": [
"sha256:c2b3dcecc565fe77920553434ed475fa21f562d4b76da6bd1a179a8b732fcc9e"
],
"index": "pypi",
"version": "==2.8.11"
},
"pyperclip": {
"hashes": [
"sha256:979325468ccf682104d5dcaf753f869868100631301d3e72f47babdea5700d1c"
],
"version": "==1.7.0"
},
"python-daemon": {
"hashes": [
"sha256:57c84f50a04d7825515e4dbf3a31c70cc44414394a71608dee6cfde469e81766",
"sha256:a0d5dc0b435a02c7e0b401e177a7c17c3f4c7b4e22e2d06271122c8fec5f8946"
],
"version": "==2.2.4"
},
"python-dateutil": {
"hashes": [
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
],
"version": "==2.8.1"
},
"six": {
"hashes": [
"sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
"sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
],
"version": "==1.14.0"
},
"tornado": {
"hashes": [
"sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d",
"sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409",
"sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f",
"sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f",
"sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5",
"sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb",
"sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444"
],
"version": "==5.1.1"
},
"wcwidth": {
"hashes": [
"sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603",
"sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"
],
"version": "==0.1.8"
}
},
"develop": {}
}

7
docs/conf.py
View File

@@ -38,12 +38,7 @@ master_doc = "index"
# Add any Sphinx extension module names here, as strings. They can be # Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones. # ones.
extensions = [ extensions = ["sphinx.ext.autodoc", "sphinx.ext.coverage", "sphinx.ext.napoleon", "sphinxarg.ext"]
"sphinx.ext.autodoc",
"sphinx.ext.coverage",
"sphinx.ext.napoleon",
"sphinxarg.ext",
]
# Add any paths that contain templates here, relative to this directory. # Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"] templates_path = ["_templates"]

23
recon-pipeline.py
View File

@@ -16,12 +16,16 @@ os.environ["PYTHONPATH"] = f"{os.environ.get('PYTHONPATH')}:{str(Path(__file__).
# suppress "You should consider upgrading via the 'pip install --upgrade pip' command." warning # suppress "You should consider upgrading via the 'pip install --upgrade pip' command." warning
os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
# in case we need pipenv, add its default --user installed directory to the PATH
sys.path.append(str(Path.home() / ".local" / "bin"))
# third party imports # third party imports
import cmd2 # noqa: E402 import cmd2 # noqa: E402
from cmd2.ansi import style # noqa: E402 from cmd2.ansi import style # noqa: E402
# project's module imports # project's module imports
from recon import get_scans, tools, scan_parser, install_parser, status_parser # noqa: F401,E402 from recon import get_scans, tools, scan_parser, install_parser, status_parser # noqa: F401,E402
from recon.config import defaults # noqa: F401,E402
# select loop, handles async stdout/stderr processing of subprocesses # select loop, handles async stdout/stderr processing of subprocesses
selector = selectors.DefaultSelector() selector = selectors.DefaultSelector()
@@ -58,6 +62,9 @@ class ReconShell(cmd2.Cmd):
self.sentry = False self.sentry = False
self.prompt = "recon-pipeline> " self.prompt = "recon-pipeline> "
self.selectorloop = None self.selectorloop = None
self.continue_install = True
Path(defaults.get("tools-dir")).mkdir(parents=True, exist_ok=True)
# register hooks to handle selector loop start and cleanup # register hooks to handle selector loop start and cleanup
self.register_preloop_hook(self._preloop_hook) self.register_preloop_hook(self._preloop_hook)
@@ -215,7 +222,7 @@ class ReconShell(cmd2.Cmd):
continue continue
self.async_alert( self.async_alert(
style(f"[!] {args.tool} has an unmet dependency; installing {dependency}", fg="yellow", bold=True,) style(f"[!] {args.tool} has an unmet dependency; installing {dependency}", fg="yellow", bold=True)
) )
# install the dependency before continuing with installation # install the dependency before continuing with installation
@@ -224,13 +231,17 @@ class ReconShell(cmd2.Cmd):
if tools.get(args.tool).get("installed"): if tools.get(args.tool).get("installed"):
return self.async_alert(style(f"[!] {args.tool} is already installed.", fg="yellow")) return self.async_alert(style(f"[!] {args.tool} is already installed.", fg="yellow"))
else: else:
# list of return values from commands run during each tool installation # list of return values from commands run during each tool installation
# used to determine whether the tool installed correctly or not # used to determine whether the tool installed correctly or not
retvals = list() retvals = list()
self.async_alert(style(f"[*] Installing {args.tool}...", fg="bright_yellow")) self.async_alert(style(f"[*] Installing {args.tool}...", fg="bright_yellow"))
addl_env_vars = tools.get(args.tool).get("environ")
if addl_env_vars is not None:
addl_env_vars.update(dict(os.environ))
for command in tools.get(args.tool).get("commands"): for command in tools.get(args.tool).get("commands"):
# run all commands required to install the tool # run all commands required to install the tool
@@ -240,11 +251,15 @@ class ReconShell(cmd2.Cmd):
if tools.get(args.tool).get("shell"): if tools.get(args.tool).get("shell"):
# go tools use subshells (cmd1 && cmd2 && cmd3 ...) during install, so need shell=True # go tools use subshells (cmd1 && cmd2 && cmd3 ...) during install, so need shell=True
proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,) proc = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=addl_env_vars
)
else: else:
# "normal" command, split up the string as usual and run it # "normal" command, split up the string as usual and run it
proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE,) proc = subprocess.Popen(
shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=addl_env_vars
)
out, err = proc.communicate() out, err = proc.communicate()

80
recon/__init__.py
View File

@@ -10,7 +10,7 @@ from collections import defaultdict
import cmd2 import cmd2
import recon import recon
from recon.config import tool_paths from recon.config import tool_paths, defaults
# tool definitions for recon-pipeline's auto-installer # tool definitions for recon-pipeline's auto-installer
tools = { tools = {
@@ -26,19 +26,32 @@ tools = {
], ],
"shell": True, "shell": True,
}, },
"luigi": {"installed": False, "dependencies": ["pipenv"], "commands": ["pipenv install luigi"],}, "luigi": {"installed": False, "dependencies": None, "commands": ["pip install luigi"]},
"pipenv": {"installed": False, "dependencies": None, "commands": ["sudo apt-get install -y -q pipenv"],}, "seclists": {
"installed": False,
"dependencies": None,
"commands": [f"git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists"],
},
"masscan": { "masscan": {
"installed": False, "installed": False,
"dependencies": None, "dependencies": None,
"commands": [ "commands": [
"git clone https://github.com/robertdavidgraham/masscan /tmp/masscan", "git clone https://github.com/robertdavidgraham/masscan /tmp/masscan",
"make -s -j -C /tmp/masscan", "make -s -j -C /tmp/masscan",
f"sudo mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}", f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}",
"rm -rf /tmp/masscan", "rm -rf /tmp/masscan",
], ],
}, },
"amass": {"installed": False, "dependencies": None, "commands": ["sudo apt-get install -y -q amass"],}, "amass": {
"installed": False,
"dependencies": ["go"],
"commands": [
f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...",
f"cp ~/go/bin/amass {tool_paths.get('amass')}",
],
"shell": True,
"environ": {"GO111MODULE": "on"},
},
"aquatone": { "aquatone": {
"installed": False, "installed": False,
"dependencies": None, "dependencies": None,
@@ -47,7 +60,7 @@ tools = {
"mkdir /tmp/aquatone", "mkdir /tmp/aquatone",
"wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip", "wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip",
"unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone", "unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone",
f"sudo mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}", f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}",
"rm -rf /tmp/aquatone", "rm -rf /tmp/aquatone",
], ],
}, },
@@ -56,17 +69,17 @@ tools = {
"dependencies": None, "dependencies": None,
"shell": True, "shell": True,
"commands": [ "commands": [
f"sudo bash -c 'if [[ -d {Path(tool_paths.get('CORScanner')).parent} ]] ; then cd {Path(tool_paths.get('CORScanner')).parent} && git pull; else git clone https://github.com/chenjj/CORScanner.git {Path(tool_paths.get('CORScanner')).parent}; fi'", f"bash -c 'if [[ -d {Path(tool_paths.get('CORScanner')).parent} ]] ; then cd {Path(tool_paths.get('CORScanner')).parent} && git fetch --all && git pull; else git clone https://github.com/chenjj/CORScanner.git {Path(tool_paths.get('CORScanner')).parent}; fi'",
f"pip install -q -r {Path(tool_paths.get('CORScanner')).parent / 'requirements.txt'}", f"pip install -r {Path(tool_paths.get('CORScanner')).parent / 'requirements.txt'}",
"pip install -q future", "pip install future",
], ],
}, },
"gobuster": { "gobuster": {
"installed": False, "installed": False,
"dependencies": ["go"], "dependencies": ["go", "seclists"],
"commands": [ "commands": [
"go get github.com/OJ/gobuster", f"{tool_paths.get('go')} get github.com/OJ/gobuster",
"(cd ~/go/src/github.com/OJ/gobuster && go build && go install)", f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
], ],
"shell": True, "shell": True,
}, },
@@ -74,35 +87,46 @@ tools = {
"installed": False, "installed": False,
"dependencies": ["go"], "dependencies": ["go"],
"commands": [ "commands": [
"go get github.com/anshumanbh/tko-subs", f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs",
"(cd ~/go/src/github.com/anshumanbh/tko-subs && go build && go install)", f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
], ],
"shell": True, "shell": True,
}, },
"subjack": { "subjack": {
"installed": False, "installed": False,
"dependencies": ["go"], "dependencies": ["go"],
"commands": ["go get github.com/haccer/subjack", "(cd ~/go/src/github.com/haccer/subjack && go install)",], "commands": [
f"{tool_paths.get('go')} get github.com/haccer/subjack",
f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)",
],
"shell": True, "shell": True,
}, },
"webanalyze": { "webanalyze": {
"installed": False, "installed": False,
"dependencies": ["go"], "dependencies": ["go"],
"commands": [ "commands": [
"go get github.com/rverton/webanalyze/...", f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...",
"(cd ~/go/src/github.com/rverton/webanalyze && go build && go install)", f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
], ],
"shell": True, "shell": True,
}, },
"recursive-gobuster": { "recursive-gobuster": {
"installed": False, "installed": False,
"dependencies": None, "dependencies": ["gobuster", "seclists"],
"shell": True, "shell": True,
"commands": [ "commands": [
f"sudo bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'", f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'"
],
},
"go": {
"installed": False,
"dependencies": None,
"commands": [
"wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz",
"sudo tar -C /usr/local -xvf /tmp/go.tar.gz",
f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'',
], ],
}, },
"go": {"installed": False, "dependencies": None, "commands": ["sudo apt-get install -y -q golang"],},
} }
@@ -167,13 +191,13 @@ scan_parser.add_argument(
help="file created by the user that defines the target's scope; list of ips/domains", help="file created by the user that defines the target's scope; list of ips/domains",
) )
scan_parser.add_argument( scan_parser.add_argument(
"--exempt-list", completer_method=cmd2.Cmd.path_complete, help="list of blacklisted ips/domains", "--exempt-list", completer_method=cmd2.Cmd.path_complete, help="list of blacklisted ips/domains"
) )
scan_parser.add_argument( scan_parser.add_argument(
"--results-dir", completer_method=cmd2.Cmd.path_complete, help="directory in which to save scan results", "--results-dir", completer_method=cmd2.Cmd.path_complete, help="directory in which to save scan results"
) )
scan_parser.add_argument( scan_parser.add_argument(
"--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster", "--wordlist", completer_method=cmd2.Cmd.path_complete, help="path to wordlist used by gobuster"
) )
scan_parser.add_argument( scan_parser.add_argument(
"--interface", "--interface",
@@ -183,11 +207,9 @@ scan_parser.add_argument(
scan_parser.add_argument("--recursive", action="store_true", help="whether or not to recursively gobust") scan_parser.add_argument("--recursive", action="store_true", help="whether or not to recursively gobust")
scan_parser.add_argument("--rate", help="rate at which masscan should scan") scan_parser.add_argument("--rate", help="rate at which masscan should scan")
scan_parser.add_argument( scan_parser.add_argument(
"--top-ports", help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)", "--top-ports", help="ports to scan as specified by nmap's list of top-ports (only meaningful to around 5000)"
)
scan_parser.add_argument(
"--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)",
) )
scan_parser.add_argument("--ports", help="port specification for masscan (all ports example: 1-65535,U:1-65535)")
scan_parser.add_argument("--threads", help="number of threads for all of the threaded applications to use") scan_parser.add_argument("--threads", help="number of threads for all of the threaded applications to use")
scan_parser.add_argument("--scan-timeout", help="scan timeout for aquatone") scan_parser.add_argument("--scan-timeout", help="scan timeout for aquatone")
scan_parser.add_argument("--proxy", help="proxy for gobuster if desired (ex. 127.0.0.1:8080)") scan_parser.add_argument("--proxy", help="proxy for gobuster if desired (ex. 127.0.0.1:8080)")
@@ -198,8 +220,8 @@ scan_parser.add_argument(
help="ppen a web browser to Luigi's central scheduler's visualization site (see how the sausage is made!)", help="ppen a web browser to Luigi's central scheduler's visualization site (see how the sausage is made!)",
) )
scan_parser.add_argument( scan_parser.add_argument(
"--local-scheduler", action="store_true", help="use the local scheduler instead of the central scheduler (luigid)", "--local-scheduler", action="store_true", help="use the local scheduler instead of the central scheduler (luigid)"
) )
scan_parser.add_argument( scan_parser.add_argument(
"--verbose", action="store_true", help="shows debug messages from luigi, useful for troubleshooting", "--verbose", action="store_true", help="shows debug messages from luigi, useful for troubleshooting"
) )

9
recon/amass.py
View File

@@ -6,6 +6,7 @@ import luigi
from luigi.util import inherits from luigi.util import inherits
from luigi.contrib.external_program import ExternalProgramTask from luigi.contrib.external_program import ExternalProgramTask
from recon.config import tool_paths
from recon.targets import TargetList from recon.targets import TargetList
@@ -77,7 +78,7 @@ class AmassScan(ExternalProgramTask):
return f"touch {self.output().path}".split() return f"touch {self.output().path}".split()
command = [ command = [
"amass", f"{tool_paths.get('amass')}",
"enum", "enum",
"-active", "-active",
"-ip", "-ip",
@@ -117,11 +118,7 @@ class ParseAmassOutput(luigi.Task):
luigi.ExternalTask - TargetList luigi.ExternalTask - TargetList
""" """
args = { args = {"target_file": self.target_file, "exempt_list": self.exempt_list, "results_dir": self.results_dir}
"target_file": self.target_file,
"exempt_list": self.exempt_list,
"results_dir": self.results_dir,
}
return AmassScan(**args) return AmassScan(**args)
def output(self): def output(self):

15
recon/config.py
View File

@@ -1,30 +1,35 @@
# flake8: noqa E231 # flake8: noqa E231
from pathlib import Path from pathlib import Path
defaults = { defaults = {
"proxy": "", "proxy": "",
"threads": "10", "threads": "10",
"masscan-rate": "1000", "masscan-rate": "1000",
"masscan-iface": "tun0", "masscan-iface": "tun0",
"tools-dir": f"{Path.home()}/.recon-tools",
"results-dir": "recon-results", "results-dir": "recon-results",
"aquatone-scan-timeout": "900", "aquatone-scan-timeout": "900",
"gobuster-extensions": "", "gobuster-extensions": "",
"gobuster-wordlist": "/usr/share/seclists/Discovery/Web-Content/common.txt",
} }
defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt"
web_ports = {"80", "443", "8080", "8000", "8443"} web_ports = {"80", "443", "8080", "8000", "8443"}
tool_paths = { tool_paths = {
"aquatone": "/usr/local/bin/aquatone", "aquatone": f"{defaults.get('tools-dir')}/aquatone",
"tko-subs": f"{Path.home()}/go/bin/tko-subs", "tko-subs": f"{Path.home()}/go/bin/tko-subs",
"tko-subs-dir": f"{Path.home()}/go/src/github.com/anshumanbh/tko-subs", "tko-subs-dir": f"{Path.home()}/go/src/github.com/anshumanbh/tko-subs",
"subjack": f"{Path.home()}/go/bin/subjack", "subjack": f"{Path.home()}/go/bin/subjack",
"subjack-fingerprints": f"{Path.home()}/go/src/github.com/haccer/subjack/fingerprints.json", "subjack-fingerprints": f"{Path.home()}/go/src/github.com/haccer/subjack/fingerprints.json",
"CORScanner": "/opt/CORScanner/cors_scan.py", "CORScanner": f"{defaults.get('tools-dir')}/CORScanner/cors_scan.py",
"gobuster": f"{Path.home()}/go/bin/gobuster", "gobuster": f"{Path.home()}/go/bin/gobuster",
"recursive-gobuster": "/opt/recursive-gobuster/recursive-gobuster.pyz", "recursive-gobuster": f"{defaults.get('tools-dir')}/recursive-gobuster/recursive-gobuster.pyz",
"webanalyze": f"{Path.home()}/go/bin/webanalyze", "webanalyze": f"{Path.home()}/go/bin/webanalyze",
"masscan": "/usr/local/bin/masscan", "masscan": f"{defaults.get('tools-dir')}/masscan",
"amass": f"{defaults.get('tools-dir')}/amass",
"go": "/usr/local/go/bin/go"
} }
top_tcp_ports = [ top_tcp_ports = [

2
recon/masscan.py
View File

@@ -105,7 +105,7 @@ class MasscanScan(luigi.Task):
if target_list.path.endswith("domains"): if target_list.path.endswith("domains"):
yield ParseAmassOutput( yield ParseAmassOutput(
target_file=self.target_file, exempt_list=self.exempt_list, results_dir=self.results_dir, target_file=self.target_file, exempt_list=self.exempt_list, results_dir=self.results_dir
) )
command = [ command = [

7
recon/web/gobuster.py
View File

@@ -114,12 +114,7 @@ class GobusterScan(luigi.Task):
for url_scheme in ("https://", "http://"): for url_scheme in ("https://", "http://"):
if self.recursive: if self.recursive:
command = [ command = [tool_paths.get("recursive-gobuster"), "-w", self.wordlist, f"{url_scheme}{target}"]
tool_paths.get("recursive-gobuster"),
"-w",
self.wordlist,
f"{url_scheme}{target}",
]
else: else:
command = [ command = [
tool_paths.get("gobuster"), tool_paths.get("gobuster"),

2
recon/web/targets.py
View File

@@ -43,7 +43,7 @@ class GatherWebTargets(luigi.Task):
return { return {
"masscan-output": ParseMasscanOutput(**args), "masscan-output": ParseMasscanOutput(**args),
"amass-output": ParseAmassOutput( "amass-output": ParseAmassOutput(
exempt_list=self.exempt_list, target_file=self.target_file, results_dir=self.results_dir, exempt_list=self.exempt_list, target_file=self.target_file, results_dir=self.results_dir
), ),
} }

6
recon/web/webanalyze.py
View File

@@ -110,11 +110,7 @@ class WebanalyzeScan(luigi.Task):
pass pass
for url_scheme in ("https://", "http://"): for url_scheme in ("https://", "http://"):
command = [ command = [tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}"]
tool_paths.get("webanalyze"),
"-host",
f"{url_scheme}{target}",
]
commands.append(command) commands.append(command)
Path(self.output().path).mkdir(parents=True, exist_ok=True) Path(self.output().path).mkdir(parents=True, exist_ok=True)

4
recon/wrappers.py
View File

@@ -9,9 +9,7 @@ from recon.web.gobuster import GobusterScan
from recon.web.webanalyze import WebanalyzeScan from recon.web.webanalyze import WebanalyzeScan
@inherits( @inherits(SearchsploitScan, AquatoneScan, TKOSubsScan, SubjackScan, CORScannerScan, GobusterScan, WebanalyzeScan)
SearchsploitScan, AquatoneScan, TKOSubsScan, SubjackScan, CORScannerScan, GobusterScan, WebanalyzeScan,
)
class FullScan(luigi.WrapperTask): class FullScan(luigi.WrapperTask):
""" Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time. """ Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time.

210
tests/test_install/test_install_command.py
View File

@@ -3,12 +3,22 @@ import importlib
import subprocess import subprocess
from pathlib import Path from pathlib import Path
from ..utils import setup_install_test, run_cmd, is_kali from recon.config import tool_paths, defaults
from recon.config import tool_paths from ..utils import setup_install_test, run_cmd
recon_pipeline = importlib.import_module("recon-pipeline") recon_pipeline = importlib.import_module("recon-pipeline")
def test_install_go():
go = Path(tool_paths.get("go"))
rs = recon_pipeline.ReconShell()
run_cmd(rs, "install go")
assert go.exists()
def test_install_masscan(): def test_install_masscan():
masscan = Path(tool_paths.get("masscan")) masscan = Path(tool_paths.get("masscan"))
@@ -16,51 +26,37 @@ def test_install_masscan():
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install masscan") run_cmd(rs, "install masscan")
assert masscan.exists() is True assert masscan.exists()
def test_install_amass(): # def test_install_amass():
setup_install_test() # amass = Path(tool_paths.get("amass"))
#
if not is_kali(): # setup_install_test(amass)
return True #
# rs = recon_pipeline.ReconShell()
if shutil.which("amass") is not None: #
subprocess.run("sudo apt remove amass -y".split()) # assert Path(defaults.get("tools-dir")).exists()
#
rs = recon_pipeline.ReconShell() # run_cmd(rs, "install amass")
#
run_cmd(rs, "install amass") # assert amass.exists()
assert shutil.which("amass") is not None
def test_install_pipenv():
setup_install_test()
if not is_kali():
return True
if shutil.which("pipenv") is not None:
subprocess.run("sudo apt remove pipenv -y".split())
rs = recon_pipeline.ReconShell()
run_cmd(rs, "install pipenv")
assert shutil.which("pipenv") is not None
def test_install_luigi(): def test_install_luigi():
setup_install_test() setup_install_test()
if shutil.which("luigi") is not None: if shutil.which("luigi") is not None:
subprocess.run("pipenv uninstall luigi".split()) subprocess.run("pip uninstall luigi".split())
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install luigi") run_cmd(rs, "install luigi")
assert shutil.which("luigi") is not None assert shutil.which("luigi") is not None
@@ -73,65 +69,75 @@ def test_install_aquatone():
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install aquatone") run_cmd(rs, "install aquatone")
assert aquatone.exists() is True assert aquatone.exists()
def test_install_gobuster(): # def test_install_gobuster():
gobuster = Path(tool_paths.get("gobuster")) # gobuster = Path(tool_paths.get("gobuster"))
#
setup_install_test(gobuster) # setup_install_test(gobuster)
#
assert shutil.which("go") is not None # assert Path(tool_paths.get("go")).exists()
#
rs = recon_pipeline.ReconShell() # rs = recon_pipeline.ReconShell()
#
run_cmd(rs, "install gobuster") # assert Path(defaults.get("tools-dir")).exists()
#
assert gobuster.exists() is True # run_cmd(rs, "install gobuster")
#
# assert gobuster.exists()
#
#
# def test_install_tkosubs():
# tkosubs = Path(tool_paths.get("tko-subs"))
#
# setup_install_test(tkosubs)
#
# assert Path(tool_paths.get("go")).exists()
#
# rs = recon_pipeline.ReconShell()
#
# assert Path(defaults.get("tools-dir")).exists()
#
# run_cmd(rs, "install tko-subs")
#
# assert tkosubs.exists()
def test_install_tkosubs(): # def test_install_subjack():
tkosubs = Path(tool_paths.get("tko-subs")) # subjack = Path(tool_paths.get("subjack"))
#
setup_install_test(tkosubs) # setup_install_test(subjack)
#
assert shutil.which("go") is not None # assert Path(tool_paths.get("go")).exists()
#
rs = recon_pipeline.ReconShell() # rs = recon_pipeline.ReconShell()
#
run_cmd(rs, "install tko-subs") # assert Path(defaults.get("tools-dir")).exists()
#
assert tkosubs.exists() is True # run_cmd(rs, "install subjack")
#
# assert subjack.exists()
def test_install_subjack(): # def test_install_webanalyze():
subjack = Path(tool_paths.get("subjack")) # webanalyze = Path(tool_paths.get("webanalyze"))
#
setup_install_test(subjack) # setup_install_test(webanalyze)
#
assert shutil.which("go") is not None # assert Path(tool_paths.get("go")).exists()
#
rs = recon_pipeline.ReconShell() # rs = recon_pipeline.ReconShell()
#
run_cmd(rs, "install subjack") # assert Path(defaults.get("tools-dir")).exists()
#
assert subjack.exists() is True # run_cmd(rs, "install webanalyze")
#
# assert webanalyze.exists()
def test_install_webanalyze():
webanalyze = Path(tool_paths.get("webanalyze"))
setup_install_test(webanalyze)
assert shutil.which("go") is not None
rs = recon_pipeline.ReconShell()
run_cmd(rs, "install webanalyze")
assert webanalyze.exists() is True
def test_install_corscanner(): def test_install_corscanner():
@@ -144,9 +150,11 @@ def test_install_corscanner():
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install corscanner") run_cmd(rs, "install corscanner")
assert corscanner.exists() is True assert corscanner.exists()
def test_update_corscanner(): def test_update_corscanner():
@@ -155,13 +163,15 @@ def test_update_corscanner():
setup_install_test() setup_install_test()
if not corscanner.parent.exists(): if not corscanner.parent.exists():
subprocess.run(f"sudo git clone https://github.com/chenjj/CORScanner.git {corscanner.parent}".split()) subprocess.run(f"git clone https://github.com/chenjj/CORScanner.git {corscanner.parent}".split())
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install corscanner") run_cmd(rs, "install corscanner")
assert corscanner.exists() is True assert corscanner.exists()
def test_install_recursive_gobuster(): def test_install_recursive_gobuster():
@@ -174,9 +184,11 @@ def test_install_recursive_gobuster():
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install recursive-gobuster") run_cmd(rs, "install recursive-gobuster")
assert recursive_gobuster.exists() is True assert recursive_gobuster.exists()
def test_update_recursive_gobuster(): def test_update_recursive_gobuster():
@@ -186,14 +198,16 @@ def test_update_recursive_gobuster():
if not recursive_gobuster.parent.exists(): if not recursive_gobuster.parent.exists():
subprocess.run( subprocess.run(
f"sudo git clone https://github.com/epi052/recursive-gobuster.git {recursive_gobuster.parent}".split() f"git clone https://github.com/epi052/recursive-gobuster.git {recursive_gobuster.parent}".split()
) )
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install recursive-gobuster") run_cmd(rs, "install recursive-gobuster")
assert recursive_gobuster.exists() is True assert recursive_gobuster.exists()
def test_install_luigi_service(): def test_install_luigi_service():
@@ -201,18 +215,18 @@ def test_install_luigi_service():
setup_install_test(luigi_service) setup_install_test(luigi_service)
proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) proc = subprocess.run("sudo systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE)
if proc.stdout.decode().strip() == "enabled": if proc.stdout.decode().strip() == "enabled":
subprocess.run("systemctl disable luigid.service".split()) subprocess.run("sudo systemctl disable luigid.service".split())
proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) proc = subprocess.run("sudo systemctl is-active luigid.service".split(), stdout=subprocess.PIPE)
if proc.stdout.decode().strip() == "active": if proc.stdout.decode().strip() == "active":
subprocess.run("systemctl stop luigid.service".split()) subprocess.run("sudo systemctl stop luigid.service".split())
if Path("/usr/local/bin/luigid").exists(): if Path("/usr/local/bin/luigid").exists():
Path("/usr/local/bin/luigid").unlink() subprocess.run("sudo rm /usr/local/bin/luigid".split())
rs = recon_pipeline.ReconShell() rs = recon_pipeline.ReconShell()
@@ -220,10 +234,10 @@ def test_install_luigi_service():
assert Path("/lib/systemd/system/luigid.service").exists() assert Path("/lib/systemd/system/luigid.service").exists()
proc = subprocess.run("systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE) proc = subprocess.run("sudo systemctl is-enabled luigid.service".split(), stdout=subprocess.PIPE)
assert proc.stdout.decode().strip() == "enabled" assert proc.stdout.decode().strip() == "enabled"
proc = subprocess.run("systemctl is-active luigid.service".split(), stdout=subprocess.PIPE) proc = subprocess.run("sudo systemctl is-active luigid.service".split(), stdout=subprocess.PIPE)
assert proc.stdout.decode().strip() == "active" assert proc.stdout.decode().strip() == "active"
assert Path("/usr/local/bin/luigid").exists() assert Path("/usr/local/bin/luigid").exists()

7
tests/test_recon/test_amass.py
View File

@@ -27,12 +27,7 @@ ips = [
"104.20.61.51", "104.20.61.51",
"104.20.60.51", "104.20.60.51",
] ]
ip6s = [ ip6s = ["2606:4700:10::6814:3c33", "2606:4700:10::6814:3d33", "2606:4700:10::6814:3d33", "2606:4700:10::6814:3c33"]
"2606:4700:10::6814:3c33",
"2606:4700:10::6814:3d33",
"2606:4700:10::6814:3d33",
"2606:4700:10::6814:3c33",
]
subdomains = [ subdomains = [
"blog.bitdiscovery.com", "blog.bitdiscovery.com",
"bitdiscovery.com", "bitdiscovery.com",

6
tests/test_recon/test_config.py
View File

@@ -1,5 +1,4 @@
from pathlib import Path from pathlib import Path
from ..utils import is_kali
from recon.config import tool_paths, defaults, web_ports, top_tcp_ports, top_udp_ports from recon.config import tool_paths, defaults, web_ports, top_tcp_ports, top_udp_ports
@@ -9,11 +8,6 @@ def test_tool_paths_absolute():
assert Path(path).is_absolute() assert Path(path).is_absolute()
def test_wordlist_exists():
if is_kali():
assert Path(defaults.get("gobuster-wordlist")).exists()
def test_threads_numeric(): def test_threads_numeric():
assert defaults.get("threads").isnumeric() assert defaults.get("threads").isnumeric()

4
tests/test_recon/test_targets.py
View File

@@ -36,7 +36,7 @@ def test_results_dir_relative(tmp_path):
targetfile = tmp_path / "test_targetlist" targetfile = tmp_path / "test_targetlist"
targetfile.write_text("stuff.com") targetfile.write_text("stuff.com")
tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / ".." / tmp_path / "recon-results")),) tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / ".." / tmp_path / "recon-results")))
out = tl.output() out = tl.output()
assert out.path == str((tmp_path / "recon-results" / "target-results" / "domains").resolve()) assert out.path == str((tmp_path / "recon-results" / "target-results" / "domains").resolve())
@@ -46,7 +46,7 @@ def test_results_dir_absolute(tmp_path):
targetfile = tmp_path / "test_targetlist" targetfile = tmp_path / "test_targetlist"
targetfile.write_text("stuff.com") targetfile.write_text("stuff.com")
tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / "recon-results").resolve()),) tl = TargetList(target_file=str(targetfile), results_dir=str((tmp_path / "recon-results").resolve()))
out = tl.output() out = tl.output()
assert out.path == str((tmp_path / "recon-results" / "target-results" / "domains").resolve()) assert out.path == str((tmp_path / "recon-results" / "target-results" / "domains").resolve())

2
tests/utils.py
View File

@@ -61,5 +61,5 @@ def setup_install_test(tool=None):
if tool is not None: if tool is not None:
try: try:
tool.unlink() tool.unlink()
except FileNotFoundError: except (FileNotFoundError, PermissionError):
pass pass