mirror of
https://github.com/aljazceru/recon-pipeline.git
synced 2025-12-19 07:14:23 +01:00
Co-authored-by: Ryan Good <usafaryangood@gmail.com> * added initial skeleton; restructured project directories * removed workers directive from luigi; changed input to tko-subs * changed masscan command to use config.tool_paths * linted __init__ files and updated docstring for get_scans * added per-file-ignores for linting * recon-pipeline linted * PoC working for amass results -> db; rudimentary db mgmt commands also * more linting * added database management commands to the shell * db_location passes through to all tasks; masscan results added to db * removed unused imports from masscan.py * added ParseNmapOutput class to handle parsing for database storage * cleaned up repeat code * searchsploit results stored in db * lint/format * gobuster scans now stored in database * fixed test_recon tests to use db_location * fixed web tests * tkosub entries recorded in db * subjack scan results stored in database * webanalyze results stored in db * refactored older commits to use newer helper functions * refactored older commits to use newer helper functions * aquatone results stored in database refactored a few scans to use dbmanager helper functions refactored db structure wrt headers/screenshots added 80/443 to web_ports in config.py * fixed a few queries and re-added webanalyze to FullScan * view targets/endpoints done * overhauled nmap parsing * print all nmap_results good, next to focus on filtering * complex nmap filters complete * nmap printing done * updated pipfile * view web-technologies complete * view searchsploit results complete * removed filesystem code from amass * targetlist moved to db only * targets,amass,masscan all cutover to full database; added view ports * nmap fully db compliant * aquatone and webtargets db compliant * gobuster uses db now * webanalyze db compliant * all scans except corscanner are db compliant * recon tests passing * web tests passing * linted files * added tests for helpers.py and parsers.py * refactored some redundant code * added tests to pre-commit * updated amass tests and pre-commit version * updated recon.targets tests * updated nmap tests * updated masscan tests * updated config tests * updated web targets tests * added gobuster tests * added aquatone tests * added subdomain takeover and webanalyze tests; updated test data * removed homegrown sqlite target in favor of the sqla implementation * added tests for recon-pipeline.py * fixed cluge function to set __package__ globally * updated amass tests * updated targets tests * updated nmap tests * updated masscan tests * updated aquatone tests * updated nmap tests to account for no searchsploit * updated nmap tests to account for no searchsploit * updated masscan tests * updated subjack/tkosub tests * updated web targets tests * updated webanalyze tests * added corscanner tests * linted DBManager a bit * fixed weird cyclic import issue that only happened during docs build; housekeeping * added models tests, removed test_install dir * updated docs a bit; sidenav is wonky * fixed readthedocs requirements.txt * fixed issue where view results werent populated directly after scan * added new tests to pipeline; working on docs * updated a few overlooked view command items * updated tests to reflect changes to shell * incremental push of docs update * documentation done * updated exploitdb install * updated exploitdb install * updated seclists install * parseamass updates db in the event of no amass output * removed corscanner * added pipenv shell to install instructions per @GreaterGoodest * added pipenv shell to install instructions per @GreaterGoodest * added check for chromium-browser during aquatone install; closes #26 * added check for old recon-tools dir; updated Path.resolve calls to Path.expanduser.resolve; fixed very specific import bug due to filesystem location * added CONTIBUTING.md; updated pre-commit hooks/README * added .gitattributes for linguist reporting * updated tests * fixed a few weird bugs found during test * updated README * updated asciinema links in README * updated README with view command video * updated other location for url scheme /status * add ability to specify single target using --target (#31) * updated a few items in docs and moved tool-dict to tools-dir * fixed issue where removing tempfile without --verbose caused scan to fail
126 lines
5.8 KiB
Python
126 lines
5.8 KiB
Python
from pathlib import Path
|
|
|
|
from .config import tool_paths, defaults
|
|
|
|
# tool definitions for recon-pipeline's auto-installer
|
|
tools = {
|
|
"luigi-service": {
|
|
"installed": False,
|
|
"dependencies": None,
|
|
"commands": [
|
|
f"sudo cp {str(Path(__file__).parents[2] / 'luigid.service')} /lib/systemd/system/luigid.service",
|
|
f"sudo cp $(which luigid) /usr/local/bin",
|
|
"sudo systemctl daemon-reload",
|
|
"sudo systemctl start luigid.service",
|
|
"sudo systemctl enable luigid.service",
|
|
],
|
|
"shell": True,
|
|
},
|
|
"seclists": {
|
|
"installed": False,
|
|
"dependencies": None,
|
|
"shell": True,
|
|
"commands": [
|
|
f"bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists {defaults.get('tools-dir')}/seclists; elif [[ -d {defaults.get('tools-dir')}/seclists ]] ; then cd {defaults.get('tools-dir')}/seclists && git fetch --all && git pull; else git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists; fi'"
|
|
],
|
|
},
|
|
"searchsploit": {
|
|
"installed": False,
|
|
"dependencies": None,
|
|
"shell": True,
|
|
"commands": [
|
|
f"bash -c 'if [[ -d /usr/share/exploitdb ]]; then ln -s /usr/share/exploitdb {defaults.get('tools-dir')}/exploitdb && sudo ln -s $(which searchsploit) {defaults.get('tools-dir')}/exploitdb/searchsploit; elif [[ -d {Path(tool_paths.get('searchsploit')).parent} ]]; then cd {Path(tool_paths.get('searchsploit')).parent} && git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git {defaults.get('tools-dir')}/exploitdb; fi'",
|
|
f"bash -c 'if [[ -f {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc ]]; then cp -n {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc {Path.home().expanduser().resolve()}; fi'",
|
|
f"bash -c 'if [[ -f {Path.home().resolve()}/.searchsploit_rc ]]; then sed -i 's#/opt#{defaults.get('tools-dir')}#g' {Path.home().resolve()}/.searchsploit_rc; fi'",
|
|
],
|
|
},
|
|
"masscan": {
|
|
"installed": False,
|
|
"dependencies": None,
|
|
"commands": [
|
|
"git clone https://github.com/robertdavidgraham/masscan /tmp/masscan",
|
|
"make -s -j -C /tmp/masscan",
|
|
f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}",
|
|
"rm -rf /tmp/masscan",
|
|
f"sudo setcap CAP_NET_RAW+ep {tool_paths.get('masscan')}",
|
|
],
|
|
},
|
|
"amass": {
|
|
"installed": False,
|
|
"dependencies": ["go"],
|
|
"commands": [
|
|
f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...",
|
|
f"cp ~/go/bin/amass {tool_paths.get('amass')}",
|
|
],
|
|
"shell": True,
|
|
"environ": {"GO111MODULE": "on"},
|
|
},
|
|
"aquatone": {
|
|
"installed": False,
|
|
"dependencies": None,
|
|
"shell": True,
|
|
"commands": [
|
|
"mkdir /tmp/aquatone",
|
|
"wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip",
|
|
"bash -c 'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'",
|
|
"unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone",
|
|
f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}",
|
|
"rm -rf /tmp/aquatone",
|
|
"bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'",
|
|
],
|
|
},
|
|
"gobuster": {
|
|
"installed": False,
|
|
"dependencies": ["go", "seclists"],
|
|
"commands": [
|
|
f"{tool_paths.get('go')} get github.com/OJ/gobuster",
|
|
f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
|
|
],
|
|
"shell": True,
|
|
},
|
|
"tko-subs": {
|
|
"installed": False,
|
|
"dependencies": ["go"],
|
|
"commands": [
|
|
f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs",
|
|
f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
|
|
],
|
|
"shell": True,
|
|
},
|
|
"subjack": {
|
|
"installed": False,
|
|
"dependencies": ["go"],
|
|
"commands": [
|
|
f"{tool_paths.get('go')} get github.com/haccer/subjack",
|
|
f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)",
|
|
],
|
|
"shell": True,
|
|
},
|
|
"webanalyze": {
|
|
"installed": False,
|
|
"dependencies": ["go"],
|
|
"commands": [
|
|
f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...",
|
|
f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
|
|
],
|
|
"shell": True,
|
|
},
|
|
"recursive-gobuster": {
|
|
"installed": False,
|
|
"dependencies": ["gobuster", "seclists"],
|
|
"shell": True,
|
|
"commands": [
|
|
f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'"
|
|
],
|
|
},
|
|
"go": {
|
|
"installed": False,
|
|
"dependencies": None,
|
|
"commands": [
|
|
"wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz",
|
|
"sudo tar -C /usr/local -xvf /tmp/go.tar.gz",
|
|
f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'',
|
|
],
|
|
},
|
|
}
|