mirror of
https://github.com/aljazceru/recon-pipeline.git
synced 2025-12-23 01:04:20 +01:00
Co-authored-by: Ryan Good <usafaryangood@gmail.com> * added initial skeleton; restructured project directories * removed workers directive from luigi; changed input to tko-subs * changed masscan command to use config.tool_paths * linted __init__ files and updated docstring for get_scans * added per-file-ignores for linting * recon-pipeline linted * PoC working for amass results -> db; rudimentary db mgmt commands also * more linting * added database management commands to the shell * db_location passes through to all tasks; masscan results added to db * removed unused imports from masscan.py * added ParseNmapOutput class to handle parsing for database storage * cleaned up repeat code * searchsploit results stored in db * lint/format * gobuster scans now stored in database * fixed test_recon tests to use db_location * fixed web tests * tkosub entries recorded in db * subjack scan results stored in database * webanalyze results stored in db * refactored older commits to use newer helper functions * refactored older commits to use newer helper functions * aquatone results stored in database refactored a few scans to use dbmanager helper functions refactored db structure wrt headers/screenshots added 80/443 to web_ports in config.py * fixed a few queries and re-added webanalyze to FullScan * view targets/endpoints done * overhauled nmap parsing * print all nmap_results good, next to focus on filtering * complex nmap filters complete * nmap printing done * updated pipfile * view web-technologies complete * view searchsploit results complete * removed filesystem code from amass * targetlist moved to db only * targets,amass,masscan all cutover to full database; added view ports * nmap fully db compliant * aquatone and webtargets db compliant * gobuster uses db now * webanalyze db compliant * all scans except corscanner are db compliant * recon tests passing * web tests passing * linted files * added tests for helpers.py and parsers.py * refactored some redundant code * added tests to pre-commit * updated amass tests and pre-commit version * updated recon.targets tests * updated nmap tests * updated masscan tests * updated config tests * updated web targets tests * added gobuster tests * added aquatone tests * added subdomain takeover and webanalyze tests; updated test data * removed homegrown sqlite target in favor of the sqla implementation * added tests for recon-pipeline.py * fixed cluge function to set __package__ globally * updated amass tests * updated targets tests * updated nmap tests * updated masscan tests * updated aquatone tests * updated nmap tests to account for no searchsploit * updated nmap tests to account for no searchsploit * updated masscan tests * updated subjack/tkosub tests * updated web targets tests * updated webanalyze tests * added corscanner tests * linted DBManager a bit * fixed weird cyclic import issue that only happened during docs build; housekeeping * added models tests, removed test_install dir * updated docs a bit; sidenav is wonky * fixed readthedocs requirements.txt * fixed issue where view results werent populated directly after scan * added new tests to pipeline; working on docs * updated a few overlooked view command items * updated tests to reflect changes to shell * incremental push of docs update * documentation done * updated exploitdb install * updated exploitdb install * updated seclists install * parseamass updates db in the event of no amass output * removed corscanner * added pipenv shell to install instructions per @GreaterGoodest * added pipenv shell to install instructions per @GreaterGoodest * added check for chromium-browser during aquatone install; closes #26 * added check for old recon-tools dir; updated Path.resolve calls to Path.expanduser.resolve; fixed very specific import bug due to filesystem location * added CONTIBUTING.md; updated pre-commit hooks/README * added .gitattributes for linguist reporting * updated tests * fixed a few weird bugs found during test * updated README * updated asciinema links in README * updated README with view command video * updated other location for url scheme /status * add ability to specify single target using --target (#31) * updated a few items in docs and moved tool-dict to tools-dir * fixed issue where removing tempfile without --verbose caused scan to fail
147 lines
5.7 KiB
Python
147 lines
5.7 KiB
Python
import shutil
|
|
import tempfile
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from pipeline.recon.web import SubjackScan, TKOSubsScan, GatherWebTargets
|
|
|
|
subjack_results = Path(__file__).parent.parent / "data" / "recon-results" / "subjack-results"
|
|
tkosubs_results = Path(__file__).parent.parent / "data" / "recon-results" / "tkosubs-results"
|
|
|
|
|
|
class TestTKOSubsScanScan:
|
|
def setup_method(self):
|
|
self.tmp_path = Path(tempfile.mkdtemp())
|
|
self.scan = TKOSubsScan(
|
|
target_file=__file__, results_dir=str(self.tmp_path), db_location=str(self.tmp_path / "testing.sqlite")
|
|
)
|
|
|
|
def teardown_method(self):
|
|
shutil.rmtree(self.tmp_path)
|
|
|
|
def test_scan_requires(self):
|
|
with patch("pipeline.recon.web.GatherWebTargets"):
|
|
retval = self.scan.requires()
|
|
assert isinstance(retval, GatherWebTargets)
|
|
|
|
def test_scan_creates_results_dir(self):
|
|
assert self.scan.results_subfolder == self.tmp_path / "tkosubs-results"
|
|
|
|
def test_scan_creates_results_file(self):
|
|
assert self.scan.output_file == self.tmp_path / "tkosubs-results" / "tkosubs.csv"
|
|
|
|
def test_scan_creates_database(self):
|
|
assert self.scan.db_mgr.location.exists()
|
|
assert self.tmp_path / "testing.sqlite" == self.scan.db_mgr.location
|
|
|
|
def test_scan_creates_results(self):
|
|
self.scan.results_subfolder = tkosubs_results
|
|
self.scan.output_file = self.scan.results_subfolder / "tkosubs.csv"
|
|
self.scan.parse_results()
|
|
assert self.scan.output().exists()
|
|
|
|
def test_parse_results(self):
|
|
myresults = self.tmp_path / "tkosubs-results" / "tkosubs.csv"
|
|
myresults.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
content = "Domain,Cname,Provider,IsVulnerable,IsTakenOver,Response\n"
|
|
content += "google.com,Cname,Provider,true,IsTakenOver,Response\n"
|
|
content += "maps.google.com,Cname,Provider,false,IsTakenOver,Response\n"
|
|
|
|
myresults.write_text(content)
|
|
|
|
self.scan.output_file = myresults
|
|
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname = MagicMock()
|
|
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.return_value = MagicMock()
|
|
self.scan.db_mgr.add = MagicMock()
|
|
|
|
self.scan.parse_results()
|
|
assert self.scan.output().exists()
|
|
assert self.scan.db_mgr.add.called
|
|
assert self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.called
|
|
|
|
@pytest.mark.parametrize("test_input", [["google.com"], None])
|
|
def test_scan_run(self, test_input):
|
|
with patch("subprocess.run") as mocked_run:
|
|
self.scan.parse_results = MagicMock()
|
|
self.scan.db_mgr.get_all_hostnames = MagicMock()
|
|
self.scan.db_mgr.get_all_hostnames.return_value = test_input
|
|
|
|
self.scan.run()
|
|
if test_input is None:
|
|
assert not mocked_run.called
|
|
assert not self.scan.parse_results.called
|
|
else:
|
|
assert mocked_run.called
|
|
assert self.scan.parse_results.called
|
|
|
|
|
|
class TestSubjackScan:
|
|
def setup_method(self):
|
|
self.tmp_path = Path(tempfile.mkdtemp())
|
|
self.scan = SubjackScan(
|
|
target_file=__file__, results_dir=str(self.tmp_path), db_location=str(self.tmp_path / "testing.sqlite")
|
|
)
|
|
|
|
def teardown_method(self):
|
|
shutil.rmtree(self.tmp_path)
|
|
|
|
def test_scan_requires(self):
|
|
with patch("pipeline.recon.web.GatherWebTargets"):
|
|
retval = self.scan.requires()
|
|
assert isinstance(retval, GatherWebTargets)
|
|
|
|
def test_scan_creates_results_dir(self):
|
|
assert self.scan.results_subfolder == self.tmp_path / "subjack-results"
|
|
|
|
def test_scan_creates_results_file(self):
|
|
assert self.scan.output_file == self.tmp_path / "subjack-results" / "subjack.txt"
|
|
|
|
def test_scan_creates_database(self):
|
|
assert self.scan.db_mgr.location.exists()
|
|
assert self.tmp_path / "testing.sqlite" == self.scan.db_mgr.location
|
|
|
|
def test_scan_creates_results(self):
|
|
self.scan.results_subfolder = subjack_results
|
|
self.scan.output_file = self.scan.results_subfolder / "subjack.txt"
|
|
self.scan.parse_results()
|
|
assert self.scan.output().exists()
|
|
|
|
def test_parse_results(self):
|
|
myresults = self.tmp_path / "subjack-results" / "subjack.txt"
|
|
myresults.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
content = "[Not Vulnerable] email.assetinventory.bugcrowd.com\n"
|
|
content += "[Vulnerable] email.assetinventory.bugcrowd.com\n"
|
|
content += "[Vulnerable] assetinventory.bugcrowd.com:8080\n"
|
|
content += "weird input\n"
|
|
|
|
myresults.write_text(content)
|
|
|
|
self.scan.output_file = myresults
|
|
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname = MagicMock()
|
|
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.return_value = MagicMock()
|
|
self.scan.db_mgr.add = MagicMock()
|
|
|
|
self.scan.parse_results()
|
|
assert self.scan.output().exists()
|
|
assert self.scan.db_mgr.add.called
|
|
assert self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.called
|
|
|
|
@pytest.mark.parametrize("test_input", [["google.com"], None])
|
|
def test_scan_run(self, test_input):
|
|
with patch("subprocess.run") as mocked_run:
|
|
self.scan.parse_results = MagicMock()
|
|
self.scan.db_mgr.get_all_hostnames = MagicMock()
|
|
self.scan.db_mgr.get_all_hostnames.return_value = test_input
|
|
|
|
self.scan.run()
|
|
if test_input is None:
|
|
assert not mocked_run.called
|
|
assert not self.scan.parse_results.called
|
|
else:
|
|
assert mocked_run.called
|
|
assert self.scan.parse_results.called
|