Files
recon-pipeline/tests/test_web/test_subdomain_takeover.py
epi052 6eb3bd8cb0 Completed store results in a database project (#32)
Co-authored-by: Ryan Good <usafaryangood@gmail.com>

* added initial skeleton; restructured project directories

* removed workers directive from luigi; changed input to tko-subs

* changed masscan command to use config.tool_paths

* linted __init__ files and updated docstring for get_scans

* added per-file-ignores for linting

* recon-pipeline linted

* PoC working for amass results -> db; rudimentary db mgmt commands also

* more linting

* added database management commands to the shell

* db_location passes through to all tasks; masscan results added to db

* removed unused imports from masscan.py

* added ParseNmapOutput class to handle parsing for database storage

* cleaned up repeat code

* searchsploit results stored in db

* lint/format

* gobuster scans now stored in database

* fixed test_recon tests to use db_location

* fixed web tests

* tkosub entries recorded in db

* subjack scan results stored in database

* webanalyze results stored in db

* refactored older commits to use newer helper functions

* refactored older commits to use newer helper functions

* aquatone results stored in database

refactored a few scans to use dbmanager helper functions
refactored db structure wrt headers/screenshots
added 80/443 to web_ports in config.py

* fixed a few queries and re-added webanalyze to FullScan

* view targets/endpoints done

* overhauled nmap parsing

* print all nmap_results good, next to focus on filtering

* complex nmap filters complete

* nmap printing done

* updated pipfile

* view web-technologies complete

* view searchsploit results complete

* removed filesystem code from amass

* targetlist moved to db only

* targets,amass,masscan all cutover to full database; added view ports

* nmap fully db compliant

* aquatone and webtargets db compliant

* gobuster uses db now

* webanalyze db compliant

* all scans except corscanner are db compliant

* recon tests passing

* web tests passing

* linted files

* added tests for helpers.py and parsers.py

* refactored some redundant code

* added tests to pre-commit

* updated amass tests and pre-commit version

* updated recon.targets tests

* updated nmap tests

* updated masscan tests

* updated config tests

* updated web targets tests

* added gobuster tests

* added aquatone tests

* added subdomain takeover and webanalyze tests; updated test data

* removed homegrown sqlite target in favor of the sqla implementation

* added tests for recon-pipeline.py

* fixed cluge function to set __package__ globally

* updated amass tests

* updated targets tests

* updated nmap tests

* updated masscan tests

* updated aquatone tests

* updated nmap tests to account for no searchsploit

* updated nmap tests to account for no searchsploit

* updated masscan tests

* updated subjack/tkosub tests

* updated web targets tests

* updated webanalyze tests

* added corscanner tests

* linted DBManager a bit

* fixed weird cyclic import issue that only happened during docs build; housekeeping

* added models tests, removed test_install dir

* updated docs a bit; sidenav is wonky

* fixed readthedocs requirements.txt

* fixed issue where view results werent populated directly after scan

* added new tests to pipeline; working on docs

* updated a few overlooked view command items

* updated tests to reflect changes to shell

* incremental push of docs update

* documentation done

* updated exploitdb install

* updated exploitdb install

* updated seclists install

* parseamass updates db in the event of no amass output

* removed corscanner

* added pipenv shell to install instructions per @GreaterGoodest

* added pipenv shell to install instructions per @GreaterGoodest

* added check for chromium-browser during aquatone install; closes #26

* added check for old recon-tools dir; updated Path.resolve calls to Path.expanduser.resolve; fixed very specific import bug due to filesystem location

* added CONTIBUTING.md; updated pre-commit hooks/README

* added .gitattributes for linguist reporting

* updated tests

* fixed a few weird bugs found during test

* updated README

* updated asciinema links in README

* updated README with view command video

* updated other location for url scheme /status

* add ability to specify single target using --target (#31)

* updated a few items in docs and moved tool-dict to tools-dir

* fixed issue where removing tempfile without --verbose caused scan to fail
2020-04-17 10:29:16 -05:00

147 lines
5.7 KiB
Python

import shutil
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from pipeline.recon.web import SubjackScan, TKOSubsScan, GatherWebTargets
subjack_results = Path(__file__).parent.parent / "data" / "recon-results" / "subjack-results"
tkosubs_results = Path(__file__).parent.parent / "data" / "recon-results" / "tkosubs-results"
class TestTKOSubsScanScan:
def setup_method(self):
self.tmp_path = Path(tempfile.mkdtemp())
self.scan = TKOSubsScan(
target_file=__file__, results_dir=str(self.tmp_path), db_location=str(self.tmp_path / "testing.sqlite")
)
def teardown_method(self):
shutil.rmtree(self.tmp_path)
def test_scan_requires(self):
with patch("pipeline.recon.web.GatherWebTargets"):
retval = self.scan.requires()
assert isinstance(retval, GatherWebTargets)
def test_scan_creates_results_dir(self):
assert self.scan.results_subfolder == self.tmp_path / "tkosubs-results"
def test_scan_creates_results_file(self):
assert self.scan.output_file == self.tmp_path / "tkosubs-results" / "tkosubs.csv"
def test_scan_creates_database(self):
assert self.scan.db_mgr.location.exists()
assert self.tmp_path / "testing.sqlite" == self.scan.db_mgr.location
def test_scan_creates_results(self):
self.scan.results_subfolder = tkosubs_results
self.scan.output_file = self.scan.results_subfolder / "tkosubs.csv"
self.scan.parse_results()
assert self.scan.output().exists()
def test_parse_results(self):
myresults = self.tmp_path / "tkosubs-results" / "tkosubs.csv"
myresults.parent.mkdir(parents=True, exist_ok=True)
content = "Domain,Cname,Provider,IsVulnerable,IsTakenOver,Response\n"
content += "google.com,Cname,Provider,true,IsTakenOver,Response\n"
content += "maps.google.com,Cname,Provider,false,IsTakenOver,Response\n"
myresults.write_text(content)
self.scan.output_file = myresults
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname = MagicMock()
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.return_value = MagicMock()
self.scan.db_mgr.add = MagicMock()
self.scan.parse_results()
assert self.scan.output().exists()
assert self.scan.db_mgr.add.called
assert self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.called
@pytest.mark.parametrize("test_input", [["google.com"], None])
def test_scan_run(self, test_input):
with patch("subprocess.run") as mocked_run:
self.scan.parse_results = MagicMock()
self.scan.db_mgr.get_all_hostnames = MagicMock()
self.scan.db_mgr.get_all_hostnames.return_value = test_input
self.scan.run()
if test_input is None:
assert not mocked_run.called
assert not self.scan.parse_results.called
else:
assert mocked_run.called
assert self.scan.parse_results.called
class TestSubjackScan:
def setup_method(self):
self.tmp_path = Path(tempfile.mkdtemp())
self.scan = SubjackScan(
target_file=__file__, results_dir=str(self.tmp_path), db_location=str(self.tmp_path / "testing.sqlite")
)
def teardown_method(self):
shutil.rmtree(self.tmp_path)
def test_scan_requires(self):
with patch("pipeline.recon.web.GatherWebTargets"):
retval = self.scan.requires()
assert isinstance(retval, GatherWebTargets)
def test_scan_creates_results_dir(self):
assert self.scan.results_subfolder == self.tmp_path / "subjack-results"
def test_scan_creates_results_file(self):
assert self.scan.output_file == self.tmp_path / "subjack-results" / "subjack.txt"
def test_scan_creates_database(self):
assert self.scan.db_mgr.location.exists()
assert self.tmp_path / "testing.sqlite" == self.scan.db_mgr.location
def test_scan_creates_results(self):
self.scan.results_subfolder = subjack_results
self.scan.output_file = self.scan.results_subfolder / "subjack.txt"
self.scan.parse_results()
assert self.scan.output().exists()
def test_parse_results(self):
myresults = self.tmp_path / "subjack-results" / "subjack.txt"
myresults.parent.mkdir(parents=True, exist_ok=True)
content = "[Not Vulnerable] email.assetinventory.bugcrowd.com\n"
content += "[Vulnerable] email.assetinventory.bugcrowd.com\n"
content += "[Vulnerable] assetinventory.bugcrowd.com:8080\n"
content += "weird input\n"
myresults.write_text(content)
self.scan.output_file = myresults
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname = MagicMock()
self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.return_value = MagicMock()
self.scan.db_mgr.add = MagicMock()
self.scan.parse_results()
assert self.scan.output().exists()
assert self.scan.db_mgr.add.called
assert self.scan.db_mgr.get_or_create_target_by_ip_or_hostname.called
@pytest.mark.parametrize("test_input", [["google.com"], None])
def test_scan_run(self, test_input):
with patch("subprocess.run") as mocked_run:
self.scan.parse_results = MagicMock()
self.scan.db_mgr.get_all_hostnames = MagicMock()
self.scan.db_mgr.get_all_hostnames.return_value = test_input
self.scan.run()
if test_input is None:
assert not mocked_run.called
assert not self.scan.parse_results.called
else:
assert mocked_run.called
assert self.scan.parse_results.called