mirror of
https://github.com/aljazceru/recon-pipeline.git
synced 2025-12-24 01:34:26 +01:00
Completed store results in a database project (#32)
Co-authored-by: Ryan Good <usafaryangood@gmail.com> * added initial skeleton; restructured project directories * removed workers directive from luigi; changed input to tko-subs * changed masscan command to use config.tool_paths * linted __init__ files and updated docstring for get_scans * added per-file-ignores for linting * recon-pipeline linted * PoC working for amass results -> db; rudimentary db mgmt commands also * more linting * added database management commands to the shell * db_location passes through to all tasks; masscan results added to db * removed unused imports from masscan.py * added ParseNmapOutput class to handle parsing for database storage * cleaned up repeat code * searchsploit results stored in db * lint/format * gobuster scans now stored in database * fixed test_recon tests to use db_location * fixed web tests * tkosub entries recorded in db * subjack scan results stored in database * webanalyze results stored in db * refactored older commits to use newer helper functions * refactored older commits to use newer helper functions * aquatone results stored in database refactored a few scans to use dbmanager helper functions refactored db structure wrt headers/screenshots added 80/443 to web_ports in config.py * fixed a few queries and re-added webanalyze to FullScan * view targets/endpoints done * overhauled nmap parsing * print all nmap_results good, next to focus on filtering * complex nmap filters complete * nmap printing done * updated pipfile * view web-technologies complete * view searchsploit results complete * removed filesystem code from amass * targetlist moved to db only * targets,amass,masscan all cutover to full database; added view ports * nmap fully db compliant * aquatone and webtargets db compliant * gobuster uses db now * webanalyze db compliant * all scans except corscanner are db compliant * recon tests passing * web tests passing * linted files * added tests for helpers.py and parsers.py * refactored some redundant code * added tests to pre-commit * updated amass tests and pre-commit version * updated recon.targets tests * updated nmap tests * updated masscan tests * updated config tests * updated web targets tests * added gobuster tests * added aquatone tests * added subdomain takeover and webanalyze tests; updated test data * removed homegrown sqlite target in favor of the sqla implementation * added tests for recon-pipeline.py * fixed cluge function to set __package__ globally * updated amass tests * updated targets tests * updated nmap tests * updated masscan tests * updated aquatone tests * updated nmap tests to account for no searchsploit * updated nmap tests to account for no searchsploit * updated masscan tests * updated subjack/tkosub tests * updated web targets tests * updated webanalyze tests * added corscanner tests * linted DBManager a bit * fixed weird cyclic import issue that only happened during docs build; housekeeping * added models tests, removed test_install dir * updated docs a bit; sidenav is wonky * fixed readthedocs requirements.txt * fixed issue where view results werent populated directly after scan * added new tests to pipeline; working on docs * updated a few overlooked view command items * updated tests to reflect changes to shell * incremental push of docs update * documentation done * updated exploitdb install * updated exploitdb install * updated seclists install * parseamass updates db in the event of no amass output * removed corscanner * added pipenv shell to install instructions per @GreaterGoodest * added pipenv shell to install instructions per @GreaterGoodest * added check for chromium-browser during aquatone install; closes #26 * added check for old recon-tools dir; updated Path.resolve calls to Path.expanduser.resolve; fixed very specific import bug due to filesystem location * added CONTIBUTING.md; updated pre-commit hooks/README * added .gitattributes for linguist reporting * updated tests * fixed a few weird bugs found during test * updated README * updated asciinema links in README * updated README with view command video * updated other location for url scheme /status * add ability to specify single target using --target (#31) * updated a few items in docs and moved tool-dict to tools-dir * fixed issue where removing tempfile without --verbose caused scan to fail
This commit is contained in:
epi052
127
pipeline/recon/wrappers.py
Normal file
127
pipeline/recon/wrappers.py
Normal file
@@ -0,0 +1,127 @@
|
||||
import luigi
|
||||
from luigi.util import inherits
|
||||
|
||||
from .nmap import SearchsploitScan
|
||||
from .web import AquatoneScan
|
||||
from .web import GobusterScan
|
||||
from .web import WebanalyzeScan
|
||||
from .web import TKOSubsScan, SubjackScan
|
||||
|
||||
|
||||
@inherits(SearchsploitScan, AquatoneScan, TKOSubsScan, SubjackScan, GobusterScan, WebanalyzeScan)
|
||||
class FullScan(luigi.WrapperTask):
|
||||
""" Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time.
|
||||
|
||||
Note:
|
||||
Because FullScan is a wrapper, it requires all Parameters for any of the Scans that it wraps.
|
||||
|
||||
Args:
|
||||
threads: number of threads for parallel gobuster command execution
|
||||
wordlist: wordlist used for forced browsing
|
||||
extensions: additional extensions to apply to each item in the wordlist
|
||||
recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly)
|
||||
proxy: protocol://ip:port proxy specification for gobuster
|
||||
exempt_list: Path to a file providing blacklisted subdomains, one per line.
|
||||
top_ports: Scan top N most popular ports
|
||||
ports: specifies the port(s) to be scanned
|
||||
interface: use the named raw network interface, such as "eth0"
|
||||
rate: desired rate for transmitting packets (packets per second)
|
||||
target_file: specifies the file on disk containing a list of ips or domains
|
||||
results_dir: specifes the directory on disk to which all Task results are written
|
||||
"""
|
||||
|
||||
def requires(self):
|
||||
""" FullScan is a wrapper, as such it requires any Tasks that it wraps. """
|
||||
args = {
|
||||
"results_dir": self.results_dir,
|
||||
"rate": self.rate,
|
||||
"target_file": self.target_file,
|
||||
"top_ports": self.top_ports,
|
||||
"interface": self.interface,
|
||||
"ports": self.ports,
|
||||
"exempt_list": self.exempt_list,
|
||||
"threads": self.threads,
|
||||
"proxy": self.proxy,
|
||||
"wordlist": self.wordlist,
|
||||
"extensions": self.extensions,
|
||||
"recursive": self.recursive,
|
||||
"db_location": self.db_location,
|
||||
}
|
||||
|
||||
yield GobusterScan(**args)
|
||||
|
||||
# remove options that are gobuster specific; if left dictionary unpacking to other scans throws an exception
|
||||
for gobuster_opt in ("proxy", "wordlist", "extensions", "recursive"):
|
||||
del args[gobuster_opt]
|
||||
|
||||
# add aquatone scan specific option
|
||||
args.update({"scan_timeout": self.scan_timeout})
|
||||
|
||||
yield AquatoneScan(**args)
|
||||
|
||||
del args["scan_timeout"]
|
||||
|
||||
yield SubjackScan(**args)
|
||||
yield SearchsploitScan(**args)
|
||||
yield WebanalyzeScan(**args)
|
||||
|
||||
del args["threads"]
|
||||
|
||||
yield TKOSubsScan(**args)
|
||||
|
||||
|
||||
@inherits(SearchsploitScan, AquatoneScan, GobusterScan, WebanalyzeScan)
|
||||
class HTBScan(luigi.WrapperTask):
|
||||
""" Wraps multiple scan types in order to run tasks on the same hierarchical level at the same time.
|
||||
|
||||
Note:
|
||||
Because HTBScan is a wrapper, it requires all Parameters for any of the Scans that it wraps.
|
||||
|
||||
Args:
|
||||
threads: number of threads for parallel gobuster command execution
|
||||
wordlist: wordlist used for forced browsing
|
||||
extensions: additional extensions to apply to each item in the wordlist
|
||||
recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly)
|
||||
proxy: protocol://ip:port proxy specification for gobuster
|
||||
exempt_list: Path to a file providing blacklisted subdomains, one per line.
|
||||
top_ports: Scan top N most popular ports
|
||||
ports: specifies the port(s) to be scanned
|
||||
interface: use the named raw network interface, such as "eth0"
|
||||
rate: desired rate for transmitting packets (packets per second)
|
||||
target_file: specifies the file on disk containing a list of ips or domains
|
||||
results_dir: specifes the directory on disk to which all Task results are written
|
||||
"""
|
||||
|
||||
def requires(self):
|
||||
""" HTBScan is a wrapper, as such it requires any Tasks that it wraps. """
|
||||
args = {
|
||||
"results_dir": self.results_dir,
|
||||
"rate": self.rate,
|
||||
"target_file": self.target_file,
|
||||
"top_ports": self.top_ports,
|
||||
"interface": self.interface,
|
||||
"ports": self.ports,
|
||||
"exempt_list": self.exempt_list,
|
||||
"threads": self.threads,
|
||||
"proxy": self.proxy,
|
||||
"db_location": self.db_location,
|
||||
"wordlist": self.wordlist,
|
||||
"extensions": self.extensions,
|
||||
"recursive": self.recursive,
|
||||
}
|
||||
|
||||
yield GobusterScan(**args)
|
||||
|
||||
# remove options that are gobuster specific; if left dictionary unpacking to other scans throws an exception
|
||||
for gobuster_opt in ("proxy", "wordlist", "extensions", "recursive"):
|
||||
del args[gobuster_opt]
|
||||
|
||||
# add aquatone scan specific option
|
||||
args.update({"scan_timeout": self.scan_timeout})
|
||||
|
||||
yield AquatoneScan(**args)
|
||||
|
||||
del args["scan_timeout"]
|
||||
|
||||
yield SearchsploitScan(**args)
|
||||
yield WebanalyzeScan(**args)
|
||||
Reference in New Issue
Block a user