aljaz/recon-pipeline
1
0
Fork 0
mirror of https://github.com/aljazceru/recon-pipeline.git synced 2025-12-24 01:34:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Modifies reference structure for tool paths (#61)

Browse Source
This commit is contained in:
Ryan Good
2020-05-11 21:07:15 -04:00
committed by GitHub
parent 1448cd037f
commit c8bb606ecc
31 changed files with 115 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pipeline/recon/__init__.py
View File

@@ -4,7 +4,7 @@ from .wrappers import FullScan, HTBScan
from .amass import AmassScan, ParseAmassOutput
from .masscan import MasscanScan, ParseMasscanOutput
from .nmap import ThreadedNmapScan, SearchsploitScan
from .config import tool_paths, top_udp_ports, top_tcp_ports, defaults, web_ports
from .config import top_udp_ports, top_tcp_ports, defaults, web_ports
from .parsers import (
install_parser,
scan_parser,

4
pipeline/recon/amass.py
View File

@@ -7,8 +7,8 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .config import tool_paths
from .targets import TargetList
from ..tools import tools
from ..models.target_model import Target
@@ -94,7 +94,7 @@ class AmassScan(luigi.Task):
return subprocess.run(f"touch {self.output().path}".split())
command = [
f"{tool_paths.get('amass')}",
tools.get("amass").get("path"),
"enum",
"-active",
"-ip",

20
pipeline/recon/config.py
View File

@@ -16,25 +16,7 @@ defaults = {
defaults["tools-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/tools"
defaults["database-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/databases"
defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt"
tool_paths = {
"aquatone": f"{defaults.get('tools-dir')}/aquatone",
"tko-subs": f"{Path.home()}/go/bin/tko-subs",
"tko-subs-dir": f"{Path.home()}/go/src/github.com/anshumanbh/tko-subs",
"subjack": f"{Path.home()}/go/bin/subjack",
"subjack-fingerprints": f"{Path.home()}/go/src/github.com/haccer/subjack/fingerprints.json",
"gobuster": f"{Path.home()}/go/bin/gobuster",
"recursive-gobuster": f"{defaults.get('tools-dir')}/recursive-gobuster/recursive-gobuster.pyz",
"webanalyze": f"{Path.home()}/go/bin/webanalyze",
"masscan": f"{defaults.get('tools-dir')}/masscan",
"amass": f"{defaults.get('tools-dir')}/amass",
"go": "/usr/local/go/bin/go",
"searchsploit": f"{defaults.get('tools-dir')}/exploitdb/searchsploit",
"luigid": str(Path(__file__).parents[2] / "luigid.service"),
"seclists": f"{defaults.get('tools-dir')}/seclists",
"exploitdb": f"{defaults.get('tools-dir')}/exploitdb",
"waybackurls": f"{Path.home()}/go/bin/waybackurls",
}
defaults["project-dir"] = str(Path(__file__).parents[2])
web_ports = {
"80",

5
pipeline/recon/masscan.py
View File

@@ -9,11 +9,12 @@ from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .targets import TargetList
from ..tools import tools
from .amass import ParseAmassOutput
from ..models.port_model import Port
from ..models.ip_address_model import IPAddress
from .config import top_tcp_ports, top_udp_ports, defaults, tool_paths, web_ports
from .config import top_tcp_ports, top_udp_ports, defaults, web_ports
@inherits(TargetList, ParseAmassOutput)
@@ -108,7 +109,7 @@ class MasscanScan(luigi.Task):
)
command = [
tool_paths.get("masscan"),
tools.get("masscan").get("path"),
"-v",
"--open",
"--banners",

5
pipeline/recon/nmap.py
View File

@@ -12,9 +12,10 @@ from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .masscan import ParseMasscanOutput
from .config import defaults, tool_paths
from .config import defaults
from .helpers import get_ip_address_version, is_ip_address
from ..tools import tools
from ..models.port_model import Port
from ..models.nse_model import NSEResult
from ..models.target_model import Target
@@ -281,7 +282,7 @@ class SearchsploitScan(luigi.Task):
""" Grabs the xml files created by ThreadedNmap and runs searchsploit --nmap on each one, saving the output. """
for entry in Path(self.input().get("localtarget").path).glob("nmap*.xml"):
proc = subprocess.run(
[tool_paths.get("searchsploit"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE
[tools.get("searchsploit").get("path"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE
)
if proc.stdout:
# change wall-searchsploit-results/nmap.10.10.10.157-tcp to 10.10.10.157

5
pipeline/recon/web/aquatone.py
View File

@@ -9,7 +9,8 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
from ...tools import tools
import pipeline.models.db_manager
from ...models.port_model import Port
@@ -250,7 +251,7 @@ class AquatoneScan(luigi.Task):
self.results_subfolder.mkdir(parents=True, exist_ok=True)
command = [
tool_paths.get("aquatone"),
tools.get("aquatone").get("path"),
"-scan-timeout",
self.scan_timeout,
"-threads",

13
pipeline/recon/web/gobuster.py
View File

@@ -11,7 +11,8 @@ from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
from ...tools import tools
from ...models.endpoint_model import Endpoint
from ..helpers import get_ip_address_version, is_ip_address
@@ -139,10 +140,16 @@ class GobusterScan(luigi.Task):
for url_scheme in ("https://", "http://"):
if self.recursive:
command = [tool_paths.get("recursive-gobuster"), "-s", "-w", self.wordlist, f"{url_scheme}{target}"]
command = [
tools.get("recursive-gobuster").get("path"),
"-s",
"-w",
self.wordlist,
f"{url_scheme}{target}",
]
else:
command = [
tool_paths.get("gobuster"),
tools.get("gobuster").get("path"),
"dir",
"-q",
"-e",

11
pipeline/recon/web/subdomain_takeover.py
View File

@@ -8,8 +8,9 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from ...tools import tools
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
@inherits(GatherWebTargets)
@@ -120,9 +121,9 @@ class TKOSubsScan(luigi.Task):
return
command = [
tool_paths.get("tko-subs"),
tools.get("tko-subs").get("path"),
f"-domain={','.join(domains)}",
f"-data={tool_paths.get('tko-subs-dir')}/providers-data.csv",
f"-data={tools.get('tko-subs').get('git_dir')}/providers-data.csv",
f"-output={self.output_file}",
]
@@ -261,7 +262,7 @@ class SubjackScan(luigi.Task):
f.write(f"{hostname}\n")
command = [
tool_paths.get("subjack"),
tools.get("subjack").get("path"),
"-w",
str(subjack_input_file),
"-t",
@@ -274,7 +275,7 @@ class SubjackScan(luigi.Task):
"-v",
"-ssl",
"-c",
tool_paths.get("subjack-fingerprints"),
tools.get("subjack").get("fingerprints"),
]
subprocess.run(command)

7
pipeline/recon/web/webanalyze.py
View File

@@ -11,8 +11,9 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from ...tools import tools
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
from ...models.technology_model import Technology
from ..helpers import get_ip_address_version, is_ip_address
@@ -153,7 +154,7 @@ class WebanalyzeScan(luigi.Task):
target = f"[{target}]"
for url_scheme in ("https://", "http://"):
command = [tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}", "-output", "csv"]
command = [tools.get("webanalyze").get("path"), "-host", f"{url_scheme}{target}", "-output", "csv"]
commands.append(command)
self.results_subfolder.mkdir(parents=True, exist_ok=True)
@@ -162,7 +163,7 @@ class WebanalyzeScan(luigi.Task):
os.chdir(self.results_subfolder)
if not Path("apps.json").exists():
subprocess.run(f"{tool_paths.get('webanalyze')} -update".split())
subprocess.run(f"{tools.get('webanalyze').get('path')} -update".split())
with ThreadPoolExecutor(max_workers=self.threads) as executor:
executor.map(self._wrapped_subprocess, commands)