aljaz/recon-pipeline
1
0
Fork 0
mirror of https://github.com/aljazceru/recon-pipeline.git synced 2025-12-19 23:34:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Modifies reference structure for tool paths (#61)

Browse Source
This commit is contained in:
Ryan Good
2020-05-11 21:07:15 -04:00
committed by GitHub
parent 1448cd037f
commit c8bb606ecc
31 changed files with 115 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@@ -12,9 +12,9 @@ A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1.
2.
3.
1.
2.
3.
**Expected behavior**
A clear and concise description of what you expected to happen.

7
docs/modifications/new_scanner.rst
View File

@@ -53,19 +53,18 @@ Dynamically creating strings and filesystem paths are handled by the following t
In order to get values out of ``pipeline.recon.config.py``, you'll need to use one of the yaml helpers listed below.
- ``!get_parent`` - (niche) get parent directory of a given tool that is defined in the ``pipeline.recon.config.tool_paths`` dictionary
- ``!get_default`` - get a value from the ``pipeline.recon.config.defaults`` dictionary
- ``!get_tool_path`` - get a value from the ``pipeline.recon.config.tool_paths`` dictionary
- ``!get_tool_path`` - get a path value from the ``pipeline.tools.tools`` dictionary
Simple Example Tool Definition
******************************
The example below needs go to be installed prior to being installed itself. It then grabs the path to the ``go`` binary from ``pipeline.recon.config.tool_paths`` by using ``!get_tool_path``. After that, it creates a command using ``!join`` that will look like ``/usr/local/go/bin/go get github.com/tomnomnom/waybackurls``. This command will be run by the ``install waybackurls`` command (or ``install all``).
The example below needs go to be installed prior to being installed itself. It then grabs the path to the ``go`` binary from ``pipeline.tools.tools`` by using ``!get_tool_path``. After that, it creates a command using ``!join`` that will look like ``/usr/local/go/bin/go get github.com/tomnomnom/waybackurls``. This command will be run by the ``install waybackurls`` command (or ``install all``).
.. code-block:: yaml
dependencies: [go]
go: &gobin !get_tool_path "{go}"
go: &gobin !get_tool_path "{go[path]}"
commands:
- !join [*gobin, get github.com/tomnomnom/waybackurls]

2
pipeline/recon-pipeline.py
View File

@@ -341,13 +341,11 @@ class ReconShell(cmd2.Cmd):
self.do_install(tool)
return
if persistent_tool_dict.exists():
tools = pickle.loads(persistent_tool_dict.read_bytes())
if tools.get(args.tool).get("dependencies"):
# get all of the requested tools dependencies
for dependency in tools.get(args.tool).get("dependencies"):
if tools.get(dependency).get("installed"):
# already installed, skip it

2
pipeline/recon/__init__.py
View File

@@ -4,7 +4,7 @@ from .wrappers import FullScan, HTBScan
from .amass import AmassScan, ParseAmassOutput
from .masscan import MasscanScan, ParseMasscanOutput
from .nmap import ThreadedNmapScan, SearchsploitScan
from .config import tool_paths, top_udp_ports, top_tcp_ports, defaults, web_ports
from .config import top_udp_ports, top_tcp_ports, defaults, web_ports
from .parsers import (
install_parser,
scan_parser,

4
pipeline/recon/amass.py
View File

@@ -7,8 +7,8 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .config import tool_paths
from .targets import TargetList
from ..tools import tools
from ..models.target_model import Target
@@ -94,7 +94,7 @@ class AmassScan(luigi.Task):
return subprocess.run(f"touch {self.output().path}".split())
command = [
f"{tool_paths.get('amass')}",
tools.get("amass").get("path"),
"enum",
"-active",
"-ip",

20
pipeline/recon/config.py
View File

@@ -16,25 +16,7 @@ defaults = {
defaults["tools-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/tools"
defaults["database-dir"] = f"{defaults.get('home')}/.local/recon-pipeline/databases"
defaults["gobuster-wordlist"] = f"{defaults.get('tools-dir')}/seclists/Discovery/Web-Content/common.txt"
tool_paths = {
"aquatone": f"{defaults.get('tools-dir')}/aquatone",
"tko-subs": f"{Path.home()}/go/bin/tko-subs",
"tko-subs-dir": f"{Path.home()}/go/src/github.com/anshumanbh/tko-subs",
"subjack": f"{Path.home()}/go/bin/subjack",
"subjack-fingerprints": f"{Path.home()}/go/src/github.com/haccer/subjack/fingerprints.json",
"gobuster": f"{Path.home()}/go/bin/gobuster",
"recursive-gobuster": f"{defaults.get('tools-dir')}/recursive-gobuster/recursive-gobuster.pyz",
"webanalyze": f"{Path.home()}/go/bin/webanalyze",
"masscan": f"{defaults.get('tools-dir')}/masscan",
"amass": f"{defaults.get('tools-dir')}/amass",
"go": "/usr/local/go/bin/go",
"searchsploit": f"{defaults.get('tools-dir')}/exploitdb/searchsploit",
"luigid": str(Path(__file__).parents[2] / "luigid.service"),
"seclists": f"{defaults.get('tools-dir')}/seclists",
"exploitdb": f"{defaults.get('tools-dir')}/exploitdb",
"waybackurls": f"{Path.home()}/go/bin/waybackurls",
}
defaults["project-dir"] = str(Path(__file__).parents[2])
web_ports = {
"80",

5
pipeline/recon/masscan.py
View File

@@ -9,11 +9,12 @@ from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .targets import TargetList
from ..tools import tools
from .amass import ParseAmassOutput
from ..models.port_model import Port
from ..models.ip_address_model import IPAddress
from .config import top_tcp_ports, top_udp_ports, defaults, tool_paths, web_ports
from .config import top_tcp_ports, top_udp_ports, defaults, web_ports
@inherits(TargetList, ParseAmassOutput)
@@ -108,7 +109,7 @@ class MasscanScan(luigi.Task):
)
command = [
tool_paths.get("masscan"),
tools.get("masscan").get("path"),
"-v",
"--open",
"--banners",

5
pipeline/recon/nmap.py
View File

@@ -12,9 +12,10 @@ from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .masscan import ParseMasscanOutput
from .config import defaults, tool_paths
from .config import defaults
from .helpers import get_ip_address_version, is_ip_address
from ..tools import tools
from ..models.port_model import Port
from ..models.nse_model import NSEResult
from ..models.target_model import Target
@@ -281,7 +282,7 @@ class SearchsploitScan(luigi.Task):
""" Grabs the xml files created by ThreadedNmap and runs searchsploit --nmap on each one, saving the output. """
for entry in Path(self.input().get("localtarget").path).glob("nmap*.xml"):
proc = subprocess.run(
[tool_paths.get("searchsploit"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE
[tools.get("searchsploit").get("path"), "-j", "-v", "--nmap", str(entry)], stdout=subprocess.PIPE
)
if proc.stdout:
# change wall-searchsploit-results/nmap.10.10.10.157-tcp to 10.10.10.157

5
pipeline/recon/web/aquatone.py
View File

@@ -9,7 +9,8 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
from ...tools import tools
import pipeline.models.db_manager
from ...models.port_model import Port
@@ -250,7 +251,7 @@ class AquatoneScan(luigi.Task):
self.results_subfolder.mkdir(parents=True, exist_ok=True)
command = [
tool_paths.get("aquatone"),
tools.get("aquatone").get("path"),
"-scan-timeout",
self.scan_timeout,
"-threads",

13
pipeline/recon/web/gobuster.py
View File

@@ -11,7 +11,8 @@ from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
from ...tools import tools
from ...models.endpoint_model import Endpoint
from ..helpers import get_ip_address_version, is_ip_address
@@ -139,10 +140,16 @@ class GobusterScan(luigi.Task):
for url_scheme in ("https://", "http://"):
if self.recursive:
command = [tool_paths.get("recursive-gobuster"), "-s", "-w", self.wordlist, f"{url_scheme}{target}"]
command = [
tools.get("recursive-gobuster").get("path"),
"-s",
"-w",
self.wordlist,
f"{url_scheme}{target}",
]
else:
command = [
tool_paths.get("gobuster"),
tools.get("gobuster").get("path"),
"dir",
"-q",
"-e",

11
pipeline/recon/web/subdomain_takeover.py
View File

@@ -8,8 +8,9 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from ...tools import tools
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
@inherits(GatherWebTargets)
@@ -120,9 +121,9 @@ class TKOSubsScan(luigi.Task):
return
command = [
tool_paths.get("tko-subs"),
tools.get("tko-subs").get("path"),
f"-domain={','.join(domains)}",
f"-data={tool_paths.get('tko-subs-dir')}/providers-data.csv",
f"-data={tools.get('tko-subs').get('git_dir')}/providers-data.csv",
f"-output={self.output_file}",
]
@@ -261,7 +262,7 @@ class SubjackScan(luigi.Task):
f.write(f"{hostname}\n")
command = [
tool_paths.get("subjack"),
tools.get("subjack").get("path"),
"-w",
str(subjack_input_file),
"-t",
@@ -274,7 +275,7 @@ class SubjackScan(luigi.Task):
"-v",
"-ssl",
"-c",
tool_paths.get("subjack-fingerprints"),
tools.get("subjack").get("fingerprints"),
]
subprocess.run(command)

7
pipeline/recon/web/webanalyze.py
View File

@@ -11,8 +11,9 @@ from luigi.util import inherits
from luigi.contrib.sqla import SQLAlchemyTarget
import pipeline.models.db_manager
from ...tools import tools
from .targets import GatherWebTargets
from ..config import tool_paths, defaults
from ..config import defaults
from ...models.technology_model import Technology
from ..helpers import get_ip_address_version, is_ip_address
@@ -153,7 +154,7 @@ class WebanalyzeScan(luigi.Task):
target = f"[{target}]"
for url_scheme in ("https://", "http://"):
command = [tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}", "-output", "csv"]
command = [tools.get("webanalyze").get("path"), "-host", f"{url_scheme}{target}", "-output", "csv"]
commands.append(command)
self.results_subfolder.mkdir(parents=True, exist_ok=True)
@@ -162,7 +163,7 @@ class WebanalyzeScan(luigi.Task):
os.chdir(self.results_subfolder)
if not Path("apps.json").exists():
subprocess.run(f"{tool_paths.get('webanalyze')} -update".split())
subprocess.run(f"{tools.get('webanalyze').get('path')} -update".split())
with ThreadPoolExecutor(max_workers=self.threads) as executor:
executor.map(self._wrapped_subprocess, commands)

7
pipeline/tools/amass.yaml
View File

@@ -1,10 +1,11 @@
installed: false
dependencies: [go]
go: &gotool !get_tool_path "{go}"
amass: &amass !get_tool_path "{amass}"
tools: &tools !get_default "{tools-dir}"
go: &gotool !get_tool_path "{go[path]}"
path: &amass !join_path [*tools, "amass"]
commands:
- !join [*gotool, get -u github.com/OWASP/Amass/v3/...]
- !join [*gotool, get github.com/OWASP/Amass/v3/...]
- !join [cp ~/go/bin/amass, *amass]
shell: true

6
pipeline/tools/aquatone.yaml
View File

@@ -1,6 +1,6 @@
installed: false
dependencies:
aquatone: &aqua !get_tool_path "{aquatone}"
tools: &tools !get_default "{tools-dir}"
path: &aqua !join_path [*tools, aquatone]
commands:
- mkdir /tmp/aquatone
@@ -10,5 +10,3 @@ commands:
- !join [mv /tmp/aquatone/aquatone, *aqua]
- rm -rf /tmp/aquatone
- bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'
shell: true

3
pipeline/tools/exploitdb.yaml Normal file
View File

@@ -0,0 +1,3 @@
installed: true
tools: &tools !get_default "{tools-dir}"
path: !join_path [*tools, exploitdb]

7
pipeline/tools/go.yaml
View File

@@ -1,12 +1,9 @@
installed: false
dependencies:
home: &home !get_default "{home}"
go: &gotool !get_tool_path "{go}"
path: &gotool /usr/local/go/bin/go
bashrc: &bashrc !join_path [*home, "/.bashrc;"]
commands:
- wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz
- wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -O /tmp/go.tar.gz
- sudo tar -C /usr/local -xvf /tmp/go.tar.gz
- !join ["bash -c 'if [ ! $(echo ${PATH} | grep $(dirname", *gotool, ")) ]; then echo PATH=${PATH}:/usr/local/go/bin >>", *bashrc, "fi'"]
shell: true

3
pipeline/tools/gobuster.yaml
View File

@@ -1,7 +1,8 @@
installed: false
dependencies: [go, seclists]
home: &home !get_default "{home}"
go: &gotool !get_tool_path "{go}"
path: !join_path [*home, go/bin/gobuster]
go: &gotool !get_tool_path "{go[path]}"
go_home: &gohome !join_path [*home, "/go/src/github.com/OJ/gobuster &&"]
commands:

29
pipeline/tools/loader.py
View File

@@ -1,9 +1,10 @@
import yaml
from pathlib import Path
from ..recon.config import tool_paths, defaults
from ..recon.config import defaults
definitions = Path(__file__).parent
tools = {}
def join(loader, node):
@@ -30,16 +31,10 @@ def get_default(loader, node):
return py_str.format(**defaults)
def get_parent(loader, node):
""" yaml tag handler to access tool parents """
py_str = loader.construct_python_str(node)
return Path(py_str.format(**tool_paths)).parent
def get_tool_path(loader, node):
""" yaml tag handler to access tool_paths dict at load time """
""" yaml tag handler to access tools dict at load time """
py_str = loader.construct_python_str(node)
return py_str.format(**tool_paths)
return py_str.format(**tools)
yaml.add_constructor("!join", join)
@@ -47,12 +42,20 @@ yaml.add_constructor("!join_empty", join_empty)
yaml.add_constructor("!join_path", join_path)
yaml.add_constructor("!get_default", get_default)
yaml.add_constructor("!get_tool_path", get_tool_path)
yaml.add_constructor("!get_parent", get_parent)
tools = {}
for file in definitions.iterdir():
if file.name.endswith(".yaml"):
def load_yaml(file):
try:
config = yaml.full_load(file.read_text())
tool_name = str(file.name.replace(".yaml", ""))
tools[tool_name] = config
except KeyError as error: # load dependencies first
dependency = error.args[0]
dependency_file = definitions / (dependency + ".yaml")
load_yaml(dependency_file)
load_yaml(file)
for file in definitions.iterdir():
if file.name.endswith(".yaml") and file.name.replace(".yaml", "") not in tools:
load_yaml(file)

4
pipeline/tools/luigi-service.yaml
View File

@@ -1,6 +1,6 @@
installed: false
dependencies:
service-file: &svcfile !get_tool_path "{luigid}"
project-dir: &proj !get_default "{project-dir}"
service-file: &svcfile !join_path [*proj, luigid.service]
commands:
- !join [sudo cp, *svcfile, /lib/systemd/system/luigid.service]

6
pipeline/tools/masscan.yaml
View File

@@ -1,6 +1,6 @@
installed: false
dependencies:
masscan: &masscan !get_tool_path "{masscan}"
tools: &tools !get_default "{tools-dir}"
path: &masscan !join_path [*tools, masscan]
commands:
- git clone https://github.com/robertdavidgraham/masscan /tmp/masscan
@@ -8,5 +8,3 @@ commands:
- !join [mv /tmp/masscan/bin/masscan, *masscan]
- rm -rf /tmp/masscan
- !join [sudo setcap CAP_NET_RAW+ep, *masscan]
shell: true

8
pipeline/tools/recursive-gobuster.yaml
View File

@@ -1,10 +1,10 @@
installed: false
dependencies: [go]
recursive-parent: &recpar !get_parent "{recursive-gobuster}"
tools: &tools !get_default "{tools-dir}"
path: !join_path [*tools, recursive-gobuster/recursive-gobuster.pyz]
recursive-parent: &recpar !join_path [*tools, recursive-gobuster]
commands:
- !join ["bash -c 'if [ -d", *recpar, "]; then cd", *recpar,
"&& git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git",
*recpar, ; fi']
shell: true
*recpar, ; fi']

10
pipeline/tools/searchsploit.yaml
View File

@@ -1,9 +1,9 @@
installed: false
dependencies:
dependencies: [exploitdb]
home: &home !get_default "{home}"
tools-dir: &tools !get_default "{tools-dir}"
exploitdb-file: &exploitdb !get_tool_path "{exploitdb}"
searchsploit-file: &searchsploit !get_tool_path "{searchsploit}"
tools: &tools !get_default "{tools-dir}"
exploitdb-file: &exploitdb !get_tool_path "{exploitdb[path]}"
path: &searchsploit !join_path [*tools, exploitdb/searchsploit]
searchsploit-rc: &ss_rc !join_path [*exploitdb, ".searchsploit_rc"]
homesploit: &homesploit !join_path [*home, ".searchsploit_rc"]
sed-command: &sedcom !join_empty ["'s#/opt#", *tools, "#g'"]
@@ -15,5 +15,3 @@ commands:
"&& git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git", *exploitdb, ; fi']
- !join ["bash -c 'if [ -f", *ss_rc, "]; then cp -n", *ss_rc, *home, ; fi']
- !join ["bash -c 'if [ -f", *homesploit, "]; then sed -i", *sedcom, *homesploit, ; fi']
shell: true

8
pipeline/tools/seclists.yaml
View File

@@ -1,10 +1,8 @@
installed: false
depencencies:
seclists-file: &secfile !get_tool_path "{seclists}"
tools: &tools !get_default "{tools-dir}"
path: &secfile !join_path [*tools, seclists]
commands:
- !join ["bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists",
*secfile, "; elif [[ -d", *secfile, "]] ; then cd", *secfile, "&& git fetch --all && git pull;",
else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi']
shell: true
else git clone https://github.com/danielmiessler/SecLists.git, *secfile, ; fi']

4
pipeline/tools/subjack.yaml
View File

@@ -1,8 +1,10 @@
installed: false
dependencies: [go]
go: &gotool !get_tool_path "{go}"
go: &gotool !get_tool_path "{go[path]}"
home: &home !get_default "{home}"
path: !join_path [*home, go/bin/subjack]
subjack_home: &subjhome !join_path [*home, "/go/src/github.com/haccer/subjack &&"]
fingerprints: !join_path [*home, go/src/github.com/haccer/subjack/fingerprints.json]
commands:
- !join [*gotool, get github.com/haccer/subjack]

6
pipeline/tools/tko-subs.yaml
View File

@@ -1,11 +1,13 @@
installed: false
dependencies: [go]
go: &gotool !get_tool_path "{go}"
go: &gotool !get_tool_path "{go[path]}"
home: &home !get_default "{home}"
path: !join_path [*home, go/bin/tko-subs]
tko_home: &tkohome !join_path [*home, "go/src/github.com/anshumanbh/tko-subs &&"]
git_dir: !join_path [*home, go/src/github.com/anshumanbh/tko-subs]
commands:
- !join [*gotool, get, github.com/anshumanbh/tko-subs]
- !join [(cd, *tkohome, *gotool, "build &&", *gotool, "install)"]
shell: true
shell: true

8
pipeline/tools/waybackurls.yaml
View File

@@ -1,9 +1,7 @@
installed: false
dependencies: ["go"]
go: &gotool !get_tool_path "{go}"
dependencies: [go]
go: &gotool !get_tool_path "{go[path]}"
path: !join_path [!get_default "{home}", go, bin, waybackurls]
commands:
- !join [*gotool, get, github.com/tomnomnom/waybackurls]
shell: false
- !join [*gotool, get, github.com/tomnomnom/waybackurls]

5
pipeline/tools/webanalyze.yaml
View File

@@ -1,11 +1,12 @@
installed: false
dependencies: [go]
home: &home !get_default "{home}"
go: &gotool !get_tool_path "{go}"
go: &gotool !get_tool_path "{go[path]}"
path: !join_path [*home, go/bin/webanalyze]
webanalyze_home: &webhome !join_path [*home, "/go/src/github.com/rverton/webanalyze &&"]
commands:
- !join [*gotool, get github.com/rverton/webanalyze/...]
- !join [(cd, *webhome, *gotool, "build &&", *gotool, install)]
shell: true
shell: true

8
tests/test_recon/test_config.py
View File

@@ -2,12 +2,14 @@ from pathlib import Path
import pytest
from pipeline.recon import tool_paths, defaults, web_ports, top_tcp_ports, top_udp_ports
from pipeline.recon import defaults, web_ports, top_tcp_ports, top_udp_ports
from pipeline.tools import tools
def test_tool_paths_absolute():
for path in tool_paths.values():
assert Path(path).is_absolute()
for tool in tools.values():
if tool.get("path"):
assert Path(tool["path"]).is_absolute()
@pytest.mark.parametrize("test_input", ["database-dir", "tools-dir", "gobuster-wordlist"])

5
tests/test_recon/test_nmap.py
View File

@@ -7,7 +7,8 @@ import luigi
import pytest
from luigi.contrib.sqla import SQLAlchemyTarget
from pipeline.recon import ThreadedNmapScan, SearchsploitScan, ParseMasscanOutput, config
from pipeline.recon import ThreadedNmapScan, SearchsploitScan, ParseMasscanOutput
from pipeline.tools import tools
nmap_results = Path(__file__).parent.parent / "data" / "recon-results" / "nmap-results"
@@ -101,7 +102,7 @@ class TestSearchsploitScan:
assert len(self.scan.db_mgr.get_all_searchsploit_results()) == 0
if not Path(config.tool_paths.get("searchsploit")).exists():
if not Path(tools.get("searchsploit").get("path")).exists():
pytest.skip("exploit-db's searchsploit tool not installed")
self.scan.run()

139
tests/test_shell/test_recon_pipeline_shell.py
View File

@@ -12,7 +12,8 @@ import pytest
from pipeline.models.port_model import Port
from pipeline.models.target_model import Target
from pipeline.models.db_manager import DBManager
from pipeline.recon.config import defaults, tool_paths
from pipeline.tools import tools
from pipeline.recon.config import defaults
from pipeline.models.ip_address_model import IPAddress
recon_shell = importlib.import_module("pipeline.recon-pipeline")
@@ -294,143 +295,21 @@ class TestReconShell:
# ("all", "commands failed and may have not installed properly", 1)
# after tools moved to DB, update this test
@pytest.mark.parametrize("test_input, expected, return_code", [("all", "is already installed", 0)])
@pytest.mark.parametrize(
"test_input, expected, return_code", [("all", "is already installed", 0), ("amass", "dependency", 1)]
)
def test_do_install(self, test_input, expected, return_code, capsys, tmp_path):
process_mock = MagicMock()
attrs = {"communicate.return_value": (b"output", b"error"), "returncode": return_code}
process_mock.configure_mock(**attrs)
tool_dict = {
"luigi-service": {
"installed": False,
"dependencies": None,
"commands": [
f"sudo cp {str(Path(__file__).parents[2] / 'luigid.service')} /lib/systemd/system/luigid.service",
f"sudo cp $(which luigid) /usr/local/bin",
"sudo systemctl daemon-reload",
"sudo systemctl start luigid.service",
"sudo systemctl enable luigid.service",
],
"shell": True,
},
"seclists": {
"installed": False,
"dependencies": None,
"shell": True,
"commands": [
f"bash -c 'if [[ -d /usr/share/seclists ]]; then ln -s /usr/share/seclists {defaults.get('tools-dir')}/seclists; elif [[ -d {defaults.get('tools-dir')}/seclists ]] ; then cd {defaults.get('tools-dir')}/seclists && git fetch --all && git pull; else git clone https://github.com/danielmiessler/SecLists.git {defaults.get('tools-dir')}/seclists; fi'"
],
},
"searchsploit": {
"installed": False,
"dependencies": None,
"shell": True,
"commands": [
f"bash -c 'if [[ -d /usr/share/exploitdb ]]; then ln -s /usr/share/exploitdb {defaults.get('tools-dir')}/exploitdb && sudo ln -s $(which searchsploit) {defaults.get('tools-dir')}/exploitdb/searchsploit; elif [[ -d {Path(tool_paths.get('searchsploit')).parent} ]]; then cd {Path(tool_paths.get('searchsploit')).parent} && git fetch --all && git pull; else git clone https://github.com/offensive-security/exploitdb.git {defaults.get('tools-dir')}/exploitdb; fi'",
f"bash -c 'if [[ -f {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc ]]; then cp -n {Path(tool_paths.get('searchsploit')).parent}/.searchsploit_rc {Path.home().expanduser().resolve()}; fi'",
f"bash -c 'if [[ -f {Path.home().resolve()}/.searchsploit_rc ]]; then sed -i 's#/opt#{defaults.get('tools-dir')}#g' {Path.home().resolve()}/.searchsploit_rc; fi'",
],
},
"masscan": {
"installed": False,
"dependencies": None,
"commands": [
"git clone https://github.com/robertdavidgraham/masscan /tmp/masscan",
"make -s -j -C /tmp/masscan",
f"mv /tmp/masscan/bin/masscan {tool_paths.get('masscan')}",
"rm -rf /tmp/masscan",
f"sudo setcap CAP_NET_RAW+ep {tool_paths.get('masscan')}",
],
},
"amass": {
"installed": False,
"dependencies": ["go"],
"commands": [
f"{tool_paths.get('go')} get -u github.com/OWASP/Amass/v3/...",
f"cp ~/go/bin/amass {tool_paths.get('amass')}",
],
"shell": True,
"environ": {"GO111MODULE": "on"},
},
"aquatone": {
"installed": False,
"dependencies": None,
"shell": True,
"commands": [
"mkdir /tmp/aquatone",
"wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip",
"bash -c 'if [[ ! $(which unzip) ]]; then sudo apt install -y zip; fi'",
"unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone",
f"mv /tmp/aquatone/aquatone {tool_paths.get('aquatone')}",
"rm -rf /tmp/aquatone",
"bash -c 'found=false; for loc in {/usr/bin/google-chrome,/usr/bin/google-chrome-beta,/usr/bin/google-chrome-unstable,/usr/bin/chromium-browser,/usr/bin/chromium}; do if [[ $(which $loc) ]]; then found=true; break; fi ; done; if [[ $found = false ]]; then sudo apt install -y chromium-browser ; fi'",
],
},
"gobuster": {
"installed": False,
"dependencies": ["go", "seclists"],
"commands": [
f"{tool_paths.get('go')} get github.com/OJ/gobuster",
f"(cd ~/go/src/github.com/OJ/gobuster && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
],
"shell": True,
},
"tko-subs": {
"installed": False,
"dependencies": ["go"],
"commands": [
f"{tool_paths.get('go')} get github.com/anshumanbh/tko-subs",
f"(cd ~/go/src/github.com/anshumanbh/tko-subs && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
],
"shell": True,
},
"subjack": {
"installed": False,
"dependencies": ["go"],
"commands": [
f"{tool_paths.get('go')} get github.com/haccer/subjack",
f"(cd ~/go/src/github.com/haccer/subjack && {tool_paths.get('go')} install)",
],
"shell": True,
},
"webanalyze": {
"installed": False,
"dependencies": ["go"],
"commands": [
f"{tool_paths.get('go')} get github.com/rverton/webanalyze/...",
f"(cd ~/go/src/github.com/rverton/webanalyze && {tool_paths.get('go')} build && {tool_paths.get('go')} install)",
],
"shell": True,
},
"recursive-gobuster": {
"installed": False,
"dependencies": ["gobuster", "seclists"],
"shell": True,
"commands": [
f"bash -c 'if [[ -d {Path(tool_paths.get('recursive-gobuster')).parent} ]] ; then cd {Path(tool_paths.get('recursive-gobuster')).parent} && git fetch --all && git pull; else git clone https://github.com/epi052/recursive-gobuster.git {Path(tool_paths.get('recursive-gobuster')).parent}; fi'"
],
},
"go": {
"installed": False,
"dependencies": None,
"commands": [
"wget -q https://dl.google.com/go/go1.13.7.linux-amd64.tar.gz -O /tmp/go.tar.gz",
"sudo tar -C /usr/local -xvf /tmp/go.tar.gz",
f'bash -c \'if [[ ! $(echo "${{PATH}}" | grep $(dirname {tool_paths.get("go")})) ]]; then echo "PATH=${{PATH}}:/usr/local/go/bin" >> ~/.bashrc; fi\'',
],
},
"waybackurls": {
"installed": True,
"depencencies": ["go"],
"commands": ["/usr/local/go/bin/go get github.com/tomnomnom/waybackurls"],
"shell": True,
},
}
tooldir = tmp_path / ".local" / "recon-pipeline" / "tools"
tooldir.mkdir(parents=True, exist_ok=True)
pickle.dump(tool_dict, (tooldir / ".tool-dict.pkl").open("wb"))
tools["waybackurls"]["installed"] = True
tools["amass"]["shell"] = False
pickle.dump(tools, (tooldir / ".tool-dict.pkl").open("wb"))
with patch("subprocess.Popen", autospec=True) as mocked_popen:
mocked_popen.return_value = process_mock

4
tests/test_web/test_webanalyze.py
View File

@@ -4,7 +4,7 @@ import tempfile
from pathlib import Path
from unittest.mock import patch, MagicMock
from pipeline.recon.config import tool_paths
from pipeline.tools import tools
from pipeline.recon.web import WebanalyzeScan, GatherWebTargets
webanalyze_results = Path(__file__).parent.parent / "data" / "recon-results" / "webanalyze-results"
@@ -65,7 +65,7 @@ class TestWebanalyzeScan:
self.scan.results_subfolder.mkdir()
os.chdir(self.scan.results_subfolder)
assert len([x for x in self.scan.results_subfolder.iterdir()]) == 0
cmd = [tool_paths.get("webanalyze"), "-host", "https://google.com", "-output", "csv"]
cmd = [tools.get("webanalyze").get("path"), "-host", "https://google.com", "-output", "csv"]
self.scan._wrapped_subprocess(cmd)
assert len([x for x in self.scan.results_subfolder.iterdir()]) == 1
assert next(self.scan.results_subfolder.iterdir()).name == "webanalyze-https_google.com.csv"