mirror of
https://github.com/aljazceru/recon-pipeline.git
synced 2025-12-22 08:44:22 +01:00
updated README; included cmd2 in Pipfile; fixed displaying multiple queued tasks
This commit is contained in:
epi052
1
Pipfile
1
Pipfile
@@ -7,6 +7,7 @@ verify_ssl = true
|
|||||||
|
|
||||||
[packages]
|
[packages]
|
||||||
luigi = "*"
|
luigi = "*"
|
||||||
|
cmd2 = "*"
|
||||||
|
|
||||||
[requires]
|
[requires]
|
||||||
python_version = "3.7"
|
python_version = "3.7"
|
||||||
|
|||||||
66
Pipfile.lock
generated
66
Pipfile.lock
generated
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"_meta": {
|
"_meta": {
|
||||||
"hash": {
|
"hash": {
|
||||||
"sha256": "a3c10660fc478036a0c0e8fe132bc0df0503a47ee414e96a666bd1fd88e32771"
|
"sha256": "f6e3610c10920d9297afe9d974892f5e970a04cfb8c3f8164e874031c6274037"
|
||||||
},
|
},
|
||||||
"pipfile-spec": 6,
|
"pipfile-spec": 6,
|
||||||
"requires": {
|
"requires": {
|
||||||
@@ -16,13 +16,34 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"default": {
|
"default": {
|
||||||
|
"attrs": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||||
|
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||||
|
],
|
||||||
|
"version": "==19.3.0"
|
||||||
|
},
|
||||||
|
"cmd2": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:208812035933cdb5c1c254cf266ffd7f560ca3a075569f3a39fc4e4a4427c2a0",
|
||||||
|
"sha256:8ad12ef3cc46d03073c545b6e80a3f84a5921f6653073a60e7d9a7ff3b352c9e"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==0.9.23"
|
||||||
|
},
|
||||||
|
"colorama": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
|
||||||
|
"sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"
|
||||||
|
],
|
||||||
|
"version": "==0.4.3"
|
||||||
|
},
|
||||||
"docutils": {
|
"docutils": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0",
|
"sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
|
||||||
"sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827",
|
"sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
|
||||||
"sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"
|
|
||||||
],
|
],
|
||||||
"version": "==0.15.2"
|
"version": "==0.16"
|
||||||
},
|
},
|
||||||
"lockfile": {
|
"lockfile": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@@ -33,31 +54,37 @@
|
|||||||
},
|
},
|
||||||
"luigi": {
|
"luigi": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:ac8d6f25a417498f09bbf79ab7ea4d9f16d431cf5015ed03fbf489307f3ea661"
|
"sha256:c2b3dcecc565fe77920553434ed475fa21f562d4b76da6bd1a179a8b732fcc9e"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==2.8.9"
|
"version": "==2.8.11"
|
||||||
|
},
|
||||||
|
"pyperclip": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:979325468ccf682104d5dcaf753f869868100631301d3e72f47babdea5700d1c"
|
||||||
|
],
|
||||||
|
"version": "==1.7.0"
|
||||||
},
|
},
|
||||||
"python-daemon": {
|
"python-daemon": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:261c859be5c12ae7d4286dc6951e87e9e1a70a882a8b41fd926efc1ec4214f73",
|
"sha256:57c84f50a04d7825515e4dbf3a31c70cc44414394a71608dee6cfde469e81766",
|
||||||
"sha256:53da55aec3bb67b576e13a8091a2181f99b395c2eec32a5a0d91d347a5c420a7"
|
"sha256:a0d5dc0b435a02c7e0b401e177a7c17c3f4c7b4e22e2d06271122c8fec5f8946"
|
||||||
],
|
],
|
||||||
"version": "==2.1.2"
|
"version": "==2.2.4"
|
||||||
},
|
},
|
||||||
"python-dateutil": {
|
"python-dateutil": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
|
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
|
||||||
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
|
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
|
||||||
],
|
],
|
||||||
"version": "==2.8.0"
|
"version": "==2.8.1"
|
||||||
},
|
},
|
||||||
"six": {
|
"six": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
|
||||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
|
||||||
],
|
],
|
||||||
"version": "==1.12.0"
|
"version": "==1.13.0"
|
||||||
},
|
},
|
||||||
"tornado": {
|
"tornado": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@@ -70,6 +97,13 @@
|
|||||||
"sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444"
|
"sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444"
|
||||||
],
|
],
|
||||||
"version": "==5.1.1"
|
"version": "==5.1.1"
|
||||||
|
},
|
||||||
|
"wcwidth": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603",
|
||||||
|
"sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"
|
||||||
|
],
|
||||||
|
"version": "==0.1.8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"develop": {}
|
"develop": {}
|
||||||
|
|||||||
112
README.md
112
README.md
@@ -2,69 +2,71 @@
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
There are an [accompanying set of blog posts](https://epi052.gitlab.io/notes-to-self/blog/2019-09-01-how-to-build-an-automated-recon-pipeline-with-python-and-luigi/) detailing the development process and underpinnings of the pipeline. Feel free to check them out if you're so inclined, but they're in no way required reading to use the tool.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
> Automatic installation only tested on kali 2019.4
|
||||||
|
|
||||||
|
There are two primary phases for installation:
|
||||||
|
|
||||||
|
1. prior to [cmd2](https://github.com/python-cmd2/cmd2) being installed
|
||||||
|
2. everything else
|
||||||
|
|
||||||
|
First, the manual steps are as follows (and shown below)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
apt install pipenv
|
||||||
|
git clone https://github.com/epi052/recon-pipeline.git
|
||||||
|
cd recon-pipeline
|
||||||
|
pipenv install cmd2
|
||||||
|
```
|
||||||
|
|
||||||
|
[](https://asciinema.org/a/AxFd1SaLVx7mQdxqQBLfh6aqj)
|
||||||
|
|
||||||
|
Once manual installation of [cmd2](https://github.com/python-cmd2/cmd2) is complete, the `recon-pipeline` shell provides its own `install` command (seen below). A simple `install all` will handle all installation steps (as long as you're running a newer version of kali; all other OS's are untested, good luck!)
|
||||||
|
|
||||||
|
[](https://asciinema.org/a/293305)
|
||||||
|
|
||||||
## Command Execution
|
## Command Execution
|
||||||
|
|
||||||
### PYTHONPATH
|
Command execution is handled through the `recon-pipeline` shell (seen below).
|
||||||
To run the pipelines, you need to set your `PYTHONPATH` environment variable to the path of this project on disk. This can be accomplished in a few ways; two solutions are offered.
|
|
||||||
|
|
||||||
1. Prepend `PYTHONPATH=/path/to/recon-pipline` to any luigi pipeline command being run.
|
[](https://asciinema.org/a/293302)
|
||||||
2. Add `export PYTHONPATH=/path/to/recon-pipeline` to your `.bashrc`
|
|
||||||
|
|
||||||
### Scheduler
|
### Target File and Exempt List File (defining scope)
|
||||||
|
|
||||||
Either add `--local-scheduler` to your `luigi` command on the command line or run `systemctl start luigid` before attempting to run any `luigi` commands.
|
The pipeline expects a file that describes the target's scope to be provided as an argument to the `--target-file` option. The target file can consist of domains, ip addresses, and ip ranges, one per line.
|
||||||
|
|
||||||
#### Systemd service file for luigid
|
|
||||||
```
|
|
||||||
cat >> /lib/systemd/system/luigid.service << EOF
|
|
||||||
[Unit]
|
|
||||||
Description=Spotify Luigi server
|
|
||||||
Documentation=https://luigi.readthedocs.io/en/stable/
|
|
||||||
Requires=network.target remote-fs.target
|
|
||||||
After=network.target remote-fs.target
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
ExecStart=/usr/local/bin/luigid --background --pidfile /var/run/luigid.pid --logdir /var/log/luigi
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
### scope file
|
|
||||||
|
|
||||||
The pipeline expects a file that describes the project scope to be in the current working directory. By convention, TARGET_NAME should be something like tesla or some other target identifier.
|
|
||||||
|
|
||||||
### luigi command structure
|
|
||||||
|
|
||||||
With the `PYTHONPATH` setup, luigi commands take on the following structure (prepend `PYTHONPATH` if not exported from `.bashrc`):
|
|
||||||
|
|
||||||
`luigi --module PACKAGENAME.MODULENAME CLASSNAME *args`
|
|
||||||
|
|
||||||
You can get options for each module by running `luigi --module PACKAGENAME.MODULENAME CLASSNAME --help`
|
|
||||||
|
|
||||||
example help statement
|
|
||||||
`luigi --module recon.targets TargetList --help`
|
|
||||||
|
|
||||||
```text
|
```text
|
||||||
usage: luigi [--local-scheduler] [--module CORE_MODULE] [--help] [--help-all]
|
tesla.com
|
||||||
[--TargetList-target-file TARGETLIST_TARGET_FILE]
|
tesla.cn
|
||||||
[--target-file TARGET_FILE]
|
teslamotors.com
|
||||||
[Required root task]
|
...
|
||||||
|
|
||||||
positional arguments:
|
|
||||||
Required root task Task family to run. Is not optional.
|
|
||||||
|
|
||||||
optional arguments:
|
|
||||||
--local-scheduler Use an in-memory central scheduler. Useful for
|
|
||||||
testing.
|
|
||||||
--module CORE_MODULE Used for dynamic loading of modules
|
|
||||||
--help Show most common flags and all task-specific flags
|
|
||||||
--help-all Show all command line flags
|
|
||||||
--TargetList-target-file TARGETLIST_TARGET_FILE
|
|
||||||
--target-file TARGET_FILE
|
|
||||||
```
|
```
|
||||||
|
|
||||||
An example scope file command, where `tesla` is the name of the file and it is located in the current directory.
|
Some bug bounty scopes have expressly verboten subdomains and/or top-level domains, for that there is the `--exempt-list` option. The exempt list follows the same rules as the target file.
|
||||||
|
|
||||||
|
```text
|
||||||
|
shop.eu.teslamotors.com
|
||||||
|
energysupport.tesla.com
|
||||||
|
feedback.tesla.com
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using a Scheduler
|
||||||
|
|
||||||
|
The backbone of this pipeline is spotify's [luigi](https://github.com/spotify/luigi) batch process management framework. Luigi uses the concept of a scheduler in order to manage task execution. Two types of scheduler are available, a local scheduler and a central scheduler. The local scheduler is useful for development and debugging while the central scheduler provides the following two benefits:
|
||||||
|
|
||||||
|
- Make sure two instances of the same task are not running simultaneously
|
||||||
|
- Provide visualization of everything that’s going on
|
||||||
|
|
||||||
|
While in the `recon-pipeline` shell, running `install luigi-service` will copy the `luigid.service` file provided in the
|
||||||
|
repo to its appropriate systemd location and start/enable the service. The result is that the central scheduler is up
|
||||||
|
and running easily.
|
||||||
|
|
||||||
|
The other option is to add `--local-scheduler` to your `scan` command from within the `recon-pipeline` shell.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
`PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.targets TargetList --target-file tesla`
|
|
||||||
|
|
||||||
|
|||||||
@@ -45,14 +45,10 @@ def get_scans():
|
|||||||
# tool definitions for the auto-installer
|
# tool definitions for the auto-installer
|
||||||
tools = {
|
tools = {
|
||||||
"go": {"installed": False, "dependencies": None, "commands": ["apt-get install -y -q golang"]},
|
"go": {"installed": False, "dependencies": None, "commands": ["apt-get install -y -q golang"]},
|
||||||
"luigi": {
|
"luigi": {"installed": False, "dependencies": ["pipenv"], "commands": ["pipenv install luigi"]},
|
||||||
"installed": False,
|
|
||||||
"dependencies": ["pipenv", "luigi-service"],
|
|
||||||
"commands": ["pipenv install luigi"],
|
|
||||||
},
|
|
||||||
"luigi-service": {
|
"luigi-service": {
|
||||||
"installed": False,
|
"installed": False,
|
||||||
"dependencies": None,
|
"dependencies": ["luigi"],
|
||||||
"commands": [
|
"commands": [
|
||||||
f"cp {str(Path(__file__).parent / 'luigid.service')} /lib/systemd/system/luigid.service",
|
f"cp {str(Path(__file__).parent / 'luigid.service')} /lib/systemd/system/luigid.service",
|
||||||
f"cp $(which luigid) /usr/local/bin",
|
f"cp $(which luigid) /usr/local/bin",
|
||||||
@@ -244,17 +240,17 @@ class ReconShell(cmd2.Cmd):
|
|||||||
# block below used for printing status messages
|
# block below used for printing status messages
|
||||||
if sentry:
|
if sentry:
|
||||||
self.poutput(style(output.strip(), fg="bright_blue"))
|
self.poutput(style(output.strip(), fg="bright_blue"))
|
||||||
elif output.startswith("INFO") and output.split()[-1] == "PENDING":
|
elif output.startswith("INFO: Informed") and output.strip().endswith("PENDING"):
|
||||||
words = output.split()
|
words = output.split()
|
||||||
self.poutput(style(f"[-] {words[5].split('_')[0]} queued", fg="bright_white"))
|
self.poutput(style(f"[-] {words[5].split('_')[0]} queued", fg="bright_white"))
|
||||||
elif output.startswith("INFO") and "running" in output:
|
elif output.startswith("INFO: ") and "running" in output:
|
||||||
words = output.split()
|
words = output.split()
|
||||||
# output looks similar to , pid=3938074) running MasscanScan(
|
# output looks similar to , pid=3938074) running MasscanScan(
|
||||||
# want to grab the index of the luigi task running
|
# want to grab the index of the luigi task running
|
||||||
scantypeidx = words.index("running") + 1
|
scantypeidx = words.index("running") + 1
|
||||||
scantype = words[scantypeidx].split("(", 1)[0]
|
scantype = words[scantypeidx].split("(", 1)[0]
|
||||||
self.poutput(style(f"[*] {scantype} running...", fg="bright_yellow"))
|
self.poutput(style(f"[*] {scantype} running...", fg="bright_yellow"))
|
||||||
elif output.startswith("INFO") and output.split()[-1] == "DONE":
|
elif output.startswith("INFO: Informed") and output.strip().endswith("DONE"):
|
||||||
words = output.split()
|
words = output.split()
|
||||||
self.poutput(
|
self.poutput(
|
||||||
style(f"[+] {words[5].split('_')[0]} complete!", fg="bright_green")
|
style(f"[+] {words[5].split('_')[0]} complete!", fg="bright_green")
|
||||||
|
|||||||
Reference in New Issue
Block a user