diff --git a/Pipfile b/Pipfile index 0de105a..1d92da0 100644 --- a/Pipfile +++ b/Pipfile @@ -7,6 +7,7 @@ verify_ssl = true [packages] luigi = "*" +cmd2 = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index d8378be..90b4d46 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a3c10660fc478036a0c0e8fe132bc0df0503a47ee414e96a666bd1fd88e32771" + "sha256": "f6e3610c10920d9297afe9d974892f5e970a04cfb8c3f8164e874031c6274037" }, "pipfile-spec": 6, "requires": { @@ -16,13 +16,34 @@ ] }, "default": { + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "version": "==19.3.0" + }, + "cmd2": { + "hashes": [ + "sha256:208812035933cdb5c1c254cf266ffd7f560ca3a075569f3a39fc4e4a4427c2a0", + "sha256:8ad12ef3cc46d03073c545b6e80a3f84a5921f6653073a60e7d9a7ff3b352c9e" + ], + "index": "pypi", + "version": "==0.9.23" + }, + "colorama": { + "hashes": [ + "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", + "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" + ], + "version": "==0.4.3" + }, "docutils": { "hashes": [ - "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", - "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", - "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", + "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" ], - "version": "==0.15.2" + "version": "==0.16" }, "lockfile": { "hashes": [ @@ -33,31 +54,37 @@ }, "luigi": { "hashes": [ - "sha256:ac8d6f25a417498f09bbf79ab7ea4d9f16d431cf5015ed03fbf489307f3ea661" + "sha256:c2b3dcecc565fe77920553434ed475fa21f562d4b76da6bd1a179a8b732fcc9e" ], "index": "pypi", - "version": "==2.8.9" + "version": "==2.8.11" + }, + "pyperclip": { + "hashes": [ + "sha256:979325468ccf682104d5dcaf753f869868100631301d3e72f47babdea5700d1c" + ], + "version": "==1.7.0" }, "python-daemon": { "hashes": [ - "sha256:261c859be5c12ae7d4286dc6951e87e9e1a70a882a8b41fd926efc1ec4214f73", - "sha256:53da55aec3bb67b576e13a8091a2181f99b395c2eec32a5a0d91d347a5c420a7" + "sha256:57c84f50a04d7825515e4dbf3a31c70cc44414394a71608dee6cfde469e81766", + "sha256:a0d5dc0b435a02c7e0b401e177a7c17c3f4c7b4e22e2d06271122c8fec5f8946" ], - "version": "==2.1.2" + "version": "==2.2.4" }, "python-dateutil": { "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "version": "==2.8.0" + "version": "==2.8.1" }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "tornado": { "hashes": [ @@ -70,6 +97,13 @@ "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" ], "version": "==5.1.1" + }, + "wcwidth": { + "hashes": [ + "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", + "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" + ], + "version": "==0.1.8" } }, "develop": {} diff --git a/README.md b/README.md index ad041f7..c170b8a 100644 --- a/README.md +++ b/README.md @@ -1,70 +1,72 @@ -# Automated Reconnaissance Pipeline +# Automated Reconnaissance Pipeline ![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg) +There are an [accompanying set of blog posts](https://epi052.gitlab.io/notes-to-self/blog/2019-09-01-how-to-build-an-automated-recon-pipeline-with-python-and-luigi/) detailing the development process and underpinnings of the pipeline. Feel free to check them out if you're so inclined, but they're in no way required reading to use the tool. + +## Installation + +> Automatic installation only tested on kali 2019.4 + +There are two primary phases for installation: + +1. prior to [cmd2](https://github.com/python-cmd2/cmd2) being installed +2. everything else + +First, the manual steps are as follows (and shown below) + +```bash +apt install pipenv +git clone https://github.com/epi052/recon-pipeline.git +cd recon-pipeline +pipenv install cmd2 +``` + +[![asciicast](https://asciinema.org/a/AxFd1SaLVx7mQdxqQBLfh6aqj.svg)](https://asciinema.org/a/AxFd1SaLVx7mQdxqQBLfh6aqj) + +Once manual installation of [cmd2](https://github.com/python-cmd2/cmd2) is complete, the `recon-pipeline` shell provides its own `install` command (seen below). A simple `install all` will handle all installation steps (as long as you're running a newer version of kali; all other OS's are untested, good luck!) + + [![asciicast](https://asciinema.org/a/293305.svg)](https://asciinema.org/a/293305) + ## Command Execution -### PYTHONPATH -To run the pipelines, you need to set your `PYTHONPATH` environment variable to the path of this project on disk. This can be accomplished in a few ways; two solutions are offered. +Command execution is handled through the `recon-pipeline` shell (seen below). -1. Prepend `PYTHONPATH=/path/to/recon-pipline` to any luigi pipeline command being run. -2. Add `export PYTHONPATH=/path/to/recon-pipeline` to your `.bashrc` +[![asciicast](https://asciinema.org/a/293302.svg)](https://asciinema.org/a/293302) -### Scheduler +### Target File and Exempt List File (defining scope) -Either add `--local-scheduler` to your `luigi` command on the command line or run `systemctl start luigid` before attempting to run any `luigi` commands. - -#### Systemd service file for luigid -``` -cat >> /lib/systemd/system/luigid.service << EOF -[Unit] -Description=Spotify Luigi server -Documentation=https://luigi.readthedocs.io/en/stable/ -Requires=network.target remote-fs.target -After=network.target remote-fs.target -[Service] -Type=simple -ExecStart=/usr/local/bin/luigid --background --pidfile /var/run/luigid.pid --logdir /var/log/luigi -[Install] -WantedBy=multi-user.target -EOF -``` - -### scope file - -The pipeline expects a file that describes the project scope to be in the current working directory. By convention, TARGET_NAME should be something like tesla or some other target identifier. - -### luigi command structure - -With the `PYTHONPATH` setup, luigi commands take on the following structure (prepend `PYTHONPATH` if not exported from `.bashrc`): - -`luigi --module PACKAGENAME.MODULENAME CLASSNAME *args` - -You can get options for each module by running `luigi --module PACKAGENAME.MODULENAME CLASSNAME --help` - -example help statement -`luigi --module recon.targets TargetList --help` +The pipeline expects a file that describes the target's scope to be provided as an argument to the `--target-file` option. The target file can consist of domains, ip addresses, and ip ranges, one per line. ```text -usage: luigi [--local-scheduler] [--module CORE_MODULE] [--help] [--help-all] - [--TargetList-target-file TARGETLIST_TARGET_FILE] - [--target-file TARGET_FILE] - [Required root task] - -positional arguments: - Required root task Task family to run. Is not optional. - -optional arguments: - --local-scheduler Use an in-memory central scheduler. Useful for - testing. - --module CORE_MODULE Used for dynamic loading of modules - --help Show most common flags and all task-specific flags - --help-all Show all command line flags - --TargetList-target-file TARGETLIST_TARGET_FILE - --target-file TARGET_FILE +tesla.com +tesla.cn +teslamotors.com +... ``` -An example scope file command, where `tesla` is the name of the file and it is located in the current directory. +Some bug bounty scopes have expressly verboten subdomains and/or top-level domains, for that there is the `--exempt-list` option. The exempt list follows the same rules as the target file. + +```text +shop.eu.teslamotors.com +energysupport.tesla.com +feedback.tesla.com +... +``` + +### Using a Scheduler + +The backbone of this pipeline is spotify's [luigi](https://github.com/spotify/luigi) batch process management framework. Luigi uses the concept of a scheduler in order to manage task execution. Two types of scheduler are available, a local scheduler and a central scheduler. The local scheduler is useful for development and debugging while the central scheduler provides the following two benefits: + +- Make sure two instances of the same task are not running simultaneously +- Provide visualization of everything that’s going on + +While in the `recon-pipeline` shell, running `install luigi-service` will copy the `luigid.service` file provided in the +repo to its appropriate systemd location and start/enable the service. The result is that the central scheduler is up +and running easily. + +The other option is to add `--local-scheduler` to your `scan` command from within the `recon-pipeline` shell. + + -`PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.targets TargetList --target-file tesla` diff --git a/recon-pipeline.py b/recon-pipeline.py index a7bc597..bbfda04 100755 --- a/recon-pipeline.py +++ b/recon-pipeline.py @@ -45,14 +45,10 @@ def get_scans(): # tool definitions for the auto-installer tools = { "go": {"installed": False, "dependencies": None, "commands": ["apt-get install -y -q golang"]}, - "luigi": { - "installed": False, - "dependencies": ["pipenv", "luigi-service"], - "commands": ["pipenv install luigi"], - }, + "luigi": {"installed": False, "dependencies": ["pipenv"], "commands": ["pipenv install luigi"]}, "luigi-service": { "installed": False, - "dependencies": None, + "dependencies": ["luigi"], "commands": [ f"cp {str(Path(__file__).parent / 'luigid.service')} /lib/systemd/system/luigid.service", f"cp $(which luigid) /usr/local/bin", @@ -244,17 +240,17 @@ class ReconShell(cmd2.Cmd): # block below used for printing status messages if sentry: self.poutput(style(output.strip(), fg="bright_blue")) - elif output.startswith("INFO") and output.split()[-1] == "PENDING": + elif output.startswith("INFO: Informed") and output.strip().endswith("PENDING"): words = output.split() self.poutput(style(f"[-] {words[5].split('_')[0]} queued", fg="bright_white")) - elif output.startswith("INFO") and "running" in output: + elif output.startswith("INFO: ") and "running" in output: words = output.split() # output looks similar to , pid=3938074) running MasscanScan( # want to grab the index of the luigi task running scantypeidx = words.index("running") + 1 scantype = words[scantypeidx].split("(", 1)[0] self.poutput(style(f"[*] {scantype} running...", fg="bright_yellow")) - elif output.startswith("INFO") and output.split()[-1] == "DONE": + elif output.startswith("INFO: Informed") and output.strip().endswith("DONE"): words = output.split() self.poutput( style(f"[+] {words[5].split('_')[0]} complete!", fg="bright_green")