diff --git a/.flake8 b/.flake8
index 1fe22e5..b1a8b56 100644
--- a/.flake8
+++ b/.flake8
@@ -2,4 +2,10 @@
max-line-length = 120
select = C,E,F,W,B,B950
ignore = E203, E501, W503
-max-complexity = 13
\ No newline at end of file
+max-complexity = 13
+per-file-ignores =
+ pipeline/recon/__init__.py:F401
+ pipeline/models/__init__.py:F401
+ pipeline/recon/web/__init__.py:F401
+ pipeline/luigi_targets/__init__.py:F401
+ tests/test_recon/test_parsers.py:F405
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..ffe294f
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+docs/* linguist-documentation
diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml
index f64bb4c..0a3faf2 100644
--- a/.github/workflows/pythonapp.yml
+++ b/.github/workflows/pythonapp.yml
@@ -28,14 +28,12 @@ jobs:
with:
args: ". --check"
- test-install:
+ test-shell:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- - name: Set up Golang
- uses: actions/setup-go@v1
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
@@ -47,8 +45,8 @@ jobs:
pipenv install -d
- name: Test with pytest
run: |
- pipenv install pytest cmd2 luigi
- pipenv run python -m pytest tests/test_install
+ pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
+ pipenv run python -m pytest tests/test_shell
test-recon:
@@ -67,7 +65,7 @@ jobs:
pipenv install -d
- name: Test with pytest
run: |
- pipenv install pytest cmd2 luigi
+ pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
pipenv run python -m pytest tests/test_recon
test-web:
@@ -87,5 +85,25 @@ jobs:
pipenv install -d
- name: Test with pytest
run: |
- pipenv install pytest cmd2 luigi
- pipenv run python -m pytest tests/test_web
\ No newline at end of file
+ pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
+ pipenv run python -m pytest tests/test_web
+
+ test-models:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v1
+ - name: Set up Python 3.7
+ uses: actions/setup-python@v1
+ with:
+ python-version: 3.7
+ - name: Set up pipenv
+ run: |
+ python -m pip install --upgrade pip
+ pip install pipenv
+ pipenv install -d
+ - name: Test with pytest
+ run: |
+ pipenv install pytest cmd2 luigi sqlalchemy python-libnmap
+ pipenv run python -m pytest tests/test_models
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index be20d39..6b11aba 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,9 +4,21 @@ repos:
hooks:
- id: black
language_version: python3.7
- args: ['.']
-- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v1.2.3
+ args: ['pipeline', 'tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models']
+- repo: https://gitlab.com/pycqa/flake8
+ rev: 3.7.9
hooks:
- id: flake8
-
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v2.5.0 # Use the ref you want to point at
+ hooks:
+ - id: trailing-whitespace
+ - id: debug-statements
+- repo: local
+ hooks:
+ - id: tests
+ name: run tests
+ entry: pytest
+ language: system
+ types: [python]
+ args: ['tests/test_web', 'tests/test_recon', 'tests/test_shell', 'tests/test_models']
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..c3c4db8
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,402 @@
+# Contributor's guide
+
+
+
+First of all, thank you for contributing! Please follow these steps to contribute:
+
+1. Find an issue that needs assistance by searching for the [Help Wanted](https://github.com/epi052/recon-pipeline/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) tag
+2. Let us know you're working on it by posting a comment on the issue
+3. Follow the [Contribution guidelines](#contribution-guidelines) to start working on the issue
+
+Remember to feel free to ask for help by leaving a comment within the Issue.
+
+Working on your first pull request? You can learn how from this *free* series
+[How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github).
+
+###### If you've found a bug that is not on the board, [follow these steps](README.md#found-a-bug).
+
+---
+
+## Contribution guidelines
+
+- [Prerequisites](#prerequisites)
+- [Forking the project](#forking-the-project)
+- [Creating a branch](#creating-a-branch)
+- [Setting up for recon-pipeline development](#setting-up-for-recon-pipeline-development)
+- [Making changes](#making-changes)
+- [Static code analysis](#static-code-analysis)
+- [Running the test suite](#running-the-test-suite)
+- [Squashing your commits](#squashing-your-commits)
+- [Creating a pull request](#creating-a-pull-request)
+- [How we review and merge pull requests](#how-we-review-and-merge-pull-requests)
+- [Next steps](#next-steps)
+- [Other resources](#other-resources)
+- [Advice](#advice)
+
+### Forking the project
+
+#### Setting up your system
+
+1. Install your favorite `git` client
+2. Create a parent projects directory on your system. For this guide, it will be assumed that it is `~/projects`.
+
+#### Forking recon-pipeline
+
+1. Go to the top-level recon-pipeline repository: Asset Inventory that discovers, learns, and (finally) lets you
+ secure everything.
+ Discover the asset inventory of all of your competitors to compare with your own.
+
+ Discover expired marketing campaigns, misspellings, SEO opportunities, and misuse of your
+ brand.
+
+ Know the places where PII is captured and stored, or where customer data may get exposed.
+
+ Know which of your assets have out of compliance technology, missing legal disclaimers, and
+ expired copyright notices.
+
+ Instantly see all of the internet-facing assets of your target company before acquisition to
+ identify risk
+ early.
+
+ You need to know what you own before you can secure what you own. Accurate Asset Inventory is
+ foundational to your security strategy.
+ Trusted By Founder of WhiteHat Security. World-Renowned Professional Hacker.
+ Jeremiah's career spans nearly 20 years and he has become one of the computer security industry's
+ biggest
+ names. Robert is a quarter-century veteran of infosec, spanning a career of
+ penetration testing, security architecture, security product management, and security research.
+ The world's top companies and agencies use Bit Discovery to
+ discover just about anything about their internet-facing assets. It's all there. Your domains, subdomains, exposed
+ technologies-organized into a speedy, searchable Inventory that stays updated automatically. Bit Discovery also inventories extensive technology information
+ about each asset in your inventory. Important changes to any of your assets are quickly identified.
+ Receive daily email summaries of changes. Bit Discovery is simple, fast, and easy to use. Thousands of technologies. Hundreds of ports. Zero hassle. GDPR is a prolific set of compliance mandates and has sweeping
+ implications for companies that serve customers in the European Union. A critical component of
+ compliance is starting off with a known set of assets and identifying the places where PII is
+ captured and stored, or where customer data may get exposed. Identify weak and outdated cipher suites. Locate the geographic region of your assets. See if your legal team is paying attention to the sites in question. Identify 3rd parties who may be capturing user traffic/sentiment. Find where 3rd party analytics software may be gathering PII. Capture the locations where social sites might correlate your users. Focus on the dynamic sites that are most likely to have customer data. Fast forward fraud investigations by locating sites of interest. See where your marketing team might be retargeting your users. Find where your marketing team might be gathering contacts. If you aren't patching you aren't compliant. Are you collecting customer data and on which websites? Asset Inventory that discovers, learns, and (finally) lets you
+ secure everything.
+ Discover the asset inventory of all of your competitors to compare with your own.
+
+ Discover expired marketing campaigns, misspellings, SEO opportunities, and misuse of your
+ brand.
+
+ Know the places where PII is captured and stored, or where customer data may get exposed.
+
+ Know which of your assets have out of compliance technology, missing legal disclaimers, and
+ expired copyright notices.
+
+ Instantly see all of the internet-facing assets of your target company before acquisition to
+ identify risk
+ early.
+
+ You need to know what you own before you can secure what you own. Accurate Asset Inventory is
+ foundational to your security strategy.
+ Trusted By Founder of WhiteHat Security. World-Renowned Professional Hacker.
+ Jeremiah's career spans nearly 20 years and he has become one of the computer security industry's
+ biggest
+ names. Robert is a quarter-century veteran of infosec, spanning a career of
+ penetration testing, security architecture, security product management, and security research.
+ The world's top companies and agencies use Bit Discovery to
+ discover just about anything about their internet-facing assets. It's all there. Your domains, subdomains, exposed
+ technologies-organized into a speedy, searchable Inventory that stays updated automatically. Bit Discovery also inventories extensive technology information
+ about each asset in your inventory. Important changes to any of your assets are quickly identified.
+ Receive daily email summaries of changes. Bit Discovery is simple, fast, and easy to use. Thousands of technologies. Hundreds of ports. Zero hassle. GDPR is a prolific set of compliance mandates and has sweeping
+ implications for companies that serve customers in the European Union. A critical component of
+ compliance is starting off with a known set of assets and identifying the places where PII is
+ captured and stored, or where customer data may get exposed. Identify weak and outdated cipher suites. Locate the geographic region of your assets. See if your legal team is paying attention to the sites in question. Identify 3rd parties who may be capturing user traffic/sentiment. Find where 3rd party analytics software may be gathering PII. Capture the locations where social sites might correlate your users. Focus on the dynamic sites that are most likely to have customer data. Fast forward fraud investigations by locating sites of interest. See where your marketing team might be retargeting your users. Find where your marketing team might be gathering contacts. If you aren't patching you aren't compliant. Are you collecting customer data and on which websites?Secure everything.
+
+ Competitive Analysis
+ Marketing & Brand Protection
+ GDPR Compliance
+ Legal
+ Mergers & Acquisitions
+ Information Security
+
+
+
+
+
+ Created by industry experts
+ Jeremiah Grossman
+
+ Robert Hansen
+
+ Discover
+
+
+ An inventory of everything
+ Go Beyond Asset Inventory
+ Know Instantly
+ Simple.
+
+ Technology Fingerprinting
+
+
+ GDPR Compliance
+
+
TLS Key Length & Protocol
+
+
Hosting Country
+
+
Outdated Copyright Notices
+
+
Advertising Networks
+
+
Google Analytics
+
+
Facebook Widgets
+
+
CRM
+
+
Fraud Investigation
+
+
SEO
+
+
Marketing Automation
+
+
Outdated Services
+
+
Forms
+ Secure everything.
+
+ Competitive Analysis
+ Marketing & Brand Protection
+ GDPR Compliance
+ Legal
+ Mergers & Acquisitions
+ Information Security
+
+
+
+
+
+ Created by industry experts
+ Jeremiah Grossman
+
+ Robert Hansen
+
+ Discover
+
+
+ An inventory of everything
+ Go Beyond Asset Inventory
+ Know Instantly
+ Simple.
+
+ Technology Fingerprinting
+
+
+ GDPR Compliance
+
+
TLS Key Length & Protocol
+
+
Hosting Country
+
+
Outdated Copyright Notices
+
+
Advertising Networks
+
+
Google Analytics
+
+
Facebook Widgets
+
+
CRM
+
+
Fraud Investigation
+
+
SEO
+
+
Marketing Automation
+
+
Outdated Services
+
+
Forms
+
Asset Inventory that discovers, learns, and (finally) lets you + secure everything.
+
+ + Discover the asset inventory of all of your competitors to compare with your own. +
++ Discover expired marketing campaigns, misspellings, SEO opportunities, and misuse of your + brand. +
++ Know the places where PII is captured and stored, or where customer data may get exposed. +
++ Know which of your assets have out of compliance technology, missing legal disclaimers, and + expired copyright notices. +
++ Instantly see all of the internet-facing assets of your target company before acquisition to + identify risk + early. +
++ You need to know what you own before you can secure what you own. Accurate Asset Inventory is + foundational to your security strategy. +
+Trusted By
+
+
+
+
+
+ Founder of WhiteHat Security. World-Renowned Professional Hacker. + Jeremiah's career spans nearly 20 years and he has become one of the computer security industry's + biggest + names.
+ More about Jeremiah > +
+ Robert is a quarter-century veteran of infosec, spanning a career of + penetration testing, security architecture, security product management, and security research. +
+ More about Robert > +
+ The world's top companies and agencies use Bit Discovery to + discover just about anything about their internet-facing assets.
+
+ It's all there. Your domains, subdomains, exposed + technologies-organized into a speedy, searchable Inventory that stays updated automatically.
+Bit Discovery also inventories extensive technology information + about each asset in your inventory.
+Important changes to any of your assets are quickly identified. + Receive daily email summaries of changes.
+Bit Discovery is simple, fast, and easy to use.
+
+ Thousands of technologies. Hundreds of ports. Zero hassle.
+
+ GDPR is a prolific set of compliance mandates and has sweeping + implications for companies that serve customers in the European Union. A critical component of + compliance is starting off with a known set of assets and identifying the places where PII is + captured and stored, or where customer data may get exposed.
+Identify weak and outdated cipher suites.
+Locate the geographic region of your assets.
+See if your legal team is paying attention to the sites in question.
+Identify 3rd parties who may be capturing user traffic/sentiment.
+Find where 3rd party analytics software may be gathering PII.
+Capture the locations where social sites might correlate your users.
+Focus on the dynamic sites that are most likely to have customer data.
+Fast forward fraud investigations by locating sites of interest.
+See where your marketing team might be retargeting your users.
+Find where your marketing team might be gathering contacts.
+If you aren't patching you aren't compliant.
+Are you collecting customer data and on which websites?
+diff --git a/tests/data/recon-results/aquatone-results/html/http__52_53_89_219__42099b4af021e53f.html b/tests/data/recon-results/aquatone-results/html/http__52_53_89_219__42099b4af021e53f.html new file mode 100644 index 0000000..a6825e9 --- /dev/null +++ b/tests/data/recon-results/aquatone-results/html/http__52_53_89_219__42099b4af021e53f.html @@ -0,0 +1,511 @@ + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + +
+ +