implemented run; mostly done with parse

pipeline/recon/helpers.py

@@ -13,7 +13,6 @@ from ..tools import tools
 
 def meets_requirements(requirements, exception):
     """ Determine if tools required to perform task are installed. """
-    print(tools.items())
     for tool in requirements:
         if not tools.get(tool).get("installed"):
             if exception:

pipeline/recon/web/__init__.py

@@ -1,3 +1,4 @@
+from .nuclei import NucleiScan
 from .aquatone import AquatoneScan
 from .gobuster import GobusterScan
 from .targets import GatherWebTargets

pipeline/recon/web/nuclei.py

@@ -1,3 +1,5 @@
+import json
+import logging
 import subprocess
 from pathlib import Path
 from urllib.parse import urlparse
@@ -6,10 +8,11 @@ import luigi
 from luigi.util import inherits
 from luigi.contrib.sqla import SQLAlchemyTarget
 
-from .targets import GatherWebTargets
 from ...tools import tools
+from ..config import defaults
+from .targets import GatherWebTargets
 from ..helpers import meets_requirements
-from ...models.endpoint_model import Endpoint
+from ...models.technology_model import Technology
 
 import pipeline.models.db_manager
 
@@ -46,6 +49,7 @@ class NucleiScan(luigi.Task):
 
     requirements = ["go", "nuclei", "masscan"]
     exception = True
+    threads = luigi.Parameter(default=defaults.get("threads"))
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -80,13 +84,98 @@ class NucleiScan(luigi.Task):
         Returns:
             luigi.contrib.sqla.SQLAlchemyTarget
         """
-        raise NotImplementedError
+        return {
+            "sqltarget": SQLAlchemyTarget(
+                connection_string=self.db_mgr.connection_string, target_table="technology", update_id=self.task_id
+            ),
+            "localtarget": luigi.LocalTarget(str(self.results_subfolder / "nuclei-results.json")),
+        }
 
     def run(self):
         """ Defines the options/arguments sent to nuclei after processing. """
         self.results_subfolder.mkdir(parents=True, exist_ok=True)
-        raise NotImplementedError
+
+        try:
+            self.threads = abs(int(self.threads))
+        except (TypeError, ValueError):
+            return logging.error("The value supplied to --threads must be a non-negative integer.")
+
+        scans = [
+            "panels",
+            # "subdomain-takeover",
+            # "tokens",
+            # "security-misconfiguration",
+            # "default-credentials",
+            # "dns",
+            # "cves",
+            # "vulnerabilities",
+            "technologies",
+            # "files",
+            # "workflows",
+            # "generic-detections",
+        ]
+
+        input_file = self.results_subfolder / "input-from-webtargets"
+
+        with open(input_file, "w") as f:
+            for target in self.db_mgr.get_all_hostnames():
+                for url_scheme in ("https://", "http://"):
+                    f.write(f"{url_scheme}{target}\n")
+
+        command = [
+            tools.get("nuclei").get("path"),
+            "-silent",
+            "-json",
+            "-c",
+            str(self.threads),
+            "-l",
+            str(input_file),
+            "-o",
+            self.output().get("localtarget").path,
+        ]
+
+        for scan in scans:
+            path = Path(defaults.get("tools-dir")) / "nuclei-templates" / scan
+            command.append("-t")
+            command.append(path)
+
+        subprocess.run(command)
+
+        input_file.unlink()
+
+        self.parse_output()
 
     def parse_output(self):
         """ Read nuclei .json results and add entries into specified database """
-        raise NotImplementedError
+        """ example data
+        
+        {"template":"tech-detect","type":"http","matched":"https://staging.bitdiscovery.com/","matcher_name":"jsdelivr","severity":"info","author":"hakluke","description":""}
+        {"template":"swagger-panel","type":"http","matched":"https://staging.bitdiscovery.com/api/swagger.yaml","severity":"info","author":"Ice3man","description":""}
+        """
+        tgt = None
+
+        with self.output().get("localtarget").open() as f:
+            for line in f:
+                try:
+                    entry = json.loads(line.strip())
+                except json.JSONDecodeError:
+                    continue
+
+                parsed_url = urlparse(entry.get("matched"))
+
+                if tgt is None:
+                    # should only hit the first line of each file
+                    tgt = self.db_mgr.get_or_create_target_by_ip_or_hostname(parsed_url.hostname)
+
+                if entry.get("template") == "tech-detect":
+                    technology = self.db_mgr.get_or_create(
+                        Technology, type=entry.get("type"), text=entry.get("matcher_name")
+                    )
+                    if technology not in tgt.technologies:
+                        tgt.technologies.append(technology)
+
+            # if tgt is not None:
+            #     self.db_mgr.add(tgt)
+            #     self.output().get("sqltarget").touch()
+        #
+        # self.db_mgr.close()