diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 33a48a1..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,12 +0,0 @@ -# These are supported funding model platforms - -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username -custom: PayPal.Me/codingo diff --git a/.gitignore b/.gitignore index 3ebf85d..884dd95 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ wheels/ *.egg-info/ .installed.cfg *.egg +*.test MANIFEST # PyInstaller diff --git a/Interlace/interlace.py b/Interlace/interlace.py index 4a0b75e..a112786 100644 --- a/Interlace/interlace.py +++ b/Interlace/interlace.py @@ -1,16 +1,24 @@ #!/usr/bin/python3 import sys + from Interlace.lib.core.input import InputParser, InputHelper from Interlace.lib.core.output import OutputHelper, Level -from Interlace.lib.threader import Pool +from Interlace.lib.threader import Pool, TaskBlock + + +def print_command(level, command, message, output): + if isinstance(command, TaskBlock): + for c in command: + print_command(level, c, message, output) + else: + output.terminal(Level.THREAD, command.name(), "Added to Queue") def build_queue(arguments, output): - queue = list() - for command in InputHelper.process_commands(arguments): - output.terminal(Level.THREAD, command, "Added to Queue") - queue.append(command) - return queue + task_list = InputHelper.process_commands(arguments) + for task in task_list: + print_command(Level.THREAD, task, "Added to Queue", output) + return task_list def main(): diff --git a/Interlace/lib/core/__version__.py b/Interlace/lib/core/__version__.py index 0bb84ff..c24ed73 100644 --- a/Interlace/lib/core/__version__.py +++ b/Interlace/lib/core/__version__.py @@ -1 +1 @@ -__version__ = '1.5.3' +__version__ = '1.5.4' diff --git a/Interlace/lib/core/input.py b/Interlace/lib/core/input.py index 22823a3..e6566fb 100644 --- a/Interlace/lib/core/input.py +++ b/Interlace/lib/core/input.py @@ -1,12 +1,12 @@ -from argparse import ArgumentParser -from netaddr import IPNetwork, IPRange, IPGlob -from Interlace.lib.core.output import OutputHelper, Level import os.path -from os import access, W_OK import sys -from re import compile -from random import sample +from argparse import ArgumentParser from math import ceil +from os import access, W_OK +from random import sample + +from netaddr import IPNetwork, IPRange, IPGlob +from Interlace.lib.threader import TaskBlock, Task class InputHelper(object): @@ -62,84 +62,125 @@ class InputHelper(object): return ips @staticmethod - def _replace_variable_for_commands(commands, variable, replacements): - tmp_commands = set() + def _process_port(port_type): + if "," in port_type: + return port_type.split(",") + elif "-" in port_type: + tmp = port_type.split("-") + begin_range = int(tmp[0]) + end_range = int(tmp[1]) + if begin_range >= end_range: + raise Exception("Invalid range provided") + return list(range(begin_range, end_range + 1)) + return [port_type] - test = list() - - for replacement in replacements: - for command in commands: - test.append(str(command).replace(variable, str(replacement))) - - tmp_commands.update(test) - return tmp_commands - @staticmethod - def _replace_variable_array(commands, variable, replacement): - tmp_commands = set() - counter = 0 + def _pre_process_commands(command_list, task_name): + task_block = TaskBlock(task_name) + parent_task = None + for command in command_list: + command = str(command).strip() + if not command: + continue + if command.startswith('_block:') and command.endswith('_'): + new_task_name = command.split('_block:')[1][:-1].strip() + if task_name == new_task_name: + return task_block + task = InputHelper._pre_process_commands(command_list, new_task_name) + else: + task = Task(command) + if command == '_blocker_': + parent_task = task_block.last() + parent_task.set_lock() + continue + if parent_task: + task.wait_for(parent_task.get_lock()) + task_block.add_task(task) + return task_block - test = list() + @staticmethod + def _pre_process_hosts(host_ranges, destination_set, arguments): + for host in host_ranges: + host = host.replace(" ", "") + for ips in host.split(","): + # check if it is a domain name + if ips.split(".")[-1][0].isalpha(): + destination_set.add(ips) + continue + # checking for CIDR + if not arguments.nocidr and "/" in ips: + destination_set.update(InputHelper._get_cidr_to_ips(ips)) + # checking for IPs in a range + elif "-" in ips: + destination_set.update(InputHelper._get_ips_from_range(ips)) + # checking for glob ranges + elif "*" in ips: + destination_set.update(InputHelper._get_ips_from_glob(ips)) + else: + destination_set.add(ips) - if not variable in sample(commands, 1)[0]: - return commands + @staticmethod + def _replace_variable_with_commands(commands, variable, replacements): + foo = [] + + def add_task(t): + if t not in set(foo): + foo.append(t) for command in commands: - test.append(str(command).replace(variable, str(replacement[counter]))) - counter += 1 + is_task = not isinstance(command, TaskBlock) + for replacement in replacements: + if is_task and command.name().find(variable) != -1: + new_task = command.clone() + new_task.replace(variable, replacement) + add_task(new_task) + elif is_task and command not in set(foo): + add_task(command) + elif not is_task: + tasks = [task for task in command.get_tasks()] + command.clear_tasks() + for r in InputHelper._replace_variable_with_commands(tasks, variable, replacements): + command.add_task(r) + add_task(command) + return foo - tmp_commands.update(test) - return tmp_commands + @staticmethod + def _replace_variable_array(commands, variable, replacement): + # TODO + if variable not in sample(commands, 1)[0]: + return + for counter, command in enumerate(commands): + if isinstance(command, TaskBlock): + InputHelper._replace_variable_array(command, variable, replacement) + else: + command.replace(variable, str(replacement[counter])) @staticmethod def process_commands(arguments): - commands = set() + commands = list() ranges = set() targets = set() exclusions_ranges = set() exclusions = set() - final_commands = set() - output = OutputHelper(arguments) - # checking for whether output is writable and whether it exists - if arguments.output: - if not access(arguments.output, W_OK): - raise Exception("Directory provided isn't writable") + if arguments.output and arguments.output[-1] == "/": + arguments.output = arguments.output[:-1] if arguments.port: - if "," in arguments.port: - ports = arguments.port.split(",") - elif "-" in arguments.port: - tmp_ports = arguments.port.split("-") - if int(tmp_ports[0]) >= int(tmp_ports[1]): - raise Exception("Invalid range provided") - ports = list(range(int(tmp_ports[0]), int(tmp_ports[1]) + 1)) - else: - ports = [arguments.port] + ports = InputHelper._process_port(arguments.port) if arguments.realport: - if "," in arguments.realport: - real_ports = arguments.realport.split(",") - elif "-" in arguments.realport: - tmp_ports = arguments.realport.split("-") - if int(tmp_ports[0]) >= int(tmp_ports[1]): - raise Exception("Invalid range provided") - real_ports = list(range(int(tmp_ports[0]), int(tmp_ports[1]) + 1)) - else: - real_ports = [arguments.realport] - + real_ports = InputHelper._process_port(arguments.realport) # process targets first if arguments.target: ranges.add(arguments.target) else: - targetFile = arguments.target_list + target_file = arguments.target_list if not sys.stdin.isatty(): - targetFile = sys.stdin - for target in targetFile: - if target.strip(): - ranges.add(target.strip()) + target_file = sys.stdin + ranges.update([target.strip() for target in target_file if target.strip()]) # process exclusions first if arguments.exclusions: @@ -147,50 +188,13 @@ class InputHelper(object): else: if arguments.exclusions_list: for exclusion in arguments.exclusions_list: - exclusions_ranges.add(target.strip()) + exclusion = exclusion.strip() + if exclusion: + exclusions.add(exclusion) # removing elements that may have spaces (helpful for easily processing comma notation) - for target in ranges: - target = target.replace(" ", "") - - for ips in target.split(","): - - # check if it is a domain name - if ips.split(".")[-1][0].isalpha(): - targets.add(ips) - continue - # checking for CIDR - if not arguments.nocidr and "/" in ips: - targets.update(InputHelper._get_cidr_to_ips(ips)) - # checking for IPs in a range - elif "-" in ips: - targets.update(InputHelper._get_ips_from_range(ips)) - # checking for glob ranges - elif "*" in ips: - targets.update(InputHelper._get_ips_from_glob(ips)) - else: - targets.add(ips) - - # removing elements that may have spaces (helpful for easily processing comma notation) - for exclusion in exclusions_ranges: - exclusion = exclusion.replace(" ", "") - - for ips in exclusion.split(","): - # check if it is a domain name - if ips.split(".")[-1][0].isalpha(): - targets.add(ips) - continue - # checking for CIDR - if not arguments.nocidr and "/" in ips: - exclusions.update(InputHelper._get_cidr_to_ips(ips)) - # checking for IPs in a range - elif "-" in ips: - exclusions.update(InputHelper._get_ips_from_range(ips)) - # checking for glob ranges - elif "*" in ips: - exclusions.update(InputHelper._get_ips_from_glob(ips)) - else: - exclusions.add(ips) + InputHelper._pre_process_hosts(ranges, targets, arguments) + InputHelper._pre_process_hosts(exclusions_ranges, exclusions, arguments) # difference operation targets -= exclusions @@ -199,43 +203,38 @@ class InputHelper(object): raise Exception("No target provided, or empty target list") if arguments.command: - commands.add(arguments.command.rstrip('\n')) + commands.append(arguments.command.rstrip('\n')) else: - for command in arguments.command_list: - commands.add(command.rstrip('\n')) + tasks = InputHelper._pre_process_commands(arguments.command_list, '') + commands = tasks.get_tasks() - final_commands = InputHelper._replace_variable_for_commands(commands, "_target_", targets) - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_host_", targets) + commands = InputHelper._replace_variable_with_commands(commands, "_target_", targets) + commands = InputHelper._replace_variable_with_commands(commands, "_host_", targets) if arguments.port: - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_port_", ports) + commands = InputHelper._replace_variable_with_commands(commands, "_port_", ports) if arguments.realport: - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_realport_", real_ports) + commands = InputHelper._replace_variable_with_commands(commands, "_realport_", real_ports) if arguments.output: - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_output_", [arguments.output]) + commands = InputHelper._replace_variable_with_commands(commands, "_output_", [arguments.output]) if arguments.proto: if "," in arguments.proto: protocols = arguments.proto.split(",") else: protocols = arguments.proto - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_proto_", protocols) - + commands = InputHelper._replace_variable_with_commands(commands, "_proto_", protocols) + # process proxies if arguments.proxy_list: - proxy_list = list() - for proxy in arguments.proxy_list: - if proxy.strip(): - proxy_list.append(proxy.strip()) + proxy_list = [proxy for proxy in arguments.proxy_list if proxy.strip()] + if len(proxy_list) < len(commands): + proxy_list = ceil(len(commands) / len(proxy_list)) * proxy_list - if len(proxy_list) < len(final_commands): - proxy_list = ceil(len(final_commands) / len(proxy_list)) * proxy_list - - final_commands = InputHelper._replace_variable_array(final_commands, "_proxy_", proxy_list) - - return final_commands + InputHelper._replace_variable_array(commands, "_proxy_", proxy_list) + return commands class InputParser(object): diff --git a/Interlace/lib/core/output.py b/Interlace/lib/core/output.py index a5caf5d..9e5d46d 100644 --- a/Interlace/lib/core/output.py +++ b/Interlace/lib/core/output.py @@ -1,7 +1,9 @@ -from colorclass import Color -from colorclass import disable_all_colors, enable_all_colors, is_enabled -from time import localtime, strftime from enum import IntEnum +from time import localtime, strftime + +from colorclass import Color +from colorclass import disable_all_colors + from Interlace.lib.core.__version__ import __version__ @@ -40,7 +42,7 @@ class OutputHelper(object): 'target': target, 'command': command, 'message': message, - 'leader':leader + 'leader': leader } if not self.silent: diff --git a/Interlace/lib/threader.py b/Interlace/lib/threader.py index f33b882..a88cbf9 100644 --- a/Interlace/lib/threader.py +++ b/Interlace/lib/threader.py @@ -1,12 +1,104 @@ -import threading import subprocess -import os +from multiprocessing import Event +from threading import Thread + from tqdm import tqdm +class Task(object): + def __init__(self, command): + self.task = command + self._lock = None + self._waiting_for_task = False + + def __cmp__(self, other): + return self.name() == other.name() + + def __hash__(self): + return self.task.__hash__() + + def clone(self): + new_task = Task(self.task) + new_task._lock = self._lock + new_task._waiting_for_task = self._waiting_for_task + return new_task + + def replace(self, old, new): + self.task = self.task.replace(old, new) + + def run(self, t=False): + if not self._waiting_for_task: + self._run_task(t) + if self._lock: + self._lock.set() + else: + self._lock.wait() + self._run_task(t) + + def wait_for(self, lock): + self._lock = lock + self._waiting_for_task = True + + def set_lock(self): + if not self._lock: + self._lock = Event() + self._lock.clear() + + def name(self): + return self.task + + def get_lock(self): + return self._lock + + def _run_task(self, t=False): + if t: + s = subprocess.Popen(self.task, shell=True, stdout=subprocess.PIPE) + t.write(s.stdout.readline().decode("utf-8")) + else: + subprocess.Popen(self.task, shell=True) + + +class TaskBlock(Task): + def __init__(self, name): + super().__init__('') + self._name = name + self.tasks = [] + + def name(self): + return self._name + + def add_task(self, task): + self.tasks.append(task) + + def clear_tasks(self): + self.tasks.clear() + + def __len__(self): + return len(self.tasks) + + def __hash__(self): + hash_value = 0 + for t in self.tasks: + hash_value ^= t.__hash__() + return hash_value + + def __iter__(self): + return self.tasks.__iter__() + + def last(self): + return self.tasks[-1] + + def _run_task(self, t=False): + for task in self.tasks: + task._run_task(t) + + def get_tasks(self): + return self.tasks + + class Worker(object): - def __init__(self, queue, timeout, output, tqdm): - self.queue = queue + def __init__(self, task_queue, timeout, output, tqdm): + self.queue = task_queue self.timeout = timeout self.output = output self.tqdm = tqdm @@ -19,24 +111,16 @@ class Worker(object): if isinstance(self.tqdm, tqdm): self.tqdm.update(1) # run task - self.run_task(task, self.tqdm) + task.run(self.tqdm) else: - self.run_task(task) + task.run() except IndexError: break - @staticmethod - def run_task(task, t=False): - if t: - s = subprocess.Popen(task, shell=True, stdout=subprocess.PIPE) - t.write(s.stdout.readline().decode("utf-8")) - else: - subprocess.Popen(task, shell=True) - class Pool(object): - def __init__(self, max_workers, queue, timeout, output, progress_bar): - + def __init__(self, max_workers, task_queue, timeout, output, progress_bar): + # convert stdin input to integer max_workers = int(max_workers) @@ -45,28 +129,26 @@ class Pool(object): raise ValueError("Workers must be >= 1") # check if the queue is empty - if not queue: + if not task_queue: raise ValueError("The queue is empty") - self.queue = queue + self.queue = task_queue self.timeout = timeout self.output = output - self.max_workers = max_workers + self.max_workers = min(len(task_queue), max_workers) if not progress_bar: - self.tqdm = tqdm(total=len(queue)) + self.tqdm = tqdm(total=len(task_queue)) else: self.tqdm = True def run(self): - workers = [Worker(self.queue, self.timeout, self.output, self.tqdm) for w in range(self.max_workers)] threads = [] - # run for worker in workers: - thread = threading.Thread(target=worker) + thread = Thread(target=worker) thread.start() threads.append(thread) @@ -74,6 +156,7 @@ class Pool(object): for thread in threads: thread.join() + # test harness if __name__ == "__main__": tasks = ["sleep 1", diff --git a/setup.py b/setup.py index a8ace8b..00041d2 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def dependencies(imported_file): with open("README.md") as file: - num_installed = False + num_installed = True try: import numpy num_installed = True @@ -35,7 +35,5 @@ with open("README.md") as file: ] }, install_requires=dependencies('requirements.txt'), - setup_requires=['pytest-runner', - '' if num_installed else 'numpy==1.16.0'], tests_require=dependencies('test-requirements.txt'), include_package_data=True)