Fixed issue 52

Interlace/interlace.py

@@ -2,15 +2,24 @@
 import sys
 from Interlace.lib.core.input import InputParser, InputHelper
 from Interlace.lib.core.output import OutputHelper, Level
-from Interlace.lib.threader import Pool
+from Interlace.lib.threader import Pool, TaskBlock
+
+
+def print_command(level, command, message, output):
+    if isinstance(command, TaskBlock):
+        for c in command:
+            print_command(level, c, message, output)
+    else:
+        output.terminal(Level.THREAD, command.name(), "Added to Queue")
 
 
 def build_queue(arguments, output):
-    queue = list()
+    task_queue = InputHelper.process_commands(arguments)
-    for command in InputHelper.process_commands(arguments):
+    task_list = []
-        output.terminal(Level.THREAD, command, "Added to Queue")
+    for task in task_queue:
-        queue.append(command)
+        print_command(Level.THREAD, task, "Added to Queue", output)
-    return queue
+        task_list.append(task)
+    return task_list
 
 
 def main():

Interlace/lib/core/input.py

@@ -1,12 +1,12 @@
-from argparse import ArgumentParser
-from netaddr import IPNetwork, IPRange, IPGlob
-from Interlace.lib.core.output import OutputHelper, Level
 import os.path
-from os import access, W_OK
 import sys
-from re import compile
+from argparse import ArgumentParser
-from random import sample
 from math import ceil
+from os import access, W_OK
+from random import sample
+
+from netaddr import IPNetwork, IPRange, IPGlob
+from Interlace.lib.threader import TaskBlock, Task
 
 
 class InputHelper(object):
@@ -62,35 +62,83 @@ class InputHelper(object):
         return ips
 
     @staticmethod
-    def _replace_variable_for_commands(commands, variable, replacements):
+    def _process_port(port_type):
-        tmp_commands = set()
+        if "," in port_type:
+            return port_type.split(",")
+        elif "-" in port_type:
+            tmp = port_type.split("-")
+            begin_range = int(tmp[0])
+            end_range = int(tmp[1])
+            if begin_range >= end_range:
+                raise Exception("Invalid range provided")
+            return list(range(begin_range, end_range + 1))
+        return [port_type]
 
-        test = list()
+    @staticmethod
+    def _pre_process_commands(command_list, task_name):
+        task_block = TaskBlock(task_name)
+        parent_task = None
+        for command in command_list:
+            command = str(command).strip()
+            if not command:
+                continue
+            if command.startswith('_block:') and command.endswith('_'):
+                new_task_name = command.split('_block:')[1][:-1].strip()
+                if task_name == new_task_name:
+                    return task_block
+                task = InputHelper._pre_process_commands(command_list, new_task_name)
+            else:
+                task = Task(command)
+                if command == '_blocker_':
+                    parent_task = task_block.last()
+                    parent_task.set_lock()
+                    continue
+            if parent_task:
+                task.wait_for(parent_task.get_lock())
+            task_block.add_task(task)
+        return task_block
 
+    @staticmethod
+    def _pre_process_hosts(host_ranges, destination_set, arguments):
+        for host in host_ranges:
+            host = host.replace(" ", "")
+            for ips in host.split(","):
+                # check if it is a domain name
+                if ips.split(".")[-1][0].isalpha():
+                    destination_set.add(ips)
+                    continue
+                # checking for CIDR
+                if not arguments.nocidr and "/" in ips:
+                    destination_set.update(InputHelper._get_cidr_to_ips(ips))
+                # checking for IPs in a range
+                elif "-" in ips:
+                    destination_set.update(InputHelper._get_ips_from_range(ips))
+                # checking for glob ranges
+                elif "*" in ips:
+                    destination_set.update(InputHelper._get_ips_from_glob(ips))
+                else:
+                    destination_set.add(ips)
+
+    @staticmethod
+    def _replace_variable_with_commands(commands, variable, replacements):
         for replacement in replacements:
             for command in commands:
-                test.append(str(command).replace(variable, str(replacement)))
+                if isinstance(command, TaskBlock):
+                    InputHelper._replace_variable_with_commands(command, variable, replacements)
+                else:
+                    command.replace(variable, str(replacement))
 
-        tmp_commands.update(test)
-        return tmp_commands
-        
     @staticmethod
     def _replace_variable_array(commands, variable, replacement):
-        tmp_commands = set()
+        # TODO
-        counter = 0
+        if variable not in sample(commands, 1)[0]:
-
+            return
-        test = list()
-
-        if not variable in sample(commands, 1)[0]:
-            return commands
-
-        for command in commands:
-            test.append(str(command).replace(variable, str(replacement[counter])))
-            counter += 1
-
-        tmp_commands.update(test)
-        return tmp_commands
 
+        for counter, command in enumerate(commands):
+            if isinstance(command, TaskBlock):
+                InputHelper._replace_variable_array(command, variable, replacement)
+            else:
+                command.replace(variable, str(replacement[counter]))
 
     @staticmethod
     def process_commands(arguments):
@@ -99,98 +147,37 @@ class InputHelper(object):
         targets = set()
         exclusions_ranges = set()
         exclusions = set()
-        final_commands = set()
-        output = OutputHelper(arguments)
 
-        # checking for whether output is writable and whether it exists
+        # checking if output is writable and exists
         if arguments.output:
             if not access(arguments.output, W_OK):
                 raise Exception("Directory provided isn't writable")
 
         if arguments.port:
-            if "," in arguments.port:
+            ports = InputHelper._process_port(arguments.port)
-                ports = arguments.port.split(",")
-            elif "-" in arguments.port:
-                tmp_ports = arguments.port.split("-")
-                if int(tmp_ports[0]) >= int(tmp_ports[1]):
-                    raise Exception("Invalid range provided")
-                ports = list(range(int(tmp_ports[0]), int(tmp_ports[1]) + 1))
-            else:
-                ports = [arguments.port]
 
         if arguments.realport:
-            if "," in arguments.realport:
+            real_ports = InputHelper._process_port(arguments.realport)
-                real_ports = arguments.realport.split(",")
-            elif "-" in arguments.realport:
-                tmp_ports = arguments.realport.split("-")
-                if int(tmp_ports[0]) >= int(tmp_ports[1]):
-                    raise Exception("Invalid range provided")
-                real_ports = list(range(int(tmp_ports[0]), int(tmp_ports[1]) + 1))
-            else:
-                real_ports = [arguments.realport]
-
 
         # process targets first
         if arguments.target:
             ranges.add(arguments.target)
         else:
-            targetFile = arguments.target_list
+            target_file = arguments.target_list
             if not sys.stdin.isatty():
-                targetFile = sys.stdin
+                target_file = sys.stdin
-            for target in targetFile:
+            ranges.update([target for target in target_file if target.strip()])
-                if target.strip():
-                    ranges.add(target.strip())          
 
         # process exclusions first
         if arguments.exclusions:
             exclusions_ranges.add(arguments.exclusions)
         else:
             if arguments.exclusions_list:
-                for exclusion in arguments.exclusions_list:
+                exclusions_ranges.update([exclusion for exclusion in arguments.exclusions_list if exclusion.strip()])
-                    exclusions_ranges.add(target.strip())
 
         # removing elements that may have spaces (helpful for easily processing comma notation)
-        for target in ranges:
+        InputHelper._pre_process_hosts(ranges, targets, arguments)
-            target = target.replace(" ", "")
+        InputHelper._pre_process_hosts(exclusions_ranges, exclusions, arguments)
-
-            for ips in target.split(","):
-
-                # check if it is a domain name
-                if ips.split(".")[-1][0].isalpha():
-                    targets.add(ips)
-                    continue
-                # checking for CIDR
-                if not arguments.nocidr and "/" in ips:
-                    targets.update(InputHelper._get_cidr_to_ips(ips))
-                # checking for IPs in a range 
-                elif "-" in ips:
-                    targets.update(InputHelper._get_ips_from_range(ips))
-                # checking for glob ranges
-                elif "*" in ips:
-                    targets.update(InputHelper._get_ips_from_glob(ips))
-                else:
-                    targets.add(ips)
-
-        # removing elements that may have spaces (helpful for easily processing comma notation)
-        for exclusion in exclusions_ranges:
-            exclusion = exclusion.replace(" ", "")
-
-            for ips in exclusion.split(","):
-                # check if it is a domain name
-                if ips.split(".")[-1][0].isalpha():
-                    targets.add(ips)
-                    continue
-                # checking for CIDR
-                if not arguments.nocidr and "/" in ips:
-                    exclusions.update(InputHelper._get_cidr_to_ips(ips))
-                # checking for IPs in a range 
-                elif "-" in ips:
-                    exclusions.update(InputHelper._get_ips_from_range(ips))
-                # checking for glob ranges
-                elif "*" in ips:
-                    exclusions.update(InputHelper._get_ips_from_glob(ips))
-                else:
-                    exclusions.add(ips)
 
         # difference operation
         targets -= exclusions
@@ -201,41 +188,36 @@ class InputHelper(object):
         if arguments.command:
             commands.add(arguments.command.rstrip('\n'))
         else:
-            for command in arguments.command_list:
+            tasks = InputHelper._pre_process_commands(arguments.command_list, '')
-                commands.add(command.rstrip('\n'))
+            commands.update(tasks.get_tasks())
 
-        final_commands = InputHelper._replace_variable_for_commands(commands, "_target_", targets)
+        InputHelper._replace_variable_with_commands(commands, "_target_", targets)
-        final_commands = InputHelper._replace_variable_for_commands(final_commands, "_host_", targets)
+        InputHelper._replace_variable_with_commands(commands, "_host_", targets)
 
         if arguments.port:
-            final_commands = InputHelper._replace_variable_for_commands(final_commands, "_port_", ports)
+            InputHelper._replace_variable_with_commands(commands, "_port_", ports)
 
         if arguments.realport:
-            final_commands = InputHelper._replace_variable_for_commands(final_commands, "_realport_", real_ports)
+            InputHelper._replace_variable_with_commands(commands, "_realport_", real_ports)
 
         if arguments.output:
-            final_commands = InputHelper._replace_variable_for_commands(final_commands, "_output_", [arguments.output])
+            InputHelper._replace_variable_with_commands(commands, "_output_", [arguments.output])
 
         if arguments.proto:
             if "," in arguments.proto:
                 protocols = arguments.proto.split(",")
             else:
                 protocols = arguments.proto
-            final_commands = InputHelper._replace_variable_for_commands(final_commands, "_proto_", protocols)
+            InputHelper._replace_variable_with_commands(commands, "_proto_", protocols)
-        
+
         # process proxies
         if arguments.proxy_list:
-            proxy_list = list()
+            proxy_list = [proxy for proxy in arguments.proxy_list if proxy.strip()]
-            for proxy in arguments.proxy_list:
+            if len(proxy_list) < len(commands):
-                if proxy.strip():
+                proxy_list = ceil(len(commands) / len(proxy_list)) * proxy_list
-                    proxy_list.append(proxy.strip())
 
-            if len(proxy_list) < len(final_commands):
+            InputHelper._replace_variable_array(commands, "_proxy_", proxy_list)
-                proxy_list = ceil(len(final_commands) / len(proxy_list)) * proxy_list
+        return commands
-
-            final_commands = InputHelper._replace_variable_array(final_commands, "_proxy_", proxy_list)
-
-        return final_commands
 
 
 class InputParser(object):

Interlace/lib/core/output.py

@@ -1,7 +1,9 @@
-from colorclass import Color
-from colorclass import disable_all_colors, enable_all_colors, is_enabled
-from time import localtime, strftime
 from enum import IntEnum
+from time import localtime, strftime
+
+from colorclass import Color
+from colorclass import disable_all_colors
+
 from Interlace.lib.core.__version__ import __version__
 
 
@@ -40,7 +42,7 @@ class OutputHelper(object):
            'target': target,
            'command': command,
            'message': message,
-            'leader':leader
+            'leader': leader
         }
 
         if not self.silent:

Interlace/lib/threader.py

@@ -1,12 +1,92 @@
-import threading
 import subprocess
-import os
+from multiprocessing import Event
+from threading import Thread
+
 from tqdm import tqdm
 
 
+class Task(object):
+    def __init__(self, command):
+        self.task = command
+        self._lock = None
+        self._waiting_for_task = False
+
+    def __hash__(self):
+        return self.task.__hash__()
+
+    def replace(self, old, new):
+        self.task = self.task.replace(old, new)
+
+    def run(self, t=False):
+        if not self._waiting_for_task:
+            self._run_task(t)
+            if self._lock:
+                self._lock.set()
+        else:
+            self._lock.wait()
+            self._run_task(t)
+
+    def wait_for(self, lock):
+        self._lock = lock
+        self._waiting_for_task = True
+
+    def set_lock(self):
+        if not self._lock:
+            self._lock = Event()
+            self._lock.clear()
+
+    def name(self):
+        return self.task
+
+    def get_lock(self):
+        return self._lock
+
+    def _run_task(self, t=False):
+        if t:
+            s = subprocess.Popen(self.task, shell=True, stdout=subprocess.PIPE)
+            t.write(s.stdout.readline().decode("utf-8"))
+        else:
+            subprocess.Popen(self.task, shell=True)
+
+
+class TaskBlock(Task):
+    def __init__(self, name):
+        super().__init__('')
+        self._name = name
+        self.tasks = []
+
+    def name(self):
+        return self._name
+
+    def add_task(self, task):
+        self.tasks.append(task)
+
+    def __len__(self):
+        return len(self.tasks)
+
+    def __hash__(self):
+        hash_value = 0
+        for t in self.tasks:
+            hash_value ^= t.__hash__()
+        return hash_value
+
+    def __iter__(self):
+        return self.tasks.__iter__()
+
+    def last(self):
+        return self.tasks[-1]
+
+    def _run_task(self, t=False):
+        for task in self.tasks:
+            task._run_task(t)
+
+    def get_tasks(self):
+        return self.tasks
+
+
 class Worker(object):
-    def __init__(self, queue, timeout, output, tqdm):
+    def __init__(self, task_queue, timeout, output, tqdm):
-        self.queue = queue
+        self.queue = task_queue
         self.timeout = timeout
         self.output = output
         self.tqdm = tqdm
@@ -19,24 +99,16 @@ class Worker(object):
                 if isinstance(self.tqdm, tqdm):
                     self.tqdm.update(1)
                     # run task
-                    self.run_task(task, self.tqdm)
+                    task.run(self.tqdm)
                 else:
-                    self.run_task(task)
+                    task.run()
             except IndexError:
                 break
 
-    @staticmethod
-    def run_task(task, t=False):
-        if t:
-            s = subprocess.Popen(task, shell=True, stdout=subprocess.PIPE)
-            t.write(s.stdout.readline().decode("utf-8"))
-        else:
-            subprocess.Popen(task, shell=True)
-
 
 class Pool(object):
-    def __init__(self, max_workers, queue, timeout, output, progress_bar):
+    def __init__(self, max_workers, task_queue, timeout, output, progress_bar):
-        
+
         # convert stdin input to integer
         max_workers = int(max_workers)
 
@@ -45,28 +117,26 @@ class Pool(object):
             raise ValueError("Workers must be >= 1")
 
         # check if the queue is empty
-        if not queue:
+        if not task_queue:
             raise ValueError("The queue is empty")
 
-        self.queue = queue
+        self.queue = task_queue
         self.timeout = timeout
         self.output = output
-        self.max_workers = max_workers
+        self.max_workers = min(len(task_queue), max_workers)
 
         if not progress_bar:
-            self.tqdm = tqdm(total=len(queue))
+            self.tqdm = tqdm(total=len(task_queue))
         else:
             self.tqdm = True
 
     def run(self):
-
         workers = [Worker(self.queue, self.timeout, self.output, self.tqdm) for w in range(self.max_workers)]
         threads = []
 
-
         # run
         for worker in workers:
-            thread = threading.Thread(target=worker)
+            thread = Thread(target=worker)
             thread.start()
             threads.append(thread)
 
@@ -74,6 +144,7 @@ class Pool(object):
         for thread in threads:
             thread.join()
 
+
 # test harness
 if __name__ == "__main__":
     tasks = ["sleep 1",