diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 33a48a1..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,12 +0,0 @@ -# These are supported funding model platforms - -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username -custom: PayPal.Me/codingo diff --git a/Interlace/interlace.py b/Interlace/interlace.py index 4a0b75e..4f84115 100644 --- a/Interlace/interlace.py +++ b/Interlace/interlace.py @@ -1,16 +1,17 @@ #!/usr/bin/python3 + import sys + from Interlace.lib.core.input import InputParser, InputHelper from Interlace.lib.core.output import OutputHelper, Level from Interlace.lib.threader import Pool def build_queue(arguments, output): - queue = list() - for command in InputHelper.process_commands(arguments): - output.terminal(Level.THREAD, command, "Added to Queue") - queue.append(command) - return queue + task_list = InputHelper.process_commands(arguments) + for task in task_list: + output.terminal(Level.THREAD, task.name(), "Added to Queue") + return task_list def main(): diff --git a/Interlace/lib/core/input.py b/Interlace/lib/core/input.py index 440e513..21b36dd 100644 --- a/Interlace/lib/core/input.py +++ b/Interlace/lib/core/input.py @@ -1,12 +1,12 @@ -from argparse import ArgumentParser -from netaddr import IPNetwork, IPRange, IPGlob -from Interlace.lib.core.output import OutputHelper, Level import os.path -from os import access, W_OK import sys -from re import compile -from random import sample, choice +from argparse import ArgumentParser from math import ceil +from random import sample, choice + +from netaddr import IPNetwork, IPRange, IPGlob + +from Interlace.lib.threader import Task class InputHelper(object): @@ -78,83 +78,134 @@ def _get_cidr_to_ips(cidr_range): return ips @staticmethod - def _replace_variable_for_commands(commands, variable, replacements): - tmp_commands = set() + def _process_port(port_type): + if "," in port_type: + return port_type.split(",") + elif "-" in port_type: + tmp = port_type.split("-") + begin_range = int(tmp[0]) + end_range = int(tmp[1]) + if begin_range >= end_range: + raise Exception("Invalid range provided") + return list(range(begin_range, end_range + 1)) + return [port_type] - test = list() - - for replacement in replacements: - for command in commands: - test.append(str(command).replace(variable, str(replacement))) - - tmp_commands.update(test) - return tmp_commands - @staticmethod - def _replace_variable_array(commands, variable, replacement): - tmp_commands = set() - counter = 0 + def _pre_process_commands(command_list, task_name, is_global_task=True): + """ + :param command_list: + :param task_name: all tasks have 'scope' and all scopes have unique names, global scope defaults '' + :param is_global_task: when True, signifies that all global tasks are meant to be run concurrently + :return: list of possibly re-adjusted commands + """ + task_block = [] + sibling = None + global_task = None + for command in command_list: + command = str(command).strip() + if not command: + continue + # the start or end of a command block + if command.startswith('_block:') and command.endswith('_'): + new_task_name = command.split('_block:')[1][:-1].strip() + # if this is the end of a block, then we're done + if task_name == new_task_name: + return task_block + # otherwise pre-process all the commands in this new `new_task_name` block + for task in InputHelper._pre_process_commands(command_list, new_task_name, False): + task_block.append(task) + sibling = task + continue + else: + # if a blocker is encountered, all commands following the blocker must wait until the last + # command in the block is executed. All block commands are synchronous + if command == '_blocker_': + global_task = sibling + continue + task = Task(command) + # if we're in the global scope and there was a previous _blocker_ encountered, we wait for the last + # child of the block + if is_global_task and global_task: + task.wait_for(global_task.get_lock()) + # all but the first command in a block scope wait for its predecessor + elif sibling and not is_global_task: + task.wait_for(sibling.get_lock()) + task_block.append(task) + sibling = task + return task_block - test = list() + @staticmethod + def _pre_process_hosts(host_ranges, destination_set, arguments): + for host in host_ranges: + host = host.replace(" ", "") + for ips in host.split(","): + # check if it is a domain name + if ips.split(".")[-1][0].isalpha(): + destination_set.add(ips) + continue + # checking for CIDR + if not arguments.nocidr and "/" in ips: + destination_set.update(InputHelper._get_cidr_to_ips(ips)) + # checking for IPs in a range + elif "-" in ips: + destination_set.update(InputHelper._get_ips_from_range(ips)) + # checking for glob ranges + elif "*" in ips: + destination_set.update(InputHelper._get_ips_from_glob(ips)) + else: + destination_set.add(ips) - if not variable in sample(commands, 1)[0]: - return commands + @staticmethod + def _replace_variable_with_commands(commands, variable, replacements): + def add_task(t, item_list): + if t not in set(item_list): + item_list.append(t) + tasks = [] for command in commands: - test.append(str(command).replace(variable, str(replacement[counter]))) - counter += 1 + for replacement in replacements: + if command.name().find(variable) != -1: + new_task = command.clone() + new_task.replace(variable, replacement) + add_task(new_task, tasks) + else: + add_task(command, tasks) + return tasks - tmp_commands.update(test) - return tmp_commands + @staticmethod + def _replace_variable_array(commands, variable, replacement): + if variable not in sample(commands, 1)[0]: + return + for counter, command in enumerate(commands): + command.replace(variable, str(replacement[counter])) @staticmethod def process_commands(arguments): - commands = set() + commands = list() ranges = set() targets = set() exclusions_ranges = set() exclusions = set() - final_commands = set() - output = OutputHelper(arguments) # removing the trailing slash if any - if arguments.output: - if arguments.output[-1] == "/": - arguments.output = arguments.output[:-1] + if arguments.output and arguments.output[-1] == "/": + arguments.output = arguments.output[:-1] if arguments.port: - if "," in arguments.port: - ports = arguments.port.split(",") - elif "-" in arguments.port: - tmp_ports = arguments.port.split("-") - if int(tmp_ports[0]) >= int(tmp_ports[1]): - raise Exception("Invalid range provided") - ports = list(range(int(tmp_ports[0]), int(tmp_ports[1]) + 1)) - else: - ports = [arguments.port] + ports = InputHelper._process_port(arguments.port) if arguments.realport: - if "," in arguments.realport: - real_ports = arguments.realport.split(",") - elif "-" in arguments.realport: - tmp_ports = arguments.realport.split("-") - if int(tmp_ports[0]) >= int(tmp_ports[1]): - raise Exception("Invalid range provided") - real_ports = list(range(int(tmp_ports[0]), int(tmp_ports[1]) + 1)) - else: - real_ports = [arguments.realport] + real_ports = InputHelper._process_port(arguments.realport) # process targets first if arguments.target: ranges.add(arguments.target) else: - targetFile = arguments.target_list + target_file = arguments.target_list if not sys.stdin.isatty(): - targetFile = sys.stdin - for target in targetFile: - if target.strip(): - ranges.add(target.strip()) + target_file = sys.stdin + ranges.update([target.strip() for target in target_file if target.strip()]) # process exclusions first if arguments.exclusions: @@ -162,50 +213,13 @@ def process_commands(arguments): else: if arguments.exclusions_list: for exclusion in arguments.exclusions_list: - exclusions_ranges.add(target.strip()) - - # removing elements that may have spaces (helpful for easily processing comma notation) - for target in ranges: - target = target.replace(" ", "") - - for ips in target.split(","): - - # check if it is a domain name - if ips.split(".")[-1][0].isalpha(): - targets.add(ips) - continue - # checking for CIDR - if not arguments.nocidr and "/" in ips: - targets.update(InputHelper._get_cidr_to_ips(ips)) - # checking for IPs in a range - elif "-" in ips: - targets.update(InputHelper._get_ips_from_range(ips)) - # checking for glob ranges - elif "*" in ips: - targets.update(InputHelper._get_ips_from_glob(ips)) - else: - targets.add(ips) + exclusion = exclusion.strip() + if exclusion: + exclusions.add(exclusion) # removing elements that may have spaces (helpful for easily processing comma notation) - for exclusion in exclusions_ranges: - exclusion = exclusion.replace(" ", "") - - for ips in exclusion.split(","): - # check if it is a domain name - if ips.split(".")[-1][0].isalpha(): - targets.add(ips) - continue - # checking for CIDR - if not arguments.nocidr and "/" in ips: - exclusions.update(InputHelper._get_cidr_to_ips(ips)) - # checking for IPs in a range - elif "-" in ips: - exclusions.update(InputHelper._get_ips_from_range(ips)) - # checking for glob ranges - elif "*" in ips: - exclusions.update(InputHelper._get_ips_from_glob(ips)) - else: - exclusions.add(ips) + InputHelper._pre_process_hosts(ranges, targets, arguments) + InputHelper._pre_process_hosts(exclusions_ranges, exclusions, arguments) # difference operation targets -= exclusions @@ -218,46 +232,40 @@ def process_commands(arguments): random_file = choice(files) if arguments.command: - commands.add(arguments.command.rstrip('\n')) + commands.append(arguments.command.rstrip('\n')) else: - for command in arguments.command_list: - commands.add(command.rstrip('\n')) + commands = InputHelper._pre_process_commands(arguments.command_list, '') - final_commands = InputHelper._replace_variable_for_commands(commands, "_target_", targets) - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_host_", targets) + commands = InputHelper._replace_variable_with_commands(commands, "_target_", targets) + commands = InputHelper._replace_variable_with_commands(commands, "_host_", targets) if arguments.port: - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_port_", ports) + commands = InputHelper._replace_variable_with_commands(commands, "_port_", ports) if arguments.realport: - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_realport_", real_ports) + commands = InputHelper._replace_variable_with_commands(commands, "_realport_", real_ports) if arguments.random: - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_random_", [random_file]) + commands = InputHelper._replace_variable_with_commands(commands, "_random_", [random_file]) if arguments.output: - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_output_", [arguments.output]) + commands = InputHelper._replace_variable_with_commands(commands, "_output_", [arguments.output]) if arguments.proto: if "," in arguments.proto: protocols = arguments.proto.split(",") else: protocols = arguments.proto - final_commands = InputHelper._replace_variable_for_commands(final_commands, "_proto_", protocols) - + commands = InputHelper._replace_variable_with_commands(commands, "_proto_", protocols) + # process proxies if arguments.proxy_list: - proxy_list = list() - for proxy in arguments.proxy_list: - if proxy.strip(): - proxy_list.append(proxy.strip()) - - if len(proxy_list) < len(final_commands): - proxy_list = ceil(len(final_commands) / len(proxy_list)) * proxy_list - - final_commands = InputHelper._replace_variable_array(final_commands, "_proxy_", proxy_list) + proxy_list = [proxy for proxy in arguments.proxy_list if proxy.strip()] + if len(proxy_list) < len(commands): + proxy_list = ceil(len(commands) / len(proxy_list)) * proxy_list - return final_commands + InputHelper._replace_variable_array(commands, "_proxy_", proxy_list) + return commands class InputParser(object): diff --git a/Interlace/lib/core/output.py b/Interlace/lib/core/output.py index a5caf5d..9e5d46d 100644 --- a/Interlace/lib/core/output.py +++ b/Interlace/lib/core/output.py @@ -1,7 +1,9 @@ -from colorclass import Color -from colorclass import disable_all_colors, enable_all_colors, is_enabled -from time import localtime, strftime from enum import IntEnum +from time import localtime, strftime + +from colorclass import Color +from colorclass import disable_all_colors + from Interlace.lib.core.__version__ import __version__ @@ -40,7 +42,7 @@ def terminal(self, level, target, command, message=""): 'target': target, 'command': command, 'message': message, - 'leader':leader + 'leader': leader } if not self.silent: diff --git a/Interlace/lib/threader.py b/Interlace/lib/threader.py index f33b882..26d11df 100644 --- a/Interlace/lib/threader.py +++ b/Interlace/lib/threader.py @@ -1,12 +1,61 @@ -import threading import subprocess -import os +from concurrent.futures import ThreadPoolExecutor +from multiprocessing import Event + from tqdm import tqdm +class Task(object): + def __init__(self, command): + self.task = command + self.self_lock = None + self.sibling_lock = None + + def __cmp__(self, other): + return self.name() == other.name() + + def __hash__(self): + return self.task.__hash__() + + def clone(self): + new_task = Task(self.task) + new_task.self_lock = self.self_lock + new_task.sibling_lock = self.sibling_lock + return new_task + + def replace(self, old, new): + self.task = self.task.replace(old, new) + + def run(self, t=False): + if self.sibling_lock: + self.sibling_lock.wait() + self._run_task(t) + if self.self_lock: + self.self_lock.set() + + def wait_for(self, _lock): + self.sibling_lock = _lock + + def name(self): + return self.task + + def get_lock(self): + if not self.self_lock: + self.self_lock = Event() + self.self_lock.clear() + return self.self_lock + + def _run_task(self, t=False): + if t: + s = subprocess.Popen(self.task, shell=True, stdout=subprocess.PIPE) + t.write(s.stdout.readline().decode("utf-8")) + else: + subprocess.Popen(self.task, shell=True) + + class Worker(object): - def __init__(self, queue, timeout, output, tqdm): - self.queue = queue + def __init__(self, task_queue, timeout, output, tqdm): + self.queue = task_queue self.timeout = timeout self.output = output self.tqdm = tqdm @@ -19,24 +68,16 @@ def __call__(self): if isinstance(self.tqdm, tqdm): self.tqdm.update(1) # run task - self.run_task(task, self.tqdm) + task.run(self.tqdm) else: - self.run_task(task) + task.run() except IndexError: break - @staticmethod - def run_task(task, t=False): - if t: - s = subprocess.Popen(task, shell=True, stdout=subprocess.PIPE) - t.write(s.stdout.readline().decode("utf-8")) - else: - subprocess.Popen(task, shell=True) - class Pool(object): - def __init__(self, max_workers, queue, timeout, output, progress_bar): - + def __init__(self, max_workers, task_queue, timeout, output, progress_bar): + # convert stdin input to integer max_workers = int(max_workers) @@ -45,34 +86,27 @@ def __init__(self, max_workers, queue, timeout, output, progress_bar): raise ValueError("Workers must be >= 1") # check if the queue is empty - if not queue: + if not task_queue: raise ValueError("The queue is empty") - self.queue = queue + self.queue = task_queue self.timeout = timeout self.output = output - self.max_workers = max_workers + self.max_workers = min(len(task_queue), max_workers) if not progress_bar: - self.tqdm = tqdm(total=len(queue)) + self.tqdm = tqdm(total=len(task_queue)) else: self.tqdm = True def run(self): - workers = [Worker(self.queue, self.timeout, self.output, self.tqdm) for w in range(self.max_workers)] - threads = [] - # run - for worker in workers: - thread = threading.Thread(target=worker) - thread.start() - threads.append(thread) - - # wait until all workers have completed their tasks - for thread in threads: - thread.join() + with ThreadPoolExecutor(self.max_workers) as executors: + for worker in workers: + executors.submit(worker) + # test harness if __name__ == "__main__": diff --git a/bar.test b/bar.test new file mode 100644 index 0000000..b8f5fbd --- /dev/null +++ b/bar.test @@ -0,0 +1,2 @@ +localhost +localone diff --git a/foo.test b/foo.test new file mode 100644 index 0000000..441c494 --- /dev/null +++ b/foo.test @@ -0,0 +1,16 @@ +echo hello +_block:file-creation_ +echo _target_ +echo _target_/a/ +echo _target_/output/scans +_block:file-creation_ +echo "Doing this" +_blocker_ +echo "Proceeding" +_block:file_ +echo _target_/out/scans +echo _target_/b/ +echo _target_/re/ +_block:file_ +_blocker_ +echo "Done" diff --git a/setup.py b/setup.py index a8ace8b..00041d2 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def dependencies(imported_file): with open("README.md") as file: - num_installed = False + num_installed = True try: import numpy num_installed = True @@ -35,7 +35,5 @@ def dependencies(imported_file): ] }, install_requires=dependencies('requirements.txt'), - setup_requires=['pytest-runner', - '' if num_installed else 'numpy==1.16.0'], tests_require=dependencies('test-requirements.txt'), include_package_data=True)