From 6137fb35585bbaba128d5c715ed045eae8b86517 Mon Sep 17 00:00:00 2001 From: cr0hn Date: Tue, 23 Feb 2016 14:24:21 +0100 Subject: [PATCH] add: scan module with multithreading and async scans. add: new requirements add: default action for modules add: banner fix: set log level fix: cmd boolean args handler fix: log messages fix: metavar: __name__ -> __tool_name__ --- .idea/workspace.xml | 763 +++++++++++-------- enteletaor_lib/api.py | 4 +- enteletaor_lib/config.py | 32 +- enteletaor_lib/enteletaor.py | 2 +- enteletaor_lib/libs/contrib/__init__.py | 1 + enteletaor_lib/libs/contrib/inetnum.py | 63 ++ enteletaor_lib/libs/core/cmd.py | 47 +- enteletaor_lib/libs/core/config.py | 4 +- enteletaor_lib/libs/core/logger.py | 8 +- enteletaor_lib/libs/hooks/config/__init__.py | 3 +- enteletaor_lib/modules/__init__.py | 5 +- enteletaor_lib/modules/scan/patch.py | 113 +++ requirements.txt | 10 +- 13 files changed, 717 insertions(+), 338 deletions(-) create mode 100644 enteletaor_lib/libs/contrib/__init__.py create mode 100644 enteletaor_lib/libs/contrib/inetnum.py create mode 100644 enteletaor_lib/modules/scan/patch.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 83d9922..6b81eac 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,49 +2,19 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + @@ -65,6 +35,7 @@ + @@ -81,20 +52,27 @@ - - - + + + - - + + - - - - - - + + + + + + + + + + + + + @@ -118,13 +96,6 @@ @@ -180,8 +158,8 @@ @@ -209,7 +187,6 @@ - @@ -242,27 +219,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + - - - - - - - @@ -270,8 +300,15 @@ + + + + + + + - + + + - + @@ -634,8 +689,9 @@ - - + + + @@ -723,33 +779,69 @@ + + + + + + + + + - + - - - + + + + + - - - - - + + - + + + + + + + + + + + + + + + + + + + + + + + @@ -784,6 +877,21 @@ 29 - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -928,11 +935,11 @@ - - - - - + + + + + @@ -969,23 +976,6 @@ - - - - - - - - - - - - - - - - - @@ -1024,17 +1014,6 @@ - - - - - - - - - - - @@ -1070,37 +1049,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1115,43 +1067,32 @@ + + + - - - - + + + - - - - - - - - - - - - - - - + + @@ -1168,7 +1109,7 @@ - + @@ -1178,25 +1119,16 @@ - + - - - - - - - - - @@ -1213,33 +1145,250 @@ - - - - - - - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + diff --git a/enteletaor_lib/api.py b/enteletaor_lib/api.py index 0dadeb2..a3b3305 100644 --- a/enteletaor_lib/api.py +++ b/enteletaor_lib/api.py @@ -29,9 +29,9 @@ def run_console(config): if not isinstance(config, GlobalExecutionParameters): raise TypeError("Expected GlobalParameters, got '%s' instead" % type(config)) - logging.warning("Starting Enteletaor execution") + logging.error("Starting Enteletaor execution") run(config) - logging.warning("Done!") + logging.error("Done!") # ---------------------------------------------------------------------- diff --git a/enteletaor_lib/config.py b/enteletaor_lib/config.py index 999601e..7a9a3fd 100644 --- a/enteletaor_lib/config.py +++ b/enteletaor_lib/config.py @@ -1,9 +1,39 @@ # -*- coding: utf-8 -*- -__name__ = "enteletaor" +__tool_name__ = "enteletaor" __author__ = "Daniel Garcia (cr0hn) - @ggdaniel" __site__ = "https://github.com/cr0hn/enteletaor" __version__ = "1.0.0" +__banner__ = """ + `` + `````..`` + ``..-:::::--.```...----` + ````...---:://////:--.--::::. + ````...--::://///////::::://-``` `` + `.--:////////////////////////-`````` + .-///////////:::/:://///:///////-..... + `.::///////////:////////:///://+///:...--. + `--:::::///:::////////////////://////------ + `.-----:///:://////:--..--://///://///:::::: + `.....-:////////+/:-.` ``-:///:/::///////- +``````-//////::++/:.` `.://///:///////` + ```-/+/////:+++/-.` `.-////://+//+/-``` + `//////::/+///:.` `.:///::/+////-`````` + -///////:://///:-`` `.-://////////:-.....` + :::::://///://///:------://////::///:-----.` + ------//////:////////////////:::///:::::--` + .--.../+//+//:///:////////:///////////::.` + ..`..-/+/////:////::://///++++++++//-. + ``````:++////////////////++++//:--.` + `` ```:///:::://///////:::--...```` + ./:::--.--://////::---...```` + `----...```.--::::--..`` + ``..````` + `` + ____ _ _ ___ ____ _ ____ ___ ____ ____ ____ + |___ |\ | | |___ | |___ | |__| | | |__/ + |___ | \| | |___ |___ |___ | | | |__| | \\ +""" # -------------------------------------------------------------------------- # Generic global config diff --git a/enteletaor_lib/enteletaor.py b/enteletaor_lib/enteletaor.py index 28b4b0e..cd8d54b 100644 --- a/enteletaor_lib/enteletaor.py +++ b/enteletaor_lib/enteletaor.py @@ -4,7 +4,7 @@ import logging __tool__ = "enteletaor" -log = logging.getLogger(__name__) +log = logging.getLogger() # ---------------------------------------------------------------------- diff --git a/enteletaor_lib/libs/contrib/__init__.py b/enteletaor_lib/libs/contrib/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/enteletaor_lib/libs/contrib/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/enteletaor_lib/libs/contrib/inetnum.py b/enteletaor_lib/libs/contrib/inetnum.py new file mode 100644 index 0000000..36c128e --- /dev/null +++ b/enteletaor_lib/libs/contrib/inetnum.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# +# Fork of: +# +# https://github.com/UndergroundLabs/ripe-inetnum-search +# + +import six +import json +import netaddr +import requests +import logging + + +# ---------------------------------------------------------------------- +def get_inet_num(search_term): + """ + Get intetnums for a domain + + :param search_term: keywork without dots, no domains are allowed. domain.com -> invalid |---| domain -> valid + :type search_term: str + + :return: iterable with IP/CIDR notation or None + :rtype: list(str) | None + + """ + # Disable request logging + requests_log = logging.getLogger("requests") + requests_log.addHandler(logging.NullHandler()) + requests_log.propagate = False + + # Search the RIPE database + # There is an issue with RIPE. When making a request and including + # the type-filter inetnum, the JSON response also includes other types. + request = requests.get('http://rest.db.ripe.net/search.json', params={ + 'query-string': search_term, + 'type-filter': 'inetnum' + }) + + json_results = json.loads(request.text) + + try: + # Filter any object that doesn't have the type 'inetnum' + ranges = [x['primary-key']['attribute'][0]['value'] + for x in json_results['objects']['object'] + if x['type'] == 'inetnum'] + except KeyError: + return None + + # Turn the IP range string into CIDR + cidrs = [] + for _range in ranges: + _range = _range.split(' - '); + cidrs.append(netaddr.iprange_to_cidrs(_range[0], _range[1])) + + results = set() + + # Print the CIDR's + for cidr in cidrs: + results.add(str(cidr[0])) + + return results \ No newline at end of file diff --git a/enteletaor_lib/libs/core/cmd.py b/enteletaor_lib/libs/core/cmd.py index 8b574dd..7ada0f4 100644 --- a/enteletaor_lib/libs/core/cmd.py +++ b/enteletaor_lib/libs/core/cmd.py @@ -6,6 +6,7 @@ import argparse as _argparse from .structs import AppSettings from ...modules import IModule +from ...config import __banner__ from ...data import GlobalExecutionParameters log = logging.getLogger() @@ -80,7 +81,7 @@ def build_arg_parser(config=None, modules=None, parser=None): # Build command line # -------------------------------------------------------------------------- if parser is None: - parser = STBArgumentParser(description='%s' % str(AppSettings.tool_name).capitalize(), + parser = STBArgumentParser(description='%s' % __banner__, epilog=_build_examples(modules), formatter_class=_argparse.RawTextHelpFormatter) _extract_parameters(config, parser) @@ -96,26 +97,39 @@ def build_arg_parser(config=None, modules=None, parser=None): mod_parser = main_subparser.add_parser(mod_class.name, help=mod_class.description) + sub_modules_enabled = False + # If module has raw argsubparser, add it if hasattr(mod_class, "__submodules__"): - sub_module_actions = mod_parser.add_subparsers(help="%s commands:" % mod_name, dest="module_%s" % mod_name) - sub_module_actions.required = True + if len([x for x in mod_class.__submodules__ if x != "default"]) > 0: - for x, y in six.iteritems(mod_class.__submodules__): - sub_help = y['help'] - sub_action = y.get('cmd_args', None) + # New sub-module added + sub_modules_enabled = True - sub_sub_parser = sub_module_actions.add_parser(x, help=sub_help) + sub_module_actions = mod_parser.add_subparsers(help="%s commands:" % mod_name, dest="module_%s" % mod_name) + sub_module_actions.required = True - # Add module parameters - if hasattr(mod_class, "__model__"): - _extract_parameters(mod_class.__model__(), sub_sub_parser, mod_name) + for x, y in six.iteritems(mod_class.__submodules__): - if sub_action is not None: - # Add sub parser - sub_action(sub_sub_parser) - else: + # Skip default action info + if x == "default": + continue + + sub_help = y['help'] + sub_action = y.get('cmd_args', None) + + sub_sub_parser = sub_module_actions.add_parser(x, help=sub_help) + + # Add module parameters + if hasattr(mod_class, "__model__"): + _extract_parameters(mod_class.__model__(), sub_sub_parser, mod_name) + + if sub_action is not None: + # Add sub parser + sub_action(sub_sub_parser) + + if sub_modules_enabled is False: # Add module parameters if hasattr(mod_class, "__model__"): _extract_parameters(mod_class.__model__(), mod_parser, mod_name) @@ -210,7 +224,7 @@ def _resolve_action(field): 'StringField': ("store", str), 'SelectField': ("store", str), 'IntegerField': ("store", int), - 'BoolField': ("store_true", bool), + 'BoolField': ("store_true", None), 'IncrementalIntegerField': ("count", None) } @@ -218,8 +232,5 @@ def _resolve_action(field): results = type_maps[in_type] - if in_type == "BoolField": - results[0] = "store_%s" % str(field.default).lower() - return results diff --git a/enteletaor_lib/libs/core/config.py b/enteletaor_lib/libs/core/config.py index 66d9674..e629fb6 100644 --- a/enteletaor_lib/libs/core/config.py +++ b/enteletaor_lib/libs/core/config.py @@ -22,13 +22,13 @@ def load_config(): """ try: - from config import __author__, __name__, __site__, __version__ + from config import __author__, __tool_name__, __site__, __version__ except ImportError: __author__ = __name__ = __site__ = __version__ = "unknown" from .structs import AppSettings AppSettings.author = __author__ - AppSettings.tool_name = __name__ + AppSettings.tool_name = __tool_name__ AppSettings.project_site = __site__ AppSettings.version = __version__ diff --git a/enteletaor_lib/libs/core/logger.py b/enteletaor_lib/libs/core/logger.py index 02e7398..f90086a 100644 --- a/enteletaor_lib/libs/core/logger.py +++ b/enteletaor_lib/libs/core/logger.py @@ -11,17 +11,17 @@ def setup_logging(): """ Setup initial logging configuration """ - from ...config import __name__, DEBUG_LEVEL + from ...config import __tool_name__, DEBUG_LEVEL # Init logger - logger = logging.getLogger('') + logger = logging.getLogger() # Set log level - logger.setLevel(abs(DEBUG_LEVEL * 10) % 50) + logger.setLevel(abs(50 - (DEBUG_LEVEL if DEBUG_LEVEL < 5 else 5) * 10)) # Set file log format file_format = logging.Formatter('[%(levelname)s] %(asctime)s - %(message)s', "%Y-%m-%d %H:%M:%S") - log_file = logging.FileHandler(filename="%s.log" % __name__) + log_file = logging.FileHandler(filename="%s.log" % __tool_name__) log_file.setFormatter(file_format) # Handler: console diff --git a/enteletaor_lib/libs/hooks/config/__init__.py b/enteletaor_lib/libs/hooks/config/__init__.py index 0593369..9f8bdee 100644 --- a/enteletaor_lib/libs/hooks/config/__init__.py +++ b/enteletaor_lib/libs/hooks/config/__init__.py @@ -25,4 +25,5 @@ def set_log_level(parsed_args): """ if hasattr(parsed_args, "verbosity"): - log.setLevel(abs(parsed_args.verbosity * 10) % 50) \ No newline at end of file + # log.setLevel(abs(50 - (parsed_args.verbosity * 10))) + log.setLevel(abs(50 - ((parsed_args.verbosity if parsed_args.verbosity < 5 else 5) * 10))) diff --git a/enteletaor_lib/modules/__init__.py b/enteletaor_lib/modules/__init__.py index f274103..c332f0e 100644 --- a/enteletaor_lib/modules/__init__.py +++ b/enteletaor_lib/modules/__init__.py @@ -14,7 +14,10 @@ class IModule: def run(self, module_config): if hasattr(self, "__submodules__"): - self.__submodules__[module_config.sub_action]['action'](module_config) + try: + self.__submodules__[module_config.sub_action]['action'](module_config) + except KeyError: + self.__submodules__["default"]['action'](module_config) else: raise NotImplemented("Run method must be override") diff --git a/enteletaor_lib/modules/scan/patch.py b/enteletaor_lib/modules/scan/patch.py new file mode 100644 index 0000000..4b3b0cc --- /dev/null +++ b/enteletaor_lib/modules/scan/patch.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + + +""" +This file contains monkey patches for +""" + +from __future__ import absolute_import + + +def new_transport_init(self, host, connect_timeout): + + import errno + import re + import socket + import ssl + + # Jython does not have this attribute + try: + from socket import SOL_TCP + except ImportError: # pragma: no cover + from socket import IPPROTO_TCP as SOL_TCP # noqa + + try: + from ssl import SSLError + except ImportError: + class SSLError(Exception): # noqa + pass + + from struct import pack, unpack + + from amqp.exceptions import UnexpectedFrame + from amqp.utils import get_errno, set_cloexec + + _UNAVAIL = errno.EAGAIN, errno.EINTR, errno.ENOENT + + AMQP_PORT = 5672 + + EMPTY_BUFFER = bytes() + + # Yes, Advanced Message Queuing Protocol Protocol is redundant + AMQP_PROTOCOL_HEADER = 'AMQP\x01\x01\x00\x09'.encode('latin_1') + + # Match things like: [fe80::1]:5432, from RFC 2732 + IPV6_LITERAL = re.compile(r'\[([\.0-9a-f:]+)\](?::(\d+))?') + + # -------------------------------------------------------------------------- + # __init__ content: + # -------------------------------------------------------------------------- + self.connected = True + msg = None + port = AMQP_PORT + + m = IPV6_LITERAL.match(host) + if m: + host = m.group(1) + if m.group(2): + port = int(m.group(2)) + else: + if ':' in host: + host, port = host.rsplit(':', 1) + port = int(port) + + self.sock = None + last_err = None + for res in socket.getaddrinfo(host, port, 0, + socket.SOCK_STREAM, SOL_TCP): + af, socktype, proto, canonname, sa = res + try: + self.sock = socket.socket(af, socktype, proto) + try: + set_cloexec(self.sock, True) + except NotImplementedError: + pass + self.sock.settimeout(connect_timeout) + self.sock.connect(sa) + except socket.error as exc: + msg = exc + self.sock.close() + self.sock = None + last_err = msg + continue + break + + if not self.sock: + # Didn't connect, return the most recent error message + raise socket.error(last_err) + + try: + # self.sock.settimeout(None) + self.sock.setsockopt(SOL_TCP, socket.TCP_NODELAY, 1) + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + self._setup_transport() + + self._write(AMQP_PROTOCOL_HEADER) + except (OSError, IOError, socket.error) as exc: + if get_errno(exc) not in _UNAVAIL: + self.connected = False + raise + + +# -------------------------------------------------------------------------- +# amqlib +# -------------------------------------------------------------------------- +def patch_transport(): + """ + This function path transport constructor to fix timeout in sockets + """ + + from amqp.transport import _AbstractTransport + + _AbstractTransport.__init__ = new_transport_init diff --git a/requirements.txt b/requirements.txt index 6078b82..5fd8c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,17 @@ six flask wtforms +eventlet colorlog +ipaddress # MQ/Brokers requirements redis +kombu celery -kombu \ No newline at end of file +pyzmq +amqp + +# contrib dependencies +requests +netaddr \ No newline at end of file