From 556d9c5873e49fa0f8b6232179c45595e38b6854 Mon Sep 17 00:00:00 2001 From: cr0hn Date: Sat, 27 Feb 2016 18:50:39 +0100 Subject: [PATCH] fix: many improvements for Python 2 compatibility fix: removed unused code fix: removed un used fiels fix: improvements in Redis dump keys add: Changed 'proc' module for 'tasks' --- .idea/enteletaor.iml | 2 +- enteletaor_lib/__init__.py | 2 +- enteletaor_lib/libs/core/cmd.py | 2 +- enteletaor_lib/libs/core/config.py | 16 +- enteletaor_lib/libs/core/model_old.py | 165 ------------------ enteletaor_lib/libs/core/models.py | 9 +- enteletaor_lib/libs/core/patterns.py | 8 - enteletaor_lib/modules/__init__.py | 2 +- enteletaor_lib/modules/redis/redis_cache.py | 41 ++++- enteletaor_lib/modules/redis/redis_dump.py | 16 +- enteletaor_lib/modules/scan/scan_main.py | 2 +- .../modules/{proc => tasks}/__init__.py | 5 +- .../modules/{proc => tasks}/cmd_actions.py | 0 .../{proc => tasks}/proc_inject_process.py | 2 +- .../{proc => tasks}/proc_list_process.py | 0 .../modules/{proc => tasks}/proc_raw_dump.py | 0 .../modules/{proc => tasks}/proc_remove.py | 0 .../modules/{proc => tasks}/utils.py | 2 +- requirements.txt | 1 + 19 files changed, 64 insertions(+), 211 deletions(-) delete mode 100644 enteletaor_lib/libs/core/model_old.py delete mode 100644 enteletaor_lib/libs/core/patterns.py rename enteletaor_lib/modules/{proc => tasks}/__init__.py (96%) rename enteletaor_lib/modules/{proc => tasks}/cmd_actions.py (100%) rename enteletaor_lib/modules/{proc => tasks}/proc_inject_process.py (97%) rename enteletaor_lib/modules/{proc => tasks}/proc_list_process.py (100%) rename enteletaor_lib/modules/{proc => tasks}/proc_raw_dump.py (100%) rename enteletaor_lib/modules/{proc => tasks}/proc_remove.py (100%) rename enteletaor_lib/modules/{proc => tasks}/utils.py (99%) diff --git a/.idea/enteletaor.iml b/.idea/enteletaor.iml index 72b7ad5..bb33f2f 100644 --- a/.idea/enteletaor.iml +++ b/.idea/enteletaor.iml @@ -5,7 +5,7 @@ - + diff --git a/enteletaor_lib/__init__.py b/enteletaor_lib/__init__.py index 6a6c732..c5c2b7a 100644 --- a/enteletaor_lib/__init__.py +++ b/enteletaor_lib/__init__.py @@ -3,7 +3,7 @@ # -------------------------------------------------------------------------- # Run Boot loader # -------------------------------------------------------------------------- -if __package__ == "enteletaor_lib": +if __package__ == "enteletaor_lib" or __name__ == "enteletaor_lib": from .libs.core.bootloader import boot_loader diff --git a/enteletaor_lib/libs/core/cmd.py b/enteletaor_lib/libs/core/cmd.py index 7ada0f4..8b515e6 100644 --- a/enteletaor_lib/libs/core/cmd.py +++ b/enteletaor_lib/libs/core/cmd.py @@ -29,7 +29,7 @@ def setup_cmd(): class STBArgumentParser(_argparse.ArgumentParser): def parse_args_and_run_hooks(self, args=None, namespace=None): - parsed_args = super(STBArgumentParser, self).parse_args(args, namespace) + parsed_args = super(self.__class__, self).parse_args(args, namespace) # Run hooks self.run_hooks(parsed_args) diff --git a/enteletaor_lib/libs/core/config.py b/enteletaor_lib/libs/core/config.py index e629fb6..0fff486 100644 --- a/enteletaor_lib/libs/core/config.py +++ b/enteletaor_lib/libs/core/config.py @@ -1,18 +1,6 @@ # -*- coding: utf-8 -*- - -# ---------------------------------------------------------------------- -def get_user_config_path(): - """ - - """ - - -# ---------------------------------------------------------------------- -def get_project_config_path(): - """ - - """ +from __future__ import absolute_import # ---------------------------------------------------------------------- @@ -24,7 +12,7 @@ def load_config(): try: from config import __author__, __tool_name__, __site__, __version__ except ImportError: - __author__ = __name__ = __site__ = __version__ = "unknown" + __author__ = __tool_name__ = __site__ = __version__ = "unknown" from .structs import AppSettings diff --git a/enteletaor_lib/libs/core/model_old.py b/enteletaor_lib/libs/core/model_old.py deleted file mode 100644 index 8b16cad..0000000 --- a/enteletaor_lib/libs/core/model_old.py +++ /dev/null @@ -1,165 +0,0 @@ -# -*- coding: utf-8 -*- -from schematics.exceptions import ConversionError, ValidationError -from schematics.models import Model as _Model -from schematics.types import BaseType as _BaseType, utf8_decode, unicode -from schematics.types import (StringType, IntType, FloatType, DateTimeType, IPv4Type, - URLType, EmailType, NumberType, LongType, DecimalType, - HashType, SHA1Type, BooleanType, DateType, UUIDType) -from schematics.types.compound import ListType - - -# region Monkey Patch - - -# -------------------------------------------------------------------------- -# Monkey patch fo Schematics to add: -# - New property for types: "description" -# - Constructor of models without dicts, using instead **kwargs -# -------------------------------------------------------------------------- -def __str__(self): - return self.description - - -# ---------------------------------------------------------------------- -def new_init(self, required=False, default=None, serialized_name=None, - choices=None, validators=None, deserialize_from=None, - serialize_when_none=None, messages=None, - - # Custom parameters - description="Field description", is_file_results=False): - - # Call original constructor - _BaseType.old__init__(self, required, default, serialized_name, choices, validators, deserialize_from, - serialize_when_none, messages) - if not isinstance(description, str): - raise TypeError("Expected str, got '%s' instead" % type(description)) - if not isinstance(is_file_results, bool): - raise TypeError("Expected bool, got '%s' instead" % type(is_file_results)) - - self.description = description - self.is_file_results = is_file_results - - -# Monkey patch! -_BaseType.old__init__ = _BaseType.__init__ -_BaseType.__init__ = new_init -# _BaseType.__str__ = __str__ - - -# endregion - - -# -------------------------------------------------------------------------- -# New type -# -------------------------------------------------------------------------- -class FileType(_BaseType): - allow_casts = (int, str) - - MESSAGES = { - 'convert' : u"Couldn't interpret '{0}' as string.", - 'max_length': u"String value is too long.", - 'min_length': u"String value is too short." - } - - def __init__(self, name=None, file_type=None, path=None, max_length=None, min_length=None, **kwargs): - self.max_length = max_length - self.min_length = min_length - self.file_type = file_type - self.path = path - - super(FileType, self).__init__(**kwargs) - - def to_native(self, value, context=None): - if value is None: - return None - - if not isinstance(value, unicode): - if isinstance(value, self.allow_casts): - if not isinstance(value, str): - value = str(value) - value = utf8_decode(value) # unicode(value, 'utf-8') - else: - raise ConversionError(self.messages['convert'].format(value)) - - return value - - def validate_length(self, value): - len_of_value = len(value) if value else 0 - - if self.max_length is not None and len_of_value > self.max_length: - raise ValidationError(self.messages['max_length']) - - if self.min_length is not None and len_of_value < self.min_length: - raise ValidationError(self.messages['min_length']) - - -# -------------------------------------------------------------------------- -# STB Model class -# -------------------------------------------------------------------------- -class Model(_Model): - MESSAGES = { - "label": "Console label" - } - - BASIC_REVERSE = { - "IntType" : "int", - "URLType" : "str", - "IPv4Type" : "str", - "DateType" : "str", - "HashType" : "str", - "SHA1Type" : "str", - "FileType" : "str", - "LongType" : "int", - "EmailType" : "str", - "FloatType" : "float", - "NumberType" : "int", - "StringType" : "str", - "DecimalType" : "float", - "BooleanType" : "bool", - "DateTimeType": "str", - } - - def __init__(self, raw_data=None, deserialize_mapping=None, strict=True, **kwargs): - super(Model, self).__init__(raw_data=raw_data, deserialize_mapping=deserialize_mapping, strict=strict) - - for k, v in kwargs.items(): - if k in self.keys(): - setattr(self, k, v) - - - # ---------------------------------------------------------------------- - def get_basic_types(self): - """ - Get a dict with basic types - """ - results = {} - for name, _type in self._fields.items(): - try: - results[name] = self.BASIC_REVERSE[_type.__class__.__name__] - except KeyError: - pass - return results - - # ---------------------------------------------------------------------- - def get_field_results(self): - """ - Return the name of property that will contains the file results - - :return: a string with the name of field of file results - :rtype: str - - """ - for name, _type in self._fields.items(): - if _type.is_file_results: - return name - - return None - - -if __name__ == '__main__': - class Testing(Model): - p1 = FileType(is_file_results=True) - - m1 = Testing() - - print(m1.get_field_results()) \ No newline at end of file diff --git a/enteletaor_lib/libs/core/models.py b/enteletaor_lib/libs/core/models.py index e48cd04..eddaeca 100644 --- a/enteletaor_lib/libs/core/models.py +++ b/enteletaor_lib/libs/core/models.py @@ -55,7 +55,7 @@ def new_module_validate(self): if func.validator() is False: self._errors = {} - if type(self._fields[name]) != type(self._fields[name].__type__): + if type(self._fields[name].data) is type(self._fields[name].__type__): self._errors[name] = ("Data type incorrect or not default value " "provided. Got %s. Expected: %s" % ( type(self._fields[name].data), @@ -115,7 +115,8 @@ def _validator(self): if self.data is None: return True else: - to_check = self.default + # to_check = self.default + return False else: if not isinstance(to_check, self.__type__): return False @@ -133,7 +134,7 @@ StringField.validator = _validator # ---------------------------------------------------------------------- class IntegerField(_IntegerField): """Improved Integer data that checks types""" - __type__ = int + __type__ = six.integer_types IntegerField.validator = _validator @@ -164,5 +165,5 @@ BoolField.validator = _validator # ---------------------------------------------------------------------- class SelectField(_SelectField): """Improved bool data that checks types""" - __type__ = str + __type__ = six.text_type SelectField.validator = _validator diff --git a/enteletaor_lib/libs/core/patterns.py b/enteletaor_lib/libs/core/patterns.py deleted file mode 100644 index 07d497c..0000000 --- a/enteletaor_lib/libs/core/patterns.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- - - -class Singleton(object): - def __new__(cls,*args,**kwargs): - if '_inst' not in vars(cls): - cls._inst = super(Singleton,cls).__new__(cls) - return cls._inst \ No newline at end of file diff --git a/enteletaor_lib/modules/__init__.py b/enteletaor_lib/modules/__init__.py index c332f0e..2aeb64f 100644 --- a/enteletaor_lib/modules/__init__.py +++ b/enteletaor_lib/modules/__init__.py @@ -6,7 +6,7 @@ log = logging.getLogger(__name__) # -------------------------------------------------------------------------- -class IModule: +class IModule(object): """Interface for modules""" name = None diff --git a/enteletaor_lib/modules/redis/redis_cache.py b/enteletaor_lib/modules/redis/redis_cache.py index 38d1dd8..e49dbd9 100644 --- a/enteletaor_lib/modules/redis/redis_cache.py +++ b/enteletaor_lib/modules/redis/redis_cache.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- +import six import redis +import string import logging from lxml import etree @@ -58,15 +60,40 @@ def handle_html(config, content): # -------------------------------------------------------------------------- # Search start and end possition of HTML page # -------------------------------------------------------------------------- + pos_ini = pos_end = None for i, x in enumerate(content): - if chr(x) == "<": - pos_ini = i - break + if six.PY2: + if six.u(x) == six.u("<"): + tmp_pos = i + else: + if chr(x) == "<": + tmp_pos = i + + # Is printable? to avoid nulls and false '<' + if tmp_pos == i and len(content) != i: + if six.PY2: + if content[i + 1] in string.printable: + pos_ini = i + break + else: + if chr(content[i + 1]) in string.printable: + pos_ini = i + break + + # else: + + # pos_ini = i + # break for i, x in enumerate(content[::-1]): - if chr(x) == ">": - pos_end = len(content) - i - break + if six.PY2: + if six.u(x) == six.u("<"): + pos_end = len(content) - i + break + else: + if chr(x) == "<": + pos_end = len(content) - i + break if pos_ini is None or pos_end is None: raise ValueError("Not found HTML content into cache") @@ -74,7 +101,7 @@ def handle_html(config, content): txt_content = content[pos_ini:pos_end] # Parse input - tree = etree.fromstring(txt_content, etree.HTMLParser()) + tree = etree.fromstring(txt_content, parser=etree.HTMLParser()) doc_root = tree.getroottree() results = None diff --git a/enteletaor_lib/modules/redis/redis_dump.py b/enteletaor_lib/modules/redis/redis_dump.py index e55426d..6188224 100644 --- a/enteletaor_lib/modules/redis/redis_dump.py +++ b/enteletaor_lib/modules/redis/redis_dump.py @@ -14,7 +14,6 @@ log = logging.getLogger() # ---------------------------------------------------------------------- def dump_keys(con): - for key in con.keys('*'): key_type = con.type(key).lower() val = None @@ -39,7 +38,6 @@ def dump_keys(con): # Human parsers # -------------------------------------------------------------------------- def decode_object(key, val, ident=5): - if isinstance(val, dict): log.error(' "%s":' % key) @@ -61,7 +59,8 @@ def _decode_object(val, ident=5): # convert value to original type -> JSON try: _transformed_info = json.loads(v.decode("utf-8")) - except (binascii.Error, AttributeError): + # except (binascii.Error, AttributeError, ValueError): + except (binascii.Error, AttributeError, ValueError): _transformed_info = v # -------------------------------------------------------------------------- @@ -114,7 +113,12 @@ def _decode_object(val, ident=5): log.error("%s}" % (" " * _new_ident)) - except Exception: + except Exception as e: + + if "BadPickleGet" == e.__class__.__name__: + log.info( + " Can't decode value for key '%s' because Pickle protocol 3 o 4 used, and it's " + "incompatible with Python 2" % k) # Try again decoding in base64 try: @@ -149,6 +153,10 @@ def action_redis_dump(config): export_file = None if config.export_results: export_file = open(config.export_results, "w") + log.error(" - Storing information into '%s'" % config.export_results) + elif config.no_screen is True: + log.error(" If results will not be displayed, you must to indicate output file for results.") + return i = 0 for i, t_val in enumerate(dump_keys(con)): diff --git a/enteletaor_lib/modules/scan/scan_main.py b/enteletaor_lib/modules/scan/scan_main.py index d1b0c3d..7613335 100644 --- a/enteletaor_lib/modules/scan/scan_main.py +++ b/enteletaor_lib/modules/scan/scan_main.py @@ -201,7 +201,7 @@ def build_targets(config): # Add CDIR to result scan_target = "%s%s" % (host_ip, "/%s" % _target_cdir[1] if len(_target_cdir) > 1 else "") - results.update(str(x) for x in ipaddress.ip_network(scan_target, strict=False)) + results.update(str(x) for x in ipaddress.ip_network(six.u(scan_target), strict=False)) return results diff --git a/enteletaor_lib/modules/proc/__init__.py b/enteletaor_lib/modules/tasks/__init__.py similarity index 96% rename from enteletaor_lib/modules/proc/__init__.py rename to enteletaor_lib/modules/tasks/__init__.py index d9e2bf2..c7baf3f 100644 --- a/enteletaor_lib/modules/proc/__init__.py +++ b/enteletaor_lib/modules/tasks/__init__.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- + import logging from modules import IModule from libs.core.structs import CommonData -from libs.core.models import IntegerField, StringField, SelectField +from libs.core.models import StringField, SelectField from .cmd_actions import parser_proc_raw_dump, parser_proc_list_process, parser_proc_inject_process from .proc_remove import action_proc_remove @@ -56,7 +57,7 @@ class RemoteProcessModule(IModule): ), } - name = "proc" + name = "tasks" description = "try to discover and handle processes in remote MQ/Brokers" # ---------------------------------------------------------------------- diff --git a/enteletaor_lib/modules/proc/cmd_actions.py b/enteletaor_lib/modules/tasks/cmd_actions.py similarity index 100% rename from enteletaor_lib/modules/proc/cmd_actions.py rename to enteletaor_lib/modules/tasks/cmd_actions.py diff --git a/enteletaor_lib/modules/proc/proc_inject_process.py b/enteletaor_lib/modules/tasks/proc_inject_process.py similarity index 97% rename from enteletaor_lib/modules/proc/proc_inject_process.py rename to enteletaor_lib/modules/tasks/proc_inject_process.py index feb2460..8fa0688 100644 --- a/enteletaor_lib/modules/proc/proc_inject_process.py +++ b/enteletaor_lib/modules/tasks/proc_inject_process.py @@ -38,7 +38,7 @@ def action_proc_inject_process(config): # Fill process information # -------------------------------------------------------------------------- inject_process = { - "args": [x for x, y in six.iteritems(parameters)], + "args": [y for x, y in six.iteritems(parameters)], "callbacks": None, "chord": None, "errbacks": None, diff --git a/enteletaor_lib/modules/proc/proc_list_process.py b/enteletaor_lib/modules/tasks/proc_list_process.py similarity index 100% rename from enteletaor_lib/modules/proc/proc_list_process.py rename to enteletaor_lib/modules/tasks/proc_list_process.py diff --git a/enteletaor_lib/modules/proc/proc_raw_dump.py b/enteletaor_lib/modules/tasks/proc_raw_dump.py similarity index 100% rename from enteletaor_lib/modules/proc/proc_raw_dump.py rename to enteletaor_lib/modules/tasks/proc_raw_dump.py diff --git a/enteletaor_lib/modules/proc/proc_remove.py b/enteletaor_lib/modules/tasks/proc_remove.py similarity index 100% rename from enteletaor_lib/modules/proc/proc_remove.py rename to enteletaor_lib/modules/tasks/proc_remove.py diff --git a/enteletaor_lib/modules/proc/utils.py b/enteletaor_lib/modules/tasks/utils.py similarity index 99% rename from enteletaor_lib/modules/proc/utils.py rename to enteletaor_lib/modules/tasks/utils.py index 13c1674..a7bab9b 100644 --- a/enteletaor_lib/modules/proc/utils.py +++ b/enteletaor_lib/modules/tasks/utils.py @@ -32,7 +32,7 @@ def get_param_type(value): return "list" elif type(value) == bytes: try: - value.decode() + six.u(value) return "bytes" except Exception: diff --git a/requirements.txt b/requirements.txt index 5fd8c50..51b8190 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ six +lxml flask wtforms eventlet