fix: many improvements for Python 2 compatibility

fix: removed unused code
fix: removed un used fiels
fix: improvements in Redis dump keys
add: Changed 'proc' module for 'tasks'

.idea/enteletaor.iml

@@ -5,7 +5,7 @@
       <sourceFolder url="file://$MODULE_DIR$/enteletaor_lib" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/.idea" />
     </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Python 3.4.3 virtualenv at ~/.virtualenvs/enteletaor" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TestRunnerService">

enteletaor_lib/__init__.py

@@ -3,7 +3,7 @@
 # --------------------------------------------------------------------------
 # Run Boot loader
 # --------------------------------------------------------------------------
-if __package__ == "enteletaor_lib":
+if __package__ == "enteletaor_lib" or __name__ == "enteletaor_lib":
 
 	from .libs.core.bootloader import boot_loader
 

enteletaor_lib/libs/core/cmd.py

@@ -29,7 +29,7 @@ def setup_cmd():
 class STBArgumentParser(_argparse.ArgumentParser):
 
     def parse_args_and_run_hooks(self, args=None, namespace=None):
-        parsed_args = super(STBArgumentParser, self).parse_args(args, namespace)
+        parsed_args = super(self.__class__, self).parse_args(args, namespace)
 
         # Run hooks
         self.run_hooks(parsed_args)

enteletaor_lib/libs/core/config.py

@@ -1,18 +1,6 @@
 # -*- coding: utf-8 -*-
 
-
+from __future__ import absolute_import
-# ----------------------------------------------------------------------
-def get_user_config_path():
-	"""
-
-	"""
-
-
-# ----------------------------------------------------------------------
-def get_project_config_path():
-	"""
-
-	"""
 
 
 # ----------------------------------------------------------------------
@@ -24,7 +12,7 @@ def load_config():
 	try:
 		from config import __author__, __tool_name__, __site__, __version__
 	except ImportError:
-		__author__ = __name__ = __site__ = __version__ = "unknown"
+		__author__ = __tool_name__ = __site__ = __version__ = "unknown"
 
 	from .structs import AppSettings
 

enteletaor_lib/libs/core/model_old.py

@@ -1,165 +0,0 @@
-# -*- coding: utf-8 -*-
-from schematics.exceptions import ConversionError, ValidationError
-from schematics.models import Model as _Model
-from schematics.types import BaseType as _BaseType, utf8_decode, unicode
-from schematics.types import (StringType, IntType, FloatType, DateTimeType, IPv4Type,
-                              URLType, EmailType, NumberType, LongType, DecimalType,
-                              HashType, SHA1Type, BooleanType, DateType, UUIDType)
-from schematics.types.compound import ListType
-
-
-# region Monkey Patch
-
-
-# --------------------------------------------------------------------------
-# Monkey patch fo Schematics to add:
-# - New property for types: "description"
-# - Constructor of models without dicts, using instead **kwargs
-# --------------------------------------------------------------------------
-def __str__(self):
-    return self.description
-
-
-# ----------------------------------------------------------------------
-def new_init(self, required=False, default=None, serialized_name=None,
-             choices=None, validators=None, deserialize_from=None,
-             serialize_when_none=None, messages=None,
-
-             # Custom parameters
-             description="Field description", is_file_results=False):
-
-    # Call original constructor
-    _BaseType.old__init__(self, required, default, serialized_name, choices, validators, deserialize_from,
-                          serialize_when_none, messages)
-    if not isinstance(description, str):
-        raise TypeError("Expected str, got '%s' instead" % type(description))
-    if not isinstance(is_file_results, bool):
-        raise TypeError("Expected bool, got '%s' instead" % type(is_file_results))
-
-    self.description = description
-    self.is_file_results = is_file_results
-
-
-# Monkey patch!
-_BaseType.old__init__ = _BaseType.__init__
-_BaseType.__init__ = new_init
-# _BaseType.__str__ = __str__
-
-
-# endregion
-        
-
-# --------------------------------------------------------------------------
-# New type
-# --------------------------------------------------------------------------
-class FileType(_BaseType):
-    allow_casts = (int, str)
-
-    MESSAGES = {
-        'convert'   : u"Couldn't interpret '{0}' as string.",
-        'max_length': u"String value is too long.",
-        'min_length': u"String value is too short."
-    }
-
-    def __init__(self, name=None, file_type=None, path=None, max_length=None, min_length=None, **kwargs):
-        self.max_length = max_length
-        self.min_length = min_length
-        self.file_type = file_type
-        self.path = path
-
-        super(FileType, self).__init__(**kwargs)
-
-    def to_native(self, value, context=None):
-        if value is None:
-            return None
-
-        if not isinstance(value, unicode):
-            if isinstance(value, self.allow_casts):
-                if not isinstance(value, str):
-                    value = str(value)
-                value = utf8_decode(value)  # unicode(value, 'utf-8')
-            else:
-                raise ConversionError(self.messages['convert'].format(value))
-
-        return value
-
-    def validate_length(self, value):
-        len_of_value = len(value) if value else 0
-
-        if self.max_length is not None and len_of_value > self.max_length:
-            raise ValidationError(self.messages['max_length'])
-
-        if self.min_length is not None and len_of_value < self.min_length:
-            raise ValidationError(self.messages['min_length'])
-
-
-# --------------------------------------------------------------------------
-# STB Model class
-# --------------------------------------------------------------------------
-class Model(_Model):
-    MESSAGES = {
-        "label": "Console label"
-    }
-
-    BASIC_REVERSE = {
-        "IntType"     : "int",
-        "URLType"     : "str",
-        "IPv4Type"    : "str",
-        "DateType"    : "str",
-        "HashType"    : "str",
-        "SHA1Type"    : "str",
-        "FileType"    : "str",
-        "LongType"    : "int",
-        "EmailType"   : "str",
-        "FloatType"   : "float",
-        "NumberType"  : "int",
-        "StringType"  : "str",
-        "DecimalType" : "float",
-        "BooleanType" : "bool",
-        "DateTimeType": "str",
-    }
-
-    def __init__(self, raw_data=None, deserialize_mapping=None, strict=True, **kwargs):
-        super(Model, self).__init__(raw_data=raw_data, deserialize_mapping=deserialize_mapping, strict=strict)
-
-        for k, v in kwargs.items():
-            if k in self.keys():
-                setattr(self, k, v)
-
-
-    # ----------------------------------------------------------------------
-    def get_basic_types(self):
-        """
-		Get a dict with basic types
-		"""
-        results = {}
-        for name, _type in self._fields.items():
-            try:
-                results[name] = self.BASIC_REVERSE[_type.__class__.__name__]
-            except KeyError:
-                pass
-        return results
-
-    # ----------------------------------------------------------------------
-    def get_field_results(self):
-        """
-        Return the name of property that will contains the file results
-
-        :return: a string with the name of field of file results
-        :rtype: str
-
-        """
-        for name, _type in self._fields.items():
-            if _type.is_file_results:
-                return name
-
-        return None
-
-
-if __name__ == '__main__':
-    class Testing(Model):
-        p1 = FileType(is_file_results=True)
-
-    m1 = Testing()
-
-    print(m1.get_field_results())

enteletaor_lib/libs/core/models.py

@@ -55,7 +55,7 @@ def new_module_validate(self):
             if func.validator() is False:
                 self._errors = {}
 
-                if type(self._fields[name]) != type(self._fields[name].__type__):
+                if type(self._fields[name].data) is type(self._fields[name].__type__):
                     self._errors[name] = ("Data type incorrect or not default value "
                                           "provided. Got %s. Expected: %s" % (
                                               type(self._fields[name].data),
@@ -115,7 +115,8 @@ def _validator(self):
         if self.data is None:
             return True
         else:
-            to_check = self.default
+            # to_check = self.default
+            return False
     else:
         if not isinstance(to_check, self.__type__):
             return False
@@ -133,7 +134,7 @@ StringField.validator = _validator
 # ----------------------------------------------------------------------
 class IntegerField(_IntegerField):
     """Improved Integer data that checks types"""
-    __type__ = int
+    __type__ = six.integer_types
 IntegerField.validator = _validator
 
 
@@ -164,5 +165,5 @@ BoolField.validator = _validator
 # ----------------------------------------------------------------------
 class SelectField(_SelectField):
     """Improved bool data that checks types"""
-    __type__ = str
+    __type__ = six.text_type
 SelectField.validator = _validator

enteletaor_lib/libs/core/patterns.py

@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-
-
-class Singleton(object):
-    def __new__(cls,*args,**kwargs):
-        if '_inst' not in vars(cls):
-            cls._inst = super(Singleton,cls).__new__(cls)
-        return cls._inst

enteletaor_lib/modules/__init__.py

@@ -6,7 +6,7 @@ log = logging.getLogger(__name__)
 
 
 # --------------------------------------------------------------------------
-class IModule:
+class IModule(object):
     """Interface for modules"""
 
     name = None

enteletaor_lib/modules/redis/redis_cache.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
+import six
 import redis
+import string
 import logging
 
 from lxml import etree
@@ -58,13 +60,38 @@ def handle_html(config, content):
 	# --------------------------------------------------------------------------
 	# Search start and end possition of HTML page
 	# --------------------------------------------------------------------------
+	pos_ini = pos_end = None
 	for i, x in enumerate(content):
+		if six.PY2:
+			if six.u(x) == six.u("<"):
+				tmp_pos = i
+		else:
 			if chr(x) == "<":
+				tmp_pos = i
+
+		# Is printable? to avoid nulls and false '<'
+		if tmp_pos == i and len(content) != i:
+			if six.PY2:
+				if content[i + 1] in string.printable:
+					pos_ini = i
+					break
+			else:
+				if chr(content[i + 1]) in string.printable:
 					pos_ini = i
 					break
 
+		# else:
+
+			# pos_ini = i
+			# break
+
 	for i, x in enumerate(content[::-1]):
-		if chr(x) == ">":
+		if six.PY2:
+			if six.u(x) == six.u("<"):
+				pos_end = len(content) - i
+				break
+		else:
+			if chr(x) == "<":
 				pos_end = len(content) - i
 				break
 
@@ -74,7 +101,7 @@ def handle_html(config, content):
 	txt_content = content[pos_ini:pos_end]
 
 	# Parse input
-	tree = etree.fromstring(txt_content, etree.HTMLParser())
+	tree = etree.fromstring(txt_content, parser=etree.HTMLParser())
 	doc_root = tree.getroottree()
 
 	results = None

enteletaor_lib/modules/redis/redis_dump.py

@@ -14,7 +14,6 @@ log = logging.getLogger()
 
 # ----------------------------------------------------------------------
 def dump_keys(con):
-
 	for key in con.keys('*'):
 		key_type = con.type(key).lower()
 		val = None
@@ -39,7 +38,6 @@ def dump_keys(con):
 # Human parsers
 # --------------------------------------------------------------------------
 def decode_object(key, val, ident=5):
-
 	if isinstance(val, dict):
 
 		log.error('    "%s":' % key)
@@ -61,7 +59,8 @@ def _decode_object(val, ident=5):
 		# convert value to original type -> JSON
 		try:
 			_transformed_info = json.loads(v.decode("utf-8"))
-		except (binascii.Error, AttributeError):
+		# except (binascii.Error, AttributeError, ValueError):
+		except (binascii.Error, AttributeError, ValueError):
 			_transformed_info = v
 
 		# --------------------------------------------------------------------------
@@ -114,7 +113,12 @@ def _decode_object(val, ident=5):
 
 				log.error("%s}" % (" " * _new_ident))
 
-			except Exception:
+			except Exception as e:
+
+				if "BadPickleGet" == e.__class__.__name__:
+					log.info(
+						"   <!!> Can't decode value for key '%s' because Pickle protocol 3 o 4 used, and it's "
+						"incompatible with Python 2" % k)
 
 				# Try again decoding in base64
 				try:
@@ -149,6 +153,10 @@ def action_redis_dump(config):
 	export_file = None
 	if config.export_results:
 		export_file = open(config.export_results, "w")
+		log.error("  - Storing information into '%s'" % config.export_results)
+	elif config.no_screen is True:
+		log.error("  <!> If results will not be displayed, you must to indicate output file for results.")
+		return
 
 	i = 0
 	for i, t_val in enumerate(dump_keys(con)):

enteletaor_lib/modules/scan/scan_main.py

@@ -201,7 +201,7 @@ def build_targets(config):
 			# Add CDIR to result
 			scan_target = "%s%s" % (host_ip, "/%s" % _target_cdir[1] if len(_target_cdir) > 1 else "")
 
-			results.update(str(x) for x in ipaddress.ip_network(scan_target, strict=False))
+			results.update(str(x) for x in ipaddress.ip_network(six.u(scan_target), strict=False))
 
 	return results
 

enteletaor_lib/modules/tasks/__init__.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 
+
 import logging
 
 from modules import IModule
 
 from libs.core.structs import CommonData
-from libs.core.models import IntegerField, StringField, SelectField
+from libs.core.models import StringField, SelectField
 
 from .cmd_actions import parser_proc_raw_dump, parser_proc_list_process, parser_proc_inject_process
 from .proc_remove import action_proc_remove
@@ -56,7 +57,7 @@ class RemoteProcessModule(IModule):
 		),
 	}
 
-	name = "proc"
+	name = "tasks"
 	description = "try to discover and handle processes in remote MQ/Brokers"
 
 	# ----------------------------------------------------------------------

enteletaor_lib/modules/tasks/proc_inject_process.py

@@ -38,7 +38,7 @@ def action_proc_inject_process(config):
 		# Fill process information
 		# --------------------------------------------------------------------------
 		inject_process = {
-			"args": [x for x, y in six.iteritems(parameters)],
+			"args": [y for x, y in six.iteritems(parameters)],
 			"callbacks": None,
 			"chord": None,
 			"errbacks": None,

enteletaor_lib/modules/tasks/utils.py

@@ -32,7 +32,7 @@ def get_param_type(value):
 			return "list"
 		elif type(value) == bytes:
 			try:
-				value.decode()
+				six.u(value)
 
 				return "bytes"
 			except Exception:

requirements.txt

@@ -1,4 +1,5 @@
 six
+lxml
 flask
 wtforms
 eventlet