dash/enteletaor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add: complete documentation

Browse Source 
fix: unused options in redis
fix: compatibility between python 2-3
fix: forgot vars declarations
fix: carry conditions in listing redis keys
fix: listing redis list keys
fix: removed duplicated tasks when they are listed
fix: index number when redis DB are listed
fix:  some error levels in log
fix: renamed *proc* -> *tasks* files
fix: added the process manager backend for 'tasks' options, thinking in future  to add new process managers
This commit is contained in:
cr0hn
2016-02-29 19:14:20 +01:00
parent d1f583af2d
commit ff2abc7b65
22 changed files with 1447 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
enteletaor_lib/modules/redis/__init__.py
View File

@@ -24,7 +24,6 @@ class ModuleModel(CommonData):
target = StringField(required=True)
port = IntegerField(default=6379)
db = IntegerField(default=0)
export_results = StringField()
# ----------------------------------------------------------------------

2
enteletaor_lib/modules/redis/cmd_actions.py
View File

@@ -13,6 +13,8 @@ def parser_redis_dump(parser):
gr = parser.add_argument_group("custom raw dump options")
gr.add_argument("--no-screen", action="store_true", dest="no_screen", default=False,
help="do not show displays raw database info into screen")
gr.add_argument("-e", "--export-results", dest="export_results",
help="export dumped information results")
# ----------------------------------------------------------------------

23
enteletaor_lib/modules/redis/redis_cache.py
View File

@@ -62,6 +62,7 @@ def handle_html(config, content):
# --------------------------------------------------------------------------
pos_ini = pos_end = None
for i, x in enumerate(content):
tmp_pos = -1
if six.PY2:
if six.u(x) == six.u("<"):
tmp_pos = i
@@ -205,9 +206,19 @@ def action_redis_cache_poison(config):
for val in cache_keys:
content = dump_key(val, con)
try:
_val = val.decode(errors="ignore")
except AttributeError:
_val = val
try:
_content = content.decode(errors="ignore")
except AttributeError:
_content = content
# If key doesn't exist content will be None
if content is None:
log.error(" - Provided key '%s' not found in server" % val)
log.error(" - Provided key '%s' not found in server" % _val)
continue
# --------------------------------------------------------------------------
@@ -217,7 +228,7 @@ def action_redis_cache_poison(config):
if config.poison is True:
# Set injection
try:
modified = handle_html(config, content)
modified = handle_html(config, content) # DO NOT USE _content. Function expect bytes, not str.
except ValueError as e:
log.error(" - Can't modify cache content: " % e)
continue
@@ -232,12 +243,12 @@ def action_redis_cache_poison(config):
# Set injection into server
con.setex(val, 200, modified)
log.error(" - Poisoned cache key '%s' at server '%s'" % (val, config.target))
log.error(" - Poisoned cache key '%s' at server '%s'" % (_val, config.target))
else:
# If not poison enabled display cache keys
log.error(" -> Key: '%s' - " % val)
log.error(" -> Content:\n %s" % content)
log.error(" -> Key: '%s'" % _val)
log.error(" -> Content:\n %s" % _content)
if not cache_keys:
log.error(" - No cache keys found in server: Can't poison remote cache.")
log.error(" - No cache keys found in server.")

8
enteletaor_lib/modules/redis/redis_disconnect.py
View File

@@ -24,13 +24,13 @@ def action_redis_server_disconnect(config):
for c in clients:
con.client_kill(c)
log.error(" - Disconnected client '%s'" % c)
log.error(" - Client '%s' was disconnected" % c)
# Disconnect only one user
else:
# Check client format
if config.client is None or ":" not in config.client:
log.error("Invalid client format. Client must be format: IP:PORT, i.e: 10.211.55.2:61864")
log.error(" <!> Invalid client format. Client must be format: IP:PORT, i.e: 10.211.55.2:61864")
return
try:
@@ -38,6 +38,6 @@ def action_redis_server_disconnect(config):
con.client_kill(_c)
log.error(" - Disconnected client '%s'" % _c)
log.error(" - Client '%s' was disconnected" % _c)
except KeyError:
log.error("Client '%s' doesn't appear to be connected to server" % config.client)
log.error(" <!> Client '%s' doesn't appear to be connected to server" % config.client)

12
enteletaor_lib/modules/redis/redis_discover_db.py
View File

@@ -19,8 +19,18 @@ def action_redis_discover_dbs(config):
log.error("Discovered '%s' DBs at '%s':" % (config.target, con.config_get("databases")['databases']))
discovered_dbs = set()
for db_name, db_content in six.iteritems(con.info("keyspace")):
log.error(" - %s - %s keys" % (db_name.upper(), db_content['keys']))
discovered_dbs.add(db_name.upper())
for i in six.moves.range((int(con.config_get("databases")['databases']) - len(con.info("keyspace")))):
log.error(" - DB%s - Empty" % str(i))
_db_name = "DB%s" % i
if _db_name in discovered_dbs:
continue
log.error(" - %s - Empty" % _db_name)

30
enteletaor_lib/modules/redis/redis_dump.py
View File

@@ -19,12 +19,14 @@ def dump_keys(con):
val = None
if key_type in (b"kv", b"string"):
val = con.get(key)
if key_type in (b"hash", b"unacked", b"unacked_index"):
elif key_type in (b"hash", b"unacked", b"unacked_index"):
val = con.hgetall(key)
if key_type == b"zet":
elif key_type == b"zet":
val = con.zrange(key, 0, -1)
if key_type in (b"set", b"list"):
elif key_type in (b"set", b"list"):
val = con.mget(key)
elif key_type == b"list":
con.lrange(key, 0, -1)
if val is not None:
if isinstance(val, list):
@@ -59,7 +61,6 @@ def _decode_object(val, ident=5):
# convert value to original type -> JSON
try:
_transformed_info = json.loads(v.decode("utf-8"))
# except (binascii.Error, AttributeError, ValueError):
except (binascii.Error, AttributeError, ValueError):
_transformed_info = v
@@ -151,14 +152,20 @@ def action_redis_dump(config):
# Export results?
export_file = None
export_file_name = None
# Fix filename
if config.export_results:
export_file = open(config.export_results, "w")
log.error(" - Storing information into '%s'" % config.export_results)
export_file_name = config.export_results if ".json" in config.export_results else "%s.json" % config.export_results
if config.export_results:
export_file = open(export_file_name, "w")
log.error(" - Storing information into '%s'" % export_file_name)
elif config.no_screen is True:
log.error(" <!> If results will not be displayed, you must to indicate output file for results.")
return
i = 0
registers = False
for i, t_val in enumerate(dump_keys(con)):
key = t_val[0]
val = t_val[1]
@@ -169,10 +176,13 @@ def action_redis_dump(config):
# Dump to file?
if export_file is not None:
export_file.write(str(val))
export_file.write("%s: %s" % (key, str(val)))
if i == 0:
log.error(" - No information to dump in database")
# There are registers
registers = True
if registers is False:
log.error(" - No information to dump in database")
# Close file descriptor
if export_file is not None:

2
enteletaor_lib/modules/scan/scan_main.py
View File

@@ -172,7 +172,7 @@ def build_targets(config):
for v in val:
log.debug(" -> Detected registered network '%s'. Added for scan." % v)
results.update(str(x) for x in ipaddress.ip_network(v, strict=False))
results.update(str(x) for x in ipaddress.ip_network(six.u(v), strict=False))
except KeyError:
# Invalid domain
log.debug(" <ii> Error while try to extract domain: '%s'" % t)

26
enteletaor_lib/modules/tasks/__init__.py
View File

@@ -8,11 +8,11 @@ from .. import IModule
from ...libs.core.structs import CommonData
from ...libs.core.models import StringField, SelectField
from .cmd_actions import parser_proc_raw_dump, parser_proc_list_process, parser_proc_inject_process
from .proc_remove import action_proc_remove
from .proc_raw_dump import action_proc_raw_dump
from .proc_list_process import action_proc_list_process
from .proc_inject_process import action_proc_inject_process
from .cmd_actions import parser_proc_raw_dump, parser_proc_list_tasks, parser_taks_inject_process
from .tasks_remove import action_proc_remove
from .tasks_raw_dump import action_proc_raw_dump
from .tasks_list_process import action_proc_list_tasks
from .tasks_inject_process import action_task_inject_process
log = logging.getLogger()
@@ -21,6 +21,8 @@ log = logging.getLogger()
class ModuleModel(CommonData):
target = StringField(required=True)
db = StringField(default=None, label="only for Redis: database to use")
process_manager = SelectField(default="celery", choices=[("celery", "Celery")],
label="process manager running in backend")
broker_type = SelectField(default="redis", choices=[
("redis", "Redis server"),
("zmq", "ZeroMQ"),
@@ -40,15 +42,15 @@ class RemoteProcessModule(IModule):
cmd_args=parser_proc_raw_dump,
action=action_proc_raw_dump
),
'list-process': dict(
help="list remote process and their params",
cmd_args=parser_proc_list_process,
action=action_proc_list_process
'list-tasks': dict(
help="list remote tasks and their params",
cmd_args=parser_proc_list_tasks,
action=action_proc_list_tasks
),
'inject': dict(
help="list remote process and their params",
cmd_args=parser_proc_inject_process,
action=action_proc_inject_process
help="inject a new task into broker",
cmd_args=parser_taks_inject_process,
action=action_task_inject_process
),
'remove': dict(
help="remove remote processes in server",

5
enteletaor_lib/modules/tasks/cmd_actions.py
View File

@@ -13,10 +13,11 @@ def parser_proc_raw_dump(parser):
help="although all information be dumped do not stop")
gr.add_argument("-I", dest="interval", type=float, default=4,
help="timeout interval between tow connections")
gr.add_argument("--output", dest="output", help="store dumped information into file")
# ----------------------------------------------------------------------
def parser_proc_list_process(parser):
def parser_proc_list_tasks(parser):
parser.add_argument("-N", "--no-stream", dest="no_stream", action="store_true", default=False,
help="force to not listen until message is received")
@@ -29,7 +30,7 @@ def parser_proc_list_process(parser):
# ----------------------------------------------------------------------
def parser_proc_inject_process(parser):
def parser_taks_inject_process(parser):
gr = parser.add_argument_group("process importing options")
gr.add_argument("-f", "--function-file", dest="function_files", type=str, required=True,

42
enteletaor_lib/modules/tasks/proc_raw_dump.py
View File

@@ -1,42 +0,0 @@
# -*- coding: utf-8 -*-
import six
import logging
from time import sleep
from kombu import Connection
from .utils import list_remote_process
log = logging.getLogger()
# ----------------------------------------------------------------------
def action_proc_raw_dump(config):
log.warning(" - Trying to connect with server...")
url = '%s://%s' % (config.broker_type, config.target)
# with Connection('redis://%s' % REDIS) as conn:
with Connection(url) as conn:
in_queue = conn.SimpleQueue('celery')
while 1:
for remote_process, remote_args in list_remote_process(config, in_queue):
# Show info
log.error("Found process information:")
log.error(" - Remote process name: '%s'" % remote_process)
log.error(" - Input parameters:")
for i, x in enumerate(remote_args):
log.error(" -> P%s: %s" % (i, x))
# Queue is empty -> wait
if config.streaming_mode:
log.error("No more messages from server. Waiting for %s seconds and try again.." % config.interval)
sleep(config.interval)
else:
log.error("No more messages from server. Exiting...")
return

8
enteletaor_lib/modules/tasks/proc_inject_process.py → enteletaor_lib/modules/tasks/tasks_inject_process.py
View File

@@ -14,10 +14,10 @@ log = logging.getLogger()
# ----------------------------------------------------------------------
def action_proc_inject_process(config):
def action_task_inject_process(config):
if config.function_files is None:
log.warning(" - input .json file with process files is needed")
log.error(" - input .json file with process files is needed")
return
# --------------------------------------------------------------------------
@@ -26,7 +26,7 @@ def action_proc_inject_process(config):
with open(config.function_files, "r") as f:
f_info = json.load(f)
log.warning(" - Building process...")
log.error(" - Building process...")
# Search and inject process
injections = []
@@ -68,7 +68,7 @@ def action_proc_inject_process(config):
with Connection(url) as conn:
in_queue = conn.SimpleQueue('celery')
log.warning(" - Sending processes to '%s'" % config.target)
log.error(" - Sending processes to '%s'" % config.target)
for i, e in enumerate(injections, 1):
log.warning(" %s) %s" % (i, e['task']))

4
enteletaor_lib/modules/tasks/proc_list_process.py → enteletaor_lib/modules/tasks/tasks_list_process.py
View File

@@ -14,7 +14,7 @@ log = logging.getLogger()
# ----------------------------------------------------------------------
def action_proc_list_process(config):
def action_proc_list_tasks(config):
log.warning(" - Trying to connect with server...")
@@ -29,7 +29,7 @@ def action_proc_list_process(config):
# Get remote process
first_msg = True
while 1:
for remote_process, remote_args in list_remote_process(config, in_queue):
for remote_process, remote_args, _ in list_remote_process(config, in_queue):
if remote_process not in process_info:
process_info[remote_process] = remote_args

83
enteletaor_lib/modules/tasks/tasks_raw_dump.py Normal file
View File

@@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
import six
import csv
import logging
from time import sleep
from kombu import Connection
from .utils import list_remote_process
log = logging.getLogger()
# ----------------------------------------------------------------------
def action_proc_raw_dump(config):
log.warning(" - Trying to connect with server...")
url = '%s://%s' % (config.broker_type, config.target)
f_output = None
csv_output = None
if config.output is not None:
fixed_f = "%s.csv" % config.output if ".csv" not in config.output else config.output
f_output = open(fixed_f, "a")
csv_output = csv.writer(f_output)
log.error(" - Storing results at '%s'" % fixed_f)
# Write first col
csv_output.writerow([
"# Task name",
"Parameters (position#value)"
])
already_processed = set()
# with Connection('redis://%s' % REDIS) as conn:
with Connection(url) as conn:
in_queue = conn.SimpleQueue('celery')
while 1:
for remote_task, remote_args, task_id in list_remote_process(config, in_queue):
# Task already processed?
if task_id not in already_processed:
# Track
already_processed.add(task_id)
# Show info
log.error(" Found process information:")
log.error(" - Remote tasks name: '%s'" % remote_task)
log.error(" - Input parameters:")
to_csv = [remote_task]
for i, x in enumerate(remote_args):
log.error(" -> P%s: %s" % (i, x))
# Prepare to store JSON
to_csv.append("%s#%s" % (i, x))
# Store
if csv_output is not None:
csv_output.writerow(to_csv)
# Queue is empty -> wait
if config.streaming_mode:
log.error(" -> No more messages from server. Waiting for %s seconds and try again.." % config.interval)
sleep(config.interval)
else:
log.error(" -> No more messages from server. Exiting...")
return
# Close file descriptor
if f_output is not None:
f_output.close()
csv_output.close()

2
enteletaor_lib/modules/tasks/proc_remove.py → enteletaor_lib/modules/tasks/tasks_remove.py
View File

@@ -23,4 +23,4 @@ def action_proc_remove(config):
for _ in get_remote_messages(config, in_queue, False):
pass
log.error(" - All processes removed from '%s'" % config.target)
log.error(" - All tasks removed from '%s'" % config.target)

5
enteletaor_lib/modules/tasks/utils.py
View File

@@ -38,7 +38,7 @@ def get_param_type(value):
except Exception:
return "str"
elif type(value) == str:
elif type(value) in (str, unicode if six.PY2 else ""):
return "str"
else:
return "object"
@@ -138,11 +138,10 @@ def list_remote_process(config, queue):
# Read info
if msg_id not in already_processed:
# remote_process = deserialized['task'].split(".")[-1]
remote_process = deserialized['task']
remote_args = deserialized['args']
# Store as processed
already_processed.add(msg_id)
yield remote_process, remote_args
yield remote_process, remote_args, msg_id