Volans has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/382479 )
Change subject: Docstrings: use Google Style
......................................................................
Docstrings: use Google Style
* Use Google Style Python Docstrings to allow automatically
generated documentation with Sphinx.
Bug: T159308
Change-Id: Ib46a063964d5701f365c0fe3225854246c643f6b
---
M README.md
M cumin/__init__.py
M cumin/backends/__init__.py
M cumin/backends/direct.py
M cumin/backends/openstack.py
M cumin/backends/puppetdb.py
M cumin/cli.py
M cumin/grammar.py
M cumin/query.py
M cumin/tests/__init__.py
M cumin/tests/integration/test_cli.py
M cumin/transport.py
M cumin/transports/__init__.py
M cumin/transports/clustershell.py
M prospector.yaml
15 files changed, 859 insertions(+), 377 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/software/cumin
refs/changes/79/382479/1
diff --git a/README.md b/README.md
index ab7da2f..6af6f73 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@
```
Given that the `pyparsing` library defines the grammar in a BNF-like style,
for the details of the tokens not
-specified above check directly the code in `cumin/grammar.py`.
+specified above check directly the source code in `cumin/grammar.py`.
The `Query` class defined in `cumin/query.py` is the one taking care of
replacing the aliases, building and executing
the query parts with their respective backends and aggregating the results.
Once a query is executed, it returns a
@@ -173,7 +173,7 @@
```
Given that the `pyparsing` library used to define the grammar usesa BNF-like
style, for the details of the tokens not
-specified above check directly the code in `cumin/backends/direct.py`.
+specified above check directly the source code in `cumin/backends/direct.py`.
#### Transports
diff --git a/cumin/__init__.py b/cumin/__init__.py
index 2be2cfc..0d556cf 100644
--- a/cumin/__init__.py
+++ b/cumin/__init__.py
@@ -9,6 +9,7 @@
try:
__version__ = get_distribution(__name__).version
+ """:py:class:`str`: the version of the current Cumin module."""
except DistributionNotFound: # pragma: no cover - this should never happen
during tests
pass # package is not installed
@@ -34,7 +35,12 @@
def trace(self, msg, *args, **kwargs):
- """Additional logging level for development debugging."""
+ """Additional logging level for development debugging.
+
+ :Parameters:
+ according to :py:class:`logging.Logger` interface for log levels.
+
+ """
if self.isEnabledFor(LOGGING_TRACE_LEVEL_NUMBER):
self._log(LOGGING_TRACE_LEVEL_NUMBER, msg, args, **kwargs) # pragma:
no cover, pylint: disable=protected-access
@@ -58,7 +64,15 @@
Called by Python's data model for each new instantiation of the class.
Arguments:
- config -- path to the configuration file to load. [optional, default:
/etc/cumin/config.yaml]
+ config (str, optional): path to the configuration file to load.
+
+ Returns:
+ dict: the configuration dictionary.
+
+ Examples:
+ >>> import cumin
+ >>> config = cumin.Config()
+
"""
if config not in cls._instances:
cls._instances[config] = parse_config(config)
@@ -73,7 +87,14 @@
"""Parse the YAML configuration file.
Arguments:
- config_file -- the path of the configuration file to load
+ config_file (str): the path of the configuration file to load.
+
+ Returns:
+ dict: the configuration dictionary.
+
+ Raises:
+ CuminError: if unable to read or parse the configuration.
+
"""
try:
with open(config_file, 'r') as f:
diff --git a/cumin/backends/__init__.py b/cumin/backends/__init__.py
index a8d0d37..19a6c0e 100644
--- a/cumin/backends/__init__.py
+++ b/cumin/backends/__init__.py
@@ -23,15 +23,16 @@
__metaclass__ = ABCMeta
- """Derived classes must define their own pyparsing grammar and set this
class attribute accordingly."""
grammar = pyparsing.NoMatch() # This grammar will never match.
+ """:py:class:`pyparsing.ParserElement`: derived classes must define their
own pyparsing grammar and set this class
+ attribute accordingly."""
def __init__(self, config, logger=None):
"""Query constructor.
Arguments:
- config -- a dictionary with the parsed configuration file
- logger -- an optional logging.Logger instance [optional, default: None]
+ config (dict): a dictionary with the parsed configuration file.
+ logger (logging.Logger, optional): an optional logger instance.
"""
self.config = config
self.logger = logger or logging.getLogger(__name__)
@@ -39,31 +40,40 @@
name=type(self).__name__, config=config))
def execute(self, query_string):
- """Build and execute the query, return the list of FQDN hostnames that
matches.
+ """Build and execute the query, return the NodeSet of FQDN hostnames
that matches.
Arguments:
- query_string -- the query string to be parsed and executed
+ query_string (str): the query string to be parsed and executed.
+
+ Returns:
+ ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
+
"""
self._build(query_string)
return self._execute()
@abstractmethod
def _execute(self):
- """Execute the already parsed query and return the list of FQDN
hostnames that matches."""
+ """Execute the already parsed query and return the NodeSet of FQDN
hostnames that matches.
+
+ Returns:
+ ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
+
+ """
@abstractmethod
def _parse_token(self, token):
"""Recursively interpret the tokens returned by the grammar parsing.
Arguments:
- token -- a single token returned by the grammar parsing
+ token (pyparsing.ParseResults): a single token returned by the
grammar parsing.
"""
def _build(self, query_string):
"""Parse the query string according to the grammar and build the query
for later execution.
Arguments:
- query_string -- the query string to be parsed
+ query_string (str): the query string to be parsed.
"""
self.logger.trace('Parsing query: {query}'.format(query=query_string))
parsed = self.grammar.parseString(query_string.strip(), parseAll=True)
@@ -75,29 +85,41 @@
class BaseQueryAggregator(BaseQuery):
"""Query aggregator abstract class.
- Add to BaseQuery the capability of aggregating query subgroups and sub
tokens into a unified result using common
- boolean operators for sets: and, or, and not, xor.
+ Add to :py:class:`cumin.backends.BaseQuery` the capability of aggregating
query subgroups and sub tokens into a
+ unified result using common boolean operators for sets: ``and``, ``or``,
``and not`` and ``xor``.
The class has a stack-like structure that must be populated by the derived
classes while building the query.
On execution the stack is traversed and the results are aggreagated
together based on subgroups and boolean
operators.
"""
def __init__(self, config, logger=None):
- """Query aggregator constructor, initialize the stack."""
+ """Query aggregator constructor, initialize the stack.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery.__init__`.
+ """
super(BaseQueryAggregator, self).__init__(config, logger=logger)
self.stack = None
self.stack_pointer = None
def _build(self, query_string):
- """Override parent class _build method to reset the stack and log
it."""
+ """Override parent method to reset the stack and log it.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery._build`.
+ """
self.stack = self._get_stack_element()
self.stack_pointer = self.stack
super(BaseQueryAggregator, self)._build(query_string)
self.logger.trace('Query stack: {stack}'.format(stack=self.stack))
def _execute(self):
- """Required by BaseQuery."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery._execute`.
+ """
hosts = NodeSet()
self._loop_stack(hosts, self.stack) # The hosts nodeset is updated in
place while looping the stack
self.logger.debug('Found {num} hosts'.format(num=len(hosts)))
@@ -117,20 +139,29 @@
@abstractmethod
def _parse_token(self, token):
- """Required by BaseQuery."""
+ """Re-define abstract method from parent abstract class.
+
+ :Parameters:
+ according to parent
:py:meth:`cumin.backends.BaseQuery._parse_token`.
+ """
@staticmethod
def _get_stack_element():
- """Return an empty stack element."""
+ """Return an empty stack element.
+
+ Returns:
+ dict: the dictionary with an empty stack element.
+
+ """
return {'hosts': None, 'children': [], 'parent': None, 'bool': None}
def _loop_stack(self, hosts, stack_element):
"""Loop the stack generated while parsing the query and aggregate the
results.
Arguments:
- hosts -- the NodeSet of hosts to update with the current stack
element results. This object is updated
- in place by reference.
- stack_element -- the stack element to iterate
+ hosts (ClusterShell.NodeSet.NodeSet): the hosts to be updated with
the current stack element results. This
+ object is updated in place by reference.
+ stack_element (dict): the stack element to iterate.
"""
if stack_element['hosts'] is None:
element_hosts = NodeSet()
@@ -142,14 +173,15 @@
self._aggregate_hosts(hosts, element_hosts, stack_element['bool'])
def _aggregate_hosts(self, hosts, element_hosts, bool_operator):
- """.
+ """Aggregate hosts according to their boolean operator.
Arguments:
- hosts -- the NodeSet of hosts to update with the results in
element_hosts according to the
- bool_operator. This object is updated in place by
reference.
- element_hosts -- the NodeSet of additional hosts to aggregate to the
results based on the bool_operator
- bool_operator -- the boolean operator to apply while aggregating the
two NodeSet. It must be None when adding
- the first hosts.
+ hosts (ClusterShell.NodeSet.NodeSet): the hosts to update with the
results in ``element_hosts`` according
+ to the bool_operator. This object is updated in place by
reference.
+ element_hosts (ClusterShell.NodeSet.NodeSet): the additional hosts
to aggregate to the results based on
+ the ``bool_operator``.
+ bool_operator (str, None): the boolean operator to apply while
aggregating the two NodeSet. It must be
+ :py:data:`None` when adding the first hosts.
"""
self.logger.trace("Aggregating: {hosts} | {boolean} |
{element_hosts}".format(
hosts=hosts, boolean=bool_operator, element_hosts=element_hosts))
diff --git a/cumin/backends/direct.py b/cumin/backends/direct.py
index 24bb670..0a9f6e8 100644
--- a/cumin/backends/direct.py
+++ b/cumin/backends/direct.py
@@ -10,18 +10,24 @@
"""Define the query grammar.
Some query examples:
- - Simple selection: host1.domain
- - ClusterShell syntax for hosts expansion:
host10[10-42].domain,host2010.other-domain
- - A complex selection:
- host100[1-5].domain or (host10[30-40].domain and (host10[10-42].domain
and not host33.domain))
- Backus-Naur form (BNF) of the grammar:
- <grammar> ::= <item> | <item> <boolean> <grammar>
- <item> ::= <hosts> | "(" <grammar> ")"
- <boolean> ::= "and not" | "and" | "xor" | "or"
+ * Simple selection: ``host1.domain``
+ * ClusterShell syntax for hosts expansion:
``host10[10-42].domain,host2010.other-domain``
+ * A complex selection:
+ ``host100[1-5].domain or (host10[30-40].domain and (host10[10-42].domain
and not host33.domain))``
+
+ Backus-Naur form (BNF) of the grammar::
+
+ <grammar> ::= <item> | <item> <boolean> <grammar>
+ <item> ::= <hosts> | "(" <grammar> ")"
+ <boolean> ::= "and not" | "and" | "xor" | "or"
Given that the pyparsing library defines the grammar in a BNF-like style,
for the details of the tokens not
- specified above check directly the code.
+ specified above check directly the source code.
+
+ Returns:
+ pyparsing.ParserElement: the grammar parser.
+
"""
# Boolean operators
boolean = (pp.CaselessKeyword('and not').leaveWhitespace() |
pp.CaselessKeyword('and') |
@@ -46,19 +52,23 @@
class DirectQuery(BaseQueryAggregator):
"""DirectQuery query builder.
- The 'direct' backend allow to use Cumin without any external dependency
for the hosts selection.
+ The `direct` backend allow to use Cumin without any external dependency
for the hosts selection.
It allow to write arbitrarily complex queries with subgroups and boolean
operators, but each item must be either the
- hostname itself, or the using host expansion using the powerful
ClusterShell NodeSet syntax, see:
- https://clustershell.readthedocs.io/en/latest/api/NodeSet.html
+ hostname itself, or the using host expansion using the powerful
:py:class:`ClusterShell.NodeSet.NodeSet` syntax.
- The typical usage for the 'direct' backend is as a reliable alternative in
cases in which the primary host
+ The typical usage for the `direct` backend is as a reliable alternative in
cases in which the primary host
selection mechanism is not working and also for testing the transports
without any external backend dependency.
"""
grammar = grammar()
+ """:py:class:`pyparsing.ParserElement`: load the grammar parser only once
in a singleton-like way."""
def _parse_token(self, token):
- """Required by BaseQuery."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent
:py:meth:`cumin.backends.BaseQueryAggregator._parse_token`.
+ """
if not isinstance(token, pp.ParseResults): # pragma: no cover - this
should never happen
raise InvalidQueryError('Expecting ParseResults object, got
{type}: {token}'.format(
type=type(token), token=token))
@@ -85,6 +95,9 @@
raise InvalidQueryError('Got unexpected token:
{token}'.format(token=token))
-# Required by the backend auto-loader in
cumin.grammar.get_registered_backends()
GRAMMAR_PREFIX = 'D'
+""":py:class:`str`: the prefix associate to this grammar, to register this
backend into the general grammar.
+Required by the backend auto-loader in
:py:meth:`cumin.grammar.get_registered_backends`."""
+
query_class = DirectQuery # pylint: disable=invalid-name
+"""Required by the backend auto-loader in
:py:meth:`cumin.grammar.get_registered_backends`."""
diff --git a/cumin/backends/openstack.py b/cumin/backends/openstack.py
index 0c28f96..b2e1b71 100644
--- a/cumin/backends/openstack.py
+++ b/cumin/backends/openstack.py
@@ -14,22 +14,28 @@
"""Define the query grammar.
Some query examples:
- - All hosts in all OpenStack projects: `*`
- - All hosts in a specific OpenStack project: `project:project_name`
- - Filter hosts using any parameter allowed by the OpenStack list-servers
API: `name:host1 image:UUID`
- See https://developer.openstack.org/api-ref/compute/#list-servers for
more details.
- Multiple filters can be added separated by space. The value can be
enclosed in single or double quotes.
- If the `project` key is not specified the hosts will be selected from
all projects.
- - To mix multiple selections the general grammar must be used with
multiple subqueries:
- `O{project:project1} or O{project:project2}`
- Backus-Naur form (BNF) of the grammar:
- <grammar> ::= "*" | <items>
- <items> ::= <item> | <item> <whitespace> <items>
- <item> ::= <key>:<value>
+ * All hosts in all OpenStack projects: ``*``
+ * All hosts in a specific OpenStack project: ``project:project_name``
+ * Filter hosts using any parameter allowed by the OpenStack list-servers
API: ``name:host1 image:UUID``
+ See `OpenStack Compute API list-servers
<https://developer.openstack.org/api-ref/compute/#list-servers>`_ for
+ more details. Multiple filters can be added separated by space. The
value can be enclosed in single or double
+ quotes. If the ``project`` key is not specified the hosts will be
selected from all projects.
+ * To mix multiple selections the general grammar must be used with
multiple subqueries:
+ ``O{project:project1} or O{project:project2}``
+
+ Backus-Naur form (BNF) of the grammar::
+
+ <grammar> ::= "*" | <items>
+ <items> ::= <item> | <item> <whitespace> <items>
+ <item> ::= <key>:<value>
Given that the pyparsing library defines the grammar in a BNF-like style,
for the details of the tokens not
- specified above check directly the code.
+ specified above check directly the source code.
+
+ Returns:
+ pyparsing.ParserElement: the grammar parser.
+
"""
quoted_string = pp.quotedString.copy().addParseAction(pp.removeQuotes) #
Both single and double quotes are allowed
@@ -49,8 +55,12 @@
"""Return a new keystone session based on configuration.
Arguments:
- config -- a dictionary with the session configuration: auth_url,
username, password
- project -- a project to scope the session to. [optional, default: None]
+ config (dict): a dictionary with the session configuration keys:
``auth_url``, ``username``, ``password``.
+ project (str, optional): a project to scope the session to.
+
+ Returns:
+ keystoneauth1.session.Session: the Keystone session scoped for the
project if specified.
+
"""
auth = keystone_identity.Password(
auth_url='{auth_url}/v3'.format(auth_url=config.get('auth_url',
'http://localhost:5000')),
@@ -66,8 +76,13 @@
"""Return a new nova client tailored to the given project.
Arguments:
- config -- a dictionary with the session configuration: auth_url,
username, password, nova_api_version, timeout
- project -- a project to scope the session to. [optional, default: None]
+ config (dict): a dictionary with the session configuration keys:
``auth_url``, ``username``, ``password``,
+ ``nova_api_version``, ``timeout``.
+ project (str): the project to scope the `novaclient` session to.
+
+ Returns:
+ novaclient.client.Client: the novaclient Client instance, already
authenticated.
+
"""
return nova_client.Client(
config.get('nova_api_version', '2'),
@@ -83,11 +98,14 @@
"""
grammar = grammar()
+ """:py:class:`pyparsing.ParserElement`: load the grammar parser only once
in a singleton-like way."""
def __init__(self, config, logger=None):
- """Query constructor for the OpenStack backend.
+ """Override parent class constructor for specific setup.
- Arguments: according to BaseQuery interface
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery.__init__`.
+
"""
super(OpenStackQuery, self).__init__(config, logger=logger)
self.openstack_config = self.config.get('openstack', {})
@@ -95,7 +113,12 @@
self.search_params = self._get_default_search_params()
def _get_default_search_params(self):
- """Return the default search parameters dictionary and set the
project, if configured."""
+ """Return the default search parameters dictionary and set the
project, if configured.
+
+ Returns:
+ dict: the dictionary with the default search parameters.
+
+ """
params = {'status': 'ACTIVE', 'vm_state': 'ACTIVE'}
config_params = self.openstack_config.get('query_params', {})
@@ -106,12 +129,25 @@
return params
def _build(self, query_string):
- """Override parent class _build method to reset search parameters."""
+ """Override parent class _build method to reset the search parameters.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery._build`.
+
+ """
self.search_params = self._get_default_search_params()
super(OpenStackQuery, self)._build(query_string)
def _execute(self):
- """Required by BaseQuery."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery._execute`.
+
+ Returns:
+ ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
+
+ """
if self.search_project is None:
hosts = NodeSet()
for project in self._get_projects():
@@ -122,7 +158,15 @@
return hosts
def _parse_token(self, token):
- """Required by BaseQuery."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent
:py:meth:`cumin.backends.BaseQuery._parse_token`.
+
+ Raises:
+ cumin.backends.InvalidQueryError: on internal parsing error.
+
+ """
if not isinstance(token, pp.ParseResults): # pragma: no cover - this
should never happen
raise InvalidQueryError('Expecting ParseResults object, got
{type}: {token}'.format(
type=type(token), token=token))
@@ -141,7 +185,12 @@
raise InvalidQueryError('Got unexpected token:
{token}'.format(token=token))
def _get_projects(self):
- """Yield the project names for all projects (except admin) from
keystone API."""
+ """Get all the project names from keystone API, filtering out the
special `admin` project. Is a `generator`.
+
+ Yields:
+ str: the project name for all the selected projects.
+
+ """
client = keystone_client.Client(
session=_get_keystone_session(self.openstack_config),
timeout=self.openstack_config.get('timeout', 10))
return (project.name for project in client.projects.list(enabled=True)
if project.name != 'admin')
@@ -150,7 +199,11 @@
"""Return a NodeSet with the list of matching hosts based for the
project based on the search parameters.
Arguments:
- project -- the project name where to get the list of hosts
+ project (str): the project name where to get the list of hosts.
+
+ Returns:
+ ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
+
"""
client = _get_nova_client(self.openstack_config, project)
@@ -166,6 +219,9 @@
for server in
client.servers.list(search_opts=self.search_params))
-# Required by the backend auto-loader in
cumin.grammar.get_registered_backends()
GRAMMAR_PREFIX = 'O'
+""":py:class:`str`: the prefix associate to this grammar, to register this
backend into the general grammar.
+Required by the backend auto-loader in
:py:meth:`cumin.grammar.get_registered_backends`."""
+
query_class = OpenStackQuery # pylint: disable=invalid-name
+"""Required by the backend auto-loader in
:py:meth:`cumin.grammar.get_registered_backends`."""
diff --git a/cumin/backends/puppetdb.py b/cumin/backends/puppetdb.py
index 877a1e7..190a3d8 100644
--- a/cumin/backends/puppetdb.py
+++ b/cumin/backends/puppetdb.py
@@ -12,46 +12,54 @@
from cumin.backends import BaseQuery, InvalidQueryError
-# Available categories
CATEGORIES = (
'F', # Fact
'R', # Resource
)
+""":py:func:`tuple`: available categories in the grammar."""
-# Available operators
OPERATORS = ('=', '>=', '<=', '<', '>', '~')
+""":py:func:`tuple`: available operators in the grammar, the same available in
PuppetDB API."""
def grammar():
"""Define the query grammar.
Some query examples:
- - All hosts: `*`
- - Hosts globbing: `host10*`
- - ClusterShell's NodeSet syntax (see
https://clustershell.readthedocs.io/en/latest/api/NodeSet.html) for hosts
- expansion: `host10[10-42].domain`
- - Category based key-value selection:
- - `R:Resource::Name`: query all the hosts that have a resource of type
`Resource::Name`.
- - `R:Resource::Name = 'resource-title'`: query all the hosts that have a
resource of type `Resource::Name` whose
- title is `resource-title`. For example `R:Class = MyModule::MyClass`.
- - `R:Resource::Name@field = 'some-value'`: query all the hosts that have
a resource of type `Resource::Name`
- whose field `field` has the value `some-value`. The valid fields are:
`tag`, `certname`, `type`, `title`,
- `exported`, `file`, `line`. The previous syntax is a shortcut for this
one with the field `title`.
- - `R:Resource::Name%param = 'some-value'`: query all the hosts that have
a resource of type `Resource::Name`
- whose parameter `param` has the value `some-value`.
- - Mixed facts/resources queries are not supported, but the same result
can be achieved by the main grammar using
- multiple subqueries.
- - A complex selection for facts:
- `host10[10-42].*.domain or (not F:key1 = value1 and host10*) or (F:key2
> value2 and F:key3 ~ '^value[0-9]+')`
- Backus-Naur form (BNF) of the grammar:
+ * All hosts: ``*``
+ * Hosts globbing: ``host10*``
+ * :py:class:`ClusterShell.NodeSet.NodeSet` syntax for hosts expansion:
``host10[10-42].domain``
+ * Category based key-value selection:
+
+ * ``R:Resource::Name``: query all the hosts that have a resource of type
`Resource::Name`.
+ * ``R:Resource::Name = 'resource-title'``: query all the hosts that have
a resource of type `Resource::Name`
+ whose title is ``resource-title``. For example ``R:Class =
MyModule::MyClass``.
+ * ``R:Resource::Name@field = 'some-value'``: query all the hosts that
have a resource of type ``Resource::Name``
+ whose field ``field`` has the value ``some-value``. The valid fields
are: ``tag``, ``certname``, ``type``,
+ ``title``, ``exported``, ``file``, ``line``. The previous syntax is a
shortcut for this one with the field
+ ``title``.
+ * ``R:Resource::Name%param = 'some-value'``: query all the hosts that
have a resource of type ``Resource::Name``
+ whose parameter ``param`` has the value ``some-value``.
+ * Mixed facts/resources queries are not supported, but the same result
can be achieved by the main grammar using
+ multiple subqueries.
+
+ * A complex selection for facts:
+ ``host10[10-42].*.domain or (not F:key1 = value1 and host10*) or (F:key2
> value2 and F:key3 ~ '^value[0-9]+')``
+
+ Backus-Naur form (BNF) of the grammar::
+
<grammar> ::= <item> | <item> <and_or> <grammar>
<item> ::= [<neg>] <query-token> | [<neg>] "(" <grammar> ")"
<query-token> ::= <token> | <hosts>
<token> ::= <category>:<key> [<operator> <value>]
Given that the pyparsing library defines the grammar in a BNF-like style,
for the details of the tokens not
- specified above check directly the code.
+ specified above check directly the source code.
+
+ Returns:
+ pyparsing.ParserElement: the grammar parser.
+
"""
# Boolean operators
and_or = (pp.CaselessKeyword('and') | pp.CaselessKeyword('or'))('bool')
@@ -92,19 +100,31 @@
class PuppetDBQuery(BaseQuery):
"""PuppetDB query builder.
- The 'puppetdb' backend allow to use an existing PuppetDB instance for the
hosts selection.
+ The `puppetdb` backend allow to use an existing PuppetDB instance for the
hosts selection.
At the moment only PuppetDB v3 API are implemented.
"""
base_url_template = 'https://{host}:{port}/v3/'
+ """:py:class:`str`: string template in the :py:meth:`str.format` style
used to generate the base URL of the
+ PuppetDB server."""
+
endpoints = {'R': 'resources', 'F': 'nodes'}
+ """:py:class:`dict`: dictionary with the mapping of the available
categories in the grammar to the PuppetDB API
+ endpoints."""
+
hosts_keys = {'R': 'certname', 'F': 'name'}
+ """:py:class:`dict`: dictionary with the mapping of the available
categories in the grammar to the PuppetDB API
+ field to query to get the hostname."""
+
grammar = grammar()
+ """:py:class:`pyparsing.ParserElement`: load the grammar parser only once
in a singleton-like way."""
def __init__(self, config, logger=None):
"""Query constructor for the PuppetDB backend.
- Arguments: according to BaseQuery interface
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery.__init__`.
+
"""
super(PuppetDBQuery, self).__init__(config, logger=logger)
self.grouped_tokens = None
@@ -120,16 +140,24 @@
@property
def category(self):
- """Getter for the property category with a default value."""
+ """Category for the current query.
+
+ :Getter:
+ Returns the current `category` or a default value if not set.
+
+ :Setter:
+ :py:class:`str`: the value to set the `category` to.
+
+ Raises:
+ cumin.backends.InvalidQueryError: if trying to set it to an
invalid `category` or mixing categories in a
+ single query.
+
+ """
return self._category or 'F'
@category.setter
def category(self, value):
- """Setter for the property category with validation.
-
- Arguments:
- value -- the value to set the category to
- """
+ """Setter for the `category` property. The relative documentation is
in the getter."""
if value not in self.endpoints:
raise InvalidQueryError("Invalid value '{category}' for category
property".format(category=value))
if self._category is not None and value != self._category:
@@ -150,18 +178,36 @@
@staticmethod
def _get_grouped_tokens():
- """Return an empty grouped tokens structure."""
+ """Return an empty grouped tokens structure.
+
+ Returns:
+ dict: the dictionary with the empty grouped tokens structure.
+
+ """
return {'parent': None, 'bool': None, 'tokens': []}
def _build(self, query_string):
- """Override parent class _build method to reset tokens and add
logging."""
+ """Override parent class _build method to reset tokens and add logging.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery._build`.
+
+ """
self.grouped_tokens = PuppetDBQuery._get_grouped_tokens()
self.current_group = self.grouped_tokens
super(PuppetDBQuery, self)._build(query_string)
self.logger.trace('Query tokens:
{tokens}'.format(tokens=self.grouped_tokens))
def _execute(self):
- """Required by BaseQuery."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.backends.BaseQuery._execute`.
+
+ Returns:
+ ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
+
+ """
query =
self._get_query_string(group=self.grouped_tokens).format(host_key=self.hosts_keys[self.category])
hosts = self._api_call(query, self.endpoints[self.category])
unique_hosts = NodeSet.fromlist([host[self.hosts_keys[self.category]]
for host in hosts])
@@ -174,11 +220,15 @@
"""Add a category token to the query 'F:key = value'.
Arguments:
- category -- the category of the token, one of CATEGORIES excluding the
alias one.
- key -- the key for this category
- value -- the value to match, if not specified the key itself will
be matched [optional, default: None]
- operator -- the comparison operator to use, one of
cumin.grammar.OPERATORS [optional: default: =]
- neg -- whether the token must be negated [optional, default:
False]
+ category (str): the category of the token, one of
:py:const:`CATEGORIES`.
+ key (str): the key for this category.
+ value (str, optional): the value to match, if not specified the
key itself will be matched.
+ operator (str, optional): the comparison operator to use, one of
:py:const:`OPERATORS`.
+ neg (bool, optional): whether the token must be negated.
+
+ Raises:
+ cumin.backends.InvalidQueryError: on internal parsing error.
+
"""
self.category = category
if operator == '~':
@@ -201,8 +251,8 @@
"""Add a list of hosts to the query.
Arguments:
- hosts -- a list of hosts to match
- neg -- whether the token must be negated [optional, default: False]
+ hosts (ClusterShell.NodeSet.NodeSet): with the list of hosts to
search.
+ neg (bool, optional): whether the token must be negated.
"""
if not hosts:
return
@@ -224,7 +274,15 @@
self.current_group['tokens'].append(query)
def _parse_token(self, token):
- """Required by BaseQuery."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent
:py:meth:`cumin.backends.BaseQuery._parse_token`.
+
+ Raises:
+ cumin.backends.InvalidQueryError: on internal parsing error.
+
+ """
if isinstance(token, str):
return
@@ -252,12 +310,19 @@
"No valid key found in token, one of bool|hosts|category
expected: {token}".format(token=token_dict))
def _get_resource_query(self, key, value=None, operator='='): # pylint:
disable=no-self-use
- """Build a resource query based on the parameters, resolving the
special cases for %params and @field.
+ """Build a resource query based on the parameters, resolving the
special cases for ``%params`` and ``@field``.
Arguments:
- key -- the key of the resource
- value -- the value to match, if not specified the key itself will
be matched [optional, default: None]
- operator -- the comparison operator to use, one of
cumin.grammar.OPERATORS [optional: default: =]
+ key (str): the key of the resource.
+ value (str, optional): the value to match, if not specified the
key itself will be matched.
+ operator (str, optional): the comparison operator to use, one of
:py:const:`OPERATORS`.
+
+ Returns:
+ str: the resource query.
+
+ Raises:
+ cumin.backends.InvalidQueryError: on invalid combinations of
parameters.
+
"""
if all(char in key for char in ('%', '@')):
raise InvalidQueryError(("Resource key cannot contain both '%'
(query a resource's parameter) and '@' "
@@ -293,7 +358,11 @@
"""Recursively build and return the PuppetDB query string.
Arguments:
- group -- a dictionary with the grouped tokens
+ group (dict): a dictionary with the grouped tokens.
+
+ Returns:
+ str: the query string for the PuppetDB API.
+
"""
if group['bool']:
query = '["{bool}", '.format(bool=group['bool'])
@@ -319,7 +388,11 @@
"""Add a boolean AND or OR query block to the query and validate logic.
Arguments:
- bool_op -- the boolean operator (and|or) to add to the query
+ bool_op (str): the boolean operator to add to the query: ``and``,
``or``.
+
+ Raises:
+ cumin.backends.InvalidQueryError: if an invalid boolean operator
was found.
+
"""
if self.current_group['bool'] is None:
self.current_group['bool'] = bool_op
@@ -333,14 +406,21 @@
"""Execute a query to PuppetDB API and return the parsed JSON.
Arguments:
- query -- the query parameter to send to the PuppetDB API
- endpoint -- the endpoint of the PuppetDB API to call
+ query (str): the query parameter to send to the PuppetDB API.
+ endpoint (str): the endpoint of the PuppetDB API to call.
+
+ Raises:
+ requests.HTTPError: if the PuppetDB API call fails.
+
"""
resources = requests.get(self.url + endpoint, params={'query': query},
verify=True)
resources.raise_for_status()
return resources.json()
-# Required by the backend auto-loader in
cumin.grammar.get_registered_backends()
GRAMMAR_PREFIX = 'P'
+""":py:class:`str`: the prefix associate to this grammar, to register this
backend into the general grammar.
+Required by the backend auto-loader in
:py:meth:`cumin.grammar.get_registered_backends`."""
+
query_class = PuppetDBQuery # pylint: disable=invalid-name
+"""Required by the backend auto-loader in
:py:meth:`cumin.grammar.get_registered_backends`."""
diff --git a/cumin/cli.py b/cumin/cli.py
index f9e16ce..86209b5 100644
--- a/cumin/cli.py
+++ b/cumin/cli.py
@@ -22,23 +22,25 @@
logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+"""logging.Logger: The logging instance."""
OUTPUT_FORMATS = ('txt', 'json')
+"""tuple: A tuple with the possible output formats."""
INTERACTIVE_BANNER = """===== Cumin Interactive REPL =====
# Press Ctrl+d or type exit() to exit the program.
= Available variables =
-# hosts -- the ClusterShell NodeSet of targeted hosts.
-# worker -- the instance of the Transport worker that was used for the
execution.
-# args -- the parsed command line arguments, an argparse.Namespace
instance.
-# config -- the cofiguration dictionary.
-# exit_code -- the return code of the execution, that will be used as exit
code.
+# hosts: the ClusterShell NodeSet of targeted hosts.
+# worker: the instance of the Transport worker that was used for the
execution.
+# args: the parsed command line arguments, an argparse.Namespace instance.
+# config: the cofiguration dictionary.
+# exit_code: the return code of the execution, that will be used as exit
code.
= Useful functions =
-# worker.get_results() -- generator that yields the tuple (nodes, output) for
each grouped result, where:
-# - nodes -- is a ClusterShell.NodeSet.NodeSet
instance
-# - output -- is a ClusterShell.MsgTree.MsgTreeElem
instance
-# h() -- print this help message.
-# help(object) -- Python default interactive help and documentation of
the given object.
+# worker.get_results(): generator that yields the tuple (nodes, output)
for each grouped result, where:
+# - nodes: is a ClusterShell.NodeSet.NodeSet
instance
+# - output: is a ClusterShell.MsgTree.MsgTreeElem
instance
+# h(): print this help message.
+# help(object): Python default interactive help and documentation of the
given object.
= Example usage:
for nodes, output in worker.get_results():
@@ -46,6 +48,7 @@
print(output)
print('-----')
"""
+"""str: The message to print when entering the intractive REPL mode."""
class KeyboardInterruptError(cumin.CuminError):
@@ -56,7 +59,7 @@
"""Parse command line arguments and return them.
Arguments:
- argv -- the list of arguments to use. If None, the command line ones are
used [optional, default: None]
+ argv: the list of arguments to use. If None, the command line ones are
used [optional, default: None]
"""
sync_mode = 'sync'
async_mode = 'async'
@@ -157,8 +160,8 @@
"""Setup the logger instance.
Arguments:
- filename -- the filename of the log file
- debug -- whether to set logging level to DEBUG [optional, default:
False]
+ filename: the filename of the log file
+ debug: whether to set logging level to DEBUG [optional, default: False]
"""
file_path = os.path.dirname(filename)
if not os.path.exists(file_path):
@@ -183,8 +186,8 @@
"""Signal handler for Ctrl+c / SIGINT, raises KeyboardInterruptError.
Arguments (as defined in https://docs.python.org/2/library/signal.html):
- signum -- the signal number
- frame -- the current stack frame
+ signum: the signal number
+ frame: the current stack frame
"""
if not sys.stdout.isatty(): # pylint: disable=no-member
logger.warning('Execution interrupted by Ctrl+c/SIGINT')
@@ -224,8 +227,8 @@
r"""Print a message to stderr and flush.
Arguments:
- message -- the message to print to sys.stderr
- end -- the character to use at the end of the message. [optional,
default: \n]
+ message: the message to print to sys.stderr
+ end: the character to use at the end of the message. [optional,
default: \n]
"""
tqdm.write('{color}{message}{reset}'.format(
color=colorama.Fore.YELLOW, message=message,
reset=colorama.Style.RESET_ALL), file=sys.stderr, end=end)
@@ -235,8 +238,8 @@
"""Resolve the hosts selection into a list of hosts and return it. Raises
KeyboardInterruptError.
Arguments:
- args -- ArgumentParser instance with parsed command line arguments
- config -- a dictionary with the parsed configuration file
+ args: ArgumentParser instance with parsed command line arguments
+ config: a dictionary with the parsed configuration file
"""
hosts = query.Query(config, logger=logger).execute(args.hosts)
@@ -280,8 +283,8 @@
"""Print the execution results in a specific format.
Arguments:
- output_format -- the output format to use, one of: 'txt', 'json'.
- worker -- the Transport worker instance to retrieve the results
from.
+ output_format: the output format to use, one of: 'txt', 'json'.
+ worker: the Transport worker instance to retrieve the results from.
"""
if output_format not in OUTPUT_FORMATS:
raise cumin.CuminError("Got invalid output format '{fmt}', expected
one of {allowed}".format(
@@ -306,8 +309,8 @@
"""Execute the commands on the selected hosts and print the results.
Arguments:
- args -- ArgumentParser instance with parsed command line arguments
- config -- a dictionary with the parsed configuration file
+ args: ArgumentParser instance with parsed command line arguments
+ config: a dictionary with the parsed configuration file
"""
hosts = get_hosts(args, config)
if not hosts:
@@ -345,7 +348,7 @@
"""CLI entry point. Execute commands on hosts according to arguments.
Arguments:
- argv -- the list of arguments to use. If None, the command line ones are
used [optional, default: None]
+ argv: the list of arguments to use. If None, the command line ones are
used [optional, default: None]
"""
signal.signal(signal.SIGINT, sigint_handler)
colorama.init()
diff --git a/cumin/grammar.py b/cumin/grammar.py
index 40104cb..364adc8 100644
--- a/cumin/grammar.py
+++ b/cumin/grammar.py
@@ -6,15 +6,29 @@
import pyparsing as pp
-
from cumin import backends, CuminError
-# Backend object
+
Backend = namedtuple('Backend', ['keyword', 'name', 'cls'])
+""":py:func:`collections.namedtuple` that define a Backend object.
+
+Keyword Arguments:
+ keyword (str): The backend keyword to be used in the grammar.
+ name (str): The backend name.
+ cls (BaseQuery): The backend class object.
+"""
def get_registered_backends():
- """Return a list of Backend objects for all the registered backends."""
+ """Get a mapping of all the registered backends with their keyword.
+
+ Returns:
+ dict: A dictionary with a ``{keyword: Backend object}`` mapping for
each available backend.
+
+ Raises:
+ cumin.CuminError: If unable to register a backend because the ey is
already used by another backend.
+
+ """
available_backends = {}
backend_names = [name for _, name, ispkg in
pkgutil.iter_modules(backends.__path__) if not ispkg]
@@ -30,32 +44,38 @@
return available_backends
-# Register all the backends only once
REGISTERED_BACKENDS = get_registered_backends()
+""":py:class:`dict`: Hold the dictionary of available backends generated at
load time mapped by their keyword."""
def grammar():
"""Define the main multi-query grammar.
Cumin provides a user-friendly generic query language that allows to
combine the results of subqueries for multiple
- backends.
+ backends:
- - Each query part can be composed with the others using boolean operators
(`and`, `or`, `and not`, `xor`).
- - Multiple query parts can be grouped together with parentheses (`(`, `)`).
- - Specific backend query (`I{backend-specific query syntax}`, where `I` is
an identifier for the specific backend).
- - Alias replacement, according to aliases defined in the configuration
file (`A:group1`).
- - The identifier `A` is reserved for the aliases replacement and cannot be
used to identify a backend.
- - A complex query example: `(D{host1 or host2} and (P{R:Class =
Role::MyClass} and not A:group1)) or D{host3}`
+ * Each query part can be composed with the others using boolean operators
``and``, ``or``, ``and not``, ``xor``.
+ * Multiple query parts can be grouped together with parentheses ``(``,
``)``.
+ * Specific backend query ``I{backend-specific query syntax}``, where ``I``
is an identifier for the specific
+ backend.
+ * Alias replacement, according to aliases defined in the configuration
file ``A:group1``.
+ * The identifier ``A`` is reserved for the aliases replacement and cannot
be used to identify a backend.
+ * A complex query example: ``(D{host1 or host2} and (P{R:Class =
Role::MyClass} and not A:group1)) or D{host3}``
- Backus-Naur form (BNF) of the grammar:
- <grammar> ::= <item> | <item> <boolean> <grammar>
- <item> ::= <backend_query> | <alias> | "(" <grammar> ")"
- <backend_query> ::= <backend> "{" <query> "}"
- <alias> ::= A:<alias_name>
- <boolean> ::= "and not" | "and" | "xor" | "or"
+ Backus-Naur form (BNF) of the grammar::
+
+ <grammar> ::= <item> | <item> <boolean> <grammar>
+ <item> ::= <backend_query> | <alias> | "(" <grammar> ")"
+ <backend_query> ::= <backend> "{" <query> "}"
+ <alias> ::= A:<alias_name>
+ <boolean> ::= "and not" | "and" | "xor" | "or"
Given that the pyparsing library defines the grammar in a BNF-like style,
for the details of the tokens not
- specified above check directly the code.
+ specified above check directly the source code.
+
+ Returns:
+ pyparsing.ParserElement: the grammar parser.
+
"""
# Boolean operators
boolean = (pp.CaselessKeyword('and not').leaveWhitespace() |
pp.CaselessKeyword('and') |
diff --git a/cumin/query.py b/cumin/query.py
index 9f80af6..cdd3f75 100644
--- a/cumin/query.py
+++ b/cumin/query.py
@@ -9,20 +9,37 @@
"""Cumin main query class.
It has multi-query capability and allow to use a default backend, if set,
without additional syntax.
- If a default_backend is set in the configuration, it will try to execute
the query string first with the default
- backend and only if the query is not parsable with that backend will try
to execute it with the multi-query grammar.
+ If a ``default_backend`` is set in the configuration, it will try to
execute the query string first with the
+ default backend and only if the query is not parsable with that backend it
will try to execute it with the
+ multi-query grammar.
- When a query is executed, a ClusterShell's NodeSet with the FQDN of the
matched hosts is returned.
+ When a query is executed, a :py:class:`ClusterShell.NodeSet.NodeSet` with
the FQDN of the matched hosts is
+ returned.
- Typical usage:
- >>> config = cumin.Config(args.config)
- >>> hosts = query.Query(config, logger=logger).execute(query_string)
+ Examples:
+ >>> import cumin
+ >>> from cumin.query import Query
+ >>> config = cumin.Config()
+ >>> hosts = Query(config, logger=logger).execute(query_string)
+
"""
grammar = grammar()
+ """:py:class:`pyparsing.ParserElement`: Load the grammar parser only once
in a singleton-like way."""
def execute(self, query_string):
- """Override parent class execute method to implement the multi-query
capability."""
+ """Override parent class execute method to implement the multi-query
capability.
+
+ :Parameters:
+ according to parent
:py:meth:`cumin.backends.BaseQueryAggregator.execute`.
+
+ Returns:
+ ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
+
+ Raises:
+ cumin.backends.InvalidQueryError: if unable to parse the query.
+
+ """
if 'default_backend' not in self.config:
try: # No default backend set, using directly the global grammar
return super(Query, self).execute(query_string)
@@ -47,7 +64,14 @@
"""Execute the query with the default backend, according to the
configuration.
Arguments:
- query_string -- the query string to be parsed and executed with the
default backend
+ query_string (str): the query string to be parsed and executed
with the default backend.
+
+ Returns:
+ ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
+
+ Raises:
+ cumin.backends.InvalidQueryError: if unable to get the default
backend from the registered backends.
+
"""
for registered_backend in REGISTERED_BACKENDS.values():
if registered_backend.name == self.config['default_backend']:
@@ -62,7 +86,15 @@
return query.execute(query_string)
def _parse_token(self, token):
- """Required by BaseQuery."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent
:py:meth:`cumin.backends.BaseQueryAggregator._parse_token`.
+
+ Raises:
+ cumin.backends.InvalidQueryError: on internal parsing error.
+
+ """
if not isinstance(token, ParseResults): # pragma: no cover - this
should never happen
raise InvalidQueryError('Expecting ParseResults object, got
{type}: {token}'.format(
type=type(token), token=token))
@@ -95,10 +127,15 @@
def _replace_alias(self, token_dict):
"""Replace any alias in the query in a recursive way, alias can
reference other aliases.
- Return True if a replacement was made, False otherwise. Raise
InvalidQueryError on failure.
-
Arguments:
- token_dict -- the dictionary of the parsed token returned by the
grammar parsing
+ token_dict (dict): the dictionary of the parsed token returned by
the grammar parsing.
+
+ Returns:
+ bool: :py:data:`True` if a replacement was made, :py:data`False`
otherwise.
+
+ Raises:
+ cumin.backends.InvalidQueryError: if unable to replace an alias.
+
"""
if 'alias' not in token_dict:
return False
diff --git a/cumin/tests/__init__.py b/cumin/tests/__init__.py
index de0b2a3..69f9ab5 100644
--- a/cumin/tests/__init__.py
+++ b/cumin/tests/__init__.py
@@ -11,8 +11,8 @@
"""Return the content of a fixture file.
Arguments:
- path -- the relative path to the test's fixture directory to be
opened.
- as_string -- return the content as a multiline string instead of a list of
lines [optional, default: False]
+ path: the relative path to the test's fixture directory to be opened.
+ as_string: return the content as a multiline string instead of a list
of lines [optional, default: False]
"""
with open(get_fixture_path(path)) as f:
if as_string:
@@ -27,6 +27,6 @@
"""Return the absolute path of the given fixture.
Arguments:
- path -- the relative path to the test's fixture directory.
+ path: the relative path to the test's fixture directory.
"""
return os.path.join(_TESTS_BASE_PATH, 'fixtures', path)
diff --git a/cumin/tests/integration/test_cli.py
b/cumin/tests/integration/test_cli.py
index 3ae90cd..90593e1 100644
--- a/cumin/tests/integration/test_cli.py
+++ b/cumin/tests/integration/test_cli.py
@@ -185,7 +185,7 @@
"""Return the expected return code based on the parameters.
Arguments:
- params -- a dictionary with all the parameters passed to the
variant_function
+ params: a dictionary with all the parameters passed to the
variant_function
"""
return_value = 2
if '-p' in params['params'] and '--global-timeout' not in params['params']:
@@ -198,7 +198,7 @@
"""Return a list of expected lines labels for global timeout-based tests.
Arguments:
- params -- a dictionary with all the parameters passed to the
variant_function
+ params: a dictionary with all the parameters passed to the
variant_function
"""
expected = []
if '--global-timeout' not in params['params']:
@@ -216,7 +216,7 @@
"""Return a list of expected lines labels for timeout-based tests.
Arguments:
- params -- a dictionary with all the parameters passed to the
variant_function
+ params: a dictionary with all the parameters passed to the
variant_function
"""
expected = []
if '-t' not in params['params']:
@@ -242,7 +242,7 @@
"""Return a list of expected lines labels for the date command based on
parameters.
Arguments:
- params -- a dictionary with all the parameters passed to the
variant_function
+ params: a dictionary with all the parameters passed to the
variant_function
"""
expected = []
if 'ls -la /tmp/non_existing' in params['commands']:
@@ -263,7 +263,7 @@
"""Return a list of expected lines labels for the ls command based on the
parameters.
Arguments:
- params -- a dictionary with all the parameters passed to the
variant_function
+ params: a dictionary with all the parameters passed to the
variant_function
"""
expected = []
if 'ls -la /tmp' in params['commands']:
@@ -311,7 +311,7 @@
"""Return the query for the nodes selection.
Arguments:
- nodes - a string with the ClusterShell NodeSet nodes selection
+ nodes: a string with the ClusterShell NodeSet nodes selection
"""
if nodes is None:
return self.all_nodes
diff --git a/cumin/transport.py b/cumin/transport.py
index c482075..bdca28a 100644
--- a/cumin/transport.py
+++ b/cumin/transport.py
@@ -10,12 +10,21 @@
@staticmethod
def new(config, target, logger=None):
- """Return an instance of the worker class for the configured transport.
+ """Create a transport worker class based on the configuration
(`factory`).
Arguments:
- config -- the configuration dictionary
- target -- a Target instance
- logger -- an optional logging instance [optional, default: None]
+ config (dict): the configuration dictionary.
+ target (cumin.transports.Target): a Target instance.
+ logger (logging.Logger, optional): an optional logger instance.
+
+ Returns:
+ BaseWorker: the created worker instance for the configured
transport.
+
+ Raises:
+ cumin.CuminError: if the configuration is missing the required
``transport`` key.
+ exceptions.ImportError: if unable to import the transport module.
+ exceptions.AttributeError: if the transport module is missing the
required ``worker_class`` attribute.
+
"""
if 'transport' not in config:
raise CuminError("Missing required parameter 'transport' in the
configuration dictionary")
diff --git a/cumin/transports/__init__.py b/cumin/transports/__init__.py
index a51347b..153f177 100644
--- a/cumin/transports/__init__.py
+++ b/cumin/transports/__init__.py
@@ -29,11 +29,11 @@
"""Command constructor.
Arguments:
- command -- the command to execute.
- timeout -- the command's timeout in seconds. [optional, default: None]
- ok_codes -- a list of exit codes to be considered successful for the
command. The exit code 0 is considered
- successful by default, if this option is set it override
it. If set to an empty list it means
- that any code is considered successful. [optional,
default: None]
+ command (str): the command to execute.
+ timeout (int, optional): the command's timeout in seconds.
+ ok_codes (list, optional): a list of exit codes to be considered
successful for the command.
+ The exit code zero is considered successful by default, if
this option is set it override it. If set
+ to an empty list ``[]``, it means that any code is considered
successful.
"""
self.command = command
self._timeout = None
@@ -46,7 +46,14 @@
self.ok_codes = ok_codes
def __repr__(self):
- """Repr of the command, allow to instantiate a Command with the same
properties."""
+ """Return the representation of the :py:class:`Command`.
+
+ The representation allow to instantiate a new :py:class:`Command`
instance with the same properties.
+
+ Returns:
+ str: the representation of the object.
+
+ """
params = ["'{command}'".format(command=self.command.replace("'",
r'\''))]
for field in ('_timeout', '_ok_codes'):
@@ -57,15 +64,27 @@
return 'cumin.transports.Command({params})'.format(params=',
'.join(params))
def __str__(self):
- """String representation of the command."""
+ """Return the string representation of the command.
+
+ Returns:
+ str: the string representation of the object.
+
+ """
return self.command
def __eq__(self, other):
- """Equality operation. Allow to directly compare a Command object to
another or a string.
+ """Equality operation. Allow to directly compare a :py:class:`Command`
object to another or a string.
- Raises ValueError if the comparing object is not an instance of
Command or a string.
+ :Parameters:
+ according to Python's Data model :py:meth:`object.__eq__`.
- Arguments: according to Python's datamodel documentation
+ Returns:
+ bool: :py:data:`True` if the `other` object is equal to this one,
:py:data:`False` otherwise.
+
+ Raises:
+ exceptions.ValueError: if the comparing object is not an instance
of :py:class:`Command` or a
+ :py:class:`str`.
+
"""
if isinstance(other, str):
other_command = other
@@ -81,25 +100,40 @@
def __ne__(self, other):
"""Inequality operation. Allow to directly compare a Command object to
another or a string.
- Raises ValueError if the comparing object is not an instance of
Command or a string.
+ :Parameters:
+ according to Python's Data model :py:meth:`object.__ne__`.
- Arguments: according to Python's datamodel documentation
+ Returns:
+ bool: :py:data:`True` if the `other` object is different to this
one, :py:data:`False` otherwise.
+
+ Raises:
+ exceptions.ValueError: if the comparing object is not an instance
of :py:class:`Command` or a
+ :py:class:`str`.
+
"""
return not self == other
@property
def timeout(self):
- """Getter for the command's timeout property, return None if not
set."""
+ """Timeout of the :py:class:`Command`.
+
+ :Getter:
+ Returns the current `timeout` or :py:data:`None` if not set.
+
+ :Setter:
+ :py:class:`float`, :py:class:`int`, :py:data:`None`: the `timeout`
in seconds for the execution of the
+ `command` on each host. Both :py:class:`float` and :py:class:`int`
are accepted and converted internally to
+ :py:class:`float`. If :py:data:`None` the `timeout` is reset to
its default value.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it to an invalid
value.
+
+ """
return self._timeout
@timeout.setter
def timeout(self, value):
- """Setter for the command's timeout property with validation, raise
WorkerError if not valid.
-
- Arguments:
- value -- the command's timeout in seconds for it's execution on each
host. Must be a positive float or a
- positive integer, or None to unset it.
- """
+ """Setter for the timeout property. The relative documentation is in
the getter."""
if isinstance(value, int):
value = float(value)
@@ -108,7 +142,22 @@
@property
def ok_codes(self):
- """Getter for the command's ok_codes property, return a list with only
the element 0 if not set."""
+ """List of exit codes to be considered successful for the execution of
the :py:class:`Command`.
+
+ :Getter:
+ Returns the current `ok_codes` or a :py:class:`list` with the
element ``0`` if not set.
+
+ :Setter:
+ :py:class:`list[int]`, :py:data:`None`: list of exit codes to be
considered successful for the execution of
+ the `command` on each host. Must be a :py:class:`list` of
:py:class:`int` in the range ``0-255`` included,
+ or :py:data:`None` to unset it. The exit code ``0`` is considered
successful by default, but it can be
+ overriden setting this property. Set it to an empty
:py:class:`list` to consider any
+ exit code successful.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it to an invalid
value.
+
+ """
ok_codes = self._ok_codes
if ok_codes is None:
ok_codes = [0]
@@ -117,13 +166,7 @@
@ok_codes.setter
def ok_codes(self, value):
- """Setter for the command's ok_codes property with validation, raise
WorkerError if not valid.
-
- Arguments:
- value -- the command's list of exit codes to be considered successful
for the execution. Must be a list of
- integers in the range 0-255 or None to unset it. The exit
code 0 is considered successful by default,
- but it can be overriden setting this property. An empty list
is also accepted.
- """
+ """Setter for the ok_codes property. The relative documentation is in
the getter."""
if value is None:
self._ok_codes = value
return
@@ -137,17 +180,31 @@
class State(object):
- """State machine for the state of a host."""
+ """State machine for the state of a host.
- # Valid states indexes
+ .. attribute:: current
+
+ :py:class:`int`: the current `state`.
+
+ .. attribute:: pending, scheduled, running, success, failed, timeout
+
+ :py:class:`int`: the available valid states, according to
:py:attr:`valid_states`.
+
+ .. attribute:: is_pending, is_scheduled, is_running, is_success,
is_failed, is_timeout
+
+ :py:class:`bool`: :py:data:`True` if this is the current `state`,
:py:data:`False` otherwise.
+
+ """
+
valid_states = range(6)
- # Valid states
+ """:py:class:`list`: valid states indexes."""
+
pending, scheduled, running, success, failed, timeout = valid_states
+ """Valid states."""
- # String representation of the valid states
states_representation = ('pending', 'scheduled', 'running', 'success',
'failed', 'timeout')
+ """:py:func:`tuple`: tuple with the string representations of the valid
states."""
- # Dictionary of tuples of valid states to which the transition is allowed
from the current state
allowed_state_transitions = {
pending: (scheduled, ),
scheduled: (running, ),
@@ -156,15 +213,18 @@
failed: (),
timeout: (),
}
+ """:py:class:`dict`: dictionary with ``{valid state: tuple of valid
states}`` mapping of allowed transitions for
+ any valid state."""
def __init__(self, init=None):
- """State constructor. The initial state is set to pending it not
provided.
-
- Raises InvalidStateError if init is an invalid state.
+ """State constructor. The initial state is set to `pending` it not
provided.
Arguments:
- init -- the initial state from where to start. If not specified, the
State will start in the pending state.
- [optional, default: None]
+ init (int, optional): the initial state from where to start. The
`pending` state will be used if not set.
+
+ Raises:
+ cumin.transports.InvalidStateError: if `init` is an invalid state.
+
"""
if init is None:
self._state = self.pending
@@ -177,10 +237,17 @@
def __getattr__(self, name):
"""Attribute accessor.
- Returns the current state and dynamically a bool for variables named
'is_{valid_state_name}'.
- Raises AttributeError otherwise.
+ :Accessible properties:
+ - `current` (:py:class:`int`): retuns the current state.
+ - `is_{valid_state_name}` (:py:class:`bool`): for each valid state
name, returns :py:data:`True` if the
+ current state matches the state in the variable name.
:py:data:`False` otherwise.
- Arguments: according to Python's datamodel documentation
+ :Parameters:
+ according to Python's Data model :py:meth:`object.__getattr__`.
+
+ Raises:
+ exceptions.AttributeError: if the attribute name is not available.
+
"""
if name == 'current':
return self._state
@@ -190,19 +257,41 @@
raise AttributeError("'State' object has no attribute
'{name}'".format(name=name))
def __repr__(self):
- """Repr of the state, allow to instantiate a State in the same
state."""
+ """Return the representation of the :py:class:`State`.
+
+ The representation allow to instantiate a new :py:class:`State`
instance with the same properties.
+
+ Returns:
+ str: the representation of the object.
+
+ """
return 'cumin.transports.State(init={state})'.format(state=self._state)
def __str__(self):
- """String representation of the state."""
+ """Return the string representation of the state.
+
+ Returns:
+ str: the string representation of the object.
+
+ """
return self.states_representation[self._state]
def __cmp__(self, other):
- """Comparison operation. Allow to directly compare a state object to
another or to an integer.
+ """Comparison operator.
- Raises ValueError if the comparing object is not an instance of State
or an integer.
+ Allow to directly compare a :py:class:`State` object to another or to
an :py:class:`int`.
- Arguments: according to Python's datamodel documentation
+ :Parameters:
+ according to Python's Data model :py:meth:`object.__cmp__`.
+
+ Returns:
+ int: a negative integer if `self` is lesser than `other`, zero if
`self` is equal to `other`, a positive
+ integer if `self` is greater than `other`.
+
+ Raises:
+ exceptions.ValueError: if the comparing object is not an instance
of :py:class:`State` or a
+ :py:class:`int`.
+
"""
if isinstance(other, int):
return self._state - other
@@ -214,10 +303,13 @@
def update(self, new):
"""Transition the state from the current state to the new one, if the
transition is allowed.
- Raises StateTransitionError if the transition is not allowed, see
allowed_state_transitions.
-
Arguments:
- new -- the new state to set. Only specific state transitions are
allowed.
+ new (int): the new state to set. Only specific state transitions
are allowed.
+
+ Raises:
+ cumin.transports.StateTransitionError: if the transition is not
allowed, see
+ :py:attr:`allowed_state_transitions`.
+
"""
if new not in self.valid_states:
raise ValueError("State must be one of {valid}, got
'{new}'".format(valid=self.valid_states, new=new))
@@ -237,13 +329,19 @@
"""Constructor, inizialize the Target with the list of hosts and
additional parameters.
Arguments:
- hosts -- a ClusterShell's NodeSet or a list of hosts that will
be targeted
- batch_size -- set the batch size so that no more that this number of
hosts are targeted at any given time.
- If greater than the number of hosts it will be
auto-resized to the number of hosts. It must be
- a positive integer or None to unset it. [optional,
default: None]
- batch_sleep -- sleep time in seconds between the end of execution of
one host in the batch and the start in
- the next host. It must be a positive float or None to
unset it. [optional, default: None]
- logger -- a logging.Logger instance [optional, default: None]
+ hosts (ClusterShell.NodeSet.NodeSet, list): hosts that will be
targeted, both
+ :py:class:`ClusterShell.NodeSet.NodeSet` and :py:class:`list`
are accepted and converted automatically
+ to :py:class:`ClusterShell.NodeSet.NodeSet` internally.
+ batch_size (int, optional): set the batch size so that no more
that this number of hosts are targeted
+ at any given time. If greater than the number of hosts it will
be auto-resized to the number of hosts.
+ It must be a positive integer or :py:data:`None` to unset it.
+ batch_sleep (int, optional): sleep time in seconds between the end
of execution of one host in the
+ batch and the start in the next host. It must be a positive
float or None to unset it.
+ logger (logging.Logger, optional): a logger instance.
+
+ Raises:
+ cumin.transports.WorkerError: if the `hosts` parameter is invalid.
+
"""
self.logger = logger or logging.getLogger(__name__)
@@ -259,16 +357,26 @@
@property
def first_batch(self):
- """Extract the first batch of hosts to execute."""
+ """First batch of the hosts to target.
+
+ :Getter:
+ Returns a :py:class:`ClusterShell.NodeSet.NodeSet` of the first
batch of hosts, according to the
+ `batch_size`.
+ """
return self.hosts[:self.batch_size]
def _compute_batch_size(self, batch_size, hosts):
"""Compute the batch_size based on the hosts size and return the value
to be used.
Arguments:
- batch_size -- a positive integer to indicate the batch_size to apply
when executing the worker or None to get
- its default value. If greater than the number of hosts,
the number of hosts will be used as value.
- hosts -- the list of hosts to use to calculate the batch size.
+ batch_size (int, None): a positive integer to indicate the
batch_size to apply when executing the worker or
+ :py:data:`None` to get its default value. If greater than the
number of hosts, the number of hosts
+ will be used as value instead.
+ hosts (ClusterShell.NodeSet.NodeSet): the list of hosts to use to
calculate the batch size.
+
+ Returns:
+ int: the effective `batch_size` to use.
+
"""
validate_positive_integer('batch_size', batch_size)
hosts_size = len(hosts)
@@ -287,8 +395,12 @@
"""Validate batch_sleep and return its value or a default value.
Arguments:
- batch_sleep -- a positive float indicating the sleep in seconds to
apply between one batched host and the next,
- or None to get its default value.
+ batch_sleep(float, None): a positive float indicating the sleep in
seconds to apply between one batched
+ host and the next, or :py:data:`None` to get its default value.
+
+ Returns:
+ float: the effective `batch_sleep` to use.
+
"""
validate_positive_float('batch_sleep', batch_sleep)
return batch_sleep or 0.0
@@ -303,9 +415,9 @@
"""Worker constructor. Setup environment variables and initialize
properties.
Arguments:
- config -- a dictionary with the parsed configuration file
- target -- a Target instance
- logger -- an optional logger instance [optional, default: None]
+ config (dict): a dictionary with the parsed configuration file.
+ target (Target): a Target instance.
+ logger (logging.Logger, optional): an optional logger instance.
"""
self.config = config
self.target = target
@@ -324,25 +436,43 @@
@abstractmethod
def execute(self):
- """Execute the task as configured. Return 0 on success, an int > 0 on
failure."""
+ """Execute the task as configured.
+
+ Returns:
+ int: ``0`` on success, a positive integer on failure.
+
+ """
@abstractmethod
def get_results(self):
- """Generator that yields tuples '(node_name, result)' with the results
of the current execution."""
+ """Iterate over the results (`generator`).
+
+ Yields:
+ tuple: with ``(hosts, result)`` for each host(s) of the current
execution.
+
+ """
@property
def commands(self):
- """Getter for the commands property with a default value."""
+ """Commands for the current execution.
+
+ :Getter:
+ Returns the current `command` :py:class:`list` or an empty
:py:class:`list` if not set.
+
+ :Setter:
+ :py:class:`list[Command]`, :py:class:`list[str]`: a
:py:class:`list` of :py:class:`Command` objects or
+ :py:class:`str` to be executed in the hosts. The elements are
converted to :py:class:`Command`
+ automatically.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it with invalid
data.
+
+ """
return self._commands or []
@commands.setter
def commands(self, value):
- """Setter for the commands property with validation, raise WorkerError
if not valid.
-
- Arguments:
- value -- a list of Command objects or strings with the commands to be
executed on the hosts. If a list of
- strings is passed, it will be automatically converted to a
list of Command objects.
- """
+ """Setter for the `commands` property. The relative documentation is
in the getter."""
if value is None:
self._commands = value
return
@@ -362,40 +492,63 @@
@abstractproperty
@property
def handler(self):
- """Getter for the handler property."""
+ """Get and set the `handler` for the current execution.
+
+ :Getter:
+ Returns the current `handler` or :py:data:`None` if not set.
+
+ :Setter:
+ :py:class:`str`, :py:class:`EventHandler`, :py:data:`None`: an
event handler to be notified of the progress
+ during execution. Its interface depends on the actual transport
chosen. Accepted values are:
+ * None => don't use an event handler (default)
+ * str => a string label to choose one of the available default
EventHandler classes in that transport,
+ * an event handler class object (not instance)
+ """
@abstractproperty
@handler.setter
def handler(self, value):
- """Setter for the handler property with validation, can raise
WorkerError if not valid.
-
- Arguments:
- value -- an event handler to be notified of the progress during
execution. It's interface depends on the
- actual transport chosen. Accepted values are:
- - None => don't use an event handler (default)
- - str => a string label to choose one of the available
default EventHandler classes in that transport,
- - an event handler class object (not instance)
- [optional, default: None]
- """
+ """Setter for the `handler` property. The relative documentation is in
the getter."""
@property
def timeout(self):
- """Getter for the global timeout property, default to 0 (unlimited) if
not set."""
+ """Global timeout for the current execution.
+
+ :Getter:
+ int: returns the current `timeout` or ``0`` (no timeout) if not
set.
+
+ :Setter:
+ :py:class:`int`, :py:data:`None`: timeout for the current
execution in seconds. Must be a positive integer
+ or :py:data:`None` to reset it.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it to an invalid
value.
+
+ """
return self._timeout or 0
@timeout.setter
def timeout(self, value):
- """Setter for the global timeout property with validation, raise
WorkerError if not valid.
-
- Arguments:
- value -- the global timeout in seconds for the whole execution. Must
be a positive integer or None to unset it.
- """
+ """Setter for the global `timeout` property. The relative
documentation is in the getter."""
validate_positive_integer('timeout', value)
self._timeout = value
@property
def success_threshold(self):
- """Getter for the success_threshold property with a default value."""
+ """Success threshold for the current execution.
+
+ :Getter:
+ float: returns the current `success_threshold` or ``1.0`` (`100%`)
if not set.
+
+ :Setter:
+ :py:class:`float`, :py:data:`None`: The success ratio threshold
that must be reached to consider the run
+ successful. A :py:class:`float` between ``0`` and ``1`` or
:py:data:`None` to reset it. The specific
+ meaning might change based on the chosen transport.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it to an invalid
value.
+
+ """
success_threshold = self._success_threshold
if success_threshold is None:
success_threshold = 1.0
@@ -404,12 +557,7 @@
@success_threshold.setter
def success_threshold(self, value):
- """Setter for the success_threshold property with validation, raise
WorkerError if not valid.
-
- Arguments:
- value -- The success ratio threshold that must be reached to consider
the run successful. A float between 0
- and 1 or None. The specific meaning might change based on the
chosen transport. [default: 1]
- """
+ """Setter for the `success_threshold` property. The relative
documentation is in the getter."""
if value is not None and (not isinstance(value, float) or
not (0.0 <= value <= 1.0)): # pylint:
disable=superfluous-parens
raise WorkerError("success_threshold must be a float beween 0 and
1, got '{value_type}': {value}".format(
@@ -419,11 +567,16 @@
def validate_list(property_name, value, allow_empty=False):
- """Helper to validate a list, raise WorkerError otherwise.
+ """Validate a list.
Arguments:
- property_name -- the name of the property to validate
- value -- the value to validate
+ property_name (str): the name of the property to validate.
+ value (list): the value to validate.
+ allow_empty (bool, optional): whether to consider an empty list valid.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it to an invalid value.
+
"""
if not isinstance(value, list):
raise_error(property_name, 'must be a list', value)
@@ -433,34 +586,42 @@
def validate_positive_integer(property_name, value):
- """Helper to validate a positive integer or None, raise WorkerError
otherwise.
+ """Validate a positive integer or :py:data:`None`.
Arguments:
- property_name -- the name of the property to validate
- value -- the value to validate
+ property_name (str): the name of the property to validate.
+ value (int, None): the value to validate.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it to an invalid value.
+
"""
if value is not None and (not isinstance(value, int) or value <= 0):
raise_error(property_name, 'must be a positive integer or None', value)
def validate_positive_float(property_name, value):
- """Helper to validate a positive float or None, raise WorkerError
otherwise.
+ """Validate a positive float or :py:data:`None`.
Arguments:
- property_name -- the name of the property to validate
- value -- the value to validate
+ property_name (str): the name of the property to validate.
+ value (float, None): the value to validate.
+
+ Raises:
+ cumin.transports.WorkerError: if trying to set it to an invalid value.
+
"""
if value is not None and (not isinstance(value, float) or value <= 0):
raise_error(property_name, 'must be a positive float or None', value)
def raise_error(property_name, message, value):
- """Helper to raise a WorkerError exception.
+ """Raise a :py:class:`WorkerError` exception.
Arguments:
- property_name -- the name of the property that raised the exception
- message -- the message to use for the exception
- value -- the value that raised the exception
+ property_name (str): the name of the property that raised the
exception.
+ message (str): the message to use for the exception.
+ value (mixed): the value that raised the exception.
"""
raise WorkerError("{property_name} {message}, got '{value_type}':
{value}".format(
property_name=property_name, message=message, value_type=type(value),
value=value))
diff --git a/cumin/transports/clustershell.py b/cumin/transports/clustershell.py
index ea75518..d62432b 100644
--- a/cumin/transports/clustershell.py
+++ b/cumin/transports/clustershell.py
@@ -19,7 +19,8 @@
def __init__(self, config, target, logger=None):
"""Worker ClusterShell constructor.
- Arguments: according to BaseWorker
+ :Parameters:
+ according to parent
:py:meth:`cumin.transports.BaseWorker.__init__`.
"""
super(ClusterShellWorker, self).__init__(config, target, logger)
self.task = Task.task_self() # Initialize a ClusterShell task
@@ -33,7 +34,11 @@
self.task.set_info(key, value)
def execute(self):
- """Required by BaseWorker."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent :py:meth:`cumin.transports.BaseWorker.execute`.
+ """
if not self.commands:
self.logger.warning('No commands provided')
return
@@ -72,21 +77,32 @@
return return_value
def get_results(self):
- """Required by BaseWorker."""
+ """Concrete implementation of parent abstract method.
+
+ :Parameters:
+ according to parent
:py:meth:`cumin.transports.BaseWorker.get_results`.
+ """
for output, nodelist in self.task.iter_buffers():
yield NodeSet.NodeSet.fromlist(nodelist), output
@property
def handler(self):
- """Getter for the handler property."""
+ """Concrete implementation of parent abstract getter and setter.
+
+ Accepted values for the setter:
+ * an instance of a custom handler class derived from
:py:class:`BaseEventHandler`.
+ * a :py:class:`str` with one of the available default handler listed
in :py:data:`DEFAULT_HANDLERS`.
+
+ The event handler is mandatory for this transport.
+
+ :Parameters:
+ according to parent :py:attr:`cumin.transports.BaseWorker.handler`.
+ """
return self._handler
@handler.setter
def handler(self, value):
- """Required by BaseTask.
-
- The available default handlers are defined in DEFAULT_HANDLERS.
- """
+ """Setter for the `handler` property. The relative documentation is in
the getter."""
if isinstance(value, type) and issubclass(value, BaseEventHandler):
self._handler = value
elif value in DEFAULT_HANDLERS:
@@ -106,8 +122,8 @@
"""Node class constructor with default values.
Arguments:
- name -- the hostname of the node.
- commands -- a list of Command objects to be executed on the node.
+ name (str): the hostname of the node.
+ commands (list): a list of :py:class:`cumin.transports.Command`
objects to be executed on the node.
"""
self.name = name
self.commands = commands
@@ -116,27 +132,27 @@
class BaseEventHandler(Event.EventHandler):
- """ClusterShell event handler extension base class.
+ """ClusterShell event handler base class.
- Inherit from ClusterShell's EventHandler class and define a base
EventHandler class to be used in Cumin.
- It can be subclassed to generate custom EventHandler classes while taking
advantage of some common
+ Inherit from :py:class:`ClusterShell.Event.EventHandler` class and define
a base `EventHandler` class to be used
+ in Cumin. It can be subclassed to generate custom `EventHandler` classes
while taking advantage of some common
functionalities.
"""
- short_command_length = 35 # For logging and printing the commands are
shortened to reach at most this length
+ short_command_length = 35
+ """:py:class:`int`: the length to which a command should be shortened in
various outputs."""
def __init__(self, target, commands, success_threshold=1.0, logger=None,
**kwargs):
"""Event handler ClusterShell extension constructor.
- If subclasses defines a self.pbar_ko tqdm progress bar, it will be
updated on timeout.
+ If subclasses defines a ``self.pbar_ko`` `tqdm` progress bar, it will
be updated on timeout.
Arguments:
- target -- a Target instance.
- commands -- the list of Command objects that has to be
executed on the nodes.
- success_threshold -- the success threshold, a float between 0 and 1,
to consider the execution successful.
- [optional, default: 1.0]
- **kwargs -- additional keyword arguments that might be used
by classes that extend this base class.
- [optional]
+ target (cumin.transports.Target): a Target instance.
+ commands (list): the list of Command objects that has to be
executed on the nodes.
+ success_threshold (float, optional): the success threshold, a
:py:class:`float` between ``0`` and ``1``,
+ to consider the execution successful.
+ **kwargs (optional): additional keyword arguments that might be
used by derived classes.
"""
super(BaseEventHandler, self).__init__()
self.success_threshold = success_threshold
@@ -171,17 +187,18 @@
"""Additional method called at the end of the whole execution, useful
for reporting and final actions.
Arguments:
- task -- a ClusterShell Task instance
+ task (ClusterShell.Task.Task): a ClusterShell Task instance.
"""
raise NotImplementedError
def on_timeout(self, task):
- """Callback called by the ClusterShellWorker when a Task.TimeoutError
is raised.
+ """Update the state of the nodes and the timeout counter.
- The whole execution timed out, update the state of the nodes and the
timeout counter accordingly.
+ Callback called by the :py:class:`ClusterShellWorker` when a
:py:exc:`ClusterShell.Task.TimeoutError` is
+ raised. It means that the whole execution timed out.
Arguments:
- task -- a ClusterShell Task instance
+ task (ClusterShell.Task.Task): a ClusterShell Task instance.
"""
num_timeout = task.num_timeout()
self.logger.error('global timeout was triggered while {num} nodes were
executing a command'.format(
@@ -207,9 +224,10 @@
def ev_pickup(self, worker):
"""Command execution started on a node, remove the command from the
node's queue.
- This callback is triggered by ClusterShell for each node when it
starts executing a command.
+ This callback is triggered by the `ClusterShell` library for each node
when it starts executing a command.
- Arguments: according to EventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`ClusterShell.Event.EventHandler.ev_pickup`.
"""
self.logger.debug("node={node}, command='{command}'".format(
node=worker.current_node, command=worker.command))
@@ -237,7 +255,8 @@
This callback is triggered by ClusterShell for each node when output
is available.
- Arguments: according to EventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`ClusterShell.Event.EventHandler.ev_read`.
"""
if self.deduplicate_output:
return
@@ -249,7 +268,8 @@
This callback is triggered by ClusterShell when the execution has
timed out.
- Arguments: according to EventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`ClusterShell.Event.EventHandler.ev_timeout`.
"""
delta_timeout = worker.task.num_timeout() - self.counters['timeout']
self.logger.debug("command='{command}', delta_timeout={num}".format(
@@ -269,14 +289,16 @@
worker.task.timer(self.target.batch_sleep, worker.eh)
def _get_log_message(self, num, message, nodes=None):
- """Helper to get a pre-formatted message suitable for logging or
printing.
-
- Returns a tuple of two strings: a logging message, the affected nodes
in NodeSet format
+ """Get a pre-formatted message suitable for logging or printing.
Arguments:
- num - the number of affecte nodes
- message - the message to print
- nodes - the list of nodes affected [optional, default: None]
+ num (int): the number of affecte nodes.
+ message (str): the message to print.
+ nodes (list, optional): the list of nodes affected.
+
+ Returns:
+ tuple: a tuple of ``(logging message, NodeSet of the affected
nodes)``.
+
"""
if nodes is None:
nodes_string = ''
@@ -292,12 +314,12 @@
return (log_message, str(nodes_string))
def _print_report_line(self, message, color=colorama.Fore.RED,
nodes_string=''): # pylint: disable=no-self-use
- """Helper to print a tqdm-friendly colored status line with
success/failure ratio and optional list of nodes.
+ """Print a tqdm-friendly colored status line with success/failure
ratio and optional list of nodes.
Arguments:
- message -- the message to print
- color -- the message color [optional, default:
colorama.Fore.RED]
- nodes_string -- the string representation of the affected nodes
[optional, default: '']
+ message (str): the message to print.
+ color (str, optional): the message color.
+ nodes_string (str, optional): the string representation of the
affected nodes.
"""
tqdm.write('{color}{message}{nodes_color}{nodes_string}{reset}'.format(
color=color, message=message, nodes_color=colorama.Fore.CYAN,
@@ -307,17 +329,22 @@
"""Return a shortened representation of a command omitting the central
part.
Arguments:
- command - the command to be shortened
+ command (str): the command to be shortened, if needed.
+
+ Returns:
+ str: the string representation of the command, shortened if too
long.
+
"""
sublen = (self.short_command_length - 3) // 2 # The -3 is for the
ellipsis
return (command[:sublen] + '...' + command[-sublen:]) if len(command)
> self.short_command_length else command
def _commands_output_report(self, buffer_iterator, command=None):
- """Helper to print the commands output in a colored and tqdm-friendly
way.
+ """Print the commands output in a colored and tqdm-friendly way.
Arguments:
- buffer_iterator - any ClusterShell object that implements
iter_buffers() like Task and Worker objects.
- command - command the output is referring to [optional,
default: None]
+ buffer_iterator (mixed): any `ClusterShell` object that implements
``iter_buffers()`` like
+ :py:class:`ClusterShell.Task.Task` and all the `Worker`
objects.
+ command (str, optional): the command the output is referring to.
"""
if not self.deduplicate_output:
tqdm.write(colorama.Fore.BLUE + '================' +
colorama.Style.RESET_ALL, file=sys.stdout)
@@ -345,7 +372,7 @@
tqdm.write(colorama.Fore.BLUE + message + colorama.Style.RESET_ALL,
file=sys.stdout)
def _global_timeout_nodes_report(self):
- """Helper to print the nodes that were caught by the global timeout in
a colored and tqdm-friendly way."""
+ """Print the nodes that were caught by the global timeout in a colored
and tqdm-friendly way."""
if not self.global_timedout:
return
@@ -362,10 +389,11 @@
self._print_report_line(not_run_message, nodes_string=not_run_nodes)
def _failed_commands_report(self, filter_command_index=-1):
- """Helper to print the nodes that failed to execute commands in a
colored and tqdm-friendly way.
+ """Print the nodes that failed to execute commands in a colored and
tqdm-friendly way.
Arguments:
- filter_command - print only the nodes that failed to execute this
specific command [optional, default: None]
+ filter_command_index (int, optional): print only the nodes that
failed to execute the command specified by
+ this command index.
"""
for state in (State.failed, State.timeout):
failed_commands = defaultdict(list)
@@ -385,7 +413,11 @@
self._print_report_line(log_message, nodes_string=nodes_string)
def _success_nodes_report(self, command=None):
- """Helper to print how many nodes succesfully executed all commands in
a colored and tqdm-friendly way."""
+ """Print how many nodes succesfully executed all commands in a colored
and tqdm-friendly way.
+
+ Arguments:
+ command (str, optional): the command the report is referring to.
+ """
if self.global_timedout and command is None:
num = sum(1 for node in self.nodes.itervalues() if
node.state.is_success and
node.running_command_index == (len(self.commands) - 1))
@@ -432,21 +464,23 @@
"""Custom ClusterShell event handler class that execute commands
synchronously.
The implemented logic is:
- - execute command #N on all nodes where command #N-1 was successful
according to batch_size
- - the success ratio is checked at each command completion on every node,
and will abort if not met, however
- nodes already scheduled for execution with ClusterShell will execute the
command anyway. The use of the
- batch_size allow to control this aspect.
- - if the execution of command #N is completed and the success ratio is
greater than the success threshold,
- re-start from the top with N=N+1
+
+ * execute command `#N` on all nodes where command #`N-1` was successful
according to `batch_size`.
+ * the success ratio is checked at each command completion on every node,
and will abort if not met, however
+ nodes already scheduled for execution with `ClusterShell` will execute
the command anyway. The use of the
+ `batch_size` allow to control this aspect.
+ * if the execution of command `#N` is completed and the success ratio is
greater than the success threshold,
+ re-start from the top with `N=N+1`.
The typical use case is to orchestrate some operation across a fleet,
ensuring that each command is completed by
enough nodes before proceeding with the next one.
"""
def __init__(self, target, commands, success_threshold=1.0, logger=None,
**kwargs):
- """Custom ClusterShell synchronous event handler constructor.
+ """Define a custom ClusterShell event handler to execute commands
synchronously.
- Arguments: according to BaseEventHandler interface
+ :Parameters:
+ according to parent :py:meth:`BaseEventHandler.__init__`.
"""
super(SyncEventHandler, self).__init__(
target, commands, success_threshold=success_threshold,
logger=logger, **kwargs)
@@ -460,8 +494,7 @@
Executed at the start of each command.
Arguments:
- schedule -- boolean to decide if the next command should be sent to
ClusterShell for execution or not.
- [optional, default: False]
+ schedule (bool, optional): whether the next command should be sent
to ClusterShell for execution or not.
"""
self.counters['success'] = 0
@@ -497,6 +530,10 @@
"""Command terminated, print the result and schedule the next command
if criteria are met.
Executed at the end of each command inside a lock.
+
+ Returns:
+ bool: :py:data:`True` if the next command should be scheduled,
:py:data:`False` otherwise.
+
"""
self._commands_output_report(Task.task_self(),
command=self.commands[self.current_command_index].command)
@@ -526,22 +563,23 @@
return True
def on_timeout(self, task):
- """Callback called by the ClusterShellWorker when a Task.TimeoutError
is raised.
+ """Override parent class `on_timeout` method to run `end_command`.
- Arguments: according to BaseEventHandler interface
+ :Parameters:
+ according to parent :py:meth:`BaseEventHandler.on_timeout`.
"""
super(SyncEventHandler, self).on_timeout(task)
self.end_command()
def ev_hup(self, worker):
- """Command execution completed.
+ """Command execution completed on a node.
This callback is triggered by ClusterShell for each node when it
completes the execution of a command.
-
Update the progress bars and keep track of nodes based on the
success/failure of the command's execution.
Schedule a timer for further decisions.
- Arguments: according to EventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`ClusterShell.Event.EventHandler.ev_hup`.
"""
self.logger.debug("node={node}, rc={rc}, command='{command}'".format(
node=worker.current_node, rc=worker.current_rc,
command=worker.command))
@@ -570,9 +608,10 @@
def ev_timer(self, timer):
"""Schedule the current command on the next node or the next command
on the first batch of nodes.
- This callback is triggered by ClusterShell when a scheduled
Task.timer() goes off.
+ This callback is triggered by `ClusterShell` when a scheduled
`Task.timer()` goes off.
- Arguments: according to EventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`ClusterShell.Event.EventHandler.ev_timer`.
"""
success_ratio = 1 - (float(self.counters['failed'] +
self.counters['timeout']) / self.counters['total'])
@@ -639,9 +678,10 @@
self.start_command(schedule=True)
def close(self, task):
- """Print a final summary report line.
+ """Concrete implementation of parent abstract method to print the
success nodes report.
- Arguments: according to BaseEventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`cumin.transports.BaseEventHandler.close`.
"""
self._success_nodes_report()
@@ -650,21 +690,23 @@
"""Custom ClusterShell event handler class that execute commands
asynchronously.
The implemented logic is:
- - execute on all nodes independently every command in a sequence, aborting
the execution on that node if any
+
+ * execute on all nodes independently every command in a sequence, aborting
the execution on that node if any
command fails.
- - The success ratio is checked at each node completion (either because it
completed all commands or aborted
+ * The success ratio is checked at each node completion (either because it
completed all commands or aborted
earlier), however nodes already scheduled for execution with
ClusterShell will execute the commands anyway. The
use of the batch_size allows to control this aspect.
- - if the success ratio is met, schedule the execution of all commands to
the next node.
+ * if the success ratio is met, schedule the execution of all commands to
the next node.
The typical use case is to execute read-only commands to gather the status
of a fleet without any special need of
orchestration between the nodes.
"""
def __init__(self, target, commands, success_threshold=1.0, logger=None,
**kwargs):
- """Custom ClusterShell asynchronous event handler constructor.
+ """Define a custom ClusterShell event handler to execute commands
asynchronously between nodes.
- Arguments: according to BaseEventHandler interface
+ :Parameters:
+ according to parent :py:meth:`BaseEventHandler.__init__`.
"""
super(AsyncEventHandler, self).__init__(
target, commands, success_threshold=success_threshold,
logger=logger, **kwargs)
@@ -680,11 +722,11 @@
"""Command execution completed on a node.
This callback is triggered by ClusterShell for each node when it
completes the execution of a command.
+ Enqueue the next command if the success criteria are met, track the
failure otherwise. Update the progress
+ bars accordingly.
- Enqueue the next command if the success criteria are met, track the
failure otherwise
- Update the progress bars accordingly
-
- Arguments: according to EventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`ClusterShell.Event.EventHandler.ev_hup`.
"""
self.logger.debug("node={node}, rc={rc}, command='{command}'".format(
node=worker.current_node, rc=worker.current_rc,
command=worker.command))
@@ -724,9 +766,10 @@
def ev_timer(self, timer):
"""Schedule the current command on the next node or the next command
on the first batch of nodes.
- This callback is triggered by ClusterShell when a scheduled
Task.timer() goes off.
+ This callback is triggered by `ClusterShell` when a scheduled
`Task.timer()` goes off.
- Arguments: according to EventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`ClusterShell.Event.EventHandler.ev_timer`.
"""
success_ratio = 1 - (float(self.counters['failed'] +
self.counters['timeout']) / self.counters['total'])
@@ -755,9 +798,10 @@
self.logger.debug('No more nodes left')
def close(self, task):
- """Properly close all progress bars and print results.
+ """Concrete implementation of parent abstract method to print the
nodes reports and close progress bars.
- Arguments: according to BaseEventHandler interface
+ :Parameters:
+ according to parent
:py:meth:`cumin.transports.BaseEventHandler.close`.
"""
self._commands_output_report(task)
@@ -779,6 +823,8 @@
self.return_value = 1
-# Required by the auto-loader in the cumin.transport.Transport factory
worker_class = ClusterShellWorker # pylint: disable=invalid-name
-DEFAULT_HANDLERS = {'sync': SyncEventHandler, 'async': AsyncEventHandler} #
Available default EventHandler classes
+"""Required by the transport auto-loader in
:py:meth:`cumin.transport.Transport.new`."""
+
+DEFAULT_HANDLERS = {'sync': SyncEventHandler, 'async': AsyncEventHandler}
+"""dict: mapping of available default event handlers for
:py:class:`ClusterShellWorker`."""
diff --git a/prospector.yaml b/prospector.yaml
index cf93a96..d0b8c5e 100644
--- a/prospector.yaml
+++ b/prospector.yaml
@@ -15,9 +15,13 @@
max-line-length: 120
pep257:
+ explain: true
+ source: true
disable:
- D203 # 1 blank line required before class docstring, D211 (after) is
enforce instead
- D213 # Multi-line docstring summary should start at the second line,
D212 (first line) is enforced instead
+ - D406 # Section name should end with a newline, incompatible with Google
Style Python Docstrings
+ - D407 # Missing dashed underline after section, incompatible with Google
Style Python Docstrings
pylint:
disable:
--
To view, visit https://gerrit.wikimedia.org/r/382479
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib46a063964d5701f365c0fe3225854246c643f6b
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/cumin
Gerrit-Branch: master
Gerrit-Owner: Volans <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
