This is an automated email from the ASF dual-hosted git repository. av pushed a commit to branch ignite-ducktape in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push: new 330c17b Ducktape codestyle (#8098) 330c17b is described below commit 330c17b084aaf91d19591178885dd2637dc8fa21 Author: Ivan Daschinskiy <ivanda...@gmail.com> AuthorDate: Thu Jul 30 12:11:59 2020 +0300 Ducktape codestyle (#8098) --- .../services/utils/decorators.py => .pylintrc} | 24 ++---- .../utils/decorators.py => check_style.sh} | 24 +++--- modules/ducktests/tests/ignitetest/__init__.py | 1 + .../ducktests/tests/ignitetest/services/ignite.py | 27 +++++-- .../tests/ignitetest/services/ignite_app.py | 12 ++- .../tests/ignitetest/services/ignite_spark_app.py | 6 +- .../ducktests/tests/ignitetest/services/spark.py | 40 +++++++--- .../tests/ignitetest/services/utils/decorators.py | 8 ++ .../ignitetest/services/utils/ignite_aware.py | 59 +++++++++++++-- .../ignitetest/services/utils/ignite_aware_app.py | 32 ++++++-- .../ignitetest/services/utils/ignite_config.py | 21 +++++- .../tests/ignitetest/services/utils/ignite_path.py | 45 +++++++----- .../tests/ignitetest/services/utils/jmx_utils.py | 85 ++++++++++++++++++++-- .../tests/ignitetest/services/zk/zookeeper.py | 39 +++++++++- .../ignitetest/tests/add_node_rebalance_test.py | 12 ++- .../tests/ignitetest/tests/discovery_test.py | 44 ++++++++--- .../tests/ignitetest/tests/pme_free_switch_test.py | 18 ++++- .../ignitetest/tests/spark_integration_test.py | 11 ++- .../tests/ignitetest/tests/utils/ignite_test.py | 12 +++ .../tests/ignitetest/tests/utils/version.py | 10 ++- 20 files changed, 414 insertions(+), 116 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/.pylintrc similarity index 65% copy from modules/ducktests/tests/ignitetest/services/utils/decorators.py copy to modules/ducktests/tests/.pylintrc index 788ddba..73536f3 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/decorators.py +++ b/modules/ducktests/tests/.pylintrc @@ -12,21 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import functools -from threading import RLock +[BASIC] +min-public-methods=0 +# TODO: Remove after migrating to python 3 +disable=R0205 +[SIMILARITIES] +ignore-imports=yes -def memoize(func): - cache = func.cache = {} - lock = RLock() - - @functools.wraps(func) - def memoized_func(*args, **kwargs): - key = str(args) + str(kwargs) - if key not in cache: - with lock: - if key not in cache: - cache[key] = func(*args, **kwargs) - return cache[key] - - return memoized_func +[FORMAT] +max-line-length=120 diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/check_style.sh old mode 100644 new mode 100755 similarity index 65% copy from modules/ducktests/tests/ignitetest/services/utils/decorators.py copy to modules/ducktests/tests/check_style.sh index 788ddba..39405a7 --- a/modules/ducktests/tests/ignitetest/services/utils/decorators.py +++ b/modules/ducktests/tests/check_style.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -12,21 +14,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import functools -from threading import RLock - -def memoize(func): - cache = func.cache = {} - lock = RLock() +if ! command -v pylint &> /dev/null +then + echo "Please, install pylint first" + exit 1 +fi - @functools.wraps(func) - def memoized_func(*args, **kwargs): - key = str(args) + str(kwargs) - if key not in cache: - with lock: - if key not in cache: - cache[key] = func(*args, **kwargs) - return cache[key] +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - return memoized_func +pylint --rcfile="$SCRIPT_DIR"/.pylintrc "$SCRIPT_DIR"/ignitetest diff --git a/modules/ducktests/tests/ignitetest/__init__.py b/modules/ducktests/tests/ignitetest/__init__.py index 1c2357f..8cbaed8 100644 --- a/modules/ducktests/tests/ignitetest/__init__.py +++ b/modules/ducktests/tests/ignitetest/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=missing-module-docstring # This determines the version of ignitetest that can be published to PyPi and installed with pip # # Note that in development, this version name can't follow Ignite's convention of having a trailing "-SNAPSHOT" diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 05f108e..bfc96ad 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains class to start ignite cluster node. +""" + import os.path import signal @@ -20,11 +24,13 @@ from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.utils.util import wait_until from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.services.utils.ignite_config import IgniteServerConfig, IgniteClientConfig from ignitetest.tests.utils.version import DEV_BRANCH class IgniteService(IgniteAwareService): + """ + Ignite node service. + """ APP_SERVICE_CLASS = "org.apache.ignite.startup.cmdline.CommandLineStartup" HEAP_DUMP_FILE = os.path.join(IgniteAwareService.PERSISTENT_ROOT, "ignite-heap.bin") @@ -38,9 +44,11 @@ class IgniteService(IgniteAwareService): "collect_default": False} } + # pylint: disable=R0913 def __init__(self, context, num_nodes, client_mode=False, version=DEV_BRANCH, properties=""): super(IgniteService, self).__init__(context, num_nodes, client_mode, version, properties) + # pylint: disable=W0221 def start(self, timeout_sec=180): super(IgniteService, self).start() @@ -66,11 +74,17 @@ class IgniteService(IgniteAwareService): return cmd def await_node_started(self, node, timeout_sec): + """ + Await topology ready event on node start. + :param node: Node to wait + :param timeout_sec: Number of seconds to wait event. + """ self.await_event_on_node("Topology snapshot", node, timeout_sec, from_the_beginning=True) if len(self.pids(node)) == 0: raise Exception("No process ids recorded on node %s" % node.account.hostname) + # pylint: disable=W0221 def stop_node(self, node, clean_shutdown=True, timeout_sec=60): pids = self.pids(node) sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL @@ -90,17 +104,20 @@ class IgniteService(IgniteAwareService): node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) def thread_dump(self, node): + """ + Generate thread dump on node. + :param node: Ignite service node. + """ for pid in self.pids(node): try: node.account.signal(pid, signal.SIGQUIT, allow_fail=True) - except: + except RemoteCommandError: self.logger.warn("Could not dump threads on node") def pids(self, node): - """Return process ids associated with running processes on the given node.""" try: cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.APP_SERVICE_CLASS - pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] + pid_arr = list(node.account.ssh_capture(cmd, allow_fail=True, callback=int)) return pid_arr - except (RemoteCommandError, ValueError) as e: + except (RemoteCommandError, ValueError): return [] diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 64de176..0702221 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -13,17 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService -from ignitetest.tests.utils.version import DEV_BRANCH - """ -The Ignite application service allows to perform custom logic writen on java. +This module contains the ignite application service allows to perform custom logic writen on java. """ +from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService +from ignitetest.tests.utils.version import DEV_BRANCH + class IgniteApplicationService(IgniteAwareApplicationService): + """ + The Ignite application service allows to perform custom logic writen on java. + """ service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService" + # pylint: disable=R0913 def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="", timeout_sec=60): super(IgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties, diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py index 775f19c1..71014b7 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py @@ -14,13 +14,17 @@ # limitations under the License. """ -The Ignite-Spark application service. +This module contains the Ignite-Spark application service. """ from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService from ignitetest.tests.utils.version import DEV_BRANCH class SparkIgniteApplicationService(IgniteAwareApplicationService): + """ + The Ignite-Spark application service. + """ + # pylint: disable=R0913 def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="", timeout_sec=60): super(SparkIgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties, diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index fdd5c73..657a6b6 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -13,18 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains spark service class. +""" + import os.path from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.service import Service from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.services.utils.ignite_config import IgniteServerConfig - from ignitetest.tests.utils.version import DEV_BRANCH class SparkService(IgniteAwareService): + """ + Start a spark node. + """ INSTALL_DIR = "/opt/spark-{version}".format(version="2.3.4") SPARK_PERSISTENT_ROOT = "/mnt/spark" @@ -68,6 +73,7 @@ class SparkService(IgniteAwareService): return cmd + # pylint: disable=W0221 def start_node(self, node, timeout_sec=30): self.init_persistent(node) @@ -91,7 +97,7 @@ class SparkService(IgniteAwareService): if len(self.pids(node)) == 0: raise Exception("No process ids recorded on node %s" % node.account.hostname) - def stop_node(self, node, clean_shutdown=True, timeout_sec=60): + def stop_node(self, node): if node == self.nodes[0]: node.account.ssh(os.path.join(SparkService.INSTALL_DIR, "sbin", "stop-master.sh")) else: @@ -103,28 +109,40 @@ class SparkService(IgniteAwareService): node.account.ssh("sudo rm -rf -- %s" % SparkService.SPARK_PERSISTENT_ROOT, allow_fail=False) def pids(self, node): - """Return process ids associated with running processes on the given node.""" try: cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name(node) - pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] - return pid_arr - except (RemoteCommandError, ValueError) as e: + return list(node.account.ssh_capture(cmd, allow_fail=True, callback=int)) + except (RemoteCommandError, ValueError): return [] def java_class_name(self, node): + """ + :param node: Spark node. + :return: Class name depending on node type (master or slave). + """ if node == self.nodes[0]: return "org.apache.spark.deploy.master.Master" - else: - return "org.apache.spark.deploy.worker.Worker" - def master_log_path(self, node): + return "org.apache.spark.deploy.worker.Worker" + + @staticmethod + def master_log_path(node): + """ + :param node: Spark master node. + :return: Path to log file. + """ return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.master.Master-{instance}-{host}.out".format( SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT, userID=node.account.user, instance=1, host=node.account.hostname) - def slave_log_path(self, node): + @staticmethod + def slave_log_path(node): + """ + :param node: Spark slave node. + :return: Path to log file. + """ return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.worker.Worker-{instance}-{host}.out".format( SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT, userID=node.account.user, diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/ignitetest/services/utils/decorators.py index 788ddba..9c767f4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/decorators.py +++ b/modules/ducktests/tests/ignitetest/services/utils/decorators.py @@ -12,11 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains various useful decorators. +""" + import functools from threading import RLock def memoize(func): + """ + Decorate function to memoize first call to thread safe cache. + """ cache = func.cache = {} lock = RLock() diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 578d728..b4d12f0 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -12,6 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains the base class to build services aware of Ignite. +""" + import os from abc import abstractmethod @@ -22,12 +27,11 @@ from ignitetest.services.utils.ignite_config import IgniteLoggerConfig, IgniteSe from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin -""" -The base class to build services aware of Ignite. -""" - class IgniteAwareService(BackgroundThreadService): + """ + The base class to build services aware of Ignite. + """ # Root directory for persistent output PERSISTENT_ROOT = "/mnt/service" STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") @@ -41,6 +45,7 @@ class IgniteAwareService(BackgroundThreadService): "collect_default": True} } + # pylint: disable=R0913 def __init__(self, context, num_nodes, client_mode, version, properties): super(IgniteAwareService, self).__init__(context, num_nodes) @@ -66,6 +71,10 @@ class IgniteAwareService(BackgroundThreadService): ignite_jmx_mixin(node, self.pids(node)) def init_persistent(self, node): + """ + Init persistent directory. + :param node: Ignite service node. + """ node.account.mkdirs(self.PERSISTENT_ROOT) node.account.create_file(self.CONFIG_FILE, self.config().render( config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, properties=self.properties)) @@ -73,18 +82,30 @@ class IgniteAwareService(BackgroundThreadService): @abstractmethod def start_cmd(self, node): + """ + Start up command for service. + :param node: Ignite service node. + """ raise NotImplementedError @abstractmethod def pids(self, node): + """ + :param node: Ignite service node. + :return: List of service's pids. + """ raise NotImplementedError def config(self): + """ + :return: Ignite node configuration. + """ if self.client_mode: return IgniteClientConfig(self.context) - else: - return IgniteServerConfig(self.context) + return IgniteServerConfig(self.context) + + # pylint: disable=W0613 def _worker(self, idx, node): cmd = self.start_cmd(node) @@ -93,9 +114,23 @@ class IgniteAwareService(BackgroundThreadService): node.account.ssh(cmd) def alive(self, node): + """ + :param node: Ignite service node. + :return: True if node is alive. + """ return len(self.pids(node)) > 0 + # pylint: disable=R0913 def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5): + """ + Await for specific event message in a node's log file. + :param evt_message: Event message. + :param node: Ignite service node. + :param timeout_sec: Number of seconds to check the condition for before failing. + :param from_the_beginning: If True, search for message from the beggining of log file. + :param backoff_sec: Number of seconds to back off between each failure to meet the condition + before checking again. + """ with node.account.monitor_log(self.STDOUT_STDERR_CAPTURE) as monitor: if from_the_beginning: monitor.offset = 0 @@ -104,12 +139,24 @@ class IgniteAwareService(BackgroundThreadService): err_msg="Event [%s] was not triggered in %d seconds" % (evt_message, timeout_sec)) def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backoff_sec=5): + """ + Await for specific event messages on all nodes. + :param evt_message: Event message. + :param timeout_sec: Number of seconds to check the condition for before failing. + :param from_the_beginning: If True, search for message from the beggining of log file. + :param backoff_sec: Number of seconds to back off between each failure to meet the condition + before checking again. + """ assert len(self.nodes) == 1 self.await_event_on_node(evt_message, self.nodes[0], timeout_sec, from_the_beginning=from_the_beginning, backoff_sec=backoff_sec) def execute(self, command): + """ + Execute command on all nodes. + :param command: Command to execute. + """ for node in self.nodes: cmd = "%s 1>> %s 2>> %s" % \ (self.path.script(command, node), diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index fd2c860..152b5fe 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -12,19 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import re - -from ducktape.services.service import Service - -from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.services.utils.ignite_config import IgniteClientConfig """ -The base class to build Ignite aware application written on java. +This module contains the base class to build Ignite aware application written on java. """ +import re + +from ignitetest.services.utils.ignite_aware import IgniteAwareService + class IgniteAwareApplicationService(IgniteAwareService): + """ + The base class to build Ignite aware application written on java. + """ + # pylint: disable=R0913 def __init__(self, context, java_class_name, client_mode, version, properties, params, timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): super(IgniteAwareApplicationService, self).__init__(context, 1, client_mode, version, properties) @@ -53,9 +55,10 @@ class IgniteAwareApplicationService(IgniteAwareService): self.STDOUT_STDERR_CAPTURE) return cmd + # pylint: disable=W0221 def stop_node(self, node, clean_shutdown=True, timeout_sec=20): self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) - node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=True, allow_fail=True) + node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, allow_fail=True) stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ @@ -73,6 +76,9 @@ class IgniteAwareApplicationService(IgniteAwareService): node.account.ssh("rm -rf %s" % self.PERSISTENT_ROOT, allow_fail=False) def app_args(self): + """ + :return: Application arguments. + """ args = self.java_class_name + "," + IgniteAwareApplicationService.CONFIG_FILE if self.params != "": @@ -84,6 +90,9 @@ class IgniteAwareApplicationService(IgniteAwareService): return node.account.java_pids(self.servicejava_class_name) def jvm_opts(self): + """ + :return: Application JVM options. + """ return "-J-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file " + \ "-J-Dlog4j.configDebug=true " \ "-J-Xmx1G " \ @@ -91,6 +100,9 @@ class IgniteAwareApplicationService(IgniteAwareService): "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " + self.jvm_options def env(self): + """ + :return: Export string of additional environment variables. + """ return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ "export EXCLUDE_TEST_CLASSES=true; " + \ "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=self.PERSISTENT_ROOT) + \ @@ -98,6 +110,10 @@ class IgniteAwareApplicationService(IgniteAwareService): % self.path.home(self.version) def extract_result(self, name): + """ + :param name: Result parameter's name. + :return: Extracted result of application run. + """ res = "" output = self.nodes[0].account.ssh_capture( diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py index 077c39b..90b3bfa 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py @@ -14,18 +14,20 @@ # limitations under the License. """ -This module renders Ignite config and all related artifacts +This module contains ignite config classes and utilities. """ +import os from jinja2 import FileSystemLoader, Environment -import os - DEFAULT_CONFIG_PATH = os.path.dirname(os.path.abspath(__file__)) + "/config" DEFAULT_IGNITE_CONF = DEFAULT_CONFIG_PATH + "/ignite.xml.j2" class Config(object): + """ + Basic configuration. + """ def __init__(self, path): tmpl_dir = os.path.dirname(path) tmpl_file = os.path.basename(path) @@ -37,12 +39,18 @@ class Config(object): self.default_params = {} def render(self, **kwargs): + """ + Render configuration. + """ kwargs.update(self.default_params) res = self.template.render(**kwargs) return res class IgniteServerConfig(Config): + """ + Ignite server node configuration. + """ def __init__(self, context): path = DEFAULT_IGNITE_CONF if 'ignite_server_config_path' in context.globals: @@ -51,6 +59,9 @@ class IgniteServerConfig(Config): class IgniteClientConfig(Config): + """ + Ignite client node configuration. + """ def __init__(self, context): path = DEFAULT_IGNITE_CONF if 'ignite_client_config_path' in context.globals: @@ -60,6 +71,8 @@ class IgniteClientConfig(Config): class IgniteLoggerConfig(Config): + """ + Ignite logger configuration. + """ def __init__(self): super(IgniteLoggerConfig, self).__init__(DEFAULT_CONFIG_PATH + "/log4j.xml.j2") - diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index 4eab4a4..4d24d36 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -13,32 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains ignite path resolve utilities. +""" + import os from ignitetest.tests.utils.version import get_version, IgniteVersion -""" -This module provides Ignite path methods -""" - class IgnitePath: - SCRATCH_ROOT = "/mnt" - IGNITE_INSTALL_ROOT = "/opt" - """Path resolver for Ignite system tests which assumes the following layout: - /opt/ignite-dev # Current version of Ignite under test - /opt/ignite-2.7.6 # Example of an older version of Ignite installed from tarball - /opt/ignite-<version> # Other previous versions of Ignite - ... - """ + /opt/ignite-dev # Current version of Ignite under test + /opt/ignite-2.7.6 # Example of an older version of Ignite installed from tarball + /opt/ignite-<version> # Other previous versions of Ignite + ... + """ + SCRATCH_ROOT = "/mnt" + IGNITE_INSTALL_ROOT = "/opt" def __init__(self, context): self.project = context.globals.get("project", "ignite") def home(self, node_or_version, project=None): - version = self._version(node_or_version) + """ + :param node_or_version: Ignite service node or IgniteVersion instance. + :param project: Project name. + :return: Home directory. + """ + version = self.__version(node_or_version) home_dir = project or self.project if version is not None: home_dir += "-%s" % str(version) @@ -46,11 +50,18 @@ class IgnitePath: return os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) def script(self, script_name, node_or_version, project=None): - version = self._version(node_or_version) + """ + :param script_name: Script name. + :param node_or_version: Ignite service node or IgniteVersion instance. + :param project: Project name. + :return: Full path to script. + """ + version = self.__version(node_or_version) return os.path.join(self.home(version, project=project), "bin", script_name) - def _version(self, node_or_version): + @staticmethod + def __version(node_or_version): if isinstance(node_or_version, IgniteVersion): return node_or_version - else: - return get_version(node_or_version) + + return get_version(node_or_version) diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py index 6843c94..bfbab25 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -12,12 +12,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains JMX Console client and different utilities and mixins to retrieve ignite node parameters +and attributes. +""" + import re from ignitetest.services.utils.decorators import memoize def ignite_jmx_mixin(node, pids): + """ + Dynamically mixin JMX attributes to Ignite service node. + :param node: Ignite service node. + :param pids: Ignite service node pids. + """ setattr(node, 'pids', pids) base_cls = node.__class__ base_cls_name = node.__class__.__name__ @@ -25,15 +36,25 @@ def ignite_jmx_mixin(node, pids): class JmxMBean(object): + """ + Dynamically exposes JMX MBean attributes. + """ def __init__(self, client, name): self.client = client self.name = name def __getattr__(self, attr): + """ + Retrieves through JMX client MBean attributes. + :param attr: Attribute name. + :return: Attribute value. + """ return self.client.mbean_attribute(self.name, attr) class JmxClient(object): + """JMX client, invokes jmxterm on node locally. + """ jmx_util_cmd = 'java -jar /opt/jmxterm.jar -v silent -n' def __init__(self, node): @@ -42,50 +63,82 @@ class JmxClient(object): @memoize def find_mbean(self, pattern, domain='org.apache'): + """ + Find mbean by specified pattern and domain on node. + :param pattern: MBean name pattern. + :param domain: Domain of MBean + :return: JmxMBean instance + """ cmd = "echo $'open %s\\n beans -d %s \\n close' | %s | grep -o '%s'" \ % (self.pid, domain, self.jmx_util_cmd, pattern) - name = next(self.run_cmd(cmd)).strip() + name = next(self.__run_cmd(cmd)).strip() return JmxMBean(self, name) def mbean_attribute(self, mbean, attr): + """ + Get MBean attribute. + :param mbean: MBean name + :param attr: Attribute name + :return: Attribute value + """ cmd = "echo $'open %s\\n get -b %s %s \\n close' | %s | sed 's/%s = \\(.*\\);/\\1/'" \ % (self.pid, mbean, attr, self.jmx_util_cmd, attr) - return iter(s.strip() for s in self.run_cmd(cmd)) + return iter(s.strip() for s in self.__run_cmd(cmd)) - def run_cmd(self, cmd): + def __run_cmd(self, cmd): return self.node.account.ssh_capture(cmd, allow_fail=False, callback=str) class DiscoveryInfo(object): + """ Ignite service node discovery info, obtained from DiscoverySpi mbean. + """ def __init__(self, coordinator, local_raw): self._local_raw = local_raw self._coordinator = coordinator @property - def id(self): + def node_id(self): + """ + :return: Local node id. + """ return self.__find__("id=([^\\s]+),") @property def coordinator(self): + """ + :return: Coordinator node id. + """ return self._coordinator @property def consistent_id(self): + """ + :return: Node consistent id, if presents (only in TcpDiscovery). + """ return self.__find__("consistentId=([^\\s]+),") @property def is_client(self): + """ + :return: True if node is client. + """ return self.__find__("isClient=([^\\s]+),") == "true" @property def order(self): + """ + :return: Topology order. + """ return int(self.__find__("order=(\\d+),")) @property def int_order(self): + """ + :return: Internal order (TcpDiscovery). + """ val = self.__find__("intOrder=(\\d+),") return int(val) if val else -1 @@ -95,16 +148,28 @@ class DiscoveryInfo(object): class IgniteJmxMixin(object): + """ + Mixin to IgniteService node, exposing useful properties, obtained from JMX. + """ @memoize def jmx_client(self): + """ + :return: JmxClient instance. + """ # noinspection PyTypeChecker return JmxClient(self) @memoize - def id(self): + def node_id(self): + """ + :return: Local node id. + """ return next(self.kernal_mbean().LocalNodeId).strip() def discovery_info(self): + """ + :return: DiscoveryInfo instance. + """ disco_mbean = self.disco_mbean() crd = next(disco_mbean.Coordinator).strip() local = next(disco_mbean.LocalNodeFormatted).strip() @@ -112,13 +177,19 @@ class IgniteJmxMixin(object): return DiscoveryInfo(crd, local) def kernal_mbean(self): + """ + :return: IgniteKernal MBean. + """ return self.jmx_client().find_mbean('.*group=Kernal,name=IgniteKernal') @memoize def disco_mbean(self): + """ + :return: DiscoverySpi MBean. + """ disco_spi = next(self.kernal_mbean().DiscoverySpiFormatted).strip() if 'ZookeeperDiscoverySpi' in disco_spi: return self.jmx_client().find_mbean('.*group=SPIs,name=ZookeeperDiscoverySpi') - else: - return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi') + + return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi') diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py index d32cbbc..5c29c66 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -12,12 +12,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains classes and utilities to start zookeeper cluster for testing ZookeeperDiscovery. +""" + import os.path from ducktape.services.service import Service class ZookeeperSettings: + """ + Settings for zookeeper quorum nodes. + """ def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181): self.tick_time = tick_time self.init_limit = init_limit @@ -26,6 +34,9 @@ class ZookeeperSettings: class ZookeeperService(Service): + """ + Zookeeper service. + """ PERSISTENT_ROOT = "/mnt/zookeeper" CONFIG_ROOT = os.path.join(PERSISTENT_ROOT, "conf") LOG_FILE = os.path.join(PERSISTENT_ROOT, "zookeeper.log") @@ -41,16 +52,17 @@ class ZookeeperService(Service): } } - def __init__(self, context, num_nodes, settings=ZookeeperSettings()): + def __init__(self, context, num_nodes, settings=ZookeeperSettings(), start_timeout_sec=60): super(ZookeeperService, self).__init__(context, num_nodes) self.settings = settings + self.start_timeout_sec = start_timeout_sec - def start(self, timeout_sec=60): - Service.start(self) + def start(self): + super(ZookeeperService, self).start() self.logger.info("Waiting for Zookeeper quorum...") for node in self.nodes: - self.await_quorum(node, timeout_sec) + self.await_quorum(node, self.start_timeout_sec) self.logger.info("Zookeeper quorum is formed.") @@ -89,6 +101,11 @@ class ZookeeperService(Service): self.logger.info("Zookeeper node %d started on %s", idx, node.account.hostname) def await_quorum(self, node, timeout): + """ + Await quorum formed on node (leader election ready). + :param node: Zookeeper service node. + :param timeout: Wait timeout. + """ with node.account.monitor_log(self.LOG_FILE) as monitor: monitor.offset = 0 monitor.wait_until( @@ -103,12 +120,26 @@ class ZookeeperService(Service): return "org.apache.zookeeper.server.quorum.QuorumPeerMain" def pids(self, node): + """ + Get pids of zookeeper service node. + :param node: Zookeeper service node. + :return: List of pids. + """ return node.account.java_pids(self.java_class_name()) def alive(self, node): + """ + Check if zookeeper service node is alive. + :param node: Zookeeper service node. + :return: True if node is alive + """ return len(self.pids(node)) > 0 def connection_string(self): + """ + Form a connection string to zookeeper cluster. + :return: Connection string. + """ return ','.join([node.account.hostname + ":" + str(2181) for node in self.nodes]) def stop_node(self, node): diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index a3c9546..d836681 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Module contains node rebalance tests. +""" + from ducktape.mark import parametrize from ducktape.mark.resource import cluster @@ -22,16 +26,16 @@ from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH, IgniteVersion, LATEST +# pylint: disable=W0223 class AddNodeRebalanceTest(IgniteTest): + """ + Test basic rebalance scenarios. + """ NUM_NODES = 4 PRELOAD_TIMEOUT = 60 DATA_AMOUNT = 1000000 REBALANCE_TIMEOUT = 60 - """ - Test performs rebalance tests. - """ - def __init__(self, test_context): super(AddNodeRebalanceTest, self).__init__(test_context=test_context) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 01aebe4..d8d4e0b 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -12,6 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +Module contains discovery tests. +""" + import random from ducktape.mark import parametrize @@ -24,7 +29,14 @@ from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7 +# pylint: disable=W0223 class DiscoveryTest(IgniteTest): + """ + Test basic discovery scenarious (TCP and Zookeeper). + 1. Start of ignite cluster. + 2. Kill random node. + 3. Wait that survived node detects node failure. + """ NUM_NODES = 7 CONFIG_TEMPLATE = """ @@ -44,11 +56,15 @@ class DiscoveryTest(IgniteTest): def __init__(self, test_context): super(DiscoveryTest, self).__init__(test_context=test_context) - self.zk = None + self.zk_quorum = None self.servers = None @staticmethod def properties(zookeeper_settings=None): + """ + :param zookeeper_settings: ZookeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used. + :return: Rendered node's properties. + """ return Template(DiscoveryTest.CONFIG_TEMPLATE) \ .render(zookeeper_settings=zookeeper_settings) @@ -56,8 +72,8 @@ class DiscoveryTest(IgniteTest): pass def teardown(self): - if self.zk: - self.zk.stop() + if self.zk_quorum: + self.zk_quorum.stop() if self.servers: self.servers.stop() @@ -66,20 +82,26 @@ class DiscoveryTest(IgniteTest): @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) def test_tcp(self, version): + """ + Test basic discovery scenario with TcpDiscoverySpi. + """ return self.__basic_test__(version, False) @cluster(num_nodes=NUM_NODES + 3) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) def test_zk(self, version): + """ + Test basic discovery scenario with ZookeeperDiscoverySpi. + """ return self.__basic_test__(version, True) def __basic_test__(self, version, with_zk=False): if with_zk: - self.zk = ZookeeperService(self.test_context, 3) + self.zk_quorum = ZookeeperService(self.test_context, 3) self.stage("Starting Zookeper quorum") - self.zk.start() - properties = self.properties(zookeeper_settings={'connection_string': self.zk.connection_string()}) + self.zk_quorum.start() + properties = self.properties(zookeeper_settings={'connection_string': self.zk_quorum.connection_string()}) self.stage("Zookeper quorum started") else: properties = self.properties() @@ -94,19 +116,19 @@ class DiscoveryTest(IgniteTest): start = self.monotonic() self.servers.start() - data = {'Ignite cluster start time (s)': self.monotonic() - start } + data = {'Ignite cluster start time (s)': self.monotonic() - start} self.stage("Topology is ready") # Node failure detection fail_node, survived_node = self.choose_random_node_to_kill(self.servers) - data["nodes"] = [node.id() for node in self.servers.nodes] + data["nodes"] = [node.node_id() for node in self.servers.nodes] disco_infos = [] for node in self.servers.nodes: disco_info = node.discovery_info() disco_infos.append({ - "id": disco_info.id, + "id": disco_info.node_id, "consistent_id": disco_info.consistent_id, "coordinator": disco_info.coordinator, "order": disco_info.order, @@ -127,6 +149,10 @@ class DiscoveryTest(IgniteTest): @staticmethod def choose_random_node_to_kill(service): + """ + :param service: Service nodes to process. + :return: Tuple of random node to kill and survived nodes + """ idx = random.randint(0, len(service.nodes) - 1) survive = [node for i, node in enumerate(service.nodes) if i != idx] diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 0df2f84d..f0cfd77 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -12,6 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains PME free switch tests. +""" + import time from ducktape.mark import parametrize @@ -23,11 +28,18 @@ from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion +# pylint: disable=W0223 class PmeFreeSwitchTest(IgniteTest): + """ + Tests PME free switch scenarios. + """ NUM_NODES = 3 @staticmethod def properties(): + """ + :return: Rendered configuration properties. + """ return """ <property name="cacheConfiguration"> <list> @@ -53,6 +65,9 @@ class PmeFreeSwitchTest(IgniteTest): @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) def test(self, version): + """ + Test PME free scenario (node stop). + """ data = {} self.stage("Starting nodes") @@ -82,7 +97,8 @@ class PmeFreeSwitchTest(IgniteTest): single_key_tx_streamer = IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.SingleKeyTxStreamerApplication", + java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test." + "SingleKeyTxStreamerApplication", properties=self.properties(), params="test-cache,1000", version=ignite_version) diff --git a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py index 7e2b4fc..a5f188b 100644 --- a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py +++ b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains spark integration test. +""" + from ducktape.mark import parametrize from ignitetest.services.ignite import IgniteService @@ -23,6 +27,7 @@ from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH +# pylint: disable=W0223 class SparkIntegrationTest(IgniteTest): """ Test performs: @@ -45,6 +50,9 @@ class SparkIntegrationTest(IgniteTest): @parametrize(version=str(DEV_BRANCH)) def test_spark_client(self, version): + """ + Performs test scenario. + """ self.spark = SparkService(self.test_context, version=version, num_nodes=2) self.spark.start() @@ -55,7 +63,8 @@ class SparkIntegrationTest(IgniteTest): IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test.SampleDataStreamerApplication", + java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test." + "SampleDataStreamerApplication", params="cache,1000", version=version).run() diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py index a9c13d9..df0d30e 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py @@ -13,15 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains basic ignite test. +""" + from ducktape.tests.test import Test from monotonic import monotonic +# pylint: disable=W0223 class IgniteTest(Test): + """ + Basic ignite test. + """ def __init__(self, test_context): super(IgniteTest, self).__init__(test_context=test_context) def stage(self, msg): + """ + Print stage mark. + :param msg: Stage mark message. + """ self.logger.info("[TEST_STAGE] " + msg + "...") @staticmethod diff --git a/modules/ducktests/tests/ignitetest/tests/utils/version.py b/modules/ducktests/tests/ignitetest/tests/utils/version.py index 2f1f003..5173188 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/version.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/version.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Module contains ignite version utility class. +""" from distutils.version import LooseVersion from ducktape.cluster.cluster import ClusterNode @@ -49,8 +52,8 @@ class IgniteVersion(LooseVersion): def __str__(self): if self.is_dev: return "dev" - else: - return LooseVersion.__str__(self) + + return LooseVersion.__str__(self) def get_version(node=None): @@ -61,7 +64,8 @@ def get_version(node=None): if isinstance(node, ClusterNode) and hasattr(node, 'version'): return getattr(node, 'version') - if isinstance(node, str) or isinstance(node, unicode): + # pylint: disable=E0602 + if isinstance(node, (str, unicode)): return node return DEV_BRANCH