This is an automated email from the ASF dual-hosted git repository. av pushed a commit to branch ignite-ducktape in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push: new 57aa778 Add jmx client and some basic discovery stuff fo IgniteClusterNode. (#8070) 57aa778 is described below commit 57aa7780f34167f17b5365b5a77d5478d0d5daef Author: Ivan Daschinskiy <ivanda...@gmail.com> AuthorDate: Thu Jul 23 15:02:00 2020 +0300 Add jmx client and some basic discovery stuff fo IgniteClusterNode. (#8070) --- modules/ducktests/tests/docker/Dockerfile | 7 ++ .../ducktests/tests/ignitetest/services/ignite.py | 9 +- .../tests/ignitetest/services/ignite_app.py | 15 +-- .../tests/ignitetest/services/ignite_spark_app.py | 4 +- .../{ignite_spark_app.py => utils/decorators.py} | 27 ++--- .../ignitetest/services/utils/ignite_aware.py | 11 +- .../ignitetest/services/utils/ignite_aware_app.py | 5 +- .../tests/ignitetest/services/utils/jmx_utils.py | 124 +++++++++++++++++++++ .../tests/ignitetest/services/zk/zookeeper.py | 14 +-- .../tests/ignitetest/tests/discovery_test.py | 19 +++- 10 files changed, 190 insertions(+), 45 deletions(-) diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index 0baefda..fedea54 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -83,6 +83,13 @@ ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a" RUN apt-get install fuse RUN cd /opt && git clone -q https://github.com/confluentinc/kibosh.git && cd "/opt/kibosh" && git reset --hard $KIBOSH_VERSION && mkdir "/opt/kibosh/build" && cd "/opt/kibosh/build" && ../configure && make -j 2 +#Install jmxterm +ARG JMXTERM_NAME="jmxterm" +ARG JMXTERM_VERSION="1.0.1" +ARG JMXTERM_ARTIFACT="$JMXTERM_NAME-$JMXTERM_VERSION-uber.jar" +RUN cd /opt && curl -OL https://github.com/jiaqi/jmxterm/releases/download/v$JMXTERM_VERSION/$JMXTERM_ARTIFACT \ + && mv $JMXTERM_ARTIFACT $JMXTERM_NAME.jar + # Set up the ducker user. RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ /var/log/ && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile && echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers USER ducker diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index a2855e3..e68d0bb 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -17,7 +17,6 @@ import os.path import signal from ducktape.cluster.remoteaccount import RemoteCommandError -from ducktape.services.service import Service from ducktape.utils.util import wait_until from ignitetest.services.utils.ignite_aware import IgniteAwareService @@ -39,15 +38,15 @@ class IgniteService(IgniteAwareService): } def __init__(self, context, num_nodes, version=DEV_BRANCH, properties=""): - IgniteAwareService.__init__(self, context, num_nodes, version, properties) + super(IgniteService, self).__init__(context, num_nodes, version, properties) def start(self, timeout_sec=180): - Service.start(self) + super(IgniteService, self).start() self.logger.info("Waiting for Ignite(s) to start...") for node in self.nodes: - self.await_node_stated(node, timeout_sec) + self.await_node_started(node, timeout_sec) def start_cmd(self, node): jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteService.PERSISTENT_ROOT + "/success_file " @@ -64,7 +63,7 @@ class IgniteService(IgniteAwareService): IgniteService.STDOUT_STDERR_CAPTURE) return cmd - def await_node_stated(self, node, timeout_sec): + def await_node_started(self, node, timeout_sec): self.await_event_on_node("Topology snapshot", node, timeout_sec, from_the_beginning=True) if len(self.pids(node)) == 0: diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 87cb8e1..237436d 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -24,15 +24,8 @@ The Ignite application service allows to perform custom logic writen on java. class IgniteApplicationService(IgniteAwareApplicationService): - def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): - IgniteAwareApplicationService.__init__( - self, context, java_class_name, version, properties, params, timeout_sec, - service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteApplicationService") - - def start(self): - Service.start(self) + service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService" - self.logger.info("Waiting for Ignite Application (%s) to start..." % self.java_class_name) - - self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) - self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True) + def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): + super(IgniteApplicationService, self).__init__(context, java_class_name, version, properties, params, + timeout_sec, self.service_java_class_name) diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py index dffe7ac..c2f0496 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py @@ -22,8 +22,8 @@ from ignitetest.version import DEV_BRANCH class SparkIgniteApplicationService(IgniteAwareApplicationService): def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): - IgniteAwareApplicationService.__init__( - self, context, java_class_name, version, properties, params, timeout_sec) + super(SparkIgniteApplicationService, self).__init__(context, java_class_name, version, properties, params, + timeout_sec) def env(self): return IgniteAwareApplicationService.env(self) + \ diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/utils/decorators.py similarity index 52% copy from modules/ducktests/tests/ignitetest/services/ignite_spark_app.py copy to modules/ducktests/tests/ignitetest/services/utils/decorators.py index dffe7ac..788ddba 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/decorators.py @@ -12,20 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import functools +from threading import RLock -""" -The Ignite-Spark application service. -""" -from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService -from ignitetest.version import DEV_BRANCH +def memoize(func): + cache = func.cache = {} + lock = RLock() -class SparkIgniteApplicationService(IgniteAwareApplicationService): - def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): - IgniteAwareApplicationService.__init__( - self, context, java_class_name, version, properties, params, timeout_sec) + @functools.wraps(func) + def memoized_func(*args, **kwargs): + key = str(args) + str(kwargs) + if key not in cache: + with lock: + if key not in cache: + cache[key] = func(*args, **kwargs) + return cache[key] - def env(self): - return IgniteAwareApplicationService.env(self) + \ - "export EXCLUDE_MODULES=\"kubernetes,aws,gce,mesos,rest-http,web-agent,zookeeper,serializers,store," \ - "rocketmq\"; " + return memoized_func diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index dd82feb..106493c 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -12,14 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os from abc import abstractmethod from ducktape.services.background_thread import BackgroundThreadService +from ducktape.utils.util import wait_until from ignitetest.services.utils.ignite_config import IgniteConfig from ignitetest.services.utils.ignite_path import IgnitePath +from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin """ The base class to build services aware of Ignite. @@ -41,7 +42,7 @@ class IgniteAwareService(BackgroundThreadService): } def __init__(self, context, num_nodes, version, properties): - BackgroundThreadService.__init__(self, context, num_nodes) + super(IgniteAwareService, self).__init__(context, num_nodes) self.log_level = "DEBUG" self.config = IgniteConfig() @@ -55,7 +56,11 @@ class IgniteAwareService(BackgroundThreadService): def start_node(self, node): self.init_persistent(node) - BackgroundThreadService.start_node(self, node) + super(IgniteAwareService, self).start_node(node) + + wait_until(lambda: len(self.pids(node)) > 0, timeout_sec=10) + + ignite_jmx_mixin(node, self.pids(node)) def init_persistent(self, node): node.account.mkdirs(self.PERSISTENT_ROOT) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index 916d155..7b37633 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -26,7 +26,7 @@ The base class to build Ignite aware application written on java. class IgniteAwareApplicationService(IgniteAwareService): def __init__(self, context, java_class_name, version, properties, params, timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): - IgniteAwareService.__init__(self, context, 1, version, properties) + super(IgniteAwareApplicationService, self).__init__(context, 1, version, properties) self.servicejava_class_name = service_java_class_name self.java_class_name = java_class_name @@ -35,10 +35,11 @@ class IgniteAwareApplicationService(IgniteAwareService): self.params = params def start(self): - Service.start(self) + super(IgniteAwareApplicationService, self).start() self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) + self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True) def start_cmd(self, node): diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py new file mode 100644 index 0000000..6843c94 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +from ignitetest.services.utils.decorators import memoize + + +def ignite_jmx_mixin(node, pids): + setattr(node, 'pids', pids) + base_cls = node.__class__ + base_cls_name = node.__class__.__name__ + node.__class__ = type(base_cls_name, (base_cls, IgniteJmxMixin), {}) + + +class JmxMBean(object): + def __init__(self, client, name): + self.client = client + self.name = name + + def __getattr__(self, attr): + return self.client.mbean_attribute(self.name, attr) + + +class JmxClient(object): + jmx_util_cmd = 'java -jar /opt/jmxterm.jar -v silent -n' + + def __init__(self, node): + self.node = node + self.pid = node.pids[0] + + @memoize + def find_mbean(self, pattern, domain='org.apache'): + cmd = "echo $'open %s\\n beans -d %s \\n close' | %s | grep -o '%s'" \ + % (self.pid, domain, self.jmx_util_cmd, pattern) + + name = next(self.run_cmd(cmd)).strip() + + return JmxMBean(self, name) + + def mbean_attribute(self, mbean, attr): + cmd = "echo $'open %s\\n get -b %s %s \\n close' | %s | sed 's/%s = \\(.*\\);/\\1/'" \ + % (self.pid, mbean, attr, self.jmx_util_cmd, attr) + + return iter(s.strip() for s in self.run_cmd(cmd)) + + def run_cmd(self, cmd): + return self.node.account.ssh_capture(cmd, allow_fail=False, callback=str) + + +class DiscoveryInfo(object): + def __init__(self, coordinator, local_raw): + self._local_raw = local_raw + self._coordinator = coordinator + + @property + def id(self): + return self.__find__("id=([^\\s]+),") + + @property + def coordinator(self): + return self._coordinator + + @property + def consistent_id(self): + return self.__find__("consistentId=([^\\s]+),") + + @property + def is_client(self): + return self.__find__("isClient=([^\\s]+),") == "true" + + @property + def order(self): + return int(self.__find__("order=(\\d+),")) + + @property + def int_order(self): + val = self.__find__("intOrder=(\\d+),") + return int(val) if val else -1 + + def __find__(self, pattern): + res = re.search(pattern, self._local_raw) + return res.group(1) if res else None + + +class IgniteJmxMixin(object): + @memoize + def jmx_client(self): + # noinspection PyTypeChecker + return JmxClient(self) + + @memoize + def id(self): + return next(self.kernal_mbean().LocalNodeId).strip() + + def discovery_info(self): + disco_mbean = self.disco_mbean() + crd = next(disco_mbean.Coordinator).strip() + local = next(disco_mbean.LocalNodeFormatted).strip() + + return DiscoveryInfo(crd, local) + + def kernal_mbean(self): + return self.jmx_client().find_mbean('.*group=Kernal,name=IgniteKernal') + + @memoize + def disco_mbean(self): + disco_spi = next(self.kernal_mbean().DiscoverySpiFormatted).strip() + + if 'ZookeeperDiscoverySpi' in disco_spi: + return self.jmx_client().find_mbean('.*group=SPIs,name=ZookeeperDiscoverySpi') + else: + return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi') diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py index 7767ce9..d32cbbc 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -12,10 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os.path -from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.service import Service @@ -99,13 +97,13 @@ class ZookeeperService(Service): err_msg="Zookeeper quorum was not formed on %s" % node.account.hostname ) + @staticmethod + def java_class_name(): + """ The class name of the Zookeeper quorum peers. """ + return "org.apache.zookeeper.server.quorum.QuorumPeerMain" + def pids(self, node): - try: - cmd = "ps ax | grep -i zookeeper | grep java | grep -v grep | awk '{print $1}'" - pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] - return pid_arr - except (RemoteCommandError, ValueError) as e: - return [] + return node.account.java_pids(self.java_class_name()) def alive(self, node): return len(self.pids(node)) > 0 diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 42228e3..dd25205 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import random from ducktape.mark import parametrize @@ -25,6 +24,7 @@ from ignitetest.tests.utils.ignite_test import IgniteTest from jinja2 import Template + class DiscoveryTest(IgniteTest): NUM_NODES = 7 @@ -101,6 +101,23 @@ class DiscoveryTest(IgniteTest): # Node failure detection fail_node, survived_node = self.choose_random_node_to_kill(self.servers) + + data["nodes"] = [node.id() for node in self.servers.nodes] + + disco_infos = [] + for node in self.servers.nodes: + disco_info = node.discovery_info() + disco_infos.append({ + "id": disco_info.id, + "consistent_id": disco_info.consistent_id, + "coordinator": disco_info.coordinator, + "order": disco_info.order, + "int_order": disco_info.int_order, + "is_client": disco_info.is_client + }) + + data["node_disco_info"] = disco_infos + self.servers.stop_node(fail_node, clean_shutdown=False) start = self.monotonic()