This is an automated email from the ASF dual-hosted git repository.
av pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new 254bb67 Json params support (#8109)
254bb67 is described below
commit 254bb679629a14a64538233792ffa7dd353f6bb9
Author: Anton Vinogradov <a...@apache.org>
AuthorDate: Wed Aug 5 12:30:20 2020 +0300
Json params support (#8109)
---
modules/ducktests/pom.xml | 18 ++++++++++++++++++
.../ducktest/tests/DataGenerationApplication.java | 7 ++++---
.../LongTxStreamerApplication.java | 5 +++--
.../SingleKeyTxStreamerApplication.java | 7 ++++---
.../ducktest/tests/smoke_test/SimpleApplication.java | 3 ++-
.../ducktest/utils/IgniteApplicationService.java | 11 ++++++++---
.../ducktest/utils/IgniteAwareApplication.java | 14 +++++++-------
.../ducktest/utils/IgniteAwareApplicationService.java | 14 +++++++++++---
modules/ducktests/tests/ignitetest/services/ignite.py | 2 +-
.../tests/ignitetest/services/utils/ignite_aware.py | 5 ++---
.../ignitetest/services/utils/ignite_aware_app.py | 13 +++++++++++--
.../tests/ignitetest/services/utils/ignite_path.py | 6 +++---
.../tests/ignitetest/tests/add_node_rebalance_test.py | 2 +-
.../tests/ignitetest/tests/pme_free_switch_test.py | 4 ++--
modules/ducktests/tests/setup.py | 1 +
15 files changed, 78 insertions(+), 34 deletions(-)
diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml
index 7bfdc2c..26226c6 100644
--- a/modules/ducktests/pom.xml
+++ b/modules/ducktests/pom.xml
@@ -132,6 +132,24 @@
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.codehaus.woodstox</groupId>
<artifactId>stax2-api</artifactId>
<version>3.1.4</version>
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
index c1661d2..f9732b2 100644
---
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.ducktest.tests;
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
@@ -34,13 +35,13 @@ public class DataGenerationApplication extends
IgniteAwareApplication {
}
/** {@inheritDoc} */
- @Override protected void run(String[] args) {
+ @Override protected void run(JsonNode jsonNode) {
log.info("Creating cache...");
- IgniteCache<Integer, Integer> cache = ignite.createCache(args[0]);
+ IgniteCache<Integer, Integer> cache =
ignite.createCache(jsonNode.get("cacheName").asText());
try (IgniteDataStreamer<Integer, Integer> stmr =
ignite.dataStreamer(cache.getName())) {
- for (int i = 0; i < Integer.parseInt(args[1]); i++) {
+ for (int i = 0; i < jsonNode.get("range").asInt(); i++) {
stmr.addData(i, i);
if (i % 10_000 == 0)
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java
index 14b96a8..82596f8 100644
---
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java
@@ -19,6 +19,7 @@ package
org.apache.ignite.internal.ducktest.tests.pme_free_switch_test;
import java.util.Collection;
import java.util.concurrent.CountDownLatch;
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.internal.IgniteEx;
@@ -47,8 +48,8 @@ public class LongTxStreamerApplication extends
IgniteAwareApplication {
}
/** {@inheritDoc} */
- @Override public void run(String[] args) throws InterruptedException {
- IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(args[0]);
+ @Override public void run(JsonNode jsonNode) throws InterruptedException {
+ IgniteCache<Integer, Integer> cache =
ignite.getOrCreateCache(jsonNode.get("cacheName").asText());
log.info("Starting Long Tx...");
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java
index 42e4d24..f38e811 100644
---
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.ducktest.tests.pme_free_switch_test;
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
@@ -33,10 +34,10 @@ public class SingleKeyTxStreamerApplication extends
IgniteAwareApplication {
}
/** {@inheritDoc} */
- @Override public void run(String[] args) {
- IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(args[0]);
+ @Override public void run(JsonNode jsonNode) {
+ IgniteCache<Integer, Integer> cache =
ignite.getOrCreateCache(jsonNode.get("cacheName").asText());
- int warmup = Integer.parseInt(args[1]);
+ int warmup = jsonNode.get("warmup").asInt();
long max = -1;
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java
index ec4057d..379f5b1 100644
---
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.ducktest.tests.smoke_test;
import java.util.UUID;
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -36,7 +37,7 @@ public class SimpleApplication extends IgniteAwareApplication
{
}
/** {@inheritDoc} */
- @Override public void run(String[] args) {
+ @Override public void run(JsonNode jsonNode) {
IgniteCache<Integer, Integer> cache =
ignite.getOrCreateCache(UUID.randomUUID().toString());
cache.put(1, 2);
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java
index a41f27c..b4f60b5 100644
---
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java
@@ -17,7 +17,9 @@
package org.apache.ignite.internal.ducktest.utils;
-import java.util.Arrays;
+import java.util.Base64;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -55,9 +57,12 @@ public class IgniteApplicationService {
try (Ignite ignite = Ignition.start(cfg)) {
IgniteAwareApplication app =
(IgniteAwareApplication)clazz.getConstructor(Ignite.class).newInstance(ignite);
- String[] appParams = Arrays.copyOfRange(params, 2, params.length);
+ ObjectMapper mapper = new ObjectMapper();
- app.start(appParams);
+ JsonNode jsonNode = params.length > 2 ?
+ mapper.readTree(Base64.getDecoder().decode(params[2])) :
mapper.createObjectNode();
+
+ app.start(jsonNode);
}
}
}
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java
index 525c76d..28c8d20 100644
---
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.ducktest.utils;
-import java.util.Arrays;
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.ignite.Ignite;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -159,18 +159,18 @@ public abstract class IgniteAwareApplication {
}
/**
- *
+ * @param jsonNode JSON node.
*/
- protected abstract void run(String[] args) throws Exception;
+ protected abstract void run(JsonNode jsonNode) throws Exception;
/**
- * @param args Args.
+ * @param jsonNode JSON node.
*/
- public void start(String[] args) {
+ public void start(JsonNode jsonNode) {
try {
- log.info("Application params: " + Arrays.toString(args));
+ log.info("Application params: " + jsonNode);
- run(args);
+ run(jsonNode);
assert inited : "Was not properly initialized.";
assert finished : "Was not properly finished.";
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java
index a7836c2..de13ef1 100644
---
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java
@@ -17,7 +17,10 @@
package org.apache.ignite.internal.ducktest.utils;
-import java.util.Arrays;
+import java.util.Base64;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -40,8 +43,13 @@ public class IgniteAwareApplicationService {
IgniteAwareApplication app =
(IgniteAwareApplication)clazz.getConstructor().newInstance();
- String[] appParams = Arrays.copyOfRange(params, 1, params.length);
+ ObjectMapper mapper = new ObjectMapper();
- app.start(appParams);
+ JsonNode jsonNode = params.length > 2 ?
+ mapper.readTree(Base64.getDecoder().decode(params[2])) :
mapper.createObjectNode();
+
+ ((ObjectNode)jsonNode).put("cfgPath", params[1]);
+
+ app.start(jsonNode);
}
}
diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py
b/modules/ducktests/tests/ignitetest/services/ignite.py
index f5054b6..08732c6 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite.py
@@ -64,7 +64,7 @@ class IgniteService(IgniteAwareService):
cmd = "export EXCLUDE_TEST_CLASSES=true; "
cmd += "export IGNITE_LOG_DIR=" + IgniteService.PERSISTENT_ROOT + "; "
- cmd += "export USER_LIBS=%s; " % self.user_libs
+ cmd += "export USER_LIBS=%s; " % ":".join(self.user_libs)
cmd += "%s %s %s 1>> %s 2>> %s &" % \
(self.path.script("ignite.sh"),
jvm_opts,
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index 3668ffe..7fe97a9 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -26,7 +26,6 @@ from ducktape.utils.util import wait_until
from ignitetest.services.utils.ignite_config import IgniteLoggerConfig,
IgniteServerConfig, IgniteClientConfig
from ignitetest.services.utils.ignite_path import IgnitePath
from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin
-
from ignitetest.tests.utils.version import IgniteVersion
@@ -66,8 +65,8 @@ class IgniteAwareService(BackgroundThreadService):
libs = modules or []
libs.extend(["ignite-log4j"])
- libs = map(lambda m: self.path.module(m) + "/*", libs)
- self.user_libs = ":".join(libs)
+
+ self.user_libs = list(map(lambda m: self.path.module(m) + "/*", libs))
def start_node(self, node):
self.init_persistent(node)
diff --git
a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
index d2fedaa..eece2a6 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py
@@ -17,6 +17,8 @@
This module contains the base class to build Ignite aware application written
on java.
"""
+import base64
+import json
import re
from ignitetest.services.utils.ignite_aware import IgniteAwareService
@@ -26,6 +28,7 @@ class IgniteAwareApplicationService(IgniteAwareService):
"""
The base class to build Ignite aware application written on java.
"""
+
# pylint: disable=R0913
def __init__(self, context, java_class_name, modules, client_mode,
version, properties, params, timeout_sec,
service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"):
@@ -82,7 +85,7 @@ class IgniteAwareApplicationService(IgniteAwareService):
args = self.java_class_name + "," +
IgniteAwareApplicationService.CONFIG_FILE
if self.params != "":
- args += "," + self.params
+ args += "," +
str(base64.b64encode(json.dumps(self.params).encode("UTF-8")))
return args
@@ -103,10 +106,16 @@ class IgniteAwareApplicationService(IgniteAwareService):
"""
:return: Export string of additional environment variables.
"""
+ if not self.version.is_dev:
+ # Jackson requred to parse application params at java side.
Release's version should be used.
+ for line in self.nodes[0].account.ssh_capture(
+ "ls -d %s/libs/optional/ignite-aws/* | grep jackson | tr
'\n' ':' | sed 's/.$//'" % self.path.home):
+ self.user_libs.extend([line])
+
return "export MAIN_CLASS={main_class};
".format(main_class=self.servicejava_class_name) + \
"export EXCLUDE_TEST_CLASSES=true; " + \
"export IGNITE_LOG_DIR={log_dir};
".format(log_dir=self.PERSISTENT_ROOT) + \
- "export
USER_LIBS=%s:/opt/ignite-dev/modules/ducktests/target/*; " % self.user_libs
+ "export
USER_LIBS=%s:/opt/ignite-dev/modules/ducktests/target/*; " %
(":".join(self.user_libs))
def extract_result(self, name):
"""
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py
b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py
index c1217b0..2177e9b 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py
@@ -36,7 +36,7 @@ class IgnitePath:
self.project = context.globals.get("project", "ignite")
home_dir = "%s-%s" % (self.project, str(self.version))
- self._home = os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir)
+ self.home = os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir)
def module(self, module_name):
"""
@@ -48,11 +48,11 @@ class IgnitePath:
else:
module_path = os.path.join("libs", "optional", module_name)
- return os.path.join(self._home, module_path)
+ return os.path.join(self.home, module_path)
def script(self, script_name):
"""
:param script_name: name of Ignite script
:return: absolute path to the specified script
"""
- return os.path.join(self._home, "bin", script_name)
+ return os.path.join(self.home, "bin", script_name)
diff --git
a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py
b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py
index d836681..28fc4fb 100644
--- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py
@@ -69,7 +69,7 @@ class AddNodeRebalanceTest(IgniteTest):
IgniteApplicationService(self.test_context,
java_class_name="org.apache.ignite.internal.ducktest.tests.DataGenerationApplication",
version=ignite_version,
- params="test-cache,%d" % self.DATA_AMOUNT,
+ params={"cacheName": "test-cache", "range":
self.DATA_AMOUNT},
timeout_sec=self.PRELOAD_TIMEOUT).run()
ignite = IgniteService(self.test_context, num_nodes=1,
version=ignite_version)
diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py
b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py
index f0cfd77..cd9feec 100644
--- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py
@@ -88,7 +88,7 @@ class PmeFreeSwitchTest(IgniteTest):
self.test_context,
java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication",
properties=self.properties(),
- params="test-cache",
+ params={"cacheName": "test-cache"},
version=ignite_version)
long_tx_streamer.start()
@@ -100,7 +100,7 @@ class PmeFreeSwitchTest(IgniteTest):
java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test."
"SingleKeyTxStreamerApplication",
properties=self.properties(),
- params="test-cache,1000",
+ params={"cacheName": "test-cache", "warmup": 1000},
version=ignite_version)
single_key_tx_streamer.start()
diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py
index 94cb914..e82bdfb 100644
--- a/modules/ducktests/tests/setup.py
+++ b/modules/ducktests/tests/setup.py
@@ -15,6 +15,7 @@
import re
import sys
+
from setuptools import find_packages, setup
from setuptools.command.test import test as TestCommand
