This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch testcontainer in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b5f530706abb97ec98804eaadd57f04fec9d6e02 Author: xiangdong huang <[email protected]> AuthorDate: Sun Apr 18 00:03:38 2021 +0800 init --- .../resources/conf/iotdb-cluster.properties | 6 ++-- .../apache/iotdb/cluster/config/ClusterConfig.java | 24 +++++++++++--- .../iotdb/db/sql/{SqlE2E.java => ClusterE2E.java} | 33 ++++++++----------- .../db/sql/{SqlE2E.java => SingleNodeE2E.java} | 2 +- .../NoProjectNameDockerComposeContainer.java | 38 ++++++++++++++++++++++ .../docker-compose.yaml} | 4 +++ .../iotdb-cluster.properties} | 38 ++++++++++------------ .../docker-compose.yaml} | 4 +++ .../iotdb-cluster.properties} | 38 ++++++++++------------ .../docker-compose.yaml} | 3 ++ .../iotdb-cluster.properties} | 38 ++++++++++------------ 11 files changed, 138 insertions(+), 90 deletions(-) diff --git a/cluster/src/assembly/resources/conf/iotdb-cluster.properties b/cluster/src/assembly/resources/conf/iotdb-cluster.properties index 73e7b42..3819dcc 100644 --- a/cluster/src/assembly/resources/conf/iotdb-cluster.properties +++ b/cluster/src/assembly/resources/conf/iotdb-cluster.properties @@ -29,6 +29,7 @@ #-------------------------------------------IMPORTANT---------------------------------------------# # used for communication between cluster nodes, eg heartbeatăraft logs and snapshots etc. +# if this parameter is commented, then the IP that binded by the hostname will be used. internal_ip=127.0.0.1 # port for metadata service @@ -51,7 +52,8 @@ open_server_rpc_port=false # nodes that already in the cluster, unnecessary to be the nodes that were used to build the # initial cluster by start-node.sh(.bat). Several nodes will be picked randomly to send the # request, the number of nodes picked depends on the number of retries. -seed_nodes=127.0.0.1:9003,127.0.0.1:9005,127.0.0.1:9007 +#seed_nodes=127.0.0.1:9003,127.0.0.1:9005,127.0.0.1:9007 +seed_nodes=127.0.0.1:9003 # whether to use thrift compressed protocol for internal communications. If you want to change # compression settings for external clients, please modify 'rpc_thrift_compression_enable' in @@ -65,7 +67,7 @@ rpc_thrift_compression_enable=false max_concurrent_client_num=10000 # number of replications for one partition -default_replica_num=3 +default_replica_num=1 # cluster name to identify different clusters # all node's cluster_name in one cluster are the same diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java index 11cedc8..2f4215e 100644 --- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java @@ -21,27 +21,31 @@ package org.apache.iotdb.cluster.config; import org.apache.iotdb.cluster.utils.ClusterConsistent; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; public class ClusterConfig { - + private static Logger logger = LoggerFactory.getLogger(ClusterConfig.class); static final String CONFIG_NAME = "iotdb-cluster.properties"; - private String internalIp = "127.0.0.1"; + private String internalIp; private int internalMetaPort = 9003; private int internalDataPort = 40010; private int clusterRpcPort = IoTDBDescriptor.getInstance().getConfig().getRpcPort(); /** each one is a {internalIp | domain name}:{meta port} string tuple. */ - private List<String> seedNodeUrls = - Arrays.asList(String.format("%s:%d", internalIp, internalMetaPort)); + private List<String> seedNodeUrls; @ClusterConsistent private boolean isRpcThriftCompressionEnabled = false; private int maxConcurrentClientNum = 10000; - @ClusterConsistent private int replicationNum = 3; + @ClusterConsistent private int replicationNum = 1; @ClusterConsistent private String clusterName = "default"; @@ -164,6 +168,16 @@ public class ClusterConfig { private boolean openServerRpcPort = false; + public ClusterConfig() { + try { + internalIp = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + logger.error(e.getMessage()); + internalIp = "127.0.0.1"; + } + seedNodeUrls = Arrays.asList(String.format("%s:%d", internalIp, internalMetaPort)); + } + public int getSelectorNumOfClientPool() { return selectorNumOfClientPool; } diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/SqlE2E.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterE2E.java similarity index 65% copy from testcontainer/src/test/java/org/apache/iotdb/db/sql/SqlE2E.java copy to testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterE2E.java index 4fec53e..44c0d08 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/SqlE2E.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/ClusterE2E.java @@ -24,11 +24,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.NoProjectNameDockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.images.PullPolicy; -import org.testcontainers.utility.DockerImageName; import java.io.File; import java.sql.Connection; @@ -36,32 +34,29 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -public class SqlE2E { +public class ClusterE2E { private Statement statement; private Connection connection; @Rule - public GenericContainer dslContainer = - new GenericContainer(DockerImageName.parse("apache/iotdb:maven-development")) - .withImagePullPolicy(PullPolicy.defaultPolicy()) - // mount another properties for changing parameters, e.g., open 5555 port (sync module) - .withFileSystemBind( - new File("src/test/resources/iotdb-engine.properties").getAbsolutePath(), - "/iotdb/conf/iotdb-engine.properties", - BindMode.READ_ONLY) - .withExposedPorts(6667) - .waitingFor(Wait.forListeningPort()); + public DockerComposeContainer environment = + new NoProjectNameDockerComposeContainer( + "3nodes", new File("src/test/resources/3nodes/docker-compose.yaml")) + .withExposedService("iotdb-server_1", 6667, Wait.forListeningPort()) + .withExposedService("iotdb-server_2", 6667, Wait.forListeningPort()) + .withExposedService("iotdb-server_3", 6667, Wait.forListeningPort()) + .withLocalCompose(true); int rpcPort = 6667; - int syncPort = 5555; @Before public void setUp() throws Exception { - rpcPort = dslContainer.getMappedPort(6667); - syncPort = dslContainer.getMappedPort(5555); + String ip = environment.getServiceHost("iotdb-server_1", 6667); + rpcPort = environment.getServicePort("iotdb-server_1", 6667); + Class.forName(Config.JDBC_DRIVER_NAME); - connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:" + rpcPort, "root", "root"); + connection = DriverManager.getConnection("jdbc:iotdb://" + ip + ":" + rpcPort, "root", "root"); statement = connection.createStatement(); } diff --git a/testcontainer/src/test/java/org/apache/iotdb/db/sql/SqlE2E.java b/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeE2E.java similarity index 99% rename from testcontainer/src/test/java/org/apache/iotdb/db/sql/SqlE2E.java rename to testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeE2E.java index 4fec53e..7a4705a 100644 --- a/testcontainer/src/test/java/org/apache/iotdb/db/sql/SqlE2E.java +++ b/testcontainer/src/test/java/org/apache/iotdb/db/sql/SingleNodeE2E.java @@ -36,7 +36,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -public class SqlE2E { +public class SingleNodeE2E { private Statement statement; private Connection connection; diff --git a/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java b/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java new file mode 100644 index 0000000..0d1fa76 --- /dev/null +++ b/testcontainer/src/test/java/org/testcontainers/containers/NoProjectNameDockerComposeContainer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.testcontainers.containers; + +import java.io.File; +import java.lang.reflect.Field; + +public class NoProjectNameDockerComposeContainer extends DockerComposeContainer { + + public NoProjectNameDockerComposeContainer(String identifier, File... composeFiles) { + super(identifier, composeFiles); + Field project = null; + try { + project = DockerComposeContainer.class.getDeclaredField("project"); + project.setAccessible(true); + project.set(this, ""); + } catch (NoSuchFieldException | IllegalAccessException e) { + e.printStackTrace(); + } + } +} diff --git a/testcontainer/src/test/resources/composes/docker-compose-3.yaml b/testcontainer/src/test/resources/1node/docker-compose.yaml similarity index 92% copy from testcontainer/src/test/resources/composes/docker-compose-3.yaml copy to testcontainer/src/test/resources/1node/docker-compose.yaml index e345335..9803930 100644 --- a/testcontainer/src/test/resources/composes/docker-compose-3.yaml +++ b/testcontainer/src/test/resources/1node/docker-compose.yaml @@ -33,6 +33,10 @@ services: interval: 5s timeout: 60s retries: 120 + volumes: + - ./iotdb-cluster.properties:/iotdb/conf/iotdb-cluster.properties + scale: 1 + networks: iotdb: diff --git a/testcontainer/src/test/resources/composes/docker-compose-3.yaml b/testcontainer/src/test/resources/1node/iotdb-cluster.properties similarity index 61% copy from testcontainer/src/test/resources/composes/docker-compose-3.yaml copy to testcontainer/src/test/resources/1node/iotdb-cluster.properties index e345335..2df52cc 100644 --- a/testcontainer/src/test/resources/composes/docker-compose-3.yaml +++ b/testcontainer/src/test/resources/1node/iotdb-cluster.properties @@ -15,25 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -version: '3.8' - -services: - iotdb-server: - image: apache/iotdb:cluster-maven-development - expose: - - 6667 - - 9003 - - 40010 - networks: - - iotdb - healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] - interval: 5s - timeout: 60s - retries: 120 -networks: - iotdb: - driver: bridge +internal_meta_port=9003 +internal_data_port=40010 +seed_nodes=3nodes_iotdb-server_1:9003 +default_replica_num=1 +consistency_level=mid +connection_timeout_ms=20000 +write_operation_timeout_ms=30000 +read_operation_timeout_ms=30000 +catch_up_timeout_ms=300000 +use_batch_in_catch_up=true +min_num_of_logs_in_mem=1000 +max_num_of_logs_in_mem=2000 +log_deletion_check_interval_second=-1 +is_use_async_server=false +is_use_async_applier=true +is_enable_raft_log_persistence=true +open_server_rpc_port=false diff --git a/testcontainer/src/test/resources/composes/docker-compose-3.yaml b/testcontainer/src/test/resources/3nodes/docker-compose.yaml similarity index 92% copy from testcontainer/src/test/resources/composes/docker-compose-3.yaml copy to testcontainer/src/test/resources/3nodes/docker-compose.yaml index e345335..bbb18cc 100644 --- a/testcontainer/src/test/resources/composes/docker-compose-3.yaml +++ b/testcontainer/src/test/resources/3nodes/docker-compose.yaml @@ -33,6 +33,10 @@ services: interval: 5s timeout: 60s retries: 120 + volumes: + - ./iotdb-cluster.properties:/iotdb/conf/iotdb-cluster.properties + scale: 3 + networks: iotdb: diff --git a/testcontainer/src/test/resources/composes/docker-compose-3.yaml b/testcontainer/src/test/resources/3nodes/iotdb-cluster.properties similarity index 59% copy from testcontainer/src/test/resources/composes/docker-compose-3.yaml copy to testcontainer/src/test/resources/3nodes/iotdb-cluster.properties index e345335..fe6133d 100644 --- a/testcontainer/src/test/resources/composes/docker-compose-3.yaml +++ b/testcontainer/src/test/resources/3nodes/iotdb-cluster.properties @@ -15,25 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -version: '3.8' - -services: - iotdb-server: - image: apache/iotdb:cluster-maven-development - expose: - - 6667 - - 9003 - - 40010 - networks: - - iotdb - healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] - interval: 5s - timeout: 60s - retries: 120 -networks: - iotdb: - driver: bridge +internal_meta_port=9003 +internal_data_port=40010 +seed_nodes=3nodes_iotdb-server_1:9003,3nodes_iotdb-server_2:9003,3nodes_iotdb-server_3:9003 +default_replica_num=3 +consistency_level=mid +connection_timeout_ms=20000 +write_operation_timeout_ms=30000 +read_operation_timeout_ms=30000 +catch_up_timeout_ms=300000 +use_batch_in_catch_up=true +min_num_of_logs_in_mem=1000 +max_num_of_logs_in_mem=2000 +log_deletion_check_interval_second=-1 +is_use_async_server=false +is_use_async_applier=true +is_enable_raft_log_persistence=true +open_server_rpc_port=false diff --git a/testcontainer/src/test/resources/composes/docker-compose-3.yaml b/testcontainer/src/test/resources/5nodes/docker-compose.yaml similarity index 92% copy from testcontainer/src/test/resources/composes/docker-compose-3.yaml copy to testcontainer/src/test/resources/5nodes/docker-compose.yaml index e345335..d45577f 100644 --- a/testcontainer/src/test/resources/composes/docker-compose-3.yaml +++ b/testcontainer/src/test/resources/5nodes/docker-compose.yaml @@ -33,6 +33,9 @@ services: interval: 5s timeout: 60s retries: 120 + volumes: + - ./iotdb-cluster.properties:/iotdb/conf/iotdb-cluster.properties + scale: 5 networks: iotdb: diff --git a/testcontainer/src/test/resources/composes/docker-compose-3.yaml b/testcontainer/src/test/resources/5nodes/iotdb-cluster.properties similarity index 57% rename from testcontainer/src/test/resources/composes/docker-compose-3.yaml rename to testcontainer/src/test/resources/5nodes/iotdb-cluster.properties index e345335..705cca6 100644 --- a/testcontainer/src/test/resources/composes/docker-compose-3.yaml +++ b/testcontainer/src/test/resources/5nodes/iotdb-cluster.properties @@ -15,25 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -version: '3.8' - -services: - iotdb-server: - image: apache/iotdb:cluster-maven-development - expose: - - 6667 - - 9003 - - 40010 - networks: - - iotdb - healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ] - interval: 5s - timeout: 60s - retries: 120 -networks: - iotdb: - driver: bridge +internal_meta_port=9003 +internal_data_port=40010 +seed_nodes=5nodes_iotdb-server_1:9003,5nodes_iotdb-server_2:9003,5nodes_iotdb-server_3:9003,5nodes_iotdb-server_4:9003,5nodes_iotdb-server_5:9003 +default_replica_num=3 +consistency_level=mid +connection_timeout_ms=20000 +write_operation_timeout_ms=30000 +read_operation_timeout_ms=30000 +catch_up_timeout_ms=300000 +use_batch_in_catch_up=true +min_num_of_logs_in_mem=1000 +max_num_of_logs_in_mem=2000 +log_deletion_check_interval_second=-1 +is_use_async_server=false +is_use_async_applier=true +is_enable_raft_log_persistence=true +open_server_rpc_port=false
