This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch debug-acceptance-timeout
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 9685ec0cdb1f1ad5504765207110e0346886ea24
Author: Márton Elek <e...@apache.org>
AuthorDate: Thu Nov 28 17:06:45 2019 +0100
HDDS-2646. Start acceptance tests only if at least one THREE pipeline is
available
---
.../dist/src/main/compose/ozone-hdfs/docker-config | 1 +
.../dist/src/main/compose/ozone-mr/common-config | 1 +
.../src/main/compose/ozone-om-ha-s3/docker-config | 1 +
.../src/main/compose/ozone-recon/docker-config | 1 +
.../src/main/compose/ozone-topology/docker-config | 2 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../src/main/compose/ozoneblockade/docker-config | 2 +
.../dist/src/main/compose/ozoneperf/docker-config | 1 +
.../src/main/compose/ozones3-haproxy/docker-config | 1 +
.../dist/src/main/compose/ozones3/docker-config | 1 +
.../src/main/compose/ozonescripts/docker-config | 2 +
.../src/main/compose/ozonesecure-mr/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 49 ----------------------
14 files changed, 16 insertions(+), 49 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
index a8a96c3..c6295eb 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
@@ -23,6 +23,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
index ee7a773..14c33ee 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
@@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=3
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
index 4122dae..5cfa826 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
@@ -29,5 +29,6 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=3
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config
index bf7871e..0c817a9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config
@@ -27,6 +27,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=3
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
ASYNC_PROFILER_HOME=/opt/profiler
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
index ec9c590..f8c939f 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
@@ -29,6 +29,8 @@
OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.con
OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
OZONE-SITE.XML_dfs.network.topology.aware.read.enable=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
+
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
ASYNC_PROFILER_HOME=/opt/profiler
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
index d82cde2..02b532a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
@@ -25,6 +25,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=3
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
ASYNC_PROFILER_HOME=/opt/profiler
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
index 3625fc6..b854aab 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
@@ -34,6 +34,8 @@ OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
OZONE-SITE.XML_hdds.scm.replication.thread.interval=6s
OZONE-SITE.XML_hdds.scm.replication.event.timeout=10s
OZONE-SITE.XML_dfs.ratis.server.failure.duration=35s
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
+
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
index 931f112..5d68afc 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
@@ -25,6 +25,7 @@ OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_ozone.replication=3
OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
ASYNC_PROFILER_HOME=/opt/profiler
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
index 75d1977..4c08c52 100644
--- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
@@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=3
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
index 75d1977..4c08c52 100644
--- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
@@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=3
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
index 9950898..787ef2e 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
@@ -26,6 +26,8 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
+
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
index 7d6fdc4..54992d8 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
@@ -31,6 +31,7 @@
OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_ozone.s3g.keytab.file=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.s3g.authentication.kerberos.principal=HTTP/s...@example.com
OZONE-SITE.XML_ozone.administrators=*
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/s...@example.com
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index ff42249..14ef63c 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -36,6 +36,7 @@
OZONE-SITE.XML_ozone.recon.authentication.kerberos.principal=HTTP/recon@EXAMPLE.
OZONE-SITE.XML_ozone.recon.keytab.file=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_recon.om.snapshot.task.initial.delay=20s
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_ozone.acl.enabled=true
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 49274c0..f4b0fbc 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -31,54 +31,6 @@ create_results_dir() {
chmod ogu+w "$RESULT_DIR"
}
-## @description print the number of datanodes up
-## @param the docker-compose file
-count_datanodes() {
- local compose_file=$1
-
- local
jmx_url='http://scm:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo'
- if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
- docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k
HTTP/s...@example.com -t /etc/security/keytabs/HTTP.keytab && curl --negotiate
-u : -s '${jmx_url}'"
- else
- docker-compose -f "${compose_file}" exec -T scm curl -s "${jmx_url}"
- fi \
- | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' || true
-}
-
-## @description wait until datanodes are up (or 30 seconds)
-## @param the docker-compose file
-## @param number of datanodes to wait for (default: 3)
-wait_for_datanodes(){
- local compose_file=$1
- local -i datanode_count=${2:-3}
-
- #Reset the timer
- SECONDS=0
-
- #Don't give it up until 30 seconds
- while [[ $SECONDS -lt 90 ]]; do
-
- #This line checks the number of HEALTHY datanodes registered in scm over
the
- # jmx HTTP servlet
- datanodes=$(count_datanodes "${compose_file}")
- if [[ "$datanodes" ]]; then
- if [[ ${datanodes} -ge ${datanode_count} ]]; then
-
- #It's up and running. Let's return from the function.
- echo "$datanodes datanodes are up and registered to the scm"
- return
- else
-
- #Print it only if a number. Could be not a number if scm is not yet
started
- echo "$datanodes datanode is up and healthy (until now)"
- fi
- fi
-
- sleep 2
- done
- echo "WARNING! Datanodes are not started successfully. Please check the
docker-compose files"
- return 1
-}
## @description wait until safemode exit (or 30 seconds)
## @param the docker-compose file
@@ -123,7 +75,6 @@ start_docker_env(){
docker-compose -f "$COMPOSE_FILE" --no-ansi down
docker-compose -f "$COMPOSE_FILE" --no-ansi up -d --scale
datanode="${datanode_count}" \
- && wait_for_datanodes "$COMPOSE_FILE" "${datanode_count}" \
&& wait_for_safemode_exit "$COMPOSE_FILE" \
&& sleep 10
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
