This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ac0bf1b924 HDDS-8754. Test Hadoop compatibility in Ozone HA
environment (#4830)
ac0bf1b924 is described below
commit ac0bf1b924e97a828f39e4cf5b86ec0be5650399
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jun 7 14:54:48 2023 +0200
HDDS-8754. Test Hadoop compatibility in Ozone HA environment (#4830)
---
.../hadoop33/.env => common/hadoop-2.7.sh} | 15 +--
.../hadoop32/.env => common/hadoop-3.1.sh} | 15 +--
.../{ozone-ha/.env => common/hadoop-3.2.sh} | 13 ++-
.../{ozone-ha/.env => common/hadoop-3.3.sh} | 13 ++-
.../{ozone-mr/common-config => common/hadoop.conf} | 24 +----
.../{ozone-mr/hadoop27/.env => common/hadoop.yaml} | 35 +++++--
.../hadoop27/docker-config => common/hadoop2.conf} | 5 -
.../hadoop32/docker-config => common/hadoop3.conf} | 5 -
hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 1 +
.../hadoop33/test.sh => ozone-ha/hadoop-test.sh} | 19 ++--
.../src/main/compose/ozone-ha/test-hadoop-2.7.sh | 1 +
.../src/main/compose/ozone-ha/test-hadoop-3.1.sh | 1 +
.../src/main/compose/ozone-ha/test-hadoop-3.2.sh | 1 +
.../src/main/compose/ozone-ha/test-hadoop-3.3.sh | 1 +
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 104 -------------------
.../src/main/compose/ozone-mr/hadoop27/test.sh | 42 --------
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 24 -----
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 99 -------------------
.../main/compose/ozone-mr/hadoop31/docker-config | 24 -----
.../src/main/compose/ozone-mr/hadoop31/test.sh | 48 ---------
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 110 ---------------------
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 ---------------------
.../main/compose/ozone-mr/hadoop33/docker-config | 24 -----
.../dist/src/main/compose/ozone-mr/test.sh | 34 -------
hadoop-ozone/dist/src/main/compose/ozone/.env | 1 +
.../hadoop32/test.sh => ozone/hadoop-test.sh} | 18 ++--
.../dist/src/main/compose/ozone/test-hadoop-2.7.sh | 1 +
.../dist/src/main/compose/ozone/test-hadoop-3.1.sh | 1 +
.../dist/src/main/compose/ozone/test-hadoop-3.2.sh | 1 +
.../dist/src/main/compose/ozone/test-hadoop-3.3.sh | 1 +
30 files changed, 97 insertions(+), 694 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/.env
b/hadoop-ozone/dist/src/main/compose/common/hadoop-2.7.sh
old mode 100644
new mode 100755
similarity index 79%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/.env
rename to hadoop-ozone/dist/src/main/compose/common/hadoop-2.7.sh
index 65e6887d7b..41f49758b0
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/.env
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-2.7.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,9 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
[email protected]@
-HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=3.3.1
[email protected]@
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
+#suite:MR
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export HADOOP_VERSION=2.7.3
+
+source ${COMPOSE_DIR}/hadoop-test.sh
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/.env
b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.1.sh
old mode 100644
new mode 100755
similarity index 79%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/.env
rename to hadoop-ozone/dist/src/main/compose/common/hadoop-3.1.sh
index e696737692..318523e288
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/.env
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.1.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,9 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
[email protected]@
-HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=3.2.2
[email protected]@
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
+#suite:MR
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export HADOOP_VERSION=3.1.2
+
+source ${COMPOSE_DIR}/hadoop-test.sh
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.2.sh
old mode 100644
new mode 100755
similarity index 79%
copy from hadoop-ozone/dist/src/main/compose/ozone-ha/.env
copy to hadoop-ozone/dist/src/main/compose/common/hadoop-3.2.sh
index 2de359fc5d..3eee1436d7
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.2.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,7 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
+#suite:MR
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export HADOOP_VERSION=3.2.2
+
+source ${COMPOSE_DIR}/hadoop-test.sh
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.3.sh
old mode 100644
new mode 100755
similarity index 79%
copy from hadoop-ozone/dist/src/main/compose/ozone-ha/.env
copy to hadoop-ozone/dist/src/main/compose/common/hadoop-3.3.sh
index 2de359fc5d..e49f95f59d
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-3.3.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,7 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
+#suite:MR
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export HADOOP_VERSION=3.3.1
+
+source ${COMPOSE_DIR}/hadoop-test.sh
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
b/hadoop-ozone/dist/src/main/compose/common/hadoop.conf
similarity index 80%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
rename to hadoop-ozone/dist/src/main/compose/common/hadoop.conf
index d5a92ee7a9..00a334ed8b 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop.conf
@@ -14,25 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=3
-OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
-
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-
-CORE-SITE.xml_fs.defaultFS=ofs://om/
+CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
+CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
MAPRED-SITE.XML_mapreduce.framework.name=yarn
MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
@@ -79,6 +62,3 @@
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false
-
-OZONE_CONF_DIR=/etc/hadoop
-OZONE_LOG_DIR=/var/log/hadoop
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/.env
b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
similarity index 51%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/.env
rename to hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
index c13fddf477..6f2eab1769 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/.env
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
@@ -14,10 +14,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
[email protected]@
-#TODO: swich to apache/hadoop. Older versions are not supported by
apache/hadoop, yet.
-# See: HADOOP-16092 for more details.
-HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=2.7.3
[email protected]@
-OZONE_RUNNER_IMAGE=apache/ozone-runner
+version: "3.4"
+services:
+ rm:
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ hostname: rm
+ volumes:
+ - ../..:/opt/ozone
+ - ../../libexec/transformation.py:/opt/transformation.py
+ ports:
+ - 8088:8088
+ env_file:
+ - docker-config
+ - ../common/hadoop.conf
+ - ../common/hadoop${HADOOP_MAJOR_VERSION}.conf
+ command: ["yarn", "resourcemanager"]
+ nm:
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ hostname: nm
+ volumes:
+ - ../..:/opt/ozone
+ - ../../libexec/transformation.py:/opt/transformation.py
+ env_file:
+ - docker-config
+ - ../common/hadoop.conf
+ - ../common/hadoop${HADOOP_MAJOR_VERSION}.conf
+ environment:
+ WAITFOR: rm:8088
+ command: ["yarn","nodemanager"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config
b/hadoop-ozone/dist/src/main/compose/common/hadoop2.conf
similarity index 82%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config
rename to hadoop-ozone/dist/src/main/compose/common/hadoop2.conf
index 3a6bdc5f26..47fa94ce16 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop2.conf
@@ -14,11 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
-CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar
-OZONE_CLASSPATH=
-
-no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config
b/hadoop-ozone/dist/src/main/compose/common/hadoop3.conf
similarity index 82%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config
rename to hadoop-ozone/dist/src/main/compose/common/hadoop3.conf
index d2fa4923da..fd15a72e2a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/common/hadoop3.conf
@@ -14,11 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
-CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar
-OZONE_CLASSPATH=
-
-no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
index 2de359fc5d..cb11be7f0b 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/.env
@@ -15,6 +15,7 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
+HADOOP_IMAGE=flokkr/hadoop
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/hadoop-test.sh
similarity index 64%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/test.sh
rename to hadoop-ozone/dist/src/main/compose/ozone-ha/hadoop-test.sh
index 6c949807b5..e9a4965cc8 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/hadoop-test.sh
@@ -15,27 +15,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#suite:MR
+export COMPOSE_FILE=docker-compose.yaml:../common/hadoop.yaml
+export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*}
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
+export SECURITY_ENABLED=false
+export OZONE_REPLICATION_FACTOR=3
+export SCM=scm1
+export OM_SERVICE_ID=omservice
# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../../testlib.sh"
+source "$COMPOSE_DIR/../testlib.sh"
start_docker_env
-execute_robot_test scm createmrenv.robot
+execute_robot_test ${SCM} createmrenv.robot
# reinitialize the directories to use
export OZONE_DIR=/opt/ozone
# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../../testlib.sh"
+source "$COMPOSE_DIR/../testlib.sh"
for scheme in o3fs ofs; do
- execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}"
ozonefs/hadoopo3fs.robot
- execute_robot_test rm -v "SCHEME:${scheme}" -N "mapreduce-${scheme}"
mapreduce.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot
done
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-2.7.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-2.7.sh
new file mode 120000
index 0000000000..59f3b43dd8
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-2.7.sh
@@ -0,0 +1 @@
+../common/hadoop-2.7.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.1.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.1.sh
new file mode 120000
index 0000000000..73ddffe9f3
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.1.sh
@@ -0,0 +1 @@
+../common/hadoop-3.1.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.2.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.2.sh
new file mode 120000
index 0000000000..d4ac16ad07
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.2.sh
@@ -0,0 +1 @@
+../common/hadoop-3.2.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.3.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.3.sh
new file mode 120000
index 0000000000..da90e3a998
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test-hadoop-3.3.sh
@@ -0,0 +1 @@
+../common/hadoop-3.3.sh
\ No newline at end of file
diff --git
a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml
deleted file mode 100644
index 90a80847ac..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml
+++ /dev/null
@@ -1,104 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - docker-config
- - ../common-config
- om:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: om
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9874:9874
- - 9862:9862
- environment:
- WAITFOR: scm:9876
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- env_file:
- - docker-config
- - ../common-config
- command: ["/opt/hadoop/bin/ozone","om"]
- s3g:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: s3g
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9878:9878
- env_file:
- - ./docker-config
- - ../common-config
- command: ["/opt/hadoop/bin/ozone","s3g"]
- scm:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: scm
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9876:9876
- - 9860:9860
- env_file:
- - docker-config
- - ../common-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-1}"
- command: ["/opt/hadoop/bin/ozone","scm"]
- rm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: rm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- ports:
- - 8088:8088
- env_file:
- - ./docker-config
- - ../common-config
- command: ["yarn", "resourcemanager"]
- nm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: nm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- env_file:
- - ./docker-config
- - ../common-config
- environment:
- WAITFOR: rm:8088
- command: ["yarn","nodemanager"]
-# Optional section: comment out this part to get DNS resolution for all the
containers.
-# dns:
-# image: andyshinn/dnsmasq:2.76
-# ports:
-# - 53:53/udp
-# - 53:53/tcp
-# volumes:
-# - "/var/run/docker.sock:/var/run/docker.sock"
-# command:
-# - "-k"
-# - "-d"
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh
deleted file mode 100755
index 05bea2a1d9..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#suite:MR
-
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../../testlib.sh"
-
-start_docker_env
-
-execute_robot_test scm createmrenv.robot
-
-# reinitialize the directories to use
-export OZONE_DIR=/opt/ozone
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../../testlib.sh"
-
-for scheme in o3fs ofs; do
- execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}"
ozonefs/hadoopo3fs.robot
- execute_robot_test rm -v "hadoop.version:${HADOOP_VERSION}" -v
"SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
-done
-
-stop_docker_env
-
-generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/.env
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/.env
deleted file mode 100644
index 77842d03b1..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/.env
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
[email protected]@
-#TODO: swich to apache/hadoop. Older versions are not supported by
apache/hadoop, yet.
-# See: HADOOP-16092 for more details.
-HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=3.1.2
[email protected]@
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
diff --git
a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml
deleted file mode 100644
index e0d1d6c24a..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - docker-config
- - ../common-config
- environment:
- OZONE_OPTS:
- om:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: om
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9874:9874
- - 9862:9862
- environment:
- WAITFOR: scm:9876
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- OZONE_OPTS:
- env_file:
- - docker-config
- - ../common-config
- command: ["/opt/hadoop/bin/ozone","om"]
- s3g:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: s3g
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9878:9878
- env_file:
- - ./docker-config
- - ../common-config
- environment:
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","s3g"]
- scm:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: scm
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9876:9876
- - 9860:9860
- env_file:
- - docker-config
- - ../common-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-1}"
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","scm"]
- rm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: rm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- ports:
- - 8088:8088
- env_file:
- - ./docker-config
- - ../common-config
- command: ["yarn", "resourcemanager"]
- nm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: nm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- env_file:
- - ./docker-config
- - ../common-config
- environment:
- WAITFOR: rm:8088
- command: ["yarn","nodemanager"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config
deleted file mode 100644
index d2fa4923da..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
-CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
-MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
-
-HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar
-OZONE_CLASSPATH=
-
-no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
deleted file mode 100755
index 162aa0da65..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#suite:MR
-
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/.env"
-source "$COMPOSE_DIR/../../testlib.sh"
-
-start_docker_env
-
-#rm is the container name (resource manager) and not the rm command
-execute_command_in_container rm bash -c "if test -e /sbin/apk; then sudo -E
apk add --update py-pip; sudo -E pip install robotframework; fi"
-
-execute_robot_test scm createmrenv.robot
-
-# reinitialize the directories to use
-export OZONE_DIR=/opt/ozone
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../../testlib.sh"
-
-for scheme in o3fs ofs; do
- execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}"
ozonefs/hadoopo3fs.robot
- execute_robot_test rm -v "hadoop.version:${HADOOP_VERSION}" -v
"SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
-done
-
-stop_docker_env
-
-generate_report
-
-cleanup_docker_images "${HADOOP_IMAGE}:${HADOOP_VERSION}"
diff --git
a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml
deleted file mode 100644
index 9cd616f4e9..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml
+++ /dev/null
@@ -1,110 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - docker-config
- - ../common-config
- om:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: om
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9874:9874
- - 9862:9862
- environment:
- WAITFOR: scm:9876
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- OZONE_OPTS:
- env_file:
- - docker-config
- - ../common-config
- command: ["/opt/hadoop/bin/ozone","om"]
- s3g:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: s3g
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9878:9878
- env_file:
- - ./docker-config
- - ../common-config
- environment:
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","s3g"]
- scm:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: scm
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9876:9876
- - 9860:9860
- env_file:
- - docker-config
- - ../common-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-1}"
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","scm"]
- rm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: rm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- ports:
- - 8088:8088
- env_file:
- - ./docker-config
- - ../common-config
- command: ["yarn", "resourcemanager"]
- nm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: nm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- env_file:
- - ./docker-config
- - ../common-config
- environment:
- WAITFOR: rm:8088
- command: ["yarn","nodemanager"]
-# Optional section: comment out this part to get DNS resolution for all the
containers.
-# Add 127.0.0.1 (or the ip of your docker machine) to the resolv.conf to
get local DNS resolution
-# For all the containers (including resource managers and Node manager UI)
-# dns:
-# image: andyshinn/dnsmasq:2.76
-# ports:
-# - 53:53/udp
-# - 53:53/tcp
-# volumes:
-# - "/var/run/docker.sock:/var/run/docker.sock"
-# command:
-# - "-k"
-# - "-d"
diff --git
a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-compose.yaml
deleted file mode 100644
index 9cd616f4e9..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-compose.yaml
+++ /dev/null
@@ -1,110 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - docker-config
- - ../common-config
- om:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: om
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9874:9874
- - 9862:9862
- environment:
- WAITFOR: scm:9876
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- OZONE_OPTS:
- env_file:
- - docker-config
- - ../common-config
- command: ["/opt/hadoop/bin/ozone","om"]
- s3g:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: s3g
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9878:9878
- env_file:
- - ./docker-config
- - ../common-config
- environment:
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","s3g"]
- scm:
- image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
- hostname: scm
- volumes:
- - ../../..:/opt/hadoop
- ports:
- - 9876:9876
- - 9860:9860
- env_file:
- - docker-config
- - ../common-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-1}"
- OZONE_OPTS:
- command: ["/opt/hadoop/bin/ozone","scm"]
- rm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: rm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- ports:
- - 8088:8088
- env_file:
- - ./docker-config
- - ../common-config
- command: ["yarn", "resourcemanager"]
- nm:
- image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
- hostname: nm
- volumes:
- - ../../..:/opt/ozone
- - ../../../libexec/transformation.py:/opt/transformation.py
- env_file:
- - ./docker-config
- - ../common-config
- environment:
- WAITFOR: rm:8088
- command: ["yarn","nodemanager"]
-# Optional section: comment out this part to get DNS resolution for all the
containers.
-# Add 127.0.0.1 (or the ip of your docker machine) to the resolv.conf to
get local DNS resolution
-# For all the containers (including resource managers and Node manager UI)
-# dns:
-# image: andyshinn/dnsmasq:2.76
-# ports:
-# - 53:53/udp
-# - 53:53/tcp
-# volumes:
-# - "/var/run/docker.sock:/var/run/docker.sock"
-# command:
-# - "-k"
-# - "-d"
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-config
deleted file mode 100644
index d2fa4923da..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-config
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
-CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
-MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
-
-HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar
-OZONE_CLASSPATH=
-
-no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
deleted file mode 100644
index 0aef327504..0000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#suite:MR
-
-SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
-ALL_RESULT_DIR="$SCRIPT_DIR/result"
-mkdir -p "$ALL_RESULT_DIR"
-rm "$ALL_RESULT_DIR"/* || true
-source "$SCRIPT_DIR/../testlib.sh"
-
-tests=$(find_tests)
-cd "$SCRIPT_DIR"
-
-RESULT=0
-run_test_scripts ${tests} || RESULT=$?
-
-generate_report "ozone-mr" "${ALL_RESULT_DIR}"
-
-exit ${RESULT}
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env
b/hadoop-ozone/dist/src/main/compose/ozone/.env
index 2de359fc5d..cb11be7f0b 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone/.env
@@ -15,6 +15,7 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
+HADOOP_IMAGE=flokkr/hadoop
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_RUNNER_IMAGE=apache/ozone-runner
OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/hadoop-test.sh
similarity index 65%
rename from hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh
rename to hadoop-ozone/dist/src/main/compose/ozone/hadoop-test.sh
index 6c949807b5..e2b34d137f 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/hadoop-test.sh
@@ -15,27 +15,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#suite:MR
+export COMPOSE_FILE=docker-compose.yaml:../common/hadoop.yaml
+export HADOOP_MAJOR_VERSION=${HADOOP_VERSION%%.*}
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
+export SECURITY_ENABLED=false
+export OZONE_REPLICATION_FACTOR=3
+export SCM=scm
# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../../testlib.sh"
+source "$COMPOSE_DIR/../testlib.sh"
start_docker_env
-execute_robot_test scm createmrenv.robot
+execute_robot_test ${SCM} createmrenv.robot
# reinitialize the directories to use
export OZONE_DIR=/opt/ozone
# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../../testlib.sh"
+source "$COMPOSE_DIR/../testlib.sh"
for scheme in o3fs ofs; do
- execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}"
ozonefs/hadoopo3fs.robot
- execute_robot_test rm -v "SCHEME:${scheme}" -N "mapreduce-${scheme}"
mapreduce.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N
"hadoop-${HADOOP_VERSION}-mapreduce-${scheme}" mapreduce.robot
done
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-2.7.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-2.7.sh
new file mode 120000
index 0000000000..59f3b43dd8
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-2.7.sh
@@ -0,0 +1 @@
+../common/hadoop-2.7.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.1.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.1.sh
new file mode 120000
index 0000000000..73ddffe9f3
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.1.sh
@@ -0,0 +1 @@
+../common/hadoop-3.1.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.2.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.2.sh
new file mode 120000
index 0000000000..d4ac16ad07
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.2.sh
@@ -0,0 +1 @@
+../common/hadoop-3.2.sh
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.3.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.3.sh
new file mode 120000
index 0000000000..da90e3a998
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-hadoop-3.3.sh
@@ -0,0 +1 @@
+../common/hadoop-3.3.sh
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
