This is an automated email from the ASF dual-hosted git repository.
hemant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 18b28d20e5 HDDS-11312. [hsync] Added upgrade tests (#7110)
18b28d20e5 is described below
commit 18b28d20e503cd333ac610019e0adb1310d36635
Author: Hemant Kumar <[email protected]>
AuthorDate: Wed Sep 4 10:55:38 2024 -0700
HDDS-11312. [hsync] Added upgrade tests (#7110)
---
.../main/compose/upgrade/compose/ha/docker-config | 1 +
.../compose/upgrade/compose/non-ha/docker-config | 1 +
.../compose/upgrade/compose/om-ha/docker-config | 1 +
.../callbacks/1.5.0/callback.sh | 31 ++++++++++
.../debug/ozone-debug-lease-recovery.robot | 22 ++++---
.../dist/src/main/smoketest/freon/hsync.robot | 51 ++++++++++++++++
.../main/smoketest/hsync/upgrade-hsync-check.robot | 68 ++++++++++++++++++++++
.../dist/src/main/smoketest/ozone-lib/freon.robot | 6 ++
8 files changed, 173 insertions(+), 8 deletions(-)
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config
b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config
index 909b72852a..a1b6da80c4 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config
@@ -40,6 +40,7 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+OZONE-SITE.XML_ozone.fs.hsync.enabled=true
# If SCM sends container close commands as part of upgrade finalization while
# datanodes are doing a leader election, all 3 replicas may end up in the
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config
b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config
index 95ce6c0c9c..88126ddf2c 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config
@@ -37,6 +37,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+OZONE-SITE.XML_ozone.fs.hsync.enabled=true
OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config
b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config
index 1b805c9896..77fa2b40ee 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config
@@ -32,6 +32,7 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+OZONE-SITE.XML_ozone.fs.hsync.enabled=true
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
diff --git
a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh
new file mode 100644
index 0000000000..ec64d5dcd5
--- /dev/null
+++
b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.5.0/callback.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source "$TEST_DIR"/testlib.sh
+
+with_this_version_pre_finalized() {
+ # New layout features were added in this version, so OM and SCM should be
pre-finalized.
+ execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include
pre-finalized upgrade/check-finalization.robot
+ # Test that HSync is disabled when pre-finalized.
+ execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" --include
pre-finalized-hsync-tests hsync/upgrade-hsync-check.robot
+}
+
+with_this_version_finalized() {
+ execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include
finalized upgrade/check-finalization.robot
+ execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync"
debug/ozone-debug-lease-recovery.robot
+ execute_robot_test "$SCM" -N "${OUTPUT_NAME}-freon-hsync" freon/hsync.robot
+}
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot
index f867ee99f6..691769dbd7 100644
---
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot
+++
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-lease-recovery.robot
@@ -17,11 +17,13 @@
Documentation Test lease recovery of ozone filesystem
Library OperatingSystem
Resource ../lib/os.robot
+Resource ../lib/fs.robot
Resource ozone-debug.robot
Test Timeout 5 minute
Suite Setup Create volume bucket and put key
*** Variables ***
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
${VOLUME} lease-recovery-volume
${BUCKET} lease-recovery-bucket
${TESTFILE} testfile22
@@ -35,13 +37,17 @@ Create volume bucket and put key
*** Test Cases ***
Test ozone debug recover for o3fs
- ${result} = Execute Lease recovery cli
o3fs://${BUCKET}.${VOLUME}.om/${TESTFILE}
- Should Contain ${result} Lease recovery SUCCEEDED
- ${result} = Execute Lease recovery cli
o3fs://${BUCKET}.${VOLUME}.om/randomfile
- Should Contain ${result} not found
+ ${o3fs_path} = Format FS URL o3fs ${VOLUME} ${BUCKET}
${TESTFILE}
+ ${result} = Execute Lease recovery cli ${o3fs_path}
+ Should Contain ${result} Lease recovery SUCCEEDED
+ ${o3fs_path} = Format FS URL o3fs ${VOLUME} ${BUCKET}
randomfile
+ ${result} = Execute Lease recovery cli ${o3fs_path}
+ Should Contain ${result} not found
Test ozone debug recover for ofs
- ${result} = Execute Lease recovery cli
ofs://om/${VOLUME}/${BUCKET}/${TESTFILE}
- Should Contain ${result} Lease recovery SUCCEEDED
- ${result} = Execute Lease recovery cli
ofs://om/${VOLUME}/${BUCKET}/randomfile
- Should Contain ${result} not found
+ ${ofs_path} = Format FS URL ofs ${VOLUME} ${BUCKET}
${TESTFILE}
+ ${result} = Execute Lease recovery cli ${ofs_path}
+ Should Contain ${result} Lease recovery SUCCEEDED
+ ${ofs_path} = Format FS URL ofs ${VOLUME} ${BUCKET}
randomfile
+ ${result} = Execute Lease recovery cli ${ofs_path}
+ Should Contain ${result} not found
diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot
b/hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot
new file mode 100644
index 0000000000..c846212442
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/freon/hsync.robot
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test HSync via freon CLI.
+Library OperatingSystem
+Library String
+Library BuiltIn
+Resource ../ozone-lib/freon.robot
+Resource ../lib/fs.robot
+Test Timeout 10 minutes
+Suite Setup Create volume and bucket
+
+*** Variables ***
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
+${VOLUME} hsync-volume
+${BUCKET} hsync-bucket
+
+*** Keywords ***
+Create volume and bucket
+ Execute ozone sh volume create /${volume}
+ Execute ozone sh bucket create /${volume}/${bucket}
+
+*** Test Cases ***
+Generate key for o3fs by HSYNC
+ ${path} = Format FS URL o3fs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HSYNC path=${path}
+
+Generate key for o3fs by HFLUSH
+ ${path} = Format FS URL o3fs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HFLUSH path=${path}
+
+Generate key for ofs by HSYNC
+ ${path} = Format FS URL ofs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HSYNC path=${path}
+
+Generate key for ofs by HFLUSH
+ ${path} = Format FS URL ofs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HFLUSH path=${path}
diff --git
a/hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot
b/hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot
new file mode 100644
index 0000000000..1250ad1344
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/hsync/upgrade-hsync-check.robot
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test HSync during upgrade
+Library OperatingSystem
+Library String
+Library BuiltIn
+Resource ../commonlib.robot
+Resource ../lib/fs.robot
+Resource ../debug/ozone-debug.robot
+Default Tags pre-finalized-hsync-tests
+Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit
test user testuser testuser.keytab
+
+*** Variables ***
+${OM_SERVICE_ID} %{OM_SERVICE_ID}
+${VOLUME} upgrade-hsync-volume
+${BUCKET} upgrade-hsync-bucket
+${KEY} upgrade-hsync-key
+
+*** Keywords ***
+Create volume bucket and put key
+ Execute ozone sh volume create /${volume}
+ Execute ozone sh bucket create /${volume}/${bucket}
+ Execute ozone sh key put /${volume}/${bucket}/${key} /etc/hosts
+
+Freon DFSG
+ [arguments] ${prefix}=dfsg ${n}=1000 ${path}={EMPTY}
${sync}=HSYNC ${buffer}=1024 ${copy-buffer}=1024 ${size}=10240
+ ${result} = Execute and checkrc ozone freon dfsg -n ${n} --sync
${sync} -s ${size} --path ${path} --buffer ${buffer} --copy-buffer
${copy-buffer} -p ${prefix} 255
+ Should contain ${result}
NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION
+
+*** Test Cases ***
+Test HSync lease recover prior to finalization
+ Create volume bucket and put key
+ ${o3fs_path} = Format FS URL o3fs ${VOLUME} ${BUCKET}
${KEY}
+ ${result} = Execute and checkrc ozone debug recover
--path=${o3fs_path} 255
+ Should contain ${result} It belongs to the layout
feature HBASE_SUPPORT, whose layout version is 7
+ ${ofs_path} = Format FS URL ofs ${VOLUME} ${BUCKET}
${KEY}
+ ${result} = Execute and checkrc ozone debug recover
--path=${ofs_path} 255
+ Should contain ${result} It belongs to the layout
feature HBASE_SUPPORT, whose layout version is 7
+
+Generate key for o3fs by HSYNC prior to finalization
+ ${path} = Format FS URL o3fs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HSYNC path=${path}
+
+Generate key for o3fs by HFLUSH prior to finalization
+ ${path} = Format FS URL o3fs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HFLUSH path=${path}
+
+Generate key for ofs by HSYNC prior to finalization
+ ${path} = Format FS URL ofs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HSYNC path=${path}
+
+Generate key for ofs by HFLUSH prior to finalization
+ ${path} = Format FS URL ofs ${VOLUME} ${BUCKET}
+ Freon DFSG sync=HFLUSH path=${path}
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot
b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot
index 8d10cc81e9..b813c9ed41 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/freon.robot
@@ -67,3 +67,9 @@ Freon OMBR
[arguments] ${prefix}=ombg ${n}=1 ${threads}=1 ${args}=${EMPTY}
${result} = Execute ozone freon ombr ${OM_HA_PARAM} -t
${threads} -n${n} -p ${prefix} ${args}
Should contain ${result} Successful executions: ${n}
+
+Freon DFSG
+ [arguments] ${prefix}=dfsg ${n}=1000 ${path}={EMPTY}
${threads}=1 ${sync}=HSYNC ${buffer}=1024 ${copy-buffer}=1024
${size}=10240 ${args}=${EMPTY}
+ ${result} = Execute ozone freon dfsg -n ${n} --sync ${sync} -s
${size} --path ${path} --buffer ${buffer} --copy-buffer ${copy-buffer} -p
${prefix} -t ${threads} ${args}
+ Should contain ${result} Successful executions: ${n}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
