This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-10656-atomic-key-overwrite
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to
refs/heads/HDDS-10656-atomic-key-overwrite by this push:
new 8c04d72234 HDDS-10947. Add robot test for rewrite of multipart key
(#6757)
8c04d72234 is described below
commit 8c04d72234ca737bd4029eb6c72b7cdc7b4a9b75
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Sun Jun 2 08:39:41 2024 +0200
HDDS-10947. Add robot test for rewrite of multipart key (#6757)
---
.../dist/src/main/compose/common/ec-test.sh | 2 +
.../dist/src/main/smoketest/ec/rewrite.robot | 45 +++++++++++++++++++++
hadoop-ozone/dist/src/main/smoketest/lib/os.robot | 23 +++++++++--
.../dist/src/main/smoketest/ozone-lib/shell.robot | 7 ++++
.../src/main/smoketest/ozone-lib/shell_tests.robot | 2 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 47 +++++++++-------------
.../dist/src/main/smoketest/s3/commonawslib.robot | 9 +----
7 files changed, 94 insertions(+), 41 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
index 65a659563e..04df2b2787 100755
--- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
@@ -20,6 +20,8 @@ start_docker_env 5
## Exclude virtual-host tests. This is tested separately as it requires
additional config.
execute_robot_test scm -v BUCKET:erasure --exclude virtual-host s3
+execute_robot_test scm ec/rewrite.robot
+
prefix=${RANDOM}
execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot
docker-compose up -d --no-recreate --scale datanode=4
diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/rewrite.robot
b/hadoop-ozone/dist/src/main/smoketest/ec/rewrite.robot
new file mode 100644
index 0000000000..5b5df209ec
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/ec/rewrite.robot
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test EC shell commands
+Library OperatingSystem
+Resource ../commonlib.robot
+Resource ../lib/os.robot
+Resource ../ozone-lib/shell.robot
+Resource ../s3/commonawslib.robot
+Resource lib.resource
+Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit
test user testuser testuser.keytab
+
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g:9878
+
+
+*** Test Cases ***
+
+Rewrite Multipart Key
+ [setup] Setup v4 headers
+ ${bucket} = Create bucket with layout /s3v OBJECT_STORE
+ ${key} = Set Variable multipart.key
+ ${file} = Create Random File MB 12
+ Execute AWSS3Cli cp ${file} s3://${bucket}/${key}
+ Key Should Match Local File /s3v/${bucket}/${key} ${file}
+ Verify Key Replica Replication Config /s3v/${bucket}/${key} RATIS
THREE
+
+ Execute ozone sh key rewrite -t EC -r rs-3-2-1024k /s3v/${bucket}/${key}
+
+ Key Should Match Local File /s3v/${bucket}/${key} ${file}
+ Verify Key EC Replication Config /s3v/${bucket}/${key} RS 3 2
1048576
diff --git a/hadoop-ozone/dist/src/main/smoketest/lib/os.robot
b/hadoop-ozone/dist/src/main/smoketest/lib/os.robot
index cc20d6ee28..98f836ff8b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/lib/os.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/lib/os.robot
@@ -40,13 +40,28 @@ Compare files
${checksumafter} = Execute md5sum ${file2} |
awk '{print $1}'
Should Be Equal ${checksumbefore}
${checksumafter}
+Create Random File MB
+ [arguments] ${size_in_megabytes} ${path}=${EMPTY}
+ ${path} = Create Random File ${size_in_megabytes} 1048576
${path}
+ [return] ${path}
+
+Create Random File KB
+ [arguments] ${size_in_kilobytes} ${path}=${EMPTY}
+ ${path} = Create Random File ${size_in_kilobytes} 1024
${path}
+ [return] ${path}
+
Create Random File
- ${postfix} = Generate Random String 5 [NUMBERS]
+ [arguments] ${block_count} ${block_size} ${path}=${EMPTY}
+ ${path} = Run Keyword If '${path}' == '${EMPTY}' Get Random
Filename
+ ... ELSE Set Variable ${path}
+ Execute dd if=/dev/urandom of=${path} bs=${block_size}
count=${block_count} status=none
+ [return] ${path}
+
+Get Random Filename
+ ${postfix} = Generate Random String 10 [LOWER]
${tmpfile} = Set Variable /tmp/tempfile-${postfix}
File Should Not Exist ${tmpfile}
- ${content} = Set Variable "Random string"
- Create File ${tmpfile} ${content}
- [Return] ${tmpfile}
+ [return] ${tmpfile}
List All Processes
${output} = Execute ps aux
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
index b255d516da..ffefda8333 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot
@@ -56,6 +56,13 @@ Find Jars Dir
${dir} = Execute ozone envvars | grep 'HDDS_LIB_JARS_DIR' | cut -f2
-d= | sed -e "s/'//g" -e 's/"//g'
Set Environment Variable HDDS_LIB_JARS_DIR ${dir}
+Create bucket with layout
+ [Arguments] ${volume} ${layout}
+ ${postfix} = Generate Random String 10 [LOWER]
+ ${bucket} = Set Variable bucket-${postfix}
+ ${result} = Execute ozone sh bucket create --layout
${layout} ${volume}/${bucket}
+ [Return] ${bucket}
+
Create Key
[arguments] ${key} ${file} ${args}=${EMPTY}
${output} = Execute ozone sh key put ${args} ${key} ${file}
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot
b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot
index 56fbcf8b61..22805efcb1 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot
@@ -48,7 +48,7 @@ Key Should Match Local File
Key Should Match Local File o3://${OM_SERVICE_ID}/vol1/bucket/passwd
/etc/passwd
Compare Key With Local File with Different File
- ${random_file} = Create Random File
+ ${random_file} = Create Random File KB 42
${matches} = Compare Key With Local File
o3://${OM_SERVICE_ID}/vol1/bucket/passwd ${random_file}
Should Be Equal ${matches} ${FALSE}
[Teardown] Remove File ${random_file}
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
index 96feec2f81..55bc880cd6 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
@@ -22,6 +22,7 @@ Resource ../commonlib.robot
Resource commonawslib.robot
Test Timeout 5 minutes
Suite Setup Setup Multipart Tests
+Suite Teardown Teardown Multipart Tests
Test Setup Generate random prefix
*** Keywords ***
@@ -29,18 +30,19 @@ Setup Multipart Tests
Setup s3 tests
# 5MB + a bit
- Create Random File KB /tmp/part1 5121
+ Create Random File KB 5121 /tmp/part1
# 1MB - a bit
- Create Random File KB /tmp/part2 1023
+ Create Random File KB 1023 /tmp/part2
-Create Random file
- [arguments] ${size_in_megabytes}
- Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576
count=${size_in_megabytes} status=none
+ Create Random File MB 10 /tmp/10mb
+ Create Random File MB 22 /tmp/22mb
+ Create Random File KB 10 /tmp/10kb
+
+
+Teardown Multipart Tests
+ Remove Files /tmp/part1 /tmp/part2 /tmp/10mb /tmp/22mb /tmp/10kb
-Create Random File KB
- [arguments] ${file} ${size_in_kilobytes}
- Execute dd if=/dev/urandom of=${file} bs=1024
count=${size_in_kilobytes} status=none
Wait Til Date Past
[arguments] ${date}
@@ -77,11 +79,9 @@ Test Multipart Upload
# upload we get error entity too small. So, considering further complete
# multipart upload, uploading each part as 5MB file, exception is for last part
- Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id
${nextUploadID}
Should contain ${result} ETag
# override part
- Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id
${nextUploadID}
Should contain ${result} ETag
@@ -94,7 +94,6 @@ Test Multipart Upload Complete
Should contain ${result} UploadId
#upload parts
- Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket
${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1
--upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
@@ -152,13 +151,11 @@ Test Multipart Upload Complete Entity too small
Should contain ${result} UploadId
#upload parts
- Execute echo "Part1" > /tmp/part1
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/part1 --upload-id
${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/10kb --upload-id
${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
- Execute echo "Part2" > /tmp/part2
- ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/part2 --upload-id
${uploadID}
+ ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/10kb --upload-id
${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
@@ -180,7 +177,6 @@ Test Multipart Upload Complete Invalid part errors and
complete mpu with few par
${result} = Execute AWSS3APICli and checkrc
complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/multipartKey3 --multipart-upload
'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]' 255
Should contain ${result} InvalidPart
#upload parts
- Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey3 --part-number 1 --body /tmp/part1 --upload-id
${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
@@ -245,7 +241,6 @@ Test list parts
Should contain ${result} UploadId
#upload parts
- Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET}
--key ${PREFIX}/multipartKey5 --part-number 1 --body /tmp/part1 --upload-id
${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should contain ${result} ETag
@@ -279,14 +274,12 @@ Test list parts
${result} = Execute AWSS3APICli and checkrc
abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5
--upload-id ${uploadID} 0
Test Multipart Upload with the simplified aws s3 cp API
- Create Random file 22
- Execute AWSS3Cli cp /tmp/part1
s3://${BUCKET}/mpyawscli
- Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli
/tmp/part1.result
+ Execute AWSS3Cli cp /tmp/22mb
s3://${BUCKET}/mpyawscli
+ Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli
/tmp/22mb.result
Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli
- Compare files /tmp/part1
/tmp/part1.result
+ Compare files /tmp/22mb
/tmp/22mb.result
Test Multipart Upload Put With Copy
- Run Keyword Create Random file 5
${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key ${PREFIX}/copytest/source --body /tmp/part1
@@ -308,8 +301,7 @@ Test Multipart Upload Put With Copy
Compare files /tmp/part1
/tmp/part-result
Test Multipart Upload Put With Copy and range
- Run Keyword Create Random file 10
- ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/source --body /tmp/part1
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/source --body /tmp/10mb
${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/copyrange/destination
@@ -332,15 +324,14 @@ Test Multipart Upload Put With Copy and range
Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/copyrange/destination --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
Execute AWSS3APICli get-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/destination /tmp/part-result
- Compare files /tmp/part1
/tmp/part-result
+ Compare files /tmp/10mb
/tmp/part-result
Test Multipart Upload Put With Copy and range with IfModifiedSince
- Run Keyword Create Random file 10
${curDate} = Get Current Date
${beforeCreate} = Subtract Time From Date ${curDate} 1 day
${tomorrow} = Add Time To Date ${curDate} 1 day
- ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/source --body /tmp/part1
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/source --body /tmp/10mb
${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/copyrange/destination
@@ -385,7 +376,7 @@ Test Multipart Upload Put With Copy and range with
IfModifiedSince
Execute AWSS3APICli complete-multipart-upload
--upload-id ${uploadID} --bucket ${BUCKET} --key
${PREFIX}/copyrange/destination --multipart-upload
'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
Execute AWSS3APICli get-object --bucket ${BUCKET}
--key ${PREFIX}/copyrange/destination /tmp/part-result
- Compare files /tmp/part1
/tmp/part-result
+ Compare files /tmp/10mb
/tmp/part-result
Test Multipart Upload list
${result} = Execute AWSS3APICli create-multipart-upload
--bucket ${BUCKET} --key ${PREFIX}/listtest/key1
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index b20537014d..45dee9270b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -129,13 +129,6 @@ Create bucket with name
Should contain ${result} Location
Should contain ${result}
${bucket}
-Create bucket with layout
- [Arguments] ${layout}
- ${postfix} = Generate Ozone String
- ${bucket} = Set Variable bucket-${postfix}
- ${result} = Execute ozone sh bucket create --layout
${layout} s3v/${bucket}
- [Return] ${bucket}
-
Setup s3 tests
Return From Keyword if ${OZONE_S3_TESTS_SET_UP}
Run Keyword Generate random prefix
@@ -156,7 +149,7 @@ Setup links for S3 tests
Create generated bucket
[Arguments] ${layout}=OBJECT_STORE
- ${BUCKET} = Create bucket with layout ${layout}
+ ${BUCKET} = Create bucket with layout s3v ${layout}
Set Global Variable ${BUCKET}
Create encrypted bucket
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
