This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 85c9c97fc5 HDDS-10570. S3A: `fs -touch` creates directory instead of
empty file in FSO bucket (#6452)
85c9c97fc5 is described below
commit 85c9c97fc5d11018233dea7d82726b86ad0e878e
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Mar 29 09:29:56 2024 +0100
HDDS-10570. S3A: `fs -touch` creates directory instead of empty file in FSO
bucket (#6452)
---
.../dist/src/main/compose/ozone-ha/test.sh | 8 +++++---
.../dist/src/main/smoketest/s3/commonawslib.robot | 24 ++++++++++------------
.../dist/src/main/smoketest/s3/objecthead.robot | 17 +++++++--------
.../dist/src/main/smoketest/s3/objectputget.robot | 24 +++-------------------
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 4 +++-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 8 +++++---
6 files changed, 36 insertions(+), 49 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
index 1361a4c0c3..976e490d32 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
@@ -38,9 +38,11 @@ execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link
-N ozonefs-ofs-link
## Exclude virtual-host tests. This is tested separately as it requires
additional config.
exclude="--exclude virtual-host"
for bucket in generated; do
- execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
- # some tests are independent of the bucket type, only need to be run once
- exclude="--exclude virtual-host --exclude no-bucket-type"
+ for layout in OBJECT_STORE LEGACY FILE_SYSTEM_OPTIMIZED; do
+ execute_robot_test ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout}
-N s3-${layout}-${bucket} ${exclude} s3
+ # some tests are independent of the bucket type, only need to be run once
+ exclude="--exclude virtual-host --exclude no-bucket-type"
+ done
done
execute_robot_test ${SCM} freon
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index 840fb963d8..b20537014d 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -23,6 +23,7 @@ ${ENDPOINT_URL} http://s3g:9878
${OZONE_S3_HEADER_VERSION} v4
${OZONE_S3_SET_CREDENTIALS} true
${BUCKET} generated
+${BUCKET_LAYOUT} OBJECT_STORE
${KEY_NAME} key1
${OZONE_S3_TESTS_SET_UP} ${FALSE}
${OZONE_AWS_ACCESS_KEY_ID} ${EMPTY}
@@ -127,16 +128,12 @@ Create bucket with name
${result} = Execute AWSS3APICli create-bucket --bucket ${bucket}
Should contain ${result} Location
Should contain ${result}
${bucket}
-Create legacy bucket
- ${postfix} = Generate Ozone String
- ${legacy_bucket} = Set Variable legacy-bucket-${postfix}
- ${result} = Execute and checkrc ozone sh bucket create -l
LEGACY s3v/${legacy_bucket} 0
- [Return] ${legacy_bucket}
-Create obs bucket
+Create bucket with layout
+ [Arguments] ${layout}
${postfix} = Generate Ozone String
- ${bucket} = Set Variable obs-bucket-${postfix}
- ${result} = Execute and checkrc ozone sh bucket create -l
OBJECT_STORE s3v/${bucket} 0
+ ${bucket} = Set Variable bucket-${postfix}
+ ${result} = Execute ozone sh bucket create --layout
${layout} s3v/${bucket}
[Return] ${bucket}
Setup s3 tests
@@ -144,7 +141,7 @@ Setup s3 tests
Run Keyword Generate random prefix
Run Keyword Install aws cli
Run Keyword if '${OZONE_S3_SET_CREDENTIALS}' == 'true' Setup v4
headers
- Run Keyword if '${BUCKET}' == 'generated' Create generated
bucket
+ Run Keyword if '${BUCKET}' == 'generated' Create generated
bucket ${BUCKET_LAYOUT}
Run Keyword if '${BUCKET}' == 'link' Setup links for S3
tests
Run Keyword if '${BUCKET}' == 'encrypted' Create encrypted
bucket
Run Keyword if '${BUCKET}' == 'erasure' Create EC bucket
@@ -154,18 +151,19 @@ Setup links for S3 tests
${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/link
Return From Keyword If ${exists}
Execute ozone sh volume create o3://${OM_SERVICE_ID}/legacy
- Execute ozone sh bucket create
o3://${OM_SERVICE_ID}/legacy/source-bucket
+ Execute ozone sh bucket create --layout ${BUCKET_LAYOUT}
o3://${OM_SERVICE_ID}/legacy/source-bucket
Create link link
Create generated bucket
- ${BUCKET} = Create bucket
+ [Arguments] ${layout}=OBJECT_STORE
+ ${BUCKET} = Create bucket with layout ${layout}
Set Global Variable ${BUCKET}
Create encrypted bucket
Return From Keyword if '${SECURITY_ENABLED}' == 'false'
${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/encrypted
Return From Keyword If ${exists}
- Execute ozone sh bucket create -k ${KEY_NAME}
o3://${OM_SERVICE_ID}/s3v/encrypted
+ Execute ozone sh bucket create -k ${KEY_NAME} --layout
${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/s3v/encrypted
Create link
[arguments] ${bucket}
@@ -175,7 +173,7 @@ Create link
Create EC bucket
${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/erasure
Return From Keyword If ${exists}
- Execute ozone sh bucket create --replication rs-3-2-1024k
--type EC o3://${OM_SERVICE_ID}/s3v/erasure
+ Execute ozone sh bucket create --replication rs-3-2-1024k
--type EC --layout ${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/s3v/erasure
Generate random prefix
${random} = Generate Ozone String
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot
index be0582edd1..66f3461b01 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot
@@ -40,22 +40,23 @@ Head object in non existing bucket
${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET}-non-existent --key ${PREFIX}/headobject/key=value/f1 255
Should contain ${result} 404
Should contain ${result} Not Found
+
Head object where path is a directory
- ${legacy-bucket} = Create legacy bucket
- ${result} = Execute AWSS3APICli and checkrc put-object --bucket
${legacy-bucket} --key ${PREFIX}/headobject/keyvalue/f1 --body /tmp/testfile 0
- ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${legacy-bucket} --key ${PREFIX}/headobject/keyvalue/ 255
+ Pass Execution If '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED' does
not apply to FSO buckets
+ ${result} = Execute AWSS3APICli and checkrc put-object --bucket
${BUCKET} --key ${PREFIX}/headobject/keyvalue/f1 --body /tmp/testfile 0
+ ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/headobject/keyvalue/ 255
Should contain ${result} 404
Should contain ${result} Not Found
Head directory objects
- ${obs-bucket} = Create obs bucket
- ${result} = Execute AWSS3APICli and checkrc put-object --bucket
${obs-bucket} --key ${PREFIX}/mydir/ --body /tmp/testfile 0
- ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${obs-bucket} --key ${PREFIX}/mydir 255
+ Pass Execution If '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED' does
not apply to FSO buckets
+ ${result} = Execute AWSS3APICli and checkrc put-object --bucket
${BUCKET} --key ${PREFIX}/mydir/ --body /tmp/testfile 0
+ ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/mydir 255
Should contain ${result} 404
Should contain ${result} Not Found
- ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${obs-bucket} --key ${PREFIX}/mydir/ 0
+ ${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/mydir/ 0
Head non existing key
${result} = Execute AWSS3APICli and checkrc head-object
--bucket ${BUCKET} --key ${PREFIX}/non-existent 255
Should contain ${result} 404
- Should contain ${result} Not Found
\ No newline at end of file
+ Should contain ${result} Not Found
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index 1b44360d6b..05348fbcba 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -44,6 +44,8 @@ Put object to s3
Get object from s3
${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/f1 /tmp/testfile.result
Compare files /tmp/testfile /tmp/testfile.result
+ ${result} = Execute AWSS3ApiCli get-object --bucket
${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte /tmp/zerobyte.result
+ Compare files /tmp/zerobyte /tmp/zerobyte.result
#This test depends on the previous test case. Can't be executed alone
Get object with wrong signature
@@ -151,34 +153,14 @@ Incorrect values for end and start offset
Should Be Equal ${expectedData}
${actualData}
Zero byte file
- ${result} = Execute
ozone sh bucket info /s3v/${BUCKET}
- ${linked} = Execute
echo '${result}' | jq -j '.sourceVolume,"/",.sourceBucket'
- ${eval} = Evaluate
"source" in """${linked}"""
- IF ${eval} == ${True}
- ${result} = Execute
ozone sh bucket info ${linked}
- END
- ${fsolayout} = Evaluate
"OPTIMIZED" in """${result}"""
-
${result} = Execute AWSS3APICli and checkrc get-object
--bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range
bytes=0-0 /tmp/testfile2.result 255
- IF ${fsolayout} == ${True}
- Should contain ${result}
NoSuchKey
- ELSE
Should contain ${result}
InvalidRange
- END
${result} = Execute AWSS3APICli and checkrc get-object
--bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range
bytes=0-1 /tmp/testfile2.result 255
- IF ${fsolayout} == ${True}
- Should contain ${result}
NoSuchKey
- ELSE
Should contain ${result}
InvalidRange
- END
${result} = Execute AWSS3APICli and checkrc get-object
--bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range
bytes=0-10000 /tmp/testfile2.result 255
- IF ${fsolayout} == ${True}
- Should contain ${result}
NoSuchKey
- ELSE
Should contain ${result}
InvalidRange
- END
Create file with user defined metadata
Execute echo "Randomtext" >
/tmp/testfile2
@@ -257,4 +239,4 @@ Create key twice with different content and expect
different ETags
# clean up
Execute AWSS3Cli rm
s3://${BUCKET}/test_key_to_check_etag_differences
Execute rm -rf /tmp/file1
- Execute rm -rf /tmp/file2
\ No newline at end of file
+ Execute rm -rf /tmp/file2
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index f163d3eac3..8b7db9f061 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -274,7 +274,9 @@ public class ObjectEndpoint extends EndpointBase {
boolean hasAmzDecodedLengthZero = amzDecodedLength != null &&
Long.parseLong(amzDecodedLength) == 0;
if (canCreateDirectory &&
- (length == 0 || hasAmzDecodedLengthZero)) {
+ (length == 0 || hasAmzDecodedLengthZero) &&
+ StringUtils.endsWith(keyPath, "/")
+ ) {
s3GAction = S3GAction.CREATE_DIRECTORY;
getClientProtocol()
.createDirectory(volume.getName(), bucketName, keyPath);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 32bcb0a2c9..17c3cba304 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -95,7 +95,9 @@ class TestObjectPut {
ReplicationConfig ratis3 =
RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE);
ECReplicationConfig ec = new ECReplicationConfig("rs-3-2-1024K");
return Stream.of(
+ Arguments.of(0, ratis3),
Arguments.of(10, ratis3),
+ Arguments.of(0, ec),
Arguments.of(10, ec)
);
}
@@ -410,7 +412,7 @@ class TestObjectPut {
void testDirectoryCreation() throws IOException,
OS3Exception {
// GIVEN
- final String path = "dir";
+ final String path = "dir/";
// WHEN
try (Response response = objectEndpoint.put(fsoBucket.getName(), path,
@@ -426,7 +428,7 @@ class TestObjectPut {
@Test
void testDirectoryCreationOverFile() throws IOException, OS3Exception {
// GIVEN
- final String path = "dir";
+ final String path = "key";
final ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
objectEndpoint.put(FSO_BUCKET_NAME, path, CONTENT.length(), 0, "", body);
@@ -434,7 +436,7 @@ class TestObjectPut {
// WHEN
final OS3Exception exception = assertThrows(OS3Exception.class,
() -> objectEndpoint
- .put(FSO_BUCKET_NAME, path, 0, 0, "", null)
+ .put(FSO_BUCKET_NAME, path + "/", 0, 0, "", null)
.close());
// THEN
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
