This is an automated email from the ASF dual-hosted git repository.
Gargi-jais11 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3dbca34cdb4 HDDS-15606. Support S3 object metadata headers
(Cache-Control, Expires, Content-Encoding) (#10786).
3dbca34cdb4 is described below
commit 3dbca34cdb4f9114430589c7e772e8c02604c772
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Thu Jul 30 15:06:34 2026 +0530
HDDS-15606. Support S3 object metadata headers (Cache-Control, Expires,
Content-Encoding) (#10786).
---
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 106 +++++++++++++++++++++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 38 ++++++--
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 56 +++++++++--
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 5 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 19 +++-
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 26 +++++
.../s3/endpoint/TestMultipartUploadComplete.java | 22 +++++
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 20 +++-
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 89 ++++++++++++-----
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 45 +++++++++
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 20 ++++
11 files changed, 405 insertions(+), 41 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 2fe78577576..8b893919cb0 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -48,6 +48,8 @@
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
@@ -3199,6 +3201,110 @@ private void
verifyBucketOwnershipVerificationAccessDenied(Executable function)
}
}
+ static Stream<Arguments> standardObjectHeaderContentEncodingCases() {
+ return Stream.of(
+ Arguments.of("gzip", "gzip"),
+ Arguments.of("deflate, gzip", "deflate, gzip"),
+ Arguments.of("gzip, aws-chunked", "gzip"),
+ Arguments.of("aws-chunked, gzip", "gzip"),
+ Arguments.of("aws-chunked", null),
+ Arguments.of("aws-chunked, aws-chunked", null));
+ }
+
+ /**
+ * ceph s3-tests coverage for standard object headers persisted on PUT and
returned on HEAD/GET.
+ */
+ @Nested
+ class StandardObjectHeaderTests {
+
+ private static final String CONTENT = "bar";
+ private static final String CACHE_CONTROL = "public, max-age=14400";
+
+ /**
+ * s3-tests: test_object_write_cache_control.
+ */
+ @Test
+ public void testObjectWriteCacheControl() {
+ final String bucketName = getBucketName("cache-control");
+ final String keyName = getKeyName("cache-control");
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ s3Client.putObject(b ->
b.bucket(bucketName).key(keyName).cacheControl(CACHE_CONTROL),
+ RequestBody.fromString(CONTENT));
+
+ HeadObjectResponse head = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
+ assertEquals(CACHE_CONTROL, head.cacheControl());
+
+ GetObjectResponse getObject = s3Client.getObject(b ->
b.bucket(bucketName).key(keyName)).response();
+ assertEquals(CACHE_CONTROL, getObject.cacheControl());
+ }
+
+ /**
+ * s3-tests: test_object_write_expires.
+ */
+ @Test
+ public void testObjectWriteExpires() {
+ final String bucketName = getBucketName("expires");
+ final String keyName = getKeyName("expires");
+ final Instant expires =
Instant.now().plusSeconds(6000).truncatedTo(ChronoUnit.SECONDS);
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ s3Client.putObject(b ->
b.bucket(bucketName).key(keyName).expires(expires),
+ RequestBody.fromString(CONTENT));
+
+ HeadObjectResponse head = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
+ assertEquals(expires, head.expires());
+
+ GetObjectResponse getObject = s3Client.getObject(b ->
b.bucket(bucketName).key(keyName)).response();
+ assertEquals(expires, getObject.expires());
+ }
+
+ /**
+ * s3-tests: test_object_content_encoding_aws_chunked.
+ */
+ @ParameterizedTest
+
@MethodSource("org.apache.hadoop.ozone.s3.awssdk.v2.AbstractS3SDKV2Tests#standardObjectHeaderContentEncodingCases")
+ public void testObjectContentEncodingAwsChunked(String requestEncoding,
+ String expectedEncoding) {
+ final String bucketName = getBucketName("content-encoding");
+ final String keyName = getKeyName("encoding");
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ s3Client.putObject(b -> b.bucket(bucketName).key(keyName)
+ .contentEncoding(requestEncoding),
+ RequestBody.fromString(CONTENT));
+
+ HeadObjectResponse head = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
+ assertEquals(expectedEncoding, head.contentEncoding());
+
+ GetObjectResponse getObject = s3Client.getObject(b ->
b.bucket(bucketName).key(keyName)).response();
+ assertEquals(expectedEncoding, getObject.contentEncoding());
+ }
+
+ @Test
+ public void testObjectWriteContentLanguageAndDisposition() {
+ final String bucketName = getBucketName("lang-disp");
+ final String keyName = getKeyName("lang-disp");
+ final String language = "en-CA";
+ final String disposition = "attachment; filename=\"test.txt\"";
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ s3Client.putObject(b -> b.bucket(bucketName).key(keyName)
+ .contentLanguage(language)
+ .contentDisposition(disposition),
+ RequestBody.fromString(CONTENT));
+
+ HeadObjectResponse head = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
+ assertEquals(language, head.contentLanguage());
+ assertEquals(disposition, head.contentDisposition());
+
+ GetObjectResponse getObject =
+ s3Client.getObject(b ->
b.bucket(bucketName).key(keyName)).response();
+ assertEquals(language, getObject.contentLanguage());
+ assertEquals(disposition, getObject.contentDisposition());
+ }
+ }
+
/**
* Integration tests for ListBuckets (GET / ListAllMyBuckets).
*/
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 3f960cf0e67..124fed461ff 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -37,6 +37,7 @@
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_TAG_PREFIX;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.RESERVED_USER_METADATA_KEY_PREFIX;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CONFIG_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER;
@@ -122,17 +123,35 @@
*/
public abstract class EndpointBase {
- protected static final String ETAG_CUSTOM = "etag-custom";
- protected static final String CONTENT_TYPE_CUSTOM = "content-type-custom";
+ protected static final String ETAG_CUSTOM =
+ RESERVED_USER_METADATA_KEY_PREFIX + "etag";
+ protected static final String CONTENT_TYPE_CUSTOM =
+ RESERVED_USER_METADATA_KEY_PREFIX + "content-type";
+ protected static final String CACHE_CONTROL_CUSTOM =
+ RESERVED_USER_METADATA_KEY_PREFIX + "cache-control";
+ protected static final String EXPIRES_CUSTOM =
+ RESERVED_USER_METADATA_KEY_PREFIX + "expires";
+ protected static final String CONTENT_ENCODING_CUSTOM =
+ RESERVED_USER_METADATA_KEY_PREFIX + "content-encoding";
+ protected static final String CONTENT_LANGUAGE_CUSTOM =
+ RESERVED_USER_METADATA_KEY_PREFIX + "content-language";
+ protected static final String CONTENT_DISPOSITION_CUSTOM =
+ RESERVED_USER_METADATA_KEY_PREFIX + "content-disposition";
// System metadata key -> custom key. A user x-amz-meta-{etag,content-type}
// collides with the system ETag / Content-Type stored under the same key, so
// it is remapped on write and rebuilt on read; the system value is returned
// via its own ETag / Content-Type response header.
private static final Map<String, String> RESERVED_METADATA_KEYS =
- ImmutableMap.of(
- ETAG, ETAG_CUSTOM,
- HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_CUSTOM);
+ ImmutableMap.<String, String>builder()
+ .put(ETAG, ETAG_CUSTOM)
+ .put(HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_CUSTOM)
+ .put(HttpHeaders.CACHE_CONTROL, CACHE_CONTROL_CUSTOM)
+ .put(HttpHeaders.EXPIRES, EXPIRES_CUSTOM)
+ .put(HttpHeaders.CONTENT_ENCODING, CONTENT_ENCODING_CUSTOM)
+ .put(HttpHeaders.CONTENT_LANGUAGE, CONTENT_LANGUAGE_CUSTOM)
+ .put(HttpHeaders.CONTENT_DISPOSITION, CONTENT_DISPOSITION_CUSTOM)
+ .build();
// Custom key -> lower-cased header, to rebuild remapped user metadata on
read.
private static final Map<String, String> REBUILT_RESERVED_KEYS =
@@ -356,6 +375,13 @@ protected Map<String, String> getCustomMetadataFromHeaders(
for (String key : customMetadataKeys) {
String mapKey =
key.substring(CUSTOM_METADATA_HEADER_PREFIX.length());
+ if (mapKey.regionMatches(true, 0, RESERVED_USER_METADATA_KEY_PREFIX, 0,
+ RESERVED_USER_METADATA_KEY_PREFIX.length())) {
+ OS3Exception ex = newError(INVALID_ARGUMENT, key);
+ ex.setErrorMessage("User metadata keys must not start with the
reserved prefix "
+ + RESERVED_USER_METADATA_KEY_PREFIX);
+ throw ex;
+ }
List<String> values = requestHeaders.get(key);
String value = StringUtils.join(values, ",");
sizeInBytes += mapKey.getBytes(UTF_8).length;
@@ -402,7 +428,7 @@ protected void addCustomMetadataHeaders(
if (RESERVED_METADATA_KEYS.containsKey(metadataKey)) {
continue;
}
- // Rebuild a remapped user value (e.g. content-type-custom ->
content-type).
+ // Rebuild a remapped user value (e.g. ozone-s3-internal-content-type ->
content-type).
metadataKey = REBUILT_RESERVED_KEYS.getOrDefault(metadataKey,
metadataKey);
responseBuilder
.header(CUSTOM_METADATA_HEADER_PREFIX + metadataKey,
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index ed9602d0fd0..ad798e36989 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -43,6 +43,7 @@
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_DIRECTIVE_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.normalizeContentEncoding;
import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes;
import static org.apache.hadoop.ozone.s3.util.S3Utils.validateSignatureHeader;
import static org.apache.hadoop.ozone.s3.util.S3Utils.wrapInQuotes;
@@ -272,6 +273,7 @@ Response handlePutRequest(ObjectRequestContext context,
String keyPath, InputStr
Map<String, String> customMetadata =
getCustomMetadataFromHeaders(getHeaders().getRequestHeaders());
putContentType(customMetadata);
+ putStandardObjectHeaders(customMetadata);
Map<String, String> tags = getTaggingFromHeaders(getHeaders());
long putLength;
@@ -480,14 +482,8 @@ Response handleGetRequest(ObjectRequestContext context,
String keyPath)
responseBuilder.header(HttpHeaders.CONTENT_TYPE, contentType);
for (Map.Entry<String, String> entry :
overrideQueryParameter.entrySet()) {
- String headerValue = getHeaders().getHeaderString(entry.getKey());
- String queryValue = queryParams.getFirst(entry.getValue());
- if (queryValue != null) {
- headerValue = queryValue;
- }
- if (headerValue != null) {
- responseBuilder.header(entry.getKey(), headerValue);
- }
+ addStoredObjectHeader(responseBuilder, keyDetails, queryParams,
entry.getKey(),
+ entry.getValue());
}
addLastModifiedDate(responseBuilder, keyDetails);
@@ -528,6 +524,45 @@ private void putContentType(Map<String, String> metadata) {
}
}
+ /**
+ * Store standard S3 object headers from the PUT request in key metadata.
+ */
+ private void putStandardObjectHeaders(Map<String, String> metadata) {
+ for (String headerName : overrideQueryParameter.keySet()) {
+ String value = getHeaders().getHeaderString(headerName);
+ if (HttpHeaders.CONTENT_ENCODING.equals(headerName)) {
+ value = normalizeContentEncoding(value);
+ }
+ if (value != null) {
+ metadata.put(headerName, value);
+ }
+ }
+ }
+
+ private void addStoredObjectHeader(ResponseBuilder responseBuilder, OzoneKey
key,
+ MultivaluedMap<String, String> queryParams, String headerName,
+ String responseOverrideParam) {
+ String headerValue = queryParams.getFirst(responseOverrideParam);
+ if (headerValue == null) {
+ headerValue = key.getMetadata().get(headerName);
+ }
+ if (HttpHeaders.CONTENT_ENCODING.equals(headerName)) {
+ headerValue = normalizeContentEncoding(headerValue);
+ }
+ if (headerValue != null) {
+ responseBuilder.header(headerName, headerValue);
+ }
+ }
+
+ private void addStoredObjectHeaders(ResponseBuilder responseBuilder,
OzoneKey key) {
+ MultivaluedMap<String, String> queryParams =
+ getContext().getUriInfo().getQueryParameters();
+ for (Map.Entry<String, String> entry : overrideQueryParameter.entrySet()) {
+ addStoredObjectHeader(responseBuilder, key, queryParams, entry.getKey(),
+ entry.getValue());
+ }
+ }
+
/** Returns the object's stored Content-Type, or the default if absent. */
private static String contentTypeOf(OzoneKey key) {
String contentType = key.getMetadata().get(HttpHeaders.CONTENT_TYPE);
@@ -643,6 +678,7 @@ public Response head(
.header(HttpHeaders.CONTENT_TYPE, contentTypeOf(key))
.header(STORAGE_CLASS_HEADER, s3StorageType.toString());
addEntityTagHeader(response, key);
+ addStoredObjectHeaders(response, key);
addLastModifiedDate(response, key);
addTagCountIfAny(response, key);
@@ -756,6 +792,7 @@ public Response initializeMultipartUpload(
Map<String, String> customMetadata =
getCustomMetadataFromHeaders(getHeaders().getRequestHeaders());
putContentType(customMetadata);
+ putStandardObjectHeaders(customMetadata);
Map<String, String> tags = getTaggingFromHeaders(getHeaders());
@@ -1184,8 +1221,9 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
} else if (metadataCopyDirective.equals(CopyDirective.REPLACE.name())) {
// Replace the metadata with the metadata form the request headers
customMetadata =
getCustomMetadataFromHeaders(getHeaders().getRequestHeaders());
- // REPLACE: Content-Type comes from the request, not the source.
+ // REPLACE: Content-Type and standard object headers come from the
request.
putContentType(customMetadata);
+ putStandardObjectHeaders(customMetadata);
} else {
OS3Exception ex = newError(INVALID_ARGUMENT, metadataCopyDirective);
ex.setErrorMessage("An error occurred (InvalidArgument) " +
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index a2ed3a84c25..d253e526386 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -75,6 +75,11 @@ public final class S3Consts {
// Constants related to custom metadata
public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
+ /**
+ * Prefix for internal metadata keys that store remapped user x-amz-meta
values
+ * colliding with system headers. Users must not set metadata keys with this
prefix.
+ */
+ public static final String RESERVED_USER_METADATA_KEY_PREFIX =
"ozone-s3-internal-";
public static final String CUSTOM_METADATA_COPY_DIRECTIVE_HEADER =
"x-amz-metadata-directive";
public static final String STORAGE_CONFIG_HEADER = "storage-config";
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
index 4f9fe6c2a27..3379d2d244d 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
@@ -36,6 +36,7 @@
import java.util.Arrays;
import java.util.Base64;
import java.util.Objects;
+import java.util.stream.Collectors;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
@@ -73,6 +74,22 @@ public static String s3urlEncode(String str)
return urlEncode(str).replace("+", "%20");
}
+ /**
+ * Returns the persisted S3 {@code Content-Encoding} value, with {@code
aws-chunked}
+ * removed per AWS semantics.
+ */
+ public static String normalizeContentEncoding(String contentEncoding) {
+ if (contentEncoding == null || contentEncoding.isEmpty()) {
+ return null;
+ }
+ String normalized = Arrays.stream(contentEncoding.split(","))
+ .map(String::trim)
+ .filter(value -> !value.isEmpty())
+ .filter(value -> !AWS_CHUNKED.equalsIgnoreCase(value))
+ .collect(Collectors.joining(", "));
+ return normalized.isEmpty() ? null : normalized;
+ }
+
private S3Utils() {
// no instances
}
@@ -162,7 +179,7 @@ public static void validateMultiChunksUpload(HttpHeaders
headers, String amzDeco
// We are only interested on "aws-chunked"
boolean containsAwsChunked = Arrays.stream(contentEncoding.split(","))
.map(String::trim)
- .anyMatch(AWS_CHUNKED::equals);
+ .anyMatch(AWS_CHUNKED::equalsIgnoreCase);
if (!containsAwsChunked) {
OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
ex.setErrorMessage("An error occurred (InvalidArgument) for multi
chunks upload: " +
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
index 2910aa22a51..7002fbd0999 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
@@ -17,7 +17,9 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.RESERVED_USER_METADATA_KEY_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -25,11 +27,14 @@
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.Map;
+import java.util.stream.Stream;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
/**
* Tests the s3 EndpointBase class methods.
@@ -108,4 +113,25 @@ public void
testCustomMetadataHeadersWithUpperCaseHeaders() throws OS3Exception
assertEquals(value, customMetadata.get(key));
}
+ @ParameterizedTest
+ @MethodSource("reservedInternalMetadataKeyPrefixCases")
+ public void testRejectReservedInternalMetadataKeyPrefix(String metadataKey) {
+ MultivaluedMap<String, String> s3requestHeaders = new
MultivaluedHashMap<>();
+ s3requestHeaders.add(CUSTOM_METADATA_HEADER_PREFIX + metadataKey,
"user-value");
+
+ EndpointBase endpointBase = new EndpointBase() {
+ };
+
+ OS3Exception e = assertThrows(OS3Exception.class, () -> endpointBase
+ .getCustomMetadataFromHeaders(s3requestHeaders));
+ assertThat(e.getCode()).contains(INVALID_ARGUMENT.getCode());
+
assertThat(e.getErrorMessage()).contains(RESERVED_USER_METADATA_KEY_PREFIX);
+ }
+
+ private static Stream<String> reservedInternalMetadataKeyPrefixCases() {
+ return Stream.of(
+ RESERVED_USER_METADATA_KEY_PREFIX + "cache-control",
+ RESERVED_USER_METADATA_KEY_PREFIX.toUpperCase(Locale.ROOT) +
"cache-control");
+ }
+
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
index 816e8e10ddf..e95d8085c2d 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
@@ -143,6 +143,28 @@ public void testMultipartStoresContentType() throws
Exception {
.getHeaderString(HttpHeaders.CONTENT_TYPE));
}
+ @Test
+ public void testMultipartStoresStandardObjectHeaders() throws Exception {
+ String key = UUID.randomUUID().toString();
+
when(headers.getHeaderString(HttpHeaders.CACHE_CONTROL)).thenReturn("no-cache");
+ when(headers.getHeaderString(HttpHeaders.EXPIRES)).thenReturn("Wed, 21 Oct
2015 07:29:00 GMT");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_ENCODING)).thenReturn("gzip");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_LANGUAGE)).thenReturn("en-CA");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_DISPOSITION)).thenReturn("inline");
+
+ String uploadID = initiateMultipartUpload(key);
+ Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID,
"Multipart Upload 1");
+ completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key, uploadID,
singletonList(part1));
+
+ Response headResponse = rest.head(OzoneConsts.S3_BUCKET, key);
+ assertEquals("no-cache",
headResponse.getHeaderString(HttpHeaders.CACHE_CONTROL));
+ assertEquals("Wed, 21 Oct 2015 07:29:00 GMT",
+ headResponse.getHeaderString(HttpHeaders.EXPIRES));
+ assertEquals("gzip",
headResponse.getHeaderString(HttpHeaders.CONTENT_ENCODING));
+ assertEquals("en-CA",
headResponse.getHeaderString(HttpHeaders.CONTENT_LANGUAGE));
+ assertEquals("inline",
headResponse.getHeaderString(HttpHeaders.CONTENT_DISPOSITION));
+ }
+
@Test
public void testMultipartInvalidPartOrderError() throws Exception {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
index 741cd766739..d33e506362c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -259,12 +259,12 @@ public void getKeyWithTag() throws IOException,
OS3Exception {
}
@Test
- public void inheritRequestHeader() throws IOException, OS3Exception {
+ public void storedObjectHeadersOnGetAndHead() throws IOException,
OS3Exception {
setDefaultHeader();
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
+ clearRequestHeaderMocks();
Response response = get(rest, BUCKET_NAME, KEY_NAME);
-
- // Content-Type is not inherited from the request; key1 has none stored.
assertEquals("binary/octet-stream",
response.getHeaderString("Content-Type"));
assertEquals(CONTENT_LANGUAGE1,
@@ -277,11 +277,16 @@ public void inheritRequestHeader() throws IOException,
OS3Exception {
response.getHeaderString("Content-Disposition"));
assertEquals(CONTENT_ENCODING1,
response.getHeaderString("Content-Encoding"));
+
+ assertEquals(CONTENT_ENCODING1,
+ rest.head(BUCKET_NAME, KEY_NAME).getHeaderString("Content-Encoding"));
}
@Test
public void overrideResponseHeader() throws IOException, OS3Exception {
setDefaultHeader();
+ assertSucceeds(() -> put(rest, BUCKET_NAME, KEY_NAME, CONTENT));
+ clearRequestHeaderMocks();
MultivaluedMap<String, String> queryParameter =
rest.getContext().getUriInfo().getQueryParameters();
// overrider request header
@@ -411,6 +416,15 @@ private void setDefaultHeader() {
.when(headers).getHeaderString("Content-Encoding");
}
+ private void clearRequestHeaderMocks() {
+ doReturn(null).when(headers).getHeaderString("Content-Type");
+ doReturn(null).when(headers).getHeaderString("Content-Language");
+ doReturn(null).when(headers).getHeaderString("Expires");
+ doReturn(null).when(headers).getHeaderString("Cache-Control");
+ doReturn(null).when(headers).getHeaderString("Content-Disposition");
+ doReturn(null).when(headers).getHeaderString("Content-Encoding");
+ }
+
@Test
public void testGetWhenKeyIsDirectoryAndDoesNotEndWithASlash()
throws IOException {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
index ccce932d0d7..2c1b7f8699c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java
@@ -19,10 +19,17 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointBase.CACHE_CONTROL_CUSTOM;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointBase.CONTENT_DISPOSITION_CUSTOM;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointBase.CONTENT_ENCODING_CUSTOM;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointBase.CONTENT_LANGUAGE_CUSTOM;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointBase.CONTENT_TYPE_CUSTOM;
+import static org.apache.hadoop.ozone.s3.endpoint.EndpointBase.EXPIRES_CUSTOM;
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertStatus;
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put;
+import static org.apache.hadoop.ozone.s3.endpoint.TestObjectGet.EXPIRES1;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
@@ -43,6 +50,7 @@
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
+import java.util.stream.Stream;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
@@ -60,6 +68,9 @@
import org.apache.http.HttpStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
/**
* Test head object.
@@ -245,35 +256,69 @@ public void testHeadObjectIncludesTagCount()
assertEquals("2", response.getHeaderString(TAG_COUNT_HEADER));
}
- @Test
- public void testHeadSeparatesUserContentTypeMetadataFromObjectContentType()
+ @ParameterizedTest
+ @MethodSource("reservedMetadataCollisionCases")
+ public void testHeadSeparatesUserMetadataFromSystemHeader(
+ String headerName, String customKey, String systemValue, String
userValue)
throws Exception {
- String keyName = "typed-with-user-meta";
- String objectContentType = "image/jpeg";
- String userContentType = "user/custom-type";
-
- // PUT with both the object's Content-Type and a colliding user
- // x-amz-meta-content-type.
- when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE))
- .thenReturn(objectContentType);
+ String keyName = "reserved-" + customKey;
MultivaluedMap<String, String> requestHeaders = new MultivaluedHashMap<>();
- requestHeaders.putSingle(
- CUSTOM_METADATA_HEADER_PREFIX + "content-type", userContentType);
+ requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX +
headerName.toLowerCase(), userValue);
when(headers.getRequestHeaders()).thenReturn(requestHeaders);
- assertSucceeds(() -> put(keyEndpoint, bucketName, keyName,
"head-content"));
+ if (HttpHeaders.CONTENT_TYPE.equals(headerName)) {
+ when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE))
+ .thenReturn(systemValue);
+ } else {
+ when(headers.getHeaderString(headerName)).thenReturn(systemValue);
+ }
- // The user value is remapped, so the object's Content-Type is preserved.
- assertEquals(objectContentType,
- bucket.getKey(keyName).getMetadata().get(HttpHeaders.CONTENT_TYPE));
+ assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, "body"));
+
+ assertEquals(systemValue,
bucket.getKey(keyName).getMetadata().get(headerName));
+ assertEquals(userValue,
+ bucket.getKey(keyName).getMetadata().get(customKey));
- // HEAD returns the object Content-Type as the standard header and the user
- // value as x-amz-meta-content-type.
Response response = keyEndpoint.head(bucketName, keyName);
assertEquals(HttpStatus.SC_OK, response.getStatus());
- assertEquals(objectContentType,
- response.getHeaderString(HttpHeaders.CONTENT_TYPE));
- assertEquals(userContentType,
- response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX +
"content-type"));
+ assertEquals(systemValue, response.getHeaderString(headerName));
+ assertEquals(userValue,
+ response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX +
headerName.toLowerCase()));
+ }
+
+ @Test
+ public void testUserMetadataSuffixDoesNotCollideWithInternalKey() throws
Exception {
+ String keyName = "reserved-cache-control-suffix";
+ MultivaluedMap<String, String> requestHeaders = new MultivaluedHashMap<>();
+ requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "cache-control",
"user-cache");
+ requestHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX +
"cache-control-custom", "suffix-value");
+ when(headers.getRequestHeaders()).thenReturn(requestHeaders);
+
when(headers.getHeaderString(HttpHeaders.CACHE_CONTROL)).thenReturn("no-cache");
+
+ assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, "body"));
+
+ assertEquals("no-cache",
+ bucket.getKey(keyName).getMetadata().get(HttpHeaders.CACHE_CONTROL));
+ assertEquals("user-cache",
+ bucket.getKey(keyName).getMetadata().get(CACHE_CONTROL_CUSTOM));
+ assertEquals("suffix-value",
+ bucket.getKey(keyName).getMetadata().get("cache-control-custom"));
+
+ Response response = keyEndpoint.head(bucketName, keyName);
+ assertEquals("no-cache",
response.getHeaderString(HttpHeaders.CACHE_CONTROL));
+ assertEquals("user-cache",
+ response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX +
"cache-control"));
+ assertEquals("suffix-value",
+ response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX +
"cache-control-custom"));
+ }
+
+ private static Stream<Arguments> reservedMetadataCollisionCases() {
+ return Stream.of(
+ Arguments.of(HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_CUSTOM,
"image/jpeg", "user/custom-type"),
+ Arguments.of(HttpHeaders.CACHE_CONTROL, CACHE_CONTROL_CUSTOM,
"no-cache", "user-cache"),
+ Arguments.of(HttpHeaders.EXPIRES, EXPIRES_CUSTOM, EXPIRES1,
"user-expires"),
+ Arguments.of(HttpHeaders.CONTENT_ENCODING, CONTENT_ENCODING_CUSTOM,
"gzip", "user-encoding"),
+ Arguments.of(HttpHeaders.CONTENT_LANGUAGE, CONTENT_LANGUAGE_CUSTOM,
"en-CA", "user-lang"),
+ Arguments.of(HttpHeaders.CONTENT_DISPOSITION,
CONTENT_DISPOSITION_CUSTOM, "inline", "user-disp"));
}
private byte[] createKey(String keyPath) throws IOException {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 2af6cbcfdcc..6b4927d1340 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -418,6 +418,51 @@ void testContentTypeStoredAndCopied() throws Exception {
destBucket.getKey(DEST_KEY).getMetadata().get(HttpHeaders.CONTENT_TYPE));
}
+ @Test
+ void testStandardObjectHeadersCopiedAndReplaced() throws Exception {
+
when(headers.getHeaderString(HttpHeaders.CACHE_CONTROL)).thenReturn("no-cache");
+ when(headers.getHeaderString(HttpHeaders.EXPIRES)).thenReturn("Wed, 21 Oct
2015 07:29:00 GMT");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_ENCODING)).thenReturn("gzip");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_LANGUAGE)).thenReturn("en-CA");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_DISPOSITION)).thenReturn("inline");
+ when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE))
+ .thenReturn("audio/mpeg");
+
+ assertSucceeds(() -> putObject(CONTENT));
+
+ when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER))
+ .thenReturn("COPY");
+ when(headers.getHeaderString(COPY_SOURCE_HEADER))
+ .thenReturn(BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
when(headers.getHeaderString(HttpHeaders.CACHE_CONTROL)).thenReturn("max-age=0");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_ENCODING)).thenReturn("compress");
+
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+ OzoneKeyDetails copied = destBucket.getKey(DEST_KEY);
+ assertEquals("no-cache",
copied.getMetadata().get(HttpHeaders.CACHE_CONTROL));
+ assertEquals("gzip",
copied.getMetadata().get(HttpHeaders.CONTENT_ENCODING));
+ assertEquals("en-CA",
copied.getMetadata().get(HttpHeaders.CONTENT_LANGUAGE));
+
+ when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER))
+ .thenReturn("REPLACE");
+ when(headers.getRequestHeaders()).thenReturn(new MultivaluedHashMap<>());
+
when(headers.getHeaderString(HttpHeaders.CACHE_CONTROL)).thenReturn("private");
+ when(headers.getHeaderString(HttpHeaders.EXPIRES)).thenReturn("Wed, 21 Oct
2015 07:28:00 GMT");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_ENCODING)).thenReturn("deflate");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_LANGUAGE)).thenReturn("de-DE");
+
when(headers.getHeaderString(HttpHeaders.CONTENT_DISPOSITION)).thenReturn("attachment");
+
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+ OzoneKeyDetails replaced = destBucket.getKey(DEST_KEY);
+ assertEquals("private",
replaced.getMetadata().get(HttpHeaders.CACHE_CONTROL));
+ assertEquals("Wed, 21 Oct 2015 07:28:00 GMT",
+ replaced.getMetadata().get(HttpHeaders.EXPIRES));
+ assertEquals("deflate",
replaced.getMetadata().get(HttpHeaders.CONTENT_ENCODING));
+ assertEquals("de-DE",
replaced.getMetadata().get(HttpHeaders.CONTENT_LANGUAGE));
+ assertEquals("attachment",
+ replaced.getMetadata().get(HttpHeaders.CONTENT_DISPOSITION));
+ }
+
@Test
public void testCopyObjectMessageDigestResetDuringException() throws
Exception {
assertSucceeds(() -> putObject(CONTENT));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
index d4736171130..2dbf3c65ba6 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
@@ -189,4 +189,24 @@ public void testValidateContentMD5WithValidInput() throws
Exception {
assertDoesNotThrow(() -> S3Utils.validateContentMD5(md5Base64, md5Hex,
"test-resource"));
}
+ @ParameterizedTest
+ @MethodSource("contentEncodingProvider")
+ public void testNormalizeContentEncoding(String input, String expected) {
+ assertEquals(expected, S3Utils.normalizeContentEncoding(input));
+ }
+
+ private static Stream<Arguments> contentEncodingProvider() {
+ return Stream.of(
+ Arguments.of(null, null),
+ Arguments.of("", null),
+ Arguments.of("gzip", "gzip"),
+ Arguments.of("deflate, gzip", "deflate, gzip"),
+ Arguments.of("gzip, aws-chunked", "gzip"),
+ Arguments.of("aws-chunked, gzip", "gzip"),
+ Arguments.of("aws-chunked", null),
+ Arguments.of("aws-chunked, aws-chunked", null),
+ Arguments.of("gzip, AWS-CHUNKED", "gzip"),
+ Arguments.of("Aws-Chunked", null));
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
