This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 615af790157 Revert "HADOOP-19197. S3A: Support AWS KMS Encryption
Context (#7193)"
615af790157 is described below
commit 615af790157aca8cf44a66c7f17fa8864c2e4683
Author: Steve Loughran <[email protected]>
AuthorDate: Mon Jul 28 14:56:40 2025 +0100
Revert "HADOOP-19197. S3A: Support AWS KMS Encryption Context (#7193)"
This reverts commit eb656c016facfa1b60f2e9e4d95ebd5ca1b1bc0a.
This was done because the writable/serializable class
org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets
is no longer wire-compatible once context attributes
are included.
---
.../hadoop/fs/CommonConfigurationKeysPublic.java | 1 -
.../src/main/resources/core-default.xml | 10 --
.../java/org/apache/hadoop/fs/s3a/Constants.java | 10 --
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 22 +----
.../delegation/EncryptionSecretOperations.java | 16 ----
.../fs/s3a/auth/delegation/EncryptionSecrets.java | 35 +------
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 14 ---
.../apache/hadoop/fs/s3a/impl/S3AEncryption.java | 106 ---------------------
.../site/markdown/tools/hadoop-aws/encryption.md | 30 ------
.../src/site/markdown/tools/hadoop-aws/index.md | 14 ---
.../hadoop/fs/s3a/AbstractTestS3AEncryption.java | 2 -
...stS3AEncryptionSSEKMSWithEncryptionContext.java | 101 --------------------
.../apache/hadoop/fs/s3a/TestSSEConfiguration.java | 69 +++-----------
.../fs/s3a/auth/TestMarshalledCredentials.java | 3 +-
.../delegation/ITestSessionDelegationTokens.java | 6 +-
.../delegation/TestS3ADelegationTokenSupport.java | 24 +----
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 4 +-
.../hadoop/fs/s3a/impl/TestS3AEncryption.java | 77 ---------------
18 files changed, 30 insertions(+), 514 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 0b36aec318d..d01ddd30f47 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -1022,7 +1022,6 @@ public class CommonConfigurationKeysPublic {
"fs.s3a.*.server-side-encryption.key",
"fs.s3a.encryption.algorithm",
"fs.s3a.encryption.key",
- "fs.s3a.encryption.context",
"fs.azure\\.account.key.*",
"credential$",
"oauth.*secret",
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index d856a83e366..3e1351c3c88 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -742,7 +742,6 @@
fs.s3a.*.server-side-encryption.key
fs.s3a.encryption.algorithm
fs.s3a.encryption.key
- fs.s3a.encryption.context
fs.s3a.secret.key
fs.s3a.*.secret.key
fs.s3a.session.key
@@ -1780,15 +1779,6 @@
</description>
</property>
-<property>
- <name>fs.s3a.encryption.context</name>
- <description>Specific encryption context to use if
fs.s3a.encryption.algorithm
- has been set to 'SSE-KMS' or 'DSSE-KMS'. The value of this property is a
set
- of non-secret comma-separated key-value pairs of additional contextual
- information about the data that are separated by equal operator (=).
- </description>
-</property>
-
<property>
<name>fs.s3a.signing-algorithm</name>
<description>Override the default signing algorithm so legacy
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 4f120ee9555..5d4f5eb40df 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -774,16 +774,6 @@ private Constants() {
public static final String S3_ENCRYPTION_KEY =
"fs.s3a.encryption.key";
- /**
- * Set S3-SSE encryption context.
- * The value of this property is a set of non-secret comma-separated
key-value pairs
- * of additional contextual information about the data that are separated by
equal
- * operator (=).
- * value:{@value}
- */
- public static final String S3_ENCRYPTION_CONTEXT =
- "fs.s3a.encryption.context";
-
/**
* Client side encryption (CSE-CUSTOM) with custom cryptographic material
manager class name.
* Custom keyring class name for CSE-KMS.
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 7cbd0e2c241..057ed3d7a01 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -38,7 +38,6 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.s3a.impl.S3AEncryption;
import org.apache.hadoop.util.functional.RemoteIterators;
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException;
@@ -1325,7 +1324,7 @@ static void
patchSecurityCredentialProviders(Configuration conf) {
* @throws IOException on any IO problem
* @throws IllegalArgumentException bad arguments
*/
- public static String lookupBucketSecret(
+ private static String lookupBucketSecret(
String bucket,
Configuration conf,
String baseKey)
@@ -1471,8 +1470,6 @@ public static EncryptionSecrets
buildEncryptionSecrets(String bucket,
int encryptionKeyLen =
StringUtils.isBlank(encryptionKey) ? 0 : encryptionKey.length();
String diagnostics = passwordDiagnostics(encryptionKey, "key");
- String encryptionContext =
S3AEncryption.getS3EncryptionContextBase64Encoded(bucket, conf,
- encryptionMethod.requiresSecret());
switch (encryptionMethod) {
case SSE_C:
LOG.debug("Using SSE-C with {}", diagnostics);
@@ -1508,7 +1505,7 @@ public static EncryptionSecrets
buildEncryptionSecrets(String bucket,
LOG.debug("Data is unencrypted");
break;
}
- return new EncryptionSecrets(encryptionMethod, encryptionKey,
encryptionContext);
+ return new EncryptionSecrets(encryptionMethod, encryptionKey);
}
/**
@@ -1701,21 +1698,6 @@ public static Map<String, String>
getTrimmedStringCollectionSplitByEquals(
final Configuration configuration,
final String name) {
String valueString = configuration.get(name);
- return getTrimmedStringCollectionSplitByEquals(valueString);
- }
-
- /**
- * Get the equal op (=) delimited key-value pairs of the <code>name</code>
property as
- * a collection of pair of <code>String</code>s, trimmed of the leading and
trailing whitespace
- * after delimiting the <code>name</code> by comma and new line separator.
- * If no such property is specified then empty <code>Map</code> is returned.
- *
- * @param valueString the string containing the key-value pairs.
- * @return property value as a <code>Map</code> of <code>String</code>s, or
empty
- * <code>Map</code>.
- */
- public static Map<String, String> getTrimmedStringCollectionSplitByEquals(
- final String valueString) {
if (null == valueString) {
return new HashMap<>();
}
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
index ea5c0cf2078..8a55a970134 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
@@ -61,20 +61,4 @@ public static Optional<String> getSSEAwsKMSKey(final
EncryptionSecrets secrets)
return Optional.empty();
}
}
-
- /**
- * Gets the SSE-KMS context if present, else don't set it in the S3 request.
- *
- * @param secrets source of the encryption secrets.
- * @return an optional AWS KMS encryption context to attach to a request.
- */
- public static Optional<String> getSSEAwsKMSEncryptionContext(final
EncryptionSecrets secrets) {
- if ((secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS
- || secrets.getEncryptionMethod() == S3AEncryptionMethods.DSSE_KMS)
- && secrets.hasEncryptionContext()) {
- return Optional.of(secrets.getEncryptionContext());
- } else {
- return Optional.empty();
- }
- }
}
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java
index f421ecca24c..092653de557 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java
@@ -67,11 +67,6 @@ public class EncryptionSecrets implements Writable,
Serializable {
*/
private String encryptionKey = "";
- /**
- * Encryption context: base64-encoded UTF-8 string.
- */
- private String encryptionContext = "";
-
/**
* This field isn't serialized/marshalled; it is rebuilt from the
* encryptionAlgorithm field.
@@ -89,28 +84,23 @@ public EncryptionSecrets() {
* Create a pair of secrets.
* @param encryptionAlgorithm algorithm enumeration.
* @param encryptionKey key/key reference.
- * @param encryptionContext base64-encoded string with the encryption
context key-value pairs.
* @throws IOException failure to initialize.
*/
public EncryptionSecrets(final S3AEncryptionMethods encryptionAlgorithm,
- final String encryptionKey,
- final String encryptionContext) throws IOException {
- this(encryptionAlgorithm.getMethod(), encryptionKey, encryptionContext);
+ final String encryptionKey) throws IOException {
+ this(encryptionAlgorithm.getMethod(), encryptionKey);
}
/**
* Create a pair of secrets.
* @param encryptionAlgorithm algorithm name
* @param encryptionKey key/key reference.
- * @param encryptionContext base64-encoded string with the encryption
context key-value pairs.
* @throws IOException failure to initialize.
*/
public EncryptionSecrets(final String encryptionAlgorithm,
- final String encryptionKey,
- final String encryptionContext) throws IOException {
+ final String encryptionKey) throws IOException {
this.encryptionAlgorithm = encryptionAlgorithm;
this.encryptionKey = encryptionKey;
- this.encryptionContext = encryptionContext;
init();
}
@@ -124,7 +114,6 @@ public void write(final DataOutput out) throws IOException {
new LongWritable(serialVersionUID).write(out);
Text.writeString(out, encryptionAlgorithm);
Text.writeString(out, encryptionKey);
- Text.writeString(out, encryptionContext);
}
/**
@@ -143,7 +132,6 @@ public void readFields(final DataInput in) throws
IOException {
}
encryptionAlgorithm = Text.readString(in, MAX_SECRET_LENGTH);
encryptionKey = Text.readString(in, MAX_SECRET_LENGTH);
- encryptionContext = Text.readString(in);
init();
}
@@ -176,10 +164,6 @@ public String getEncryptionKey() {
return encryptionKey;
}
- public String getEncryptionContext() {
- return encryptionContext;
- }
-
/**
* Does this instance have encryption options?
* That is: is the algorithm non-null.
@@ -197,14 +181,6 @@ public boolean hasEncryptionKey() {
return StringUtils.isNotEmpty(encryptionKey);
}
- /**
- * Does this instance have an encryption context?
- * @return true if there's an encryption context.
- */
- public boolean hasEncryptionContext() {
- return StringUtils.isNotEmpty(encryptionContext);
- }
-
@Override
public boolean equals(final Object o) {
if (this == o) {
@@ -215,13 +191,12 @@ public boolean equals(final Object o) {
}
final EncryptionSecrets that = (EncryptionSecrets) o;
return Objects.equals(encryptionAlgorithm, that.encryptionAlgorithm)
- && Objects.equals(encryptionKey, that.encryptionKey)
- && Objects.equals(encryptionContext, that.encryptionContext);
+ && Objects.equals(encryptionKey, that.encryptionKey);
}
@Override
public int hashCode() {
- return Objects.hash(encryptionAlgorithm, encryptionKey, encryptionContext);
+ return Objects.hash(encryptionAlgorithm, encryptionKey);
}
/**
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
index 6feca522cb6..781a3a34dc6 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
@@ -298,8 +298,6 @@ protected void copyEncryptionParameters(HeadObjectResponse
srcom,
LOG.debug("Propagating SSE-KMS settings from source {}",
sourceKMSId);
copyObjectRequestBuilder.ssekmsKeyId(sourceKMSId);
-
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
- .ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext);
return;
}
@@ -312,15 +310,11 @@ protected void
copyEncryptionParameters(HeadObjectResponse srcom,
// Set the KMS key if present, else S3 uses AWS managed key.
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(copyObjectRequestBuilder::ssekmsKeyId);
-
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
- .ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext);
break;
case DSSE_KMS:
copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(copyObjectRequestBuilder::ssekmsKeyId);
-
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
- .ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext);
break;
case SSE_C:
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
@@ -427,15 +421,11 @@ private void
putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBu
// Set the KMS key if present, else S3 uses AWS managed key.
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(putObjectRequestBuilder::ssekmsKeyId);
-
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
- .ifPresent(putObjectRequestBuilder::ssekmsEncryptionContext);
break;
case DSSE_KMS:
putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(putObjectRequestBuilder::ssekmsKeyId);
-
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
- .ifPresent(putObjectRequestBuilder::ssekmsEncryptionContext);
break;
case SSE_C:
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
@@ -507,15 +497,11 @@ private void multipartUploadEncryptionParameters(
// Set the KMS key if present, else S3 uses AWS managed key.
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(mpuRequestBuilder::ssekmsKeyId);
-
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
- .ifPresent(mpuRequestBuilder::ssekmsEncryptionContext);
break;
case DSSE_KMS:
mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(mpuRequestBuilder::ssekmsKeyId);
-
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
- .ifPresent(mpuRequestBuilder::ssekmsEncryptionContext);
break;
case SSE_C:
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java
deleted file mode 100644
index a720d2ca100..00000000000
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.impl;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.S3AUtils;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT;
-
-/**
- * Utility methods for S3A encryption properties.
- */
-public final class S3AEncryption {
-
- private static final Logger LOG =
LoggerFactory.getLogger(S3AEncryption.class);
-
- private S3AEncryption() {
- }
-
- /**
- * Get any SSE context from a configuration/credential provider.
- * @param bucket bucket to query for
- * @param conf configuration to examine
- * @return the encryption context value or ""
- * @throws IOException if reading a JCEKS file raised an IOE
- * @throws IllegalArgumentException bad arguments.
- */
- public static String getS3EncryptionContext(String bucket, Configuration
conf)
- throws IOException {
- // look up the per-bucket value of the encryption context
- String encryptionContext = S3AUtils.lookupBucketSecret(bucket, conf,
S3_ENCRYPTION_CONTEXT);
- if (encryptionContext == null) {
- // look up the global value of the encryption context
- encryptionContext = S3AUtils.lookupPassword(null, conf,
S3_ENCRYPTION_CONTEXT);
- }
- if (encryptionContext == null) {
- // no encryption context, return ""
- return "";
- }
- return encryptionContext;
- }
-
- /**
- * Get any SSE context from a configuration/credential provider.
- * This includes converting the values to a base64-encoded UTF-8 string
- * holding JSON with the encryption context key-value pairs
- * @param bucket bucket to query for
- * @param conf configuration to examine
- * @param propagateExceptions should IO exceptions be rethrown?
- * @return the Base64 encryption context or ""
- * @throws IllegalArgumentException bad arguments.
- * @throws IOException if propagateExceptions==true and reading a JCEKS file
raised an IOE
- */
- public static String getS3EncryptionContextBase64Encoded(
- String bucket,
- Configuration conf,
- boolean propagateExceptions) throws IOException {
- try {
- final String encryptionContextValue = getS3EncryptionContext(bucket,
conf);
- if (StringUtils.isBlank(encryptionContextValue)) {
- return "";
- }
- final Map<String, String> encryptionContextMap = S3AUtils
- .getTrimmedStringCollectionSplitByEquals(encryptionContextValue);
- if (encryptionContextMap.isEmpty()) {
- return "";
- }
- final String encryptionContextJson = new
ObjectMapper().writeValueAsString(
- encryptionContextMap);
- return
Base64.encodeBase64String(encryptionContextJson.getBytes(StandardCharsets.UTF_8));
- } catch (IOException e) {
- if (propagateExceptions) {
- throw e;
- }
- LOG.warn("Cannot retrieve {} for bucket {}",
- S3_ENCRYPTION_CONTEXT, bucket, e);
- return "";
- }
- }
-}
diff --git
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md
index 82e2460d85f..b219982db5c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md
@@ -243,21 +243,6 @@ The ID of the specific key used to encrypt the data should
also be set in the pr
</property>
```
-Optionally, you can specify the encryption context in the property
`fs.s3a.encryption.context`:
-
-```xml
-<property>
- <name>fs.s3a.encryption.context</name>
- <value>
- key1=value1,
- key2=value2,
- key3=value3,
- key4=value4,
- key5=value5
- </value>
-</property>
-```
-
Organizations may define a default key in the Amazon KMS; if a default key is
set,
then it will be used whenever SSE-KMS encryption is chosen and the value of
`fs.s3a.encryption.key` is empty.
@@ -393,21 +378,6 @@ The ID of the specific key used to encrypt the data should
also be set in the pr
</property>
```
-Optionally, you can specify the encryption context in the property
`fs.s3a.encryption.context`:
-
-```xml
-<property>
- <name>fs.s3a.encryption.context</name>
- <value>
- key1=value1,
- key2=value2,
- key3=value3,
- key4=value4,
- key5=value5
- </value>
-</property>
-```
-
Organizations may define a default key in the Amazon KMS; if a default key is
set,
then it will be used whenever SSE-KMS encryption is chosen and the value of
`fs.s3a.encryption.key` is empty.
diff --git
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index afed3397f56..01ce813cbec 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -1006,15 +1006,6 @@ Here are some the S3A properties for use in production.
</description>
</property>
-<property>
- <name>fs.s3a.encryption.context</name>
- <description>Specific encryption context to use if
fs.s3a.encryption.algorithm
- has been set to 'SSE-KMS' or 'DSSE-KMS'. The value of this property is a
set
- of non-secret comma-separated key-value pairs of additional contextual
- information about the data that are separated by equal operator (=).
- </description>
-</property>
-
<property>
<name>fs.s3a.signing-algorithm</name>
<description>Override the default signing algorithm so legacy
@@ -1693,11 +1684,6 @@ For a site configuration of:
<value>unset</value>
</property>
-<property>
- <name>fs.s3a.encryption.context</name>
- <value>unset</value>
-</property>
-
```
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
index 55cebeab8ef..3a3d82d94ff 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
@@ -30,7 +30,6 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
import static
org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
@@ -70,7 +69,6 @@ protected void patchConfigurationEncryptionSettings(
removeBaseAndBucketOverrides(conf,
S3_ENCRYPTION_ALGORITHM,
S3_ENCRYPTION_KEY,
- S3_ENCRYPTION_CONTEXT,
SERVER_SIDE_ENCRYPTION_ALGORITHM,
SERVER_SIDE_ENCRYPTION_KEY);
conf.set(S3_ENCRYPTION_ALGORITHM,
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java
deleted file mode 100644
index c3d4cd41fc5..00000000000
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Set;
-
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.impl.S3AEncryption;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
-import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS;
-import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_KMS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
-
-/**
- * Concrete class that extends {@link AbstractTestS3AEncryption}
- * and tests KMS encryption with encryption context.
- * S3's HeadObject doesn't return the object's encryption context.
- * Therefore, we don't have a way to assert its value in code.
- * In order to properly test if the encryption context is being set,
- * the KMS key or the IAM User need to have a deny statements like the one
below in the policy:
- * <pre>
- * {
- * "Effect": "Deny",
- * "Principal": {
- * "AWS": "*"
- * },
- * "Action": "kms:Decrypt",
- * "Resource": "*",
- * "Condition": {
- * "StringNotEquals": {
- * "kms:EncryptionContext:project": "hadoop"
- * }
- * }
- * }
- * </pre>
- * With the statement above, S3A will fail to read the object from S3 if it
was encrypted
- * without the key-pair <code>"project": "hadoop"</code> in the encryption
context.
- */
-public class ITestS3AEncryptionSSEKMSWithEncryptionContext
- extends AbstractTestS3AEncryption {
-
- private static final Set<S3AEncryptionMethods> KMS_ENCRYPTION_ALGORITHMS =
ImmutableSet.of(
- SSE_KMS, DSSE_KMS);
-
- private S3AEncryptionMethods encryptionAlgorithm;
-
- @Override
- protected Configuration createConfiguration() {
- try {
- // get the KMS key and context for this test.
- Configuration c = new Configuration();
- final String bucketName = getTestBucketName(c);
- String kmsKey = S3AUtils.getS3EncryptionKey(bucketName, c);
- String encryptionContext =
S3AEncryption.getS3EncryptionContext(bucketName, c);
- encryptionAlgorithm = S3AUtils.getEncryptionAlgorithm(bucketName, c);
- assume("Expected a KMS encryption algorithm",
- KMS_ENCRYPTION_ALGORITHMS.contains(encryptionAlgorithm));
- if (StringUtils.isBlank(encryptionContext)) {
- skip(S3_ENCRYPTION_CONTEXT + " is not set.");
- }
- Configuration conf = super.createConfiguration();
- S3ATestUtils.removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_KEY,
S3_ENCRYPTION_CONTEXT);
- conf.set(S3_ENCRYPTION_KEY, kmsKey);
- conf.set(S3_ENCRYPTION_CONTEXT, encryptionContext);
- return conf;
-
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- @Override
- protected S3AEncryptionMethods getSSEAlgorithm() {
- return encryptionAlgorithm;
- }
-}
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java
index dcda6815519..6985fa44c3b 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java
@@ -29,11 +29,9 @@
import org.junit.rules.Timeout;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.impl.S3AEncryption;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.*;
@@ -50,9 +48,6 @@ public class TestSSEConfiguration extends Assert {
/** Bucket to use for per-bucket options. */
public static final String BUCKET = "dataset-1";
- /** Valid set of key/value pairs for the encryption context. */
- private static final String VALID_ENCRYPTION_CONTEXT = "key1=value1,
key2=value2, key3=value3";
-
@Rule
public Timeout testTimeout = new Timeout(
S3ATestConstants.S3A_TEST_TIMEOUT
@@ -63,41 +58,41 @@ public class TestSSEConfiguration extends Assert {
@Test
public void testSSECNoKey() throws Throwable {
- assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), null, null);
+ assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), null);
}
@Test
public void testSSECBlankKey() throws Throwable {
- assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), "", null);
+ assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), "");
}
@Test
public void testSSECGoodKey() throws Throwable {
- assertEquals(SSE_C, getAlgorithm(SSE_C, "sseckey", null));
+ assertEquals(SSE_C, getAlgorithm(SSE_C, "sseckey"));
}
@Test
public void testKMSGoodKey() throws Throwable {
- assertEquals(SSE_KMS, getAlgorithm(SSE_KMS, "kmskey", null));
+ assertEquals(SSE_KMS, getAlgorithm(SSE_KMS, "kmskey"));
}
@Test
public void testAESKeySet() throws Throwable {
assertGetAlgorithmFails(SSE_S3_WITH_KEY_ERROR,
- SSE_S3.getMethod(), "setkey", null);
+ SSE_S3.getMethod(), "setkey");
}
@Test
public void testSSEEmptyKey() {
// test the internal logic of the test setup code
- Configuration c = buildConf(SSE_C.getMethod(), "", null);
+ Configuration c = buildConf(SSE_C.getMethod(), "");
assertEquals("", getS3EncryptionKey(BUCKET, c));
}
@Test
public void testSSEKeyNull() throws Throwable {
// test the internal logic of the test setup code
- final Configuration c = buildConf(SSE_C.getMethod(), null, null);
+ final Configuration c = buildConf(SSE_C.getMethod(), null);
assertEquals("", getS3EncryptionKey(BUCKET, c));
intercept(IOException.class, SSE_C_NO_KEY_ERROR,
@@ -152,30 +147,28 @@ void setProviderOption(final Configuration conf,
}
/**
- * Assert that the exception text from {@link #getAlgorithm(String, String,
String)}
+ * Assert that the exception text from {@link #getAlgorithm(String, String)}
* is as expected.
* @param expected expected substring in error
* @param alg algorithm to ask for
* @param key optional key value
- * @param context optional encryption context value
* @throws Exception anything else which gets raised
*/
public void assertGetAlgorithmFails(String expected,
- final String alg, final String key, final String context) throws
Exception {
+ final String alg, final String key) throws Exception {
intercept(IOException.class, expected,
- () -> getAlgorithm(alg, key, context));
+ () -> getAlgorithm(alg, key));
}
private S3AEncryptionMethods getAlgorithm(S3AEncryptionMethods algorithm,
- String key,
- String encryptionContext)
+ String key)
throws IOException {
- return getAlgorithm(algorithm.getMethod(), key, encryptionContext);
+ return getAlgorithm(algorithm.getMethod(), key);
}
- private S3AEncryptionMethods getAlgorithm(String algorithm, String key,
String encryptionContext)
+ private S3AEncryptionMethods getAlgorithm(String algorithm, String key)
throws IOException {
- return getEncryptionAlgorithm(BUCKET, buildConf(algorithm, key,
encryptionContext));
+ return getEncryptionAlgorithm(BUCKET, buildConf(algorithm, key));
}
/**
@@ -183,11 +176,10 @@ private S3AEncryptionMethods getAlgorithm(String
algorithm, String key, String e
* and key.
* @param algorithm algorithm to use, may be null
* @param key key, may be null
- * @param encryptionContext encryption context, may be null
* @return the new config.
*/
@SuppressWarnings("deprecation")
- private Configuration buildConf(String algorithm, String key, String
encryptionContext) {
+ private Configuration buildConf(String algorithm, String key) {
Configuration conf = emptyConf();
if (algorithm != null) {
conf.set(Constants.S3_ENCRYPTION_ALGORITHM, algorithm);
@@ -201,11 +193,6 @@ private Configuration buildConf(String algorithm, String
key, String encryptionC
conf.unset(SERVER_SIDE_ENCRYPTION_KEY);
conf.unset(Constants.S3_ENCRYPTION_KEY);
}
- if (encryptionContext != null) {
- conf.set(S3_ENCRYPTION_CONTEXT, encryptionContext);
- } else {
- conf.unset(S3_ENCRYPTION_CONTEXT);
- }
return conf;
}
@@ -321,30 +308,4 @@ public void testNoEncryptionMethod() throws Throwable {
assertEquals(NONE, getMethod(" "));
}
- @Test
- public void testGoodEncryptionContext() throws Throwable {
- assertEquals(SSE_KMS, getAlgorithm(SSE_KMS, "kmskey",
VALID_ENCRYPTION_CONTEXT));
- }
-
- @Test
- public void testSSEEmptyEncryptionContext() throws Throwable {
- // test the internal logic of the test setup code
- Configuration c = buildConf(SSE_KMS.getMethod(), "kmskey", "");
- assertEquals("", S3AEncryption.getS3EncryptionContext(BUCKET, c));
- }
-
- @Test
- public void testSSEEncryptionContextNull() throws Throwable {
- // test the internal logic of the test setup code
- final Configuration c = buildConf(SSE_KMS.getMethod(), "kmskey", null);
- assertEquals("", S3AEncryption.getS3EncryptionContext(BUCKET, c));
- }
-
- @Test
- public void testSSEInvalidEncryptionContext() throws Throwable {
- intercept(IllegalArgumentException.class,
- StringUtils.STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG,
- () -> getAlgorithm(SSE_KMS.getMethod(), "kmskey", "invalid context"));
- }
-
}
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java
index 71f22f4314f..b9d547635f7 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java
@@ -80,8 +80,7 @@ public void testRoundTripNoSessionData() throws Throwable {
public void testRoundTripEncryptionData() throws Throwable {
EncryptionSecrets secrets = new EncryptionSecrets(
S3AEncryptionMethods.SSE_KMS,
- "key",
- "encryptionContext");
+ "key");
EncryptionSecrets result = S3ATestUtils.roundTrip(secrets,
new Configuration());
assertEquals("round trip", secrets, result);
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
index b58ca24aaa8..efc77596685 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
@@ -116,7 +116,7 @@ public void testCanonicalization() throws Throwable {
public void testSaveLoadTokens() throws Throwable {
File tokenFile = File.createTempFile("token", "bin");
EncryptionSecrets encryptionSecrets = new EncryptionSecrets(
- S3AEncryptionMethods.SSE_KMS, KMS_KEY, "");
+ S3AEncryptionMethods.SSE_KMS, KMS_KEY);
Token<AbstractS3ATokenIdentifier> dt
= delegationTokens.createDelegationToken(encryptionSecrets, null);
final SessionTokenIdentifier origIdentifier
@@ -171,7 +171,7 @@ public void testCreateAndUseDT() throws Throwable {
assertNull("Current User has delegation token",
delegationTokens.selectTokenFromFSOwner());
EncryptionSecrets secrets = new EncryptionSecrets(
- S3AEncryptionMethods.SSE_KMS, KMS_KEY, "");
+ S3AEncryptionMethods.SSE_KMS, KMS_KEY);
Token<AbstractS3ATokenIdentifier> originalDT
= delegationTokens.createDelegationToken(secrets, null);
assertEquals("Token kind mismatch", getTokenKind(), originalDT.getKind());
@@ -229,7 +229,7 @@ public void testCreateWithRenewer() throws Throwable {
assertNull("Current User has delegation token",
delegationTokens.selectTokenFromFSOwner());
EncryptionSecrets secrets = new EncryptionSecrets(
- S3AEncryptionMethods.SSE_KMS, KMS_KEY, "");
+ S3AEncryptionMethods.SSE_KMS, KMS_KEY);
Token<AbstractS3ATokenIdentifier> dt
= delegationTokens.createDelegationToken(secrets, renewer);
assertEquals("Token kind mismatch", getTokenKind(), dt.getKind());
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java
index a06e9ac62ff..af306cc5a9a 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java
@@ -19,12 +19,10 @@
package org.apache.hadoop.fs.s3a.auth.delegation;
import java.net.URI;
-import java.nio.charset.StandardCharsets;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding;
@@ -72,17 +70,13 @@ public void testSessionTokenIssueDate() throws Throwable {
public void testSessionTokenDecode() throws Throwable {
Text alice = new Text("alice");
Text renewer = new Text("yarn");
- String encryptionKey = "encryptionKey";
- String encryptionContextJson = "{\"key\":\"value\", \"key2\": \"value3\"}";
- String encryptionContextEncoded =
Base64.encodeBase64String(encryptionContextJson.getBytes(
- StandardCharsets.UTF_8));
AbstractS3ATokenIdentifier identifier
= new SessionTokenIdentifier(SESSION_TOKEN_KIND,
alice,
renewer,
new URI("s3a://anything/"),
new MarshalledCredentials("a", "b", ""),
- new EncryptionSecrets(S3AEncryptionMethods.SSE_S3, encryptionKey,
encryptionContextEncoded),
+ new EncryptionSecrets(S3AEncryptionMethods.SSE_S3, ""),
"origin");
Token<AbstractS3ATokenIdentifier> t1 =
new Token<>(identifier,
@@ -106,10 +100,6 @@ public void testSessionTokenDecode() throws Throwable {
assertEquals("origin", decoded.getOrigin());
assertEquals("issue date", identifier.getIssueDate(),
decoded.getIssueDate());
- EncryptionSecrets encryptionSecrets = decoded.getEncryptionSecrets();
- assertEquals(S3AEncryptionMethods.SSE_S3,
encryptionSecrets.getEncryptionMethod());
- assertEquals(encryptionKey, encryptionSecrets.getEncryptionKey());
- assertEquals(encryptionContextEncoded,
encryptionSecrets.getEncryptionContext());
}
@Test
@@ -122,19 +112,13 @@ public void testFullTokenKind() throws Throwable {
@Test
public void testSessionTokenIdentifierRoundTrip() throws Throwable {
Text renewer = new Text("yarn");
- String encryptionKey = "encryptionKey";
- String encryptionContextJson = "{\"key\":\"value\", \"key2\": \"value3\"}";
- String encryptionContextEncoded =
Base64.encodeBase64String(encryptionContextJson.getBytes(
- StandardCharsets.UTF_8));
SessionTokenIdentifier id = new SessionTokenIdentifier(
SESSION_TOKEN_KIND,
new Text(),
renewer,
externalUri,
new MarshalledCredentials("a", "b", "c"),
- new EncryptionSecrets(S3AEncryptionMethods.DSSE_KMS, encryptionKey,
- encryptionContextEncoded),
- "");
+ new EncryptionSecrets(), "");
SessionTokenIdentifier result = S3ATestUtils.roundTrip(id, null);
String ids = id.toString();
@@ -143,10 +127,6 @@ public void testSessionTokenIdentifierRoundTrip() throws
Throwable {
id.getMarshalledCredentials(),
result.getMarshalledCredentials());
assertEquals("renewer in " + ids, renewer, id.getRenewer());
- EncryptionSecrets encryptionSecrets = result.getEncryptionSecrets();
- assertEquals(S3AEncryptionMethods.DSSE_KMS,
encryptionSecrets.getEncryptionMethod());
- assertEquals(encryptionKey, encryptionSecrets.getEncryptionKey());
- assertEquals(encryptionContextEncoded,
encryptionSecrets.getEncryptionContext());
}
@Test
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
index 706a0396d6c..c7771822aca 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
@@ -87,7 +87,7 @@ public void testRequestFactoryWithEncryption() throws
Throwable {
.withBucket("bucket")
.withEncryptionSecrets(
new EncryptionSecrets(S3AEncryptionMethods.SSE_KMS,
- "kms:key", ""))
+ "kms:key"))
.build();
createFactoryObjects(factory);
}
@@ -348,7 +348,7 @@ public void
testCompleteMultipartUploadRequestWithChecksumAlgorithmAndSSEC() thr
.encodeToString(encryptionKey);
final String encryptionKeyMd5 = Md5Utils.md5AsBase64(encryptionKey);
final EncryptionSecrets encryptionSecrets = new
EncryptionSecrets(S3AEncryptionMethods.SSE_C,
- encryptionKeyBase64, null);
+ encryptionKeyBase64);
RequestFactory factory = RequestFactoryImpl.builder()
.withBucket("bucket")
.withChecksumAlgorithm(ChecksumAlgorithm.CRC32_C)
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java
deleted file mode 100644
index a9d83819fda..00000000000
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.impl;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.conf.Configuration;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT;
-
-public class TestS3AEncryption {
-
- private static final String GLOBAL_CONTEXT = " project=hadoop,
jira=HADOOP-19197 ";
- private static final String BUCKET_CONTEXT = "component=fs/s3";
-
- @Test
- public void testGetS3EncryptionContextPerBucket() throws IOException {
- Configuration configuration = new Configuration(false);
- configuration.set("fs.s3a.bucket.bucket1.encryption.context",
BUCKET_CONTEXT);
- configuration.set(S3_ENCRYPTION_CONTEXT, GLOBAL_CONTEXT);
- final String result = S3AEncryption.getS3EncryptionContext("bucket1",
configuration);
- Assert.assertEquals(BUCKET_CONTEXT, result);
- }
-
- @Test
- public void testGetS3EncryptionContextFromGlobal() throws IOException {
- Configuration configuration = new Configuration(false);
- configuration.set("fs.s3a.bucket.bucket1.encryption.context",
BUCKET_CONTEXT);
- configuration.set(S3_ENCRYPTION_CONTEXT, GLOBAL_CONTEXT);
- final String result = S3AEncryption.getS3EncryptionContext("bucket2",
configuration);
- Assert.assertEquals(GLOBAL_CONTEXT.trim(), result);
- }
-
- @Test
- public void testGetS3EncryptionContextNoSet() throws IOException {
- Configuration configuration = new Configuration(false);
- final String result = S3AEncryption.getS3EncryptionContext("bucket1",
configuration);
- Assert.assertEquals("", result);
- }
-
- @Test
- public void testGetS3EncryptionContextBase64Encoded() throws IOException {
- Configuration configuration = new Configuration(false);
- configuration.set(S3_ENCRYPTION_CONTEXT, GLOBAL_CONTEXT);
- final String result =
S3AEncryption.getS3EncryptionContextBase64Encoded("bucket",
- configuration, true);
- final String decoded = new String(Base64.decodeBase64(result),
StandardCharsets.UTF_8);
- final TypeReference<Map<String, String>> typeRef = new
TypeReference<Map<String, String>>() {};
- final Map<String, String> resultMap = new
ObjectMapper().readValue(decoded, typeRef);
- Assert.assertEquals("hadoop", resultMap.get("project"));
- Assert.assertEquals("HADOOP-19197", resultMap.get("jira"));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
