[
https://issues.apache.org/jira/browse/HADOOP-19197?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18077165#comment-18077165
]
ASF GitHub Bot commented on HADOOP-19197:
-----------------------------------------
ajfabbri commented on code in PR #8185:
URL: https://github.com/apache/hadoop/pull/8185#discussion_r3163663840
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java:
##########
@@ -134,4 +167,135 @@ public void testCredentialProviderNullURI() throws
Throwable {
credentials,
MarshalledCredentials.CredentialTypeRequired.FullOnly));
}
+
+ @Test
+ public void testUnmarshallOldEncryptionSecrets() throws Throwable {
+ try (InputStream is = getClass()
+ .getResourceAsStream("/serialized-objects/EncryptionSecrets-v1.ser");
+ ObjectInputStream ois = new ObjectInputStream(is)) {
+ EncryptionSecrets secrets = new EncryptionSecrets();
+ secrets.readFields(ois);
+ Assertions.assertThat(secrets.getEncryptionAlgorithm())
+ .describedAs("encryptionAlgorithm")
+ .isEqualTo(S3AEncryptionMethods.SSE_KMS.getMethod());
+ Assertions.assertThat(secrets .getEncryptionKey())
+ .describedAs("encryptionKey")
+ .isEqualTo(KEY);
+ Assertions.assertThat(secrets .getEncryptionContext())
+ .describedAs("encryptionContext")
+ .isEqualTo(DEFAULT_S3_ENCRYPTION_CONTEXT);
+ }
+ }
+
+ @Test
+ public void testUnmarshallUnknownEncryptionSecretsVersion() throws Throwable
{
+ try (InputStream is = getClass()
+
.getResourceAsStream("/serialized-objects/EncryptionSecrets-unknown-version.ser");
+ ObjectInputStream ois = new ObjectInputStream(is)) {
+ EncryptionSecrets secrets = new EncryptionSecrets();
+ Assertions.assertThatExceptionOfType(DelegationTokenIOException.class)
+ .isThrownBy(() -> secrets.readFields(ois))
+ .withMessageStartingWith("Incompatible EncryptionSecrets version");
+ }
+ }
+
+ /**
+ * Generate the equivalent to a marshalled EncryptionSecrets value.
+ * @param id serialization ID.
+ * @param encryptionAlgorithm algorithm.
+ * @param encryptionKey key
+ * @param encryptionContext optional context
+ * @return the input
+ * @throws IOException write failure.
+ */
+ private DataInputBuffer writeEncryptionSecrets(long id,
+ final String encryptionAlgorithm,
+ final String encryptionKey,
+ final Optional<String> encryptionContext) throws IOException {
+ DataOutputBuffer out = new DataOutputBuffer();
+ new LongWritable(id).write(out);
+ Text.writeString(out, encryptionAlgorithm);
+ Text.writeString(out, encryptionKey);
+ if (encryptionContext.isPresent()) {
+ Text.writeString(out, encryptionContext.get());
+ }
+
+ DataInputBuffer dib = new DataInputBuffer();
+ dib.reset(out.getData(), out.getLength());
+ return dib;
+ }
+
+ private EncryptionSecrets readEncryptionSecrets(DataInputBuffer dib) throws
IOException {
+ final EncryptionSecrets secrets = new EncryptionSecrets();
+ secrets.readFields(dib);
+ return secrets;
+ }
+
+ private static final String ENCRYPTION_ALGORITHM = SSE_S3.getMethod();
+
+ private static final String KEY = "key";
+
+ private static final String CONTEXT = "context";
+
+ /**
+ * Verify that the low level marshalling code works.
+ */
+ @Test
+ public void testMarshallCurrentSecrets() throws Throwable {
+ EncryptionSecrets src = new EncryptionSecrets(ENCRYPTION_ALGORITHM,
+ KEY,
+ CONTEXT);
+ final DataInputBuffer in =
+ writeEncryptionSecrets(EncryptionSecrets.SERIAL_VERSION_UID_CURRENT,
+ ENCRYPTION_ALGORITHM, KEY, Optional.of(CONTEXT));
+ final EncryptionSecrets read = readEncryptionSecrets(in);
+ Assertions.assertThat(read)
+ .isEqualTo(src);
+ }
+
+ /**
+ * Generate the layout of an old secret entry, unmarshall it to the new one.
+ */
+ @Test
+ public void testUnmarshallOldSecrets() throws Throwable {
+ final DataInputBuffer dib =
writeEncryptionSecrets(EncryptionSecrets.SERIAL_VERSION_UID_1,
+ ENCRYPTION_ALGORITHM, KEY, Optional.empty());
+ final EncryptionSecrets read = readEncryptionSecrets(dib);
+
+ // all the data has been read in
+ Assertions.assertThat(dib.read())
+ .describedAs("Input stream read() at end of unmarshalling")
+ .isEqualTo(-1);
+ Assertions.assertThat(read)
+ .matches(s -> !s.hasEncryptionContext())
+ .hasFieldOrPropertyWithValue("encryptionAlgorithm",
ENCRYPTION_ALGORITHM)
+ .hasFieldOrPropertyWithValue("getEncryptionKey", KEY);
+ }
+
+ /**
+ * Generate the layout of an old secret entry, unmarshall it to the new one.
+ */
Review Comment:
small nit: remove or update this comment.
> S3A: Support AWS KMS Encryption Context
> ---------------------------------------
>
> Key: HADOOP-19197
> URL: https://issues.apache.org/jira/browse/HADOOP-19197
> Project: Hadoop Common
> Issue Type: New Feature
> Components: fs/s3
> Affects Versions: 3.4.0
> Reporter: Raphael Azzolini
> Assignee: Raphael Azzolini
> Priority: Critical
> Labels: pull-request-available
>
> S3A properties allow users to choose the AWS KMS key
> ({_}fs.s3a.encryption.key{_}) and S3 encryption algorithm to be used
> (f{_}s.s3a.encryption.algorithm{_}). In addition to the AWS KMS Key, an
> encryption context can be used as non-secret data that adds additional
> integrity and authenticity to check the encrypted data. However, there is no
> option to specify the [AWS KMS Encryption
> Context|https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context]
> in S3A.
> In AWS SDK v2 the encryption context in S3 requests is set by the parameter
> [ssekmsEncryptionContext.|https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/model/CreateMultipartUploadRequest.Builder.html#ssekmsEncryptionContext(java.lang.String)]
> It receives a base64-encoded UTF-8 string holding JSON with the encryption
> context key-value pairs. The value of this parameter could be set by the user
> in a new property {_}*fs.s3a.encryption.context*{_}, and be stored in the
> [EncryptionSecrets|https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java]
> to later be used when setting the encryption parameters in
> [RequestFactoryImpl|https://github.com/apache/hadoop/blob/f92a8ab8ae54f11946412904973eb60404dee7ff/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java].
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
