[
https://issues.apache.org/jira/browse/HADOOP-17912?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17654791#comment-17654791
]
ASF GitHub Bot commented on HADOOP-17912:
-----------------------------------------
pranavsaxena-microsoft commented on code in PR #3440:
URL: https://github.com/apache/hadoop/pull/3440#discussion_r1062146937
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java:
##########
@@ -0,0 +1,330 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
+import org.apache.hadoop.fs.azurebfs.security.EncodingHelper;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
+import
org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
+import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider;
+import org.apache.hadoop.fs.azurebfs.extensions.MockEncryptionContextProvider;
+import org.apache.hadoop.fs.azurebfs.security.EncryptionAdapter;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
+import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
+import org.apache.hadoop.fs.azurebfs.utils.EncryptionType;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Lists;
+
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA;
+import static
org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_KEY_SHA256;
+import static
org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_SERVER_ENCRYPTED;
+import static
org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_SERVER_ENCRYPTED;
+import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ENCRYPTION_KEY_LEN;
+import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
+import static
org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters.Mode.APPEND_MODE;
+import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry;
+import static
org.apache.hadoop.fs.azurebfs.utils.EncryptionType.ENCRYPTION_CONTEXT;
+import static org.apache.hadoop.fs.azurebfs.utils.EncryptionType.GLOBAL_KEY;
+import static org.apache.hadoop.fs.azurebfs.utils.EncryptionType.NONE;
+import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
+import static org.apache.hadoop.fs.permission.AclEntryType.USER;
+import static org.apache.hadoop.fs.permission.FsAction.ALL;
+
+@RunWith(Parameterized.class)
+public class ITestAbfsCustomEncryption extends AbstractAbfsIntegrationTest {
+ private final byte[] cpk = new byte[ENCRYPTION_KEY_LEN];
+ private final String cpkSHAEncoded;
+
+ // Encryption type used by filesystem while creating file
+ @Parameterized.Parameter
+ public EncryptionType fileEncryptionType;
+
+ // Encryption type used by filesystem to call different operations
+ @Parameterized.Parameter(1)
+ public EncryptionType requestEncryptionType;
+
+ @Parameterized.Parameter(2)
+ public FSOperationType operation;
+
+ @Parameterized.Parameter(3)
+ public boolean responseHeaderServerEnc;
+
+ @Parameterized.Parameter(4)
+ public boolean responseHeaderReqServerEnc;
+
+ @Parameterized.Parameter(5)
+ public boolean isExceptionCase;
+
+ @Parameterized.Parameter(6)
+ public boolean isCpkResponseHdrExpected;
+
+
+ @Parameterized.Parameters(name = "{0} mode, {2}")
+ public static Iterable<Object[]> params() {
+ return Arrays.asList(new Object[][] {
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.READ, true,
false, false, true},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.WRITE, false,
true, false, true},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.APPEND,
false, true, false, true},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.SET_ACL,
false, false, false, false},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.GET_ATTR,
true, false, false, true},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.SET_ATTR,
false, true, false, true},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.LISTSTATUS,
false, false, false, false},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.RENAME,
false, false, false, false},
+ {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.DELETE,
false, false, false, false},
+
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.WRITE, false, false, true,
false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.GET_ATTR, true, false,
true, false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.READ, false, false, true,
false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.SET_ATTR, false, true,
true, false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.RENAME, false, false,
false, false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.LISTSTATUS, false, false,
false, false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.DELETE, false, false,
false, false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.SET_ACL, false, false,
false, false},
+ {ENCRYPTION_CONTEXT, NONE, FSOperationType.SET_PERMISSION, false,
false, false, false},
+
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.READ, true, false, false,
true},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.WRITE, false, true, false,
true},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.APPEND, false, true, false,
true},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.SET_ACL, false, false, false,
false},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.LISTSTATUS, false, false,
false, false},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.RENAME, false, false, false,
false},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.DELETE, false, false, false,
false},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.GET_ATTR, true, false, false,
true},
+ {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.SET_ATTR, false, true, false,
true},
+
+ {GLOBAL_KEY, NONE, FSOperationType.READ, true, false, true, true},
+ {GLOBAL_KEY, NONE, FSOperationType.WRITE, false, true, true, true},
+ {GLOBAL_KEY, NONE, FSOperationType.SET_ATTR, false, false, true, true},
+ {GLOBAL_KEY, NONE, FSOperationType.SET_ACL, false, false, false,
false},
+ {GLOBAL_KEY, NONE, FSOperationType.RENAME, false, false, false, false},
+ {GLOBAL_KEY, NONE, FSOperationType.LISTSTATUS, false, false, false,
false},
+ {GLOBAL_KEY, NONE, FSOperationType.DELETE, false, false, false, false},
+ {GLOBAL_KEY, NONE, FSOperationType.SET_PERMISSION, false, false,
false, false},
+ });
+ }
+
+ public ITestAbfsCustomEncryption() throws Exception {
+ super();
+ Assume.assumeTrue("Account should be HNS enabled for CPK",
+ getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT,
+ false));
+ new Random().nextBytes(cpk);
+ cpkSHAEncoded = EncodingHelper.getBase64EncodedString(
+ EncodingHelper.getSHA256Hash(cpk));
+ }
+
+ @Test
+ public void testCustomEncryptionCombinations() throws Exception {
+ AzureBlobFileSystem fs = getFS();
+ Path testPath = path("/testFile");
+ String relativePath = fs.getAbfsStore().getRelativePath(testPath);
+ MockEncryptionContextProvider ecp =
+ (MockEncryptionContextProvider) createEncryptedFile(testPath);
+ AbfsRestOperation op = callOperation(fs, new Path(relativePath), ecp);
+ if (op == null) {
+ return;
+ }
+ AbfsHttpOperation httpOp = op.getResult();
+ if (isCpkResponseHdrExpected) {
+ if (requestEncryptionType == ENCRYPTION_CONTEXT) {
+ String encryptionContext =
ecp.getEncryptionContextForTest(relativePath);
+ String expectedKeySHA = EncodingHelper.getBase64EncodedString(
+ EncodingHelper.getSHA256Hash(
+ ecp.getEncryptionKeyForTest(encryptionContext)));
+
Assertions.assertThat(httpOp.getResponseHeader(X_MS_ENCRYPTION_KEY_SHA256))
+ .isEqualTo(expectedKeySHA);
+ } else { // GLOBAL_KEY
+
Assertions.assertThat(httpOp.getResponseHeader(X_MS_ENCRYPTION_KEY_SHA256))
+ .isEqualTo(cpkSHAEncoded);
+ }
+ } else {
+
Assertions.assertThat(httpOp.getResponseHeader(X_MS_ENCRYPTION_KEY_SHA256))
+ .isEqualTo(null);
+ }
+ Assertions.assertThat(httpOp.getResponseHeader(X_MS_SERVER_ENCRYPTED))
+ .isEqualTo(responseHeaderServerEnc? "true" : null);
+
Assertions.assertThat(httpOp.getResponseHeader(X_MS_REQUEST_SERVER_ENCRYPTED))
+ .isEqualTo(responseHeaderReqServerEnc? "true" : null);
+ }
+
+ /**
+ * Executes a given operation at the AbfsClient level and returns
+ * AbfsRestOperation instance to verify response headers. Asserts excetion
+ * for combinations that should not succeed.
+ * @param fs AzureBlobFileSystem instance
+ * @param testPath path of file
+ * @param ecp EncryptionContextProvider instance to support AbfsClient
methods
+ * @return Rest op or null depending on whether the request is allowed
+ * @throws Exception error
+ */
+ private AbfsRestOperation callOperation(AzureBlobFileSystem fs,
+ Path testPath, EncryptionContextProvider ecp)
+ throws Exception {
+ AbfsClient client = fs.getAbfsClient();
+ client.setEncryptionContextProvider(ecp);
+ if (isExceptionCase) {
+ LambdaTestUtils.intercept(IOException.class, () -> {
+ switch (operation) {
+ case WRITE: try (FSDataOutputStream out = fs.append(testPath)) {
+ out.write("bytes".getBytes());
+ }
+ break;
+ case READ: try (FSDataInputStream in = fs.open(testPath)) {
+ in.read(new byte[5]);
+ }
+ break;
+ case SET_ATTR: fs.setXAttr(testPath, "attribute", "value".getBytes());
+ break;
+ case GET_ATTR: fs.getXAttr(testPath, "attribute");
Review Comment:
Added
> ABFS: Support for Encryption Context
> ------------------------------------
>
> Key: HADOOP-17912
> URL: https://issues.apache.org/jira/browse/HADOOP-17912
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.3.1
> Reporter: Sumangala Patki
> Assignee: Pranav Saxena
> Priority: Major
> Labels: pull-request-available
> Time Spent: 1h
> Remaining Estimate: 0h
>
> Support for customer-provided encryption keys at the file level, superceding
> the global (account-level) key use in HADOOP-17536.
> ABFS driver will support an "EncryptionContext" plugin for retrieving
> encryption information, the implementation for which should be provided by
> the client. The keys/context retrieved will be sent via request headers to
> the server, which will store the encryption context. Subsequent REST calls to
> server that access data/user metadata of the file will require fetching the
> encryption context through a GetFileProperties call and retrieving the key
> from the custom provider, before sending the request.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
