[ https://issues.apache.org/jira/browse/HADOOP-17912?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17654790#comment-17654790 ]
ASF GitHub Bot commented on HADOOP-17912: ----------------------------------------- pranavsaxena-microsoft commented on code in PR #3440: URL: https://github.com/apache/hadoop/pull/3440#discussion_r1062146779 ########## hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java: ########## @@ -0,0 +1,330 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; +import org.apache.hadoop.fs.azurebfs.security.EncodingHelper; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.assertj.core.api.Assertions; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; +import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters; +import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider; +import org.apache.hadoop.fs.azurebfs.extensions.MockEncryptionContextProvider; +import org.apache.hadoop.fs.azurebfs.security.EncryptionAdapter; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; +import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import org.apache.hadoop.fs.azurebfs.utils.EncryptionType; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Lists; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_KEY_SHA256; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_SERVER_ENCRYPTED; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_SERVER_ENCRYPTED; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ENCRYPTION_KEY_LEN; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters.Mode.APPEND_MODE; +import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry; +import static org.apache.hadoop.fs.azurebfs.utils.EncryptionType.ENCRYPTION_CONTEXT; +import static org.apache.hadoop.fs.azurebfs.utils.EncryptionType.GLOBAL_KEY; +import static org.apache.hadoop.fs.azurebfs.utils.EncryptionType.NONE; +import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; +import static org.apache.hadoop.fs.permission.AclEntryType.USER; +import static org.apache.hadoop.fs.permission.FsAction.ALL; + +@RunWith(Parameterized.class) +public class ITestAbfsCustomEncryption extends AbstractAbfsIntegrationTest { + private final byte[] cpk = new byte[ENCRYPTION_KEY_LEN]; + private final String cpkSHAEncoded; + + // Encryption type used by filesystem while creating file + @Parameterized.Parameter + public EncryptionType fileEncryptionType; + + // Encryption type used by filesystem to call different operations + @Parameterized.Parameter(1) + public EncryptionType requestEncryptionType; + + @Parameterized.Parameter(2) + public FSOperationType operation; + + @Parameterized.Parameter(3) + public boolean responseHeaderServerEnc; + + @Parameterized.Parameter(4) + public boolean responseHeaderReqServerEnc; + + @Parameterized.Parameter(5) + public boolean isExceptionCase; + + @Parameterized.Parameter(6) + public boolean isCpkResponseHdrExpected; + + + @Parameterized.Parameters(name = "{0} mode, {2}") + public static Iterable<Object[]> params() { + return Arrays.asList(new Object[][] { + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.READ, true, false, false, true}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.WRITE, false, true, false, true}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.APPEND, false, true, false, true}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.SET_ACL, false, false, false, false}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.GET_ATTR, true, false, false, true}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.SET_ATTR, false, true, false, true}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.LISTSTATUS, false, false, false, false}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.RENAME, false, false, false, false}, + {ENCRYPTION_CONTEXT, ENCRYPTION_CONTEXT, FSOperationType.DELETE, false, false, false, false}, + + {ENCRYPTION_CONTEXT, NONE, FSOperationType.WRITE, false, false, true, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.GET_ATTR, true, false, true, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.READ, false, false, true, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.SET_ATTR, false, true, true, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.RENAME, false, false, false, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.LISTSTATUS, false, false, false, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.DELETE, false, false, false, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.SET_ACL, false, false, false, false}, + {ENCRYPTION_CONTEXT, NONE, FSOperationType.SET_PERMISSION, false, false, false, false}, + + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.READ, true, false, false, true}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.WRITE, false, true, false, true}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.APPEND, false, true, false, true}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.SET_ACL, false, false, false, false}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.LISTSTATUS, false, false, false, false}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.RENAME, false, false, false, false}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.DELETE, false, false, false, false}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.GET_ATTR, true, false, false, true}, + {GLOBAL_KEY, GLOBAL_KEY, FSOperationType.SET_ATTR, false, true, false, true}, + + {GLOBAL_KEY, NONE, FSOperationType.READ, true, false, true, true}, + {GLOBAL_KEY, NONE, FSOperationType.WRITE, false, true, true, true}, + {GLOBAL_KEY, NONE, FSOperationType.SET_ATTR, false, false, true, true}, + {GLOBAL_KEY, NONE, FSOperationType.SET_ACL, false, false, false, false}, + {GLOBAL_KEY, NONE, FSOperationType.RENAME, false, false, false, false}, + {GLOBAL_KEY, NONE, FSOperationType.LISTSTATUS, false, false, false, false}, + {GLOBAL_KEY, NONE, FSOperationType.DELETE, false, false, false, false}, + {GLOBAL_KEY, NONE, FSOperationType.SET_PERMISSION, false, false, false, false}, + }); + } + + public ITestAbfsCustomEncryption() throws Exception { + super(); Review Comment: Removed. > ABFS: Support for Encryption Context > ------------------------------------ > > Key: HADOOP-17912 > URL: https://issues.apache.org/jira/browse/HADOOP-17912 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/azure > Affects Versions: 3.3.1 > Reporter: Sumangala Patki > Assignee: Pranav Saxena > Priority: Major > Labels: pull-request-available > Time Spent: 1h > Remaining Estimate: 0h > > Support for customer-provided encryption keys at the file level, superceding > the global (account-level) key use in HADOOP-17536. > ABFS driver will support an "EncryptionContext" plugin for retrieving > encryption information, the implementation for which should be provided by > the client. The keys/context retrieved will be sent via request headers to > the server, which will store the encryption context. Subsequent REST calls to > server that access data/user metadata of the file will require fetching the > encryption context through a GetFileProperties call and retrieving the key > from the custom provider, before sending the request. -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org