This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a2610e2 HADOOP-17183. ABFS: Enabling checkaccess on ABFS
a2610e2 is described below
commit a2610e21ed5289323d8a6f6359477a8ceb2db2eb
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Fri Aug 7 03:22:02 2020 +0530
HADOOP-17183. ABFS: Enabling checkaccess on ABFS
- Contributed by Bilahari T H
---
.../constants/FileSystemConfigurations.java | 2 +-
.../ITestAzureBlobFileSystemCheckAccess.java | 110 +++++++++++++--------
2 files changed, 72 insertions(+), 40 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index 260c496..f70d102 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -84,7 +84,7 @@ public final class FileSystemConfigurations {
public static final boolean DEFAULT_ENABLE_HTTPS = true;
public static final boolean DEFAULT_USE_UPN = false;
- public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = false;
+ public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = true;
public static final boolean DEFAULT_ABFS_LATENCY_TRACK = false;
public static final long
DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS = 120;
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
index 4189d66..45a3cbd 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
@@ -17,16 +17,18 @@
*/
package org.apache.hadoop.fs.azurebfs;
-import com.google.common.collect.Lists;
-
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.List;
+import com.google.common.collect.Lists;
import org.junit.Assume;
import org.junit.Test;
+import org.mockito.Mockito;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers;
@@ -37,6 +39,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.security.AccessControlException;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET;
@@ -44,9 +47,19 @@ import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_A
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_ID;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_SECRET;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test cases for AzureBlobFileSystem.access()
+ *
+ * Some of the tests in this class requires the following 3 configs set in the
+ * test config file.
+ * fs.azure.account.test.oauth2.client.id
+ * fs.azure.account.test.oauth2.client.secret
+ * fs.azure.check.access.testuser.guid
+ * Set the above client id, secret and guid of a service principal which has no
+ * RBAC on the account.
+ *
*/
public class ITestAzureBlobFileSystemCheckAccess
extends AbstractAbfsIntegrationTest {
@@ -66,31 +79,29 @@ public class ITestAzureBlobFileSystemCheckAccess
this.isCheckAccessEnabled = getConfiguration().isCheckAccessEnabled();
this.isHNSEnabled = getConfiguration()
.getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
+ setTestUserFs();
}
private void setTestUserFs() throws Exception {
if (this.testUserFs != null) {
return;
}
- String orgClientId = getConfiguration().get(FS_AZURE_BLOB_FS_CLIENT_ID);
- String orgClientSecret = getConfiguration()
- .get(FS_AZURE_BLOB_FS_CLIENT_SECRET);
- Boolean orgCreateFileSystemDurungInit = getConfiguration()
- .getBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
true);
- getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_ID,
- getConfiguration().get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID));
- getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_SECRET,
getConfiguration()
- .get(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET));
+ final String testClientIdConfKey =
+ FS_AZURE_BLOB_FS_CLIENT_ID + "." + getAccountName();
+ final String testClientId = getConfiguration()
+ .getString(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID, "");
+ getRawConfiguration().set(testClientIdConfKey, testClientId);
+ final String clientSecretConfKey =
+ FS_AZURE_BLOB_FS_CLIENT_SECRET + "." + getAccountName();
+ final String testClientSecret = getConfiguration()
+ .getString(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET, "");
+ getRawConfiguration().set(clientSecretConfKey, testClientSecret);
getRawConfiguration()
.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
false);
- FileSystem fs = FileSystem.newInstance(getRawConfiguration());
- getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_ID, orgClientId);
- getRawConfiguration().set(FS_AZURE_BLOB_FS_CLIENT_SECRET, orgClientSecret);
- getRawConfiguration()
- .setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
- orgCreateFileSystemDurungInit);
- this.testUserFs = fs;
+ getRawConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME,
+ AuthType.OAuth.name());
+ this.testUserFs = FileSystem.newInstance(getRawConfiguration());
}
@Test(expected = IllegalArgumentException.class)
@@ -100,15 +111,15 @@ public class ITestAzureBlobFileSystemCheckAccess
@Test(expected = NullPointerException.class)
public void testCheckAccessForFileWithNullFsAction() throws Exception {
- assumeHNSAndCheckAccessEnabled();
+ Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false",
+ isHNSEnabled);
// NPE when trying to convert null FsAction enum
superUserFs.access(new Path("test.txt"), null);
}
@Test(expected = FileNotFoundException.class)
public void testCheckAccessForNonExistentFile() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path nonExistentFile = setupTestDirectoryAndUserAccess(
"/nonExistentFile1.txt", FsAction.ALL);
superUserFs.delete(nonExistentFile, true);
@@ -153,15 +164,36 @@ public class ITestAzureBlobFileSystemCheckAccess
getConfiguration()
.getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true));
Assume.assumeTrue(FS_AZURE_ENABLE_CHECK_ACCESS + " is false",
- isCheckAccessEnabled);
- setTestUserFs();
+ isCheckAccessEnabled);
+ checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID);
+ checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET);
+ checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID);
+
+ // When the driver does not know if the account is HNS enabled or not it
+ // makes a server call and fails
+ intercept(AccessControlException.class,
+ "\"This request is not authorized to perform this operation using "
+ + "this permission.\", 403",
+ () -> testUserFs.access(new Path("/"), FsAction.READ));
+
+ // When the driver has already determined if the account is HNS enabled
+ // or not, and as the account is non HNS the AzureBlobFileSystem#access
+ // acts as noop
+ AzureBlobFileSystemStore mockAbfsStore =
+ Mockito.mock(AzureBlobFileSystemStore.class);
+ Mockito.when(mockAbfsStore.getIsNamespaceEnabled()).thenReturn(true);
+ Field abfsStoreField = AzureBlobFileSystem.class.getDeclaredField(
+ "abfsStore");
+ abfsStoreField.setAccessible(true);
+ abfsStoreField.set(testUserFs, mockAbfsStore);
testUserFs.access(new Path("/"), FsAction.READ);
+
+ superUserFs.access(new Path("/"), FsAction.READ);
}
@Test
public void testFsActionNONE() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path testFilePath = setupTestDirectoryAndUserAccess("/test2.txt",
FsAction.NONE);
assertInaccessible(testFilePath, FsAction.EXECUTE);
@@ -175,8 +207,7 @@ public class ITestAzureBlobFileSystemCheckAccess
@Test
public void testFsActionEXECUTE() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path testFilePath = setupTestDirectoryAndUserAccess("/test3.txt",
FsAction.EXECUTE);
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -191,8 +222,7 @@ public class ITestAzureBlobFileSystemCheckAccess
@Test
public void testFsActionREAD() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path testFilePath = setupTestDirectoryAndUserAccess("/test4.txt",
FsAction.READ);
assertAccessible(testFilePath, FsAction.READ);
@@ -207,8 +237,7 @@ public class ITestAzureBlobFileSystemCheckAccess
@Test
public void testFsActionWRITE() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path testFilePath = setupTestDirectoryAndUserAccess("/test5.txt",
FsAction.WRITE);
assertAccessible(testFilePath, FsAction.WRITE);
@@ -223,8 +252,7 @@ public class ITestAzureBlobFileSystemCheckAccess
@Test
public void testFsActionREADEXECUTE() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path testFilePath = setupTestDirectoryAndUserAccess("/test6.txt",
FsAction.READ_EXECUTE);
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -239,8 +267,7 @@ public class ITestAzureBlobFileSystemCheckAccess
@Test
public void testFsActionWRITEEXECUTE() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path testFilePath = setupTestDirectoryAndUserAccess("/test7.txt",
FsAction.WRITE_EXECUTE);
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -255,8 +282,7 @@ public class ITestAzureBlobFileSystemCheckAccess
@Test
public void testFsActionALL() throws Exception {
- assumeHNSAndCheckAccessEnabled();
- setTestUserFs();
+ checkPrerequisites();
Path testFilePath = setupTestDirectoryAndUserAccess("/test8.txt",
FsAction.ALL);
assertAccessible(testFilePath, FsAction.EXECUTE);
@@ -268,13 +294,19 @@ public class ITestAzureBlobFileSystemCheckAccess
assertAccessible(testFilePath, FsAction.ALL);
}
- private void assumeHNSAndCheckAccessEnabled() {
+ private void checkPrerequisites() {
Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false",
isHNSEnabled);
Assume.assumeTrue(FS_AZURE_ENABLE_CHECK_ACCESS + " is false",
isCheckAccessEnabled);
+ checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID);
+ checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET);
+ checkIfConfigIsSet(FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID);
+ }
-
Assume.assumeNotNull(getRawConfiguration().get(FS_AZURE_BLOB_FS_CLIENT_ID));
+ private void checkIfConfigIsSet(String configKey){
+ AbfsConfiguration conf = getConfiguration();
+ Assume.assumeNotNull(configKey + " config missing", conf.get(configKey));
}
private void assertAccessible(Path testFilePath, FsAction fsAction)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
