HADOOP-14416. Path starting with 'wasb:///' not resolved correctly while authorizing with WASB-Ranger. Contributed by Sivaguru Sankaridurg
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b415c6fe Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b415c6fe Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b415c6fe Branch: refs/heads/YARN-5734 Commit: b415c6fe743242acf1d1d3eb7ea7091d90d2c0d4 Parents: 8236130 Author: Mingliang Liu <lium...@apache.org> Authored: Tue May 16 11:22:11 2017 -0700 Committer: Mingliang Liu <lium...@apache.org> Committed: Tue May 16 11:22:32 2017 -0700 ---------------------------------------------------------------------- .../hadoop/fs/azure/NativeAzureFileSystem.java | 47 ++--- .../hadoop/fs/azure/AbstractWasbTestBase.java | 7 +- .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 11 ++ .../TestNativeAzureFileSystemAuthorization.java | 170 ++++++++----------- .../fs/azure/TestWasbRemoteCallHelper.java | 16 +- 5 files changed, 116 insertions(+), 135 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 03a2782..0ba47ef 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -1433,13 +1433,19 @@ public class NativeAzureFileSystem extends FileSystem { * @param operation - A string describing the operation being performed ("delete", "create" etc.). * @param originalPath - The originalPath that was being accessed */ - private void performAuthCheck(String requestingAccessForPath, WasbAuthorizationOperations accessType, - String operation, String originalPath) throws WasbAuthorizationException, IOException { + private void performAuthCheck(Path requestingAccessForPath, WasbAuthorizationOperations accessType, + String operation, Path originalPath) throws WasbAuthorizationException, IOException { + + if (azureAuthorization && this.authorizer != null) { + + requestingAccessForPath = requestingAccessForPath.makeQualified(getUri(), getWorkingDirectory()); + originalPath = originalPath.makeQualified(getUri(), getWorkingDirectory()); + + if (!this.authorizer.authorize(requestingAccessForPath.toString(), accessType.toString())) { + throw new WasbAuthorizationException(operation + + " operation for Path : " + originalPath.toString() + " not allowed"); + } - if (azureAuthorization && this.authorizer != null && - !this.authorizer.authorize(requestingAccessForPath, accessType.toString())) { - throw new WasbAuthorizationException(operation - + " operation for Path : " + originalPath + " not allowed"); } } @@ -1466,7 +1472,7 @@ public class NativeAzureFileSystem extends FileSystem { Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "append", absolutePath.toString()); + performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "append", absolutePath); String key = pathToKey(absolutePath); FileMetadata meta = null; @@ -1671,7 +1677,7 @@ public class NativeAzureFileSystem extends FileSystem { Path absolutePath = makeAbsolute(f); Path ancestor = getAncestor(absolutePath); - performAuthCheck(ancestor.toString(), WasbAuthorizationOperations.WRITE, "create", absolutePath.toString()); + performAuthCheck(ancestor, WasbAuthorizationOperations.WRITE, "create", absolutePath); String key = pathToKey(absolutePath); @@ -1685,7 +1691,7 @@ public class NativeAzureFileSystem extends FileSystem { throw new FileAlreadyExistsException("File already exists:" + f); } else { - performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "create", absolutePath.toString()); + performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "create", absolutePath); } } @@ -1800,7 +1806,7 @@ public class NativeAzureFileSystem extends FileSystem { Path absolutePath = makeAbsolute(f); Path parentPath = absolutePath.getParent(); - performAuthCheck(parentPath.toString(), WasbAuthorizationOperations.WRITE, "delete", absolutePath.toString()); + performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath); String key = pathToKey(absolutePath); @@ -2002,14 +2008,12 @@ public class NativeAzureFileSystem extends FileSystem { // NOTE: Ideally the subtree needs read-write-execute access check. // But we will simplify it to write-access check. if (metaFile.isDir()) { // the absolute-path - performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.WRITE, "delete", - absolutePath.toString()); + performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath); } for (FileMetadata meta : contents) { if (meta.isDir()) { Path subTreeDir = keyToPath(meta.getKey()); - performAuthCheck(subTreeDir.toString(), WasbAuthorizationOperations.WRITE, "delete", - absolutePath.toString()); + performAuthCheck(subTreeDir, WasbAuthorizationOperations.WRITE, "delete", absolutePath); } } } @@ -2090,8 +2094,7 @@ public class NativeAzureFileSystem extends FileSystem { // Capture the absolute path and the path to key. Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "getFileStatus", - absolutePath.toString()); + performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "getFileStatus", absolutePath); String key = pathToKey(absolutePath); if (key.length() == 0) { // root always exists @@ -2192,7 +2195,7 @@ public class NativeAzureFileSystem extends FileSystem { Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "liststatus", absolutePath.toString()); + performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "liststatus", absolutePath); String key = pathToKey(absolutePath); Set<FileStatus> status = new TreeSet<FileStatus>(); @@ -2436,7 +2439,7 @@ public class NativeAzureFileSystem extends FileSystem { Path absolutePath = makeAbsolute(f); Path ancestor = getAncestor(absolutePath); - performAuthCheck(ancestor.toString(), WasbAuthorizationOperations.WRITE, "mkdirs", absolutePath.toString()); + performAuthCheck(ancestor, WasbAuthorizationOperations.WRITE, "mkdirs", absolutePath); PermissionStatus permissionStatus = null; if(noUmask) { @@ -2482,7 +2485,7 @@ public class NativeAzureFileSystem extends FileSystem { Path absolutePath = makeAbsolute(f); - performAuthCheck(absolutePath.toString(), WasbAuthorizationOperations.READ, "read", absolutePath.toString()); + performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "read", absolutePath); String key = pathToKey(absolutePath); FileMetadata meta = null; @@ -2548,8 +2551,7 @@ public class NativeAzureFileSystem extends FileSystem { return false; } - performAuthCheck(srcParentFolder.toString(), WasbAuthorizationOperations.WRITE, "rename", - absoluteSrcPath.toString()); + performAuthCheck(srcParentFolder, WasbAuthorizationOperations.WRITE, "rename", absoluteSrcPath); String srcKey = pathToKey(absoluteSrcPath); @@ -2562,8 +2564,7 @@ public class NativeAzureFileSystem extends FileSystem { Path absoluteDstPath = makeAbsolute(dst); Path dstParentFolder = absoluteDstPath.getParent(); - performAuthCheck(dstParentFolder.toString(), WasbAuthorizationOperations.WRITE, "rename", - absoluteDstPath.toString()); + performAuthCheck(dstParentFolder, WasbAuthorizationOperations.WRITE, "rename", absoluteDstPath); String dstKey = pathToKey(absoluteDstPath); FileMetadata dstMetadata = null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java index 58d278a..6ae18fe 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java @@ -20,11 +20,9 @@ package org.apache.hadoop.fs.azure; import static org.junit.Assume.assumeNotNull; -import org.apache.hadoop.fs.FileSystem; - +import com.google.common.annotations.VisibleForTesting; import org.junit.After; import org.junit.Before; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +38,8 @@ public abstract class AbstractWasbTestBase { protected static final Logger LOG = LoggerFactory.getLogger(AbstractWasbTestBase.class); - protected FileSystem fs; + @VisibleForTesting + protected NativeAzureFileSystem fs; private AzureBlobStorageTestAccount testAccount; @Before http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java index 445bfd8..0b3422c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; /** * A mock wasb authorizer implementation. @@ -32,6 +33,14 @@ public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface { private Map<AuthorizationComponent, Boolean> authRules; + // The full qualified URL to the root directory + private String qualifiedPrefixUrl; + + public MockWasbAuthorizerImpl(NativeAzureFileSystem fs) { + qualifiedPrefixUrl = new Path("/").makeQualified(fs.getUri(), fs.getWorkingDirectory()) + .toString().replaceAll("/$", ""); + } + @Override public void init(Configuration conf) { authRules = new HashMap<AuthorizationComponent, Boolean>(); @@ -40,6 +49,8 @@ public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface { public void addAuthRule(String wasbAbsolutePath, String accessType, boolean access) { + wasbAbsolutePath = qualifiedPrefixUrl + wasbAbsolutePath; + AuthorizationComponent component = wasbAbsolutePath.endsWith("*") ? new AuthorizationComponent("^" + wasbAbsolutePath.replace("*", ".*"), accessType) : new AuthorizationComponent(wasbAbsolutePath, accessType); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java index 4e49622..f7a2eb7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java @@ -78,6 +78,15 @@ public class TestNativeAzureFileSystemAuthorization } /** + * Setup the expected exception class, and exception message that the test is supposed to fail with + */ + private void setExpectedFailureMessage(String operation, Path path) { + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage(String.format("%s operation for Path : %s not allowed", + operation, path.makeQualified(fs.getUri(), fs.getWorkingDirectory()))); + } + + /** * Positive test to verify Create access check * The file is created directly under an existing folder. * No intermediate folders need to be created. @@ -86,13 +95,10 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testCreateAccessWithoutCreateIntermediateFoldersCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -117,13 +123,10 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testCreateAccessWithCreateIntermediateFoldersCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/testCreateAccessCheckPositive/1/2/3"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -148,16 +151,12 @@ public class TestNativeAzureFileSystemAuthorization @Test // (expected=WasbAuthorizationException.class) public void testCreateAccessWithOverwriteCheckNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("create operation for Path : /test.dat not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("create", testPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -184,13 +183,10 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testCreateAccessWithOverwriteCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -218,16 +214,12 @@ public class TestNativeAzureFileSystemAuthorization @Test // (expected=WasbAuthorizationException.class) public void testCreateAccessCheckNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("create operation for Path : /testCreateAccessCheckNegative/test.dat not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/testCreateAccessCheckNegative"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("create", testPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false); fs.updateWasbAuthorizer(authorizer); @@ -249,14 +241,11 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testListAccessCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/testListAccessCheckPositive"); Path intermediateFolders = new Path(parentDir, "1/2/3/"); Path testPath = new Path(intermediateFolders, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -280,16 +269,12 @@ public class TestNativeAzureFileSystemAuthorization @Test //(expected=WasbAuthorizationException.class) public void testListAccessCheckNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("liststatus operation for Path : /testListAccessCheckNegative/test.dat not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/testListAccessCheckNegative"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("liststatus", testPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false); @@ -312,14 +297,11 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testRenameAccessCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/testRenameAccessCheckPositive"); Path srcPath = new Path(parentDir, "test1.dat"); Path dstPath = new Path(parentDir, "test2.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parentDir */ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); /* for rename */ @@ -347,16 +329,13 @@ public class TestNativeAzureFileSystemAuthorization @Test //(expected=WasbAuthorizationException.class) public void testRenameAccessCheckNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegative/test1.dat not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentDir = new Path("/testRenameAccessCheckNegative"); Path srcPath = new Path(parentDir, "test1.dat"); Path dstPath = new Path(parentDir, "test2.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("rename", srcPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */ authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false); @@ -384,17 +363,14 @@ public class TestNativeAzureFileSystemAuthorization @Test //(expected=WasbAuthorizationException.class) public void testRenameAccessCheckNegativeOnDstFolder() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegativeDst/test2.dat not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentSrcDir = new Path("/testRenameAccessCheckNegativeSrc"); Path srcPath = new Path(parentSrcDir, "test1.dat"); Path parentDstDir = new Path("/testRenameAccessCheckNegativeDst"); Path dstPath = new Path(parentDstDir, "test2.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("rename", dstPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */ authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); @@ -419,17 +395,15 @@ public class TestNativeAzureFileSystemAuthorization * Positive test to verify rename access check - the dstFolder allows rename * @throws Throwable */ - @Test //(expected=WasbAuthorizationException.class) + @Test public void testRenameAccessCheckPositiveOnDstFolder() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentSrcDir = new Path("/testRenameAccessCheckPositiveSrc"); Path srcPath = new Path(parentSrcDir, "test1.dat"); Path parentDstDir = new Path("/testRenameAccessCheckPositiveDst"); Path dstPath = new Path(parentDstDir, "test2.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dirs */ authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); @@ -461,12 +435,10 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testReadAccessCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentDir = new Path("/testReadAccessCheckPositive"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -504,15 +476,12 @@ public class TestNativeAzureFileSystemAuthorization @Test //(expected=WasbAuthorizationException.class) public void testReadAccessCheckNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("read operation for Path : /testReadAccessCheckNegative/test.dat not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); Path parentDir = new Path("/testReadAccessCheckNegative"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("read", testPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false); @@ -548,13 +517,10 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testFileDeleteAccessCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -576,16 +542,12 @@ public class TestNativeAzureFileSystemAuthorization @Test //(expected=WasbAuthorizationException.class) public void testFileDeleteAccessCheckNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("delete operation for Path : /test.dat not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/"); Path testPath = new Path(parentDir, "test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("delete", testPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -622,13 +584,10 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testFileDeleteAccessWithIntermediateFoldersCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path parentDir = new Path("/testDeleteIntermediateFolder"); Path testPath = new Path(parentDir, "1/2/test.dat"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); // for create and delete authorizer.addAuthRule("/testDeleteIntermediateFolder*", @@ -655,12 +614,9 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testGetFileStatusPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path testPath = new Path("/"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.READ.toString(), true); fs.updateWasbAuthorizer(authorizer); @@ -675,15 +631,11 @@ public class TestNativeAzureFileSystemAuthorization @Test //(expected=WasbAuthorizationException.class) public void testGetFileStatusNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("getFileStatus operation for Path : / not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path testPath = new Path("/"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("getFileStatus", testPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.READ.toString(), false); fs.updateWasbAuthorizer(authorizer); @@ -698,12 +650,9 @@ public class TestNativeAzureFileSystemAuthorization @Test public void testMkdirsCheckPositive() throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path testPath = new Path("/testMkdirsAccessCheckPositive/1/2/3"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -726,15 +675,11 @@ public class TestNativeAzureFileSystemAuthorization @Test //(expected=WasbAuthorizationException.class) public void testMkdirsCheckNegative() throws Throwable { - expectedEx.expect(WasbAuthorizationException.class); - expectedEx.expectMessage("mkdirs operation for Path : /testMkdirsAccessCheckNegative/1/2/3 not allowed"); - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - Path testPath = new Path("/testMkdirsAccessCheckNegative/1/2/3"); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + setExpectedFailureMessage("mkdirs", testPath); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); authorizer.init(null); authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false); authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); @@ -749,4 +694,23 @@ public class TestNativeAzureFileSystemAuthorization fs.delete(new Path("/testMkdirsAccessCheckNegative"), true); } } + + + /** + * Positive test triple slash format (wasb:///) access check + * @throws Throwable + */ + @Test + public void testListStatusWithTripleSlashCheckPositive() throws Throwable { + + Path testPath = new Path("/"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(fs); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + Path testPathWithTripleSlash = new Path("wasb:///" + testPath); + fs.listStatus(testPathWithTripleSlash); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b415c6fe/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java index d7c40b9..77be1b8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java @@ -32,6 +32,8 @@ import org.junit.rules.ExpectedException; import org.mockito.Mockito; import java.io.ByteArrayInputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; @@ -261,16 +263,20 @@ public class TestWasbRemoteCallHelper performop(mockHttpClient); } - private void setupExpectations() { + private void setupExpectations() throws UnsupportedEncodingException { + + String path = new Path("/").makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(); + String pathEncoded = URLEncoder.encode(path, "UTF-8"); + + String requestURI = String.format("http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%s&operation_type=write";, pathEncoded); expectedEx.expect(WasbAuthorizationException.class); expectedEx.expectMessage("org.apache.hadoop.fs.azure.WasbRemoteCallException: " - + "http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%2F&"; - + "operation_type=write:Encountered IOException while making remote call"); + + requestURI + + ":Encountered IOException while making remote call" + ); } private void performop(HttpClient mockHttpClient) throws Throwable { - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); Path testPath = new Path("/", "test.dat"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
