HADOOP-14223. Extend FileStatus#toString() to include details like Erasure Coding and Encryption. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4966a6e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4966a6e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4966a6e2 Branch: refs/heads/HDFS-7240 Commit: 4966a6e26e45d7dc36e0b270066ff7c87bcd00cc Parents: 640ba1d Author: Andrew Wang <[email protected]> Authored: Wed Mar 29 14:37:21 2017 -0700 Committer: Andrew Wang <[email protected]> Committed: Wed Mar 29 14:37:21 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/FileStatus.java | 12 ++++++++++ .../src/site/markdown/filesystem/filesystem.md | 19 ++++++---------- .../org/apache/hadoop/fs/TestFileStatus.java | 10 ++++++--- .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 3 +++ .../org/apache/hadoop/hdfs/TestFileStatus.java | 3 +++ .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 7 +++++- .../hdfs/server/namenode/FSAclBaseTest.java | 23 ++++++++++++++++++++ .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 12 ++++++++++ .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 6 +++++ .../apache/hadoop/fs/adl/TestGetFileStatus.java | 7 ++++++ 10 files changed, 86 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java index 26d3962..f5111ef 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -208,6 +208,15 @@ public class FileStatus implements Writable, Comparable<FileStatus>, } /** + * Tell whether the underlying file or directory has ACLs set. + * + * @return true if the underlying file or directory has ACLs set. + */ + public boolean hasAcl() { + return permission.getAclBit(); + } + + /** * Tell whether the underlying file or directory is encrypted or not. * * @return true if the underlying file is encrypted. @@ -399,6 +408,9 @@ public class FileStatus implements Writable, Comparable<FileStatus>, if(isSymlink()) { sb.append("; symlink=" + symlink); } + sb.append("; hasAcl=" + hasAcl()); + sb.append("; isEncrypted=" + isEncrypted()); + sb.append("; isErasureCoded=" + isErasureCoded()); sb.append("}"); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 97bc7d1..b464941 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -86,20 +86,15 @@ Get the status of a path stat.length = 0 stat.isdir = False stat.symlink = FS.Symlinks[p] - if inEncryptionZone(FS, p) : - stat.isEncrypted = True - else - stat.isEncrypted = False - if isErasureCoded(FS, p) : - stat.isErasureCoded = True - else - stat.isErasureCoded = False + stat.hasAcl = hasACL(FS, p) + stat.isEncrypted = inEncryptionZone(FS, p) + stat.isErasureCoded = isErasureCoded(FS, p) The returned `FileStatus` status of the path additionally carries details on -Encryption and Erasure Coding information. `getFileStatus(Path p).isEncrypted()` -can be queried to find if the path is Encrypted. -Likewise, `getFileStatus(Path p).isErasureCoded()` will tell if the path is -Erasure Coded or not. +ACL, encryption and erasure coding information. `getFileStatus(Path p).hasAcl()` +can be queried to find if the path has an ACL. `getFileStatus(Path p).isEncrypted()` +can be queried to find if the path is encrypted. `getFileStatus(Path p).isErasureCoded()` +will tell if the path is erasure coded or not. ### `Path getHomeDirectory()` http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java index 35f2bad..d29b1a4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java @@ -295,11 +295,15 @@ public class TestFileStatus { expected.append("permission=").append(fileStatus.getPermission()).append("; "); if(fileStatus.isSymlink()) { expected.append("isSymlink=").append(true).append("; "); - expected.append("symlink=").append(fileStatus.getSymlink()).append("}"); + expected.append("symlink=").append(fileStatus.getSymlink()).append("; "); } else { - expected.append("isSymlink=").append(false).append("}"); + expected.append("isSymlink=").append(false).append("; "); } - + expected.append("hasAcl=").append(fileStatus.hasAcl()).append("; "); + expected.append("isEncrypted=").append( + fileStatus.isEncrypted()).append("; "); + expected.append("isErasureCoded=").append( + fileStatus.isErasureCoded()).append("}"); assertEquals(expected.toString(), fileStatus.toString()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java index 449e2e3..0c31c8e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java @@ -75,6 +75,9 @@ public class TestViewfsFileStatus { FileStatus stat = vfs.getFileStatus(path); assertEquals(content.length, stat.getLen()); ContractTestUtils.assertNotErasureCoded(vfs, path); + assertTrue(path + " should have erasure coding unset in " + + "FileStatus#toString(): " + stat, + stat.toString().contains("isErasureCoded=false")); // check serialization/deserialization DataOutputBuffer dob = new DataOutputBuffer(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java index 7631e7d..c74bb63 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java @@ -139,6 +139,9 @@ public class TestFileStatus { assertEquals(file1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(), status.getPath().toString()); + assertTrue(file1 + " should have erasure coding unset in " + + "FileStatus#toString(): " + status, + status.toString().contains("isErasureCoded=false")); } /** Test the FileStatus obtained calling listStatus on a file */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java index 0e270ff..e04f957 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; @@ -85,7 +86,7 @@ public class TestFileStatusWithECPolicy { assertNotNull(ecPolicy2); assertTrue(ecPolicy1.equals(ecPolicy2)); - // test file doesn't have an EC policy + // test file with EC policy fs.create(file).close(); final ErasureCodingPolicy ecPolicy3 = fs.getClient().getFileInfo(file.toUri().getPath()) @@ -93,5 +94,9 @@ public class TestFileStatusWithECPolicy { assertNotNull(ecPolicy3); assertTrue(ecPolicy1.equals(ecPolicy3)); ContractTestUtils.assertErasureCoded(fs, file); + FileStatus status = fs.getFileStatus(file); + assertTrue(file + " should have erasure coding set in " + + "FileStatus#toString(): " + status, + status.toString().contains("isErasureCoded=true")); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java index 81270e6..60b0ab1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java @@ -50,6 +50,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -120,10 +121,16 @@ public abstract class FSAclBaseTest { aclEntry(ACCESS, OTHER, NONE), aclEntry(DEFAULT, USER, "foo", ALL)); fs.setAcl(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + aclSpec = Lists.newArrayList( aclEntry(ACCESS, USER, "foo", READ_EXECUTE), aclEntry(DEFAULT, USER, "foo", READ_EXECUTE)); fs.modifyAclEntries(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + AclStatus s = fs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(new AclEntry[] { @@ -561,8 +568,18 @@ public abstract class FSAclBaseTest { aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(ACCESS, OTHER, NONE), aclEntry(DEFAULT, USER, "foo", ALL)); + fs.setAcl(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + Assert.assertTrue(path + " should have ACLs in FileStatus#toString()!", + fs.getFileStatus(path).toString().contains("hasAcl=true")); fs.removeAcl(path); + Assert.assertFalse(path + " should not have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + Assert.assertTrue(path + " should not have ACLs in FileStatus#toString()!", + fs.getFileStatus(path).toString().contains("hasAcl=false")); + AclStatus s = fs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(new AclEntry[] { }, returned); @@ -968,8 +985,14 @@ public abstract class FSAclBaseTest { List<AclEntry> aclSpec = Lists.newArrayList( aclEntry(DEFAULT, USER, "foo", ALL)); fs.setAcl(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + Path dirPath = new Path(path, "dir1"); fs.mkdirs(dirPath); + Assert.assertTrue(dirPath + " should have ACLs in FileStatus!", + fs.getFileStatus(dirPath).hasAcl()); + AclStatus s = fs.getAclStatus(dirPath); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(new AclEntry[] { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 3c366c3..46a8a56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -547,24 +547,36 @@ public class TestWebHDFS { Assert.assertEquals(expectedECDirStatus.isErasureCoded(), actualECDirStatus.isErasureCoded()); ContractTestUtils.assertErasureCoded(dfs, ecDir); + assertTrue(ecDir+ " should have erasure coding set in " + + "FileStatus#toString(): " + actualECDirStatus, + actualECDirStatus.toString().contains("isErasureCoded=true")); FileStatus expectedECFileStatus = dfs.getFileStatus(ecFile); FileStatus actualECFileStatus = webHdfs.getFileStatus(ecFile); Assert.assertEquals(expectedECFileStatus.isErasureCoded(), actualECFileStatus.isErasureCoded()); ContractTestUtils.assertErasureCoded(dfs, ecFile); + assertTrue(ecFile+ " should have erasure coding set in " + + "FileStatus#toString(): " + actualECFileStatus, + actualECFileStatus.toString().contains("isErasureCoded=true")); FileStatus expectedNormalDirStatus = dfs.getFileStatus(normalDir); FileStatus actualNormalDirStatus = webHdfs.getFileStatus(normalDir); Assert.assertEquals(expectedNormalDirStatus.isErasureCoded(), actualNormalDirStatus.isErasureCoded()); ContractTestUtils.assertNotErasureCoded(dfs, normalDir); + assertTrue(normalDir + " should have erasure coding unset in " + + "FileStatus#toString(): " + actualNormalDirStatus, + actualNormalDirStatus.toString().contains("isErasureCoded=false")); FileStatus expectedNormalFileStatus = dfs.getFileStatus(normalFile); FileStatus actualNormalFileStatus = webHdfs.getFileStatus(normalDir); Assert.assertEquals(expectedNormalFileStatus.isErasureCoded(), actualNormalFileStatus.isErasureCoded()); ContractTestUtils.assertNotErasureCoded(dfs, normalFile); + assertTrue(normalFile + " should have erasure coding unset in " + + "FileStatus#toString(): " + actualNormalFileStatus, + actualNormalFileStatus.toString().contains("isErasureCoded=false")); } finally { if (cluster != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java index deb3963..58e4d30 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java @@ -62,6 +62,9 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest { assertEquals(meta.getContentLength(), stat.getLen()); assertEquals(meta.getLastModified().getTime(), stat.getModificationTime()); ContractTestUtils.assertNotErasureCoded(fs, path); + assertTrue(path + " should have erasure coding unset in " + + "FileStatus#toString(): " + stat, + stat.toString().contains("isErasureCoded=false")); } @Test @@ -101,6 +104,9 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest { assertEquals(fs.makeQualified(path), stat.getPath()); assertTrue(stat.isDirectory()); ContractTestUtils.assertNotErasureCoded(fs, path); + assertTrue(path + " should have erasure coding unset in " + + "FileStatus#toString(): " + stat, + stat.toString().contains("isErasureCoded=false")); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/4966a6e2/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java index df39a40..0ea4b86 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java @@ -65,6 +65,9 @@ public class TestGetFileStatus extends AdlMockWebServer { Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission()); Assert.assertEquals("NotSupportYet", fileStatus.getOwner()); Assert.assertEquals("NotSupportYet", fileStatus.getGroup()); + Assert.assertTrue(path + " should have Acl!", fileStatus.hasAcl()); + Assert.assertFalse(path + " should not be encrypted!", + fileStatus.isEncrypted()); Assert.assertFalse(path + " should not be erasure coded!", fileStatus.isErasureCoded()); } @@ -82,6 +85,8 @@ public class TestGetFileStatus extends AdlMockWebServer { LOG.debug("Time : " + (endTime - startTime)); Assert.assertTrue(fileStatus.isFile()); Assert.assertEquals(true, fileStatus.getPermission().getAclBit()); + Assert.assertEquals(fileStatus.hasAcl(), + fileStatus.getPermission().getAclBit()); // With ACLBIT set to false getMockServer().enqueue(new MockResponse().setResponseCode(200) @@ -93,5 +98,7 @@ public class TestGetFileStatus extends AdlMockWebServer { LOG.debug("Time : " + (endTime - startTime)); Assert.assertTrue(fileStatus.isFile()); Assert.assertEquals(false, fileStatus.getPermission().getAclBit()); + Assert.assertEquals(fileStatus.hasAcl(), + fileStatus.getPermission().getAclBit()); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
