This is an automated email from the ASF dual-hosted git repository. shashikant pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push: new 78ca8bf HDDS-4077. Incomplete OzoneFileSystem statistics (#1329) 78ca8bf is described below commit 78ca8bfbcdb4ed9566c7906ef41fdf0f43a8bd47 Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com> AuthorDate: Mon Aug 31 08:41:20 2020 +0200 HDDS-4077. Incomplete OzoneFileSystem statistics (#1329) --- .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 15 +++-- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 71 ++++++++++++++++++++++ .../fs/ozone/BasicRootedOzoneFileSystem.java | 71 ++++++++++++++++++++++ 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java index 06d1bd3..2b8803e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java @@ -357,15 +357,18 @@ public class TestOzoneFileInterfaces { String dirPath = RandomStringUtils.randomAlphanumeric(5); Path path = createPath("/" + dirPath); paths.add(path); + + long mkdirs = statistics.getLong( + StorageStatistics.CommonStatisticNames.OP_MKDIRS); assertTrue("Makedirs returned with false for the path " + path, fs.mkdirs(path)); + assertCounter(++mkdirs, StorageStatistics.CommonStatisticNames.OP_MKDIRS); long listObjects = statistics.getLong(Statistic.OBJECTS_LIST.getSymbol()); long omListStatus = omMetrics.getNumListStatus(); FileStatus[] statusList = fs.listStatus(createPath("/")); assertEquals(1, statusList.length); - assertEquals(++listObjects, - statistics.getLong(Statistic.OBJECTS_LIST.getSymbol()).longValue()); + assertCounter(++listObjects, Statistic.OBJECTS_LIST.getSymbol()); assertEquals(++omListStatus, omMetrics.getNumListStatus()); assertEquals(fs.getFileStatus(path), statusList[0]); @@ -374,11 +377,11 @@ public class TestOzoneFileInterfaces { paths.add(path); assertTrue("Makedirs returned with false for the path " + path, fs.mkdirs(path)); + assertCounter(++mkdirs, StorageStatistics.CommonStatisticNames.OP_MKDIRS); statusList = fs.listStatus(createPath("/")); assertEquals(2, statusList.length); - assertEquals(++listObjects, - statistics.getLong(Statistic.OBJECTS_LIST.getSymbol()).longValue()); + assertCounter(++listObjects, Statistic.OBJECTS_LIST.getSymbol()); assertEquals(++omListStatus, omMetrics.getNumListStatus()); for (Path p : paths) { assertTrue(Arrays.asList(statusList).contains(fs.getFileStatus(p))); @@ -528,4 +531,8 @@ public class TestOzoneFileInterfaces { return status; } + + private void assertCounter(long value, String key) { + assertEquals(value, statistics.getLong(key).longValue()); + } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index 49f12f0..e4acabc 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -26,13 +26,16 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; @@ -692,6 +695,7 @@ public class BasicOzoneFileSystem extends FileSystem { @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { + incrementCounter(Statistic.INVOCATION_MKDIRS); LOG.trace("mkdir() path:{} ", f); String key = pathToKey(f); if (isEmpty(key)) { @@ -735,6 +739,73 @@ public class BasicOzoneFileSystem extends FileSystem { return adapter.getDefaultReplication(); } + @Override + public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, + Path dst) throws IOException { + incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE); + super.copyFromLocalFile(delSrc, overwrite, srcs, dst); + } + + @Override + public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, + Path dst) throws IOException { + incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE); + super.copyFromLocalFile(delSrc, overwrite, src, dst); + } + + @Override + public boolean exists(Path f) throws IOException { + incrementCounter(Statistic.INVOCATION_EXISTS); + return super.exists(f); + } + + @Override + public FileChecksum getFileChecksum(Path f, long length) throws IOException { + incrementCounter(Statistic.INVOCATION_GET_FILE_CHECKSUM); + return super.getFileChecksum(f, length); + } + + @Override + public FileStatus[] globStatus(Path pathPattern) throws IOException { + incrementCounter(Statistic.INVOCATION_GLOB_STATUS); + return super.globStatus(pathPattern); + } + + @Override + public FileStatus[] globStatus(Path pathPattern, PathFilter filter) + throws IOException { + incrementCounter(Statistic.INVOCATION_GLOB_STATUS); + return super.globStatus(pathPattern, filter); + } + + @Override + @SuppressWarnings("deprecation") + public boolean isDirectory(Path f) throws IOException { + incrementCounter(Statistic.INVOCATION_IS_DIRECTORY); + return super.isDirectory(f); + } + + @Override + @SuppressWarnings("deprecation") + public boolean isFile(Path f) throws IOException { + incrementCounter(Statistic.INVOCATION_IS_FILE); + return super.isFile(f); + } + + @Override + public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive) + throws IOException { + incrementCounter(Statistic.INVOCATION_LIST_FILES); + return super.listFiles(f, recursive); + } + + @Override + public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f) + throws IOException { + incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); + return super.listLocatedStatus(f); + } + /** * Turn a path (relative or otherwise) into an Ozone key. * diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 59aec47..015621c 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -24,12 +24,15 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; @@ -715,6 +718,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem { @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { + incrementCounter(Statistic.INVOCATION_MKDIRS); LOG.trace("mkdir() path:{} ", f); String key = pathToKey(f); if (isEmpty(key)) { @@ -764,6 +768,73 @@ public class BasicRootedOzoneFileSystem extends FileSystem { return adapter.getDefaultReplication(); } + @Override + public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, + Path dst) throws IOException { + incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE); + super.copyFromLocalFile(delSrc, overwrite, srcs, dst); + } + + @Override + public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, + Path dst) throws IOException { + incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE); + super.copyFromLocalFile(delSrc, overwrite, src, dst); + } + + @Override + public boolean exists(Path f) throws IOException { + incrementCounter(Statistic.INVOCATION_EXISTS); + return super.exists(f); + } + + @Override + public FileChecksum getFileChecksum(Path f, long length) throws IOException { + incrementCounter(Statistic.INVOCATION_GET_FILE_CHECKSUM); + return super.getFileChecksum(f, length); + } + + @Override + public FileStatus[] globStatus(Path pathPattern) throws IOException { + incrementCounter(Statistic.INVOCATION_GLOB_STATUS); + return super.globStatus(pathPattern); + } + + @Override + public FileStatus[] globStatus(Path pathPattern, PathFilter filter) + throws IOException { + incrementCounter(Statistic.INVOCATION_GLOB_STATUS); + return super.globStatus(pathPattern, filter); + } + + @Override + @SuppressWarnings("deprecation") + public boolean isDirectory(Path f) throws IOException { + incrementCounter(Statistic.INVOCATION_IS_DIRECTORY); + return super.isDirectory(f); + } + + @Override + @SuppressWarnings("deprecation") + public boolean isFile(Path f) throws IOException { + incrementCounter(Statistic.INVOCATION_IS_FILE); + return super.isFile(f); + } + + @Override + public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive) + throws IOException { + incrementCounter(Statistic.INVOCATION_LIST_FILES); + return super.listFiles(f, recursive); + } + + @Override + public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f) + throws IOException { + incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); + return super.listLocatedStatus(f); + } + /** * Turn a path (relative or otherwise) into an Ozone key. * --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org