Repository: hadoop Updated Branches: refs/heads/trunk c35a5a7a8 -> aa42c7a6d
HADOOP-13305. Define common statistics names across schemes. Contributed by Mingliang Liu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa42c7a6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa42c7a6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa42c7a6 Branch: refs/heads/trunk Commit: aa42c7a6dda23f9dd686cc844b31a5aeebe7e088 Parents: c35a5a7 Author: Jitendra Pandey <jiten...@apache.org> Authored: Fri Jul 1 15:34:03 2016 -0700 Committer: Jitendra Pandey <jiten...@apache.org> Committed: Fri Jul 1 15:34:03 2016 -0700 ---------------------------------------------------------------------- .../hadoop/fs/FileSystemStorageStatistics.java | 5 + .../org/apache/hadoop/fs/StorageStatistics.java | 53 ++++++++++ .../hadoop/hdfs/DFSOpsCountStatistics.java | 102 ++++++++++--------- .../hadoop/hdfs/TestDFSOpsCountStatistics.java | 15 +++ .../hadoop/fs/s3a/S3AStorageStatistics.java | 5 + .../org/apache/hadoop/fs/s3a/Statistic.java | 24 +++-- 6 files changed, 145 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java index 7c08863..8a1eb54 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java @@ -119,6 +119,11 @@ public class FileSystemStorageStatistics extends StorageStatistics { } @Override + public String getScheme() { + return stats.getScheme(); + } + + @Override public Iterator<LongStatistic> getLongStatistics() { return new LongStatisticIterator(stats.getData()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java index 4bdef80..0971f10 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java @@ -27,6 +27,51 @@ import java.util.Iterator; */ @InterfaceAudience.Public public abstract class StorageStatistics { + + /** + * These are common statistic names. + * + * The following names are considered general and preserved across different + * StorageStatistics classes. When implementing a new StorageStatistics, it is + * highly recommended to use the common statistic names. + * + * When adding new common statistic name constants, please make them unique. + * By convention, they are implicitly unique: + * - the name of the constants are uppercase, words separated by underscores. + * - the value of the constants are lowercase of the constant names. + */ + public interface CommonStatisticNames { + // The following names are for file system operation invocations + String OP_APPEND = "op_append"; + String OP_COPY_FROM_LOCAL_FILE = "op_copy_from_local_file"; + String OP_CREATE = "op_create"; + String OP_CREATE_NON_RECURSIVE = "op_create_non_recursive"; + String OP_DELETE = "op_delete"; + String OP_EXISTS = "op_exists"; + String OP_GET_CONTENT_SUMMARY = "op_get_content_summary"; + String OP_GET_FILE_CHECKSUM = "op_get_file_checksum"; + String OP_GET_FILE_STATUS = "op_get_file_status"; + String OP_GET_STATUS = "op_get_status"; + String OP_GLOB_STATUS = "op_glob_status"; + String OP_IS_FILE = "op_is_file"; + String OP_IS_DIRECTORY = "op_is_directory"; + String OP_LIST_FILES = "op_list_files"; + String OP_LIST_LOCATED_STATUS = "op_list_located_status"; + String OP_LIST_STATUS = "op_list_status"; + String OP_MKDIRS = "op_mkdirs"; + String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries"; + String OP_OPEN = "op_open"; + String OP_REMOVE_ACL = "op_remove_acl"; + String OP_REMOVE_ACL_ENTRIES = "op_remove_acl_entries"; + String OP_REMOVE_DEFAULT_ACL = "op_remove_default_acl"; + String OP_RENAME = "op_rename"; + String OP_SET_ACL = "op_set_acl"; + String OP_SET_OWNER = "op_set_owner"; + String OP_SET_PERMISSION = "op_set_permission"; + String OP_SET_TIMES = "op_set_times"; + String OP_TRUNCATE = "op_truncate"; + } + /** * A 64-bit storage statistic. */ @@ -68,6 +113,14 @@ public abstract class StorageStatistics { } /** + * @return the associated file system scheme if this is scheme specific, + * else return null. + */ + public String getScheme() { + return null; + } + + /** * Get an iterator over all the currently tracked long statistics. * * The values returned will depend on the type of FileSystem or FileContext http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index a047d34..84fc925 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.fs.StorageStatistics; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import java.util.EnumMap; import java.util.Iterator; @@ -37,55 +38,55 @@ import java.util.concurrent.atomic.AtomicLong; */ public class DFSOpsCountStatistics extends StorageStatistics { - /** This is for counting file system operations. */ + /** This is for counting distributed file system operations. */ public enum OpType { - ALLOW_SNAPSHOT("allowSnapshot"), - APPEND("append"), - CONCAT("concat"), - COPY_FROM_LOCAL_FILE("copyFromLocalFile"), - CREATE("create"), - CREATE_NON_RECURSIVE("createNonRecursive"), - CREATE_SNAPSHOT("createSnapshot"), - CREATE_SYM_LINK("createSymlink"), - DELETE("delete"), - DELETE_SNAPSHOT("deleteSnapshot"), - DISALLOW_SNAPSHOT("disallowSnapshot"), - EXISTS("exists"), - GET_BYTES_WITH_FUTURE_GS("getBytesWithFutureGenerationStamps"), - GET_CONTENT_SUMMARY("getContentSummary"), - GET_FILE_BLOCK_LOCATIONS("getFileBlockLocations"), - GET_FILE_CHECKSUM("getFileChecksum"), - GET_FILE_LINK_STATUS("getFileLinkStatus"), - GET_FILE_STATUS("getFileStatus"), - GET_LINK_TARGET("getLinkTarget"), - GET_QUOTA_USAGE("getQuotaUsage"), - GET_STATUS("getStatus"), - GET_STORAGE_POLICIES("getStoragePolicies"), - GET_STORAGE_POLICY("getStoragePolicy"), - GET_XATTR("getXAttr"), - LIST_LOCATED_STATUS("listLocatedStatus"), - LIST_STATUS("listStatus"), - MKDIRS("mkdirs"), - MODIFY_ACL_ENTRIES("modifyAclEntries"), - OPEN("open"), - PRIMITIVE_CREATE("primitiveCreate"), - PRIMITIVE_MKDIR("primitiveMkdir"), - REMOVE_ACL("removeAcl"), - REMOVE_ACL_ENTRIES("removeAclEntries"), - REMOVE_DEFAULT_ACL("removeDefaultAcl"), - REMOVE_XATTR("removeXAttr"), - RENAME("rename"), - RENAME_SNAPSHOT("renameSnapshot"), - RESOLVE_LINK("resolveLink"), - SET_ACL("setAcl"), - SET_OWNER("setOwner"), - SET_PERMISSION("setPermission"), - SET_REPLICATION("setReplication"), - SET_STORAGE_POLICY("setStoragePolicy"), - SET_TIMES("setTimes"), - SET_XATTR("setXAttr"), - TRUNCATE("truncate"), - UNSET_STORAGE_POLICY("unsetStoragePolicy"); + ALLOW_SNAPSHOT("op_allow_snapshot"), + APPEND(CommonStatisticNames.OP_APPEND), + CONCAT("op_concat"), + COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE), + CREATE(CommonStatisticNames.OP_CREATE), + CREATE_NON_RECURSIVE(CommonStatisticNames.OP_CREATE_NON_RECURSIVE), + CREATE_SNAPSHOT("op_create_snapshot"), + CREATE_SYM_LINK("op_create_symlink"), + DELETE(CommonStatisticNames.OP_DELETE), + DELETE_SNAPSHOT("op_delete_snapshot"), + DISALLOW_SNAPSHOT("op_disallow_snapshot"), + EXISTS(CommonStatisticNames.OP_EXISTS), + GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"), + GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY), + GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"), + GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM), + GET_FILE_LINK_STATUS("op_get_file_link_status"), + GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS), + GET_LINK_TARGET("op_get_link_target"), + GET_QUOTA_USAGE("op_get_quota_usage"), + GET_STATUS(CommonStatisticNames.OP_GET_STATUS), + GET_STORAGE_POLICIES("op_get_storage_policies"), + GET_STORAGE_POLICY("op_get_storage_policy"), + GET_XATTR("op_get_xattr"), + LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS), + LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS), + MKDIRS(CommonStatisticNames.OP_MKDIRS), + MODIFY_ACL_ENTRIES(CommonStatisticNames.OP_MODIFY_ACL_ENTRIES), + OPEN(CommonStatisticNames.OP_OPEN), + PRIMITIVE_CREATE("op_primitive_create"), + PRIMITIVE_MKDIR("op_primitive_mkdir"), + REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL), + REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES), + REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL), + REMOVE_XATTR("op_remove_xattr"), + RENAME(CommonStatisticNames.OP_RENAME), + RENAME_SNAPSHOT("op_rename_snapshot"), + RESOLVE_LINK("op_resolve_link"), + SET_ACL(CommonStatisticNames.OP_SET_ACL), + SET_OWNER(CommonStatisticNames.OP_SET_OWNER), + SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION), + SET_REPLICATION("op_set_replication"), + SET_STORAGE_POLICY("op_set_storagePolicy"), + SET_TIMES(CommonStatisticNames.OP_SET_TIMES), + SET_XATTR("op_set_xattr"), + TRUNCATE(CommonStatisticNames.OP_TRUNCATE), + UNSET_STORAGE_POLICY("op_unset_storage_policy"); private final String symbol; @@ -150,6 +151,11 @@ public class DFSOpsCountStatistics extends StorageStatistics { } @Override + public String getScheme() { + return HdfsConstants.HDFS_URI_SCHEME; + } + + @Override public Iterator<LongStatistic> getLongStatistics() { return new LongIterator(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java index 7c1b018..7578930 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java @@ -31,8 +31,10 @@ import org.junit.rules.ExpectedException; import org.junit.rules.Timeout; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -66,6 +68,19 @@ public class TestDFSOpsCountStatistics { } } + /** + * This is to test the the {@link OpType} symbols are unique. + */ + @Test + public void testOpTypeSymbolsAreUnique() { + final Set<String> opTypeSymbols = new HashSet<>(); + for (OpType opType : OpType.values()) { + assertFalse(opTypeSymbols.contains(opType.getSymbol())); + opTypeSymbols.add(opType.getSymbol()); + } + assertEquals(OpType.values().length, opTypeSymbols.size()); + } + @Test public void testGetLongStatistics() { short iterations = 0; // number of the iter.hasNext() http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java index a74b864..3a90c6b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java @@ -87,6 +87,11 @@ public class S3AStorageStatistics extends StorageStatistics { } @Override + public String getScheme() { + return "s3a"; + } + + @Override public Iterator<LongStatistic> getLongStatistics() { return new LongIterator(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa42c7a6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index 8121b3b..3c205f3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.s3a; +import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames; + /** * Statistic which are collected in S3A. * These statistics are available at a low level in {@link S3AStorageStatistics} @@ -38,27 +40,27 @@ public enum Statistic { FILES_DELETED("files_deleted", "Total number of files deleted from the object store."), IGNORED_ERRORS("ignored_errors", "Errors caught and ignored"), - INVOCATION_COPY_FROM_LOCAL_FILE("invocations_copyfromlocalfile", + INVOCATION_COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE, "Calls of copyFromLocalFile()"), - INVOCATION_EXISTS("invocations_exists", + INVOCATION_EXISTS(CommonStatisticNames.OP_EXISTS, "Calls of exists()"), - INVOCATION_GET_FILE_STATUS("invocations_getfilestatus", + INVOCATION_GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS, "Calls of getFileStatus()"), - INVOCATION_GLOB_STATUS("invocations_globstatus", + INVOCATION_GLOB_STATUS(CommonStatisticNames.OP_GLOB_STATUS, "Calls of globStatus()"), - INVOCATION_IS_DIRECTORY("invocations_is_directory", + INVOCATION_IS_DIRECTORY(CommonStatisticNames.OP_IS_DIRECTORY, "Calls of isDirectory()"), - INVOCATION_IS_FILE("invocations_is_file", + INVOCATION_IS_FILE(CommonStatisticNames.OP_IS_FILE, "Calls of isFile()"), - INVOCATION_LIST_FILES("invocations_listfiles", + INVOCATION_LIST_FILES(CommonStatisticNames.OP_LIST_FILES, "Calls of listFiles()"), - INVOCATION_LIST_LOCATED_STATUS("invocations_listlocatedstatus", + INVOCATION_LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS, "Calls of listLocatedStatus()"), - INVOCATION_LIST_STATUS("invocations_liststatus", + INVOCATION_LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS, "Calls of listStatus()"), - INVOCATION_MKDIRS("invocations_mdkirs", + INVOCATION_MKDIRS(CommonStatisticNames.OP_MKDIRS, "Calls of mkdirs()"), - INVOCATION_RENAME("invocations_rename", + INVOCATION_RENAME(CommonStatisticNames.OP_RENAME, "Calls of rename()"), OBJECT_COPY_REQUESTS("object_copy_requests", "Object copy requests"), OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"), --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
