Repository: hadoop Updated Branches: refs/heads/branch-2.7 d1186dbaf -> ee830b0d2
HDFS-8046. Allow better control of getContentSummary. Contributed by Kihwal Lee. (cherry picked from commit 285b31e75e51ec8e3a796c2cb0208739368ca9b8) (cherry picked from commit 7e622076d41a85fc9a8600fb270564a085f5cd83) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfb27d7f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfb27d7f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfb27d7f Branch: refs/heads/branch-2.7 Commit: cfb27d7f780b51025f99848c5c1ad1e28806ff9a Parents: d1186db Author: Kihwal Lee <kih...@apache.org> Authored: Wed Apr 8 15:39:25 2015 -0500 Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vino...@apache.org> Committed: Thu Sep 10 10:53:19 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ .../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +++- .../server/namenode/ContentSummaryComputationContext.java | 10 +++++++--- .../hdfs/server/namenode/FSDirStatAndListingOp.java | 2 +- .../apache/hadoop/hdfs/server/namenode/FSDirectory.java | 8 ++++++++ 5 files changed, 23 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e1785d2..8758cba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -77,6 +77,10 @@ Release 2.7.1 - 2015-07-06 HDFS-7546. Document, and set an accepting default for dfs.namenode.kerberos.principal.pattern (Harsh J via aw) + HDFS-8046. Allow better control of getContentSummary (kihwal) + + OPTIMIZATIONS + HDFS-7164. Feature documentation for HDFS-6581. (Arpit Agarwal) OPTIMIZATIONS http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index a8d7159..3bb8a0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -283,7 +283,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_LIST_LIMIT = "dfs.ls.limit"; public static final int DFS_LIST_LIMIT_DEFAULT = 1000; public static final String DFS_CONTENT_SUMMARY_LIMIT_KEY = "dfs.content-summary.limit"; - public static final int DFS_CONTENT_SUMMARY_LIMIT_DEFAULT = 0; + public static final int DFS_CONTENT_SUMMARY_LIMIT_DEFAULT = 5000; + public static final String DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_KEY = "dfs.content-summary.sleep-microsec"; + public static final long DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT = 500; public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated"; public static final int DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT = 0; public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index 31f34b9..5739835 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -32,6 +32,8 @@ public class ContentSummaryComputationContext { private long nextCountLimit = 0; private long limitPerRun = 0; private long yieldCount = 0; + private long sleepMilliSec = 0; + private int sleepNanoSec = 0; /** * Constructor @@ -43,17 +45,19 @@ public class ContentSummaryComputationContext { * no limit (i.e. no yielding) */ public ContentSummaryComputationContext(FSDirectory dir, - FSNamesystem fsn, long limitPerRun) { + FSNamesystem fsn, long limitPerRun, long sleepMicroSec) { this.dir = dir; this.fsn = fsn; this.limitPerRun = limitPerRun; this.nextCountLimit = limitPerRun; this.counts = new ContentCounts.Builder().build(); + this.sleepMilliSec = sleepMicroSec/1000; + this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000); } /** Constructor for blocking computation. */ public ContentSummaryComputationContext(BlockStoragePolicySuite bsps) { - this(null, null, 0); + this(null, null, 0, 1000); this.bsps = bsps; } @@ -105,7 +109,7 @@ public class ContentSummaryComputationContext { fsn.readUnlock(); try { - Thread.sleep(1); + Thread.sleep(sleepMilliSec, sleepNanoSec); } catch (InterruptedException ie) { } finally { // reacquire http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 43c2de3..850b3bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -496,7 +496,7 @@ class FSDirStatAndListingOp { // processed. 0 means disabled. I.e. blocking for the entire duration. ContentSummaryComputationContext cscc = new ContentSummaryComputationContext(fsd, fsd.getFSNamesystem(), - fsd.getContentCountLimit()); + fsd.getContentCountLimit(), fsd.getContentSleepMicroSec()); ContentSummary cs = targetNode.computeAndConvertContentSummary(cscc); fsd.addYieldCount(cscc.getYieldCount()); return cs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index b887eb3..f74c42a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -136,6 +136,7 @@ public class FSDirectory implements Closeable { private final int maxDirItems; private final int lsLimit; // max list limit private final int contentCountLimit; // max content summary counts per run + private final long contentSleepMicroSec; private final INodeMap inodeMap; // Synchronized by dirLock private long yieldCount = 0; // keep track of lock yield count. @@ -264,6 +265,9 @@ public class FSDirectory implements Closeable { this.contentCountLimit = conf.getInt( DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_DEFAULT); + this.contentSleepMicroSec = conf.getLong( + DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_KEY, + DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT); // filesystem limits this.maxComponentLength = conf.getInt( @@ -345,6 +349,10 @@ public class FSDirectory implements Closeable { return contentCountLimit; } + long getContentSleepMicroSec() { + return contentSleepMicroSec; + } + int getInodeXAttrsLimit() { return inodeXAttrsLimit; }