Repository: hadoop Updated Branches: refs/heads/HDFS-12943 191faeb96 -> 039c158d2
HDFS-13831. Make block increment deletion number configurable. Contributed by Ryan Wu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9b964d2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9b964d2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9b964d2 Branch: refs/heads/HDFS-12943 Commit: b9b964d25335943fb15cdfcf369d123bbd7e454a Parents: a4121c7 Author: Yiqun Lin <yq...@apache.org> Authored: Mon Aug 27 14:55:46 2018 +0800 Committer: Yiqun Lin <yq...@apache.org> Committed: Mon Aug 27 14:55:46 2018 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +++++ .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 11 +++++++++-- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 10 ++++++++++ .../hdfs/server/namenode/TestLargeDirectoryDelete.java | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 5ed35b8..bd88341 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -395,6 +395,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec"; public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L; + /** Block deletion increment. */ + public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY = + "dfs.namenode.block.deletion.increment"; + public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000; + public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES = HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES; public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 06bf008..6ba0e0b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -428,12 +428,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, FSNamesystem.class.getName() + ".audit"); private final int maxCorruptFileBlocksReturn; - static int BLOCK_DELETION_INCREMENT = 1000; private final boolean isPermissionEnabled; private final UserGroupInformation fsOwner; private final String supergroup; private final boolean standbyShouldCheckpoint; private final int snapshotDiffReportLimit; + private final int blockDeletionIncrement; /** Interval between each check of lease to release. */ private final long leaseRecheckIntervalMs; @@ -909,6 +909,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, DFSConfigKeys.DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES + " must be a positive integer." ); + + this.blockDeletionIncrement = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT); + Preconditions.checkArgument(blockDeletionIncrement > 0, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY + + " must be a positive integer."); } catch(IOException e) { LOG.error(getClass().getSimpleName() + " initialization failed.", e); close(); @@ -3094,7 +3101,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, while (iter.hasNext()) { writeLock(); try { - for (int i = 0; i < BLOCK_DELETION_INCREMENT && iter.hasNext(); i++) { + for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) { blockManager.removeBlock(iter.next()); } } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index f720d0d..48ff329 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5160,4 +5160,14 @@ will throw NameNodeFormatException. </description> </property> + + <property> + <name>dfs.namenode.block.deletion.increment</name> + <value>1000</value> + <description> + The number of block deletion increment. + This setting will control the block increment deletion rate to + ensure that other waiters on the lock can get in. + </description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b964d2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java index 0882d18..3027c79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java @@ -49,6 +49,7 @@ public class TestLargeDirectoryDelete { static { CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1); CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1); + CONF.setInt(DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, 1); } /** create a file with a length of <code>filelen</code> */ @@ -137,7 +138,6 @@ public class TestLargeDirectoryDelete { threads[1].start(); final long start = Time.now(); - FSNamesystem.BLOCK_DELETION_INCREMENT = 1; mc.getFileSystem().delete(new Path("/root"), true); // recursive delete final long end = Time.now(); threads[0].endThread(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org