Revert "HDFS-6682. Add a metric to expose the timestamp of the oldest under-replicated block. (aajisaka)"
This reverts commit 02c01815eca656814febcdaca6115e5f53b9c746. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cba52d87 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cba52d87 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cba52d87 Branch: refs/heads/YARN-2928 Commit: cba52d8754999f99fbadeed48b5ec3828cee6f13 Parents: ee91d17 Author: Akira Ajisaka <aajis...@apache.org> Authored: Wed Jul 29 16:48:30 2015 +0900 Committer: Zhijie Shen <zjs...@apache.org> Committed: Mon Aug 3 17:02:08 2015 -0700 ---------------------------------------------------------------------- .../hadoop-common/src/site/markdown/Metrics.md | 1 - hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 -- .../server/blockmanagement/BlockManager.java | 4 -- .../blockmanagement/UnderReplicatedBlocks.java | 33 ++------------ .../hdfs/server/namenode/FSNamesystem.java | 9 +--- .../TestUnderReplicatedBlocks.java | 48 -------------------- 6 files changed, 5 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cba52d87/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 2e6c095..646cda5 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -201,7 +201,6 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | Name | Description | |:---- |:---- | | `MissingBlocks` | Current number of missing blocks | -| `TimeOfTheOldestBlockToBeReplicated` | The timestamp of the oldest block to be replicated. If there are no under-replicated or corrupt blocks, return 0. | | `ExpiredHeartbeats` | Total number of expired heartbeats | | `TransactionsSinceLastCheckpoint` | Total number of transactions since last checkpoint | | `TransactionsSinceLastLogRoll` | Total number of transactions since last edit log roll | http://git-wip-us.apache.org/repos/asf/hadoop/blob/cba52d87/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d30c258..9a0c6da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -747,9 +747,6 @@ Release 2.8.0 - UNRELEASED HDFS-8730. Clean up the import statements in ClientProtocol. (Takanobu Asanuma via wheat9) - HDFS-6682. Add a metric to expose the timestamp of the oldest - under-replicated block. (aajisaka) - HDFS-8735. Inotify: All events classes should implement toString() API. (Surendra Singh Lilhore via aajisaka) http://git-wip-us.apache.org/repos/asf/hadoop/blob/cba52d87/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 64603d0..7dce2a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -171,10 +171,6 @@ public class BlockManager implements BlockStatsMXBean { public int getPendingDataNodeMessageCount() { return pendingDNMessages.count(); } - /** Used by metrics. */ - public long getTimeOfTheOldestBlockToBeReplicated() { - return neededReplications.getTimeOfTheOldestBlockToBeReplicated(); - } /**replicationRecheckInterval is how often namenode checks for new replication work*/ private final long replicationRecheckInterval; http://git-wip-us.apache.org/repos/asf/hadoop/blob/cba52d87/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java index 128aae6..ebc15b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java @@ -18,15 +18,10 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; - import org.apache.hadoop.hdfs.util.LightWeightLinkedSet; import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.util.Time; /** * Keep prioritized queues of under replicated blocks. @@ -87,9 +82,6 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> { /** The number of corrupt blocks with replication factor 1 */ private int corruptReplOneBlocks = 0; - /** Keep timestamp when a block is put into the queue. */ - private final Map<BlockInfo, Long> timestampsMap = - Collections.synchronizedMap(new LinkedHashMap<BlockInfo, Long>()); /** Create an object. */ UnderReplicatedBlocks() { @@ -99,14 +91,13 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> { } /** - * Empty the queues and timestamps. + * Empty the queues. */ synchronized void clear() { for (int i = 0; i < LEVEL; i++) { priorityQueues.get(i).clear(); } corruptReplOneBlocks = 0; - timestampsMap.clear(); } /** Return the total number of under replication blocks */ @@ -129,20 +120,6 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> { return size; } - /** - * Return the smallest timestamp of the under-replicated/corrupt blocks. - * If there are no under-replicated or corrupt blocks, return 0. - */ - long getTimeOfTheOldestBlockToBeReplicated() { - synchronized (timestampsMap) { - if (timestampsMap.isEmpty()) { - return 0; - } - // Since we are using LinkedHashMap, the first value is the smallest. - return timestampsMap.entrySet().iterator().next().getValue(); - } - } - /** Return the number of corrupt blocks */ synchronized int getCorruptBlockSize() { return priorityQueues.get(QUEUE_WITH_CORRUPT_BLOCKS).size(); @@ -221,7 +198,7 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> { + " has only {} replicas and need {} replicas so is added to" + " neededReplications at priority level {}", block, curReplicas, expectedReplicas, priLevel); - timestampsMap.put(block, Time.now()); + return true; } return false; @@ -266,9 +243,8 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> { if(priLevel >= 0 && priLevel < LEVEL && priorityQueues.get(priLevel).remove(block)) { NameNode.blockStateChangeLog.debug( - "BLOCK* NameSystem.UnderReplicationBlock.remove: Removing block {}" + - " from priority queue {}", block, priLevel); - timestampsMap.remove(block); + "BLOCK* NameSystem.UnderReplicationBlock.remove: Removing block {}" + + " from priority queue {}", block, priLevel); return true; } else { // Try to remove the block from all queues if the block was @@ -278,7 +254,6 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.UnderReplicationBlock.remove: Removing block" + " {} from priority queue {}", block, priLevel); - timestampsMap.remove(block); return true; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cba52d87/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 0a2422e..0b44431 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3770,14 +3770,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, // not locking return blockManager.getMissingReplOneBlocksCount(); } - - @Metric({"TimeOfTheOldestBlockToBeReplicated", - "The timestamp of the oldest block to be replicated. If there are no" + - "under-replicated or corrupt blocks, return 0."}) - public long getTimeOfTheOldestBlockToBeReplicated() { - return blockManager.getTimeOfTheOldestBlockToBeReplicated(); - } - + @Metric({"ExpiredHeartbeats", "Number of expired heartbeats"}) public int getExpiredHeartbeats() { return datanodeStatistics.getExpiredHeartbeats(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cba52d87/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java index 7615cee..27b35f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java @@ -28,10 +28,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.util.Time; import org.junit.Test; import java.util.Iterator; @@ -148,50 +146,4 @@ public class TestUnderReplicatedBlocks { } - @Test - public void testGetTimeOfTheOldestBlockToBeReplicated() { - UnderReplicatedBlocks blocks = new UnderReplicatedBlocks(); - BlockInfo block1 = new BlockInfoContiguous(new Block(1), (short) 1); - BlockInfo block2 = new BlockInfoContiguous(new Block(2), (short) 1); - - // if there are no under-replicated or corrupt blocks, return 0 - assertEquals(blocks.getTimeOfTheOldestBlockToBeReplicated(), 0L); - - // add block1, add block2, remove block1, remove block2 - long time1 = Time.now(); - blocks.add(block1, 1, 0, 3); - long time2 = Time.now(); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() >= time1); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() <= time2); - - blocks.add(block2, 2, 0, 3); - long time3 = Time.now(); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() >= time1); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() <= time2); - - blocks.remove(block1, UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() >= time2); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() <= time3); - - blocks.remove(block2, UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED); - assertEquals(blocks.getTimeOfTheOldestBlockToBeReplicated(), 0L); - - // add block2, add block1, remove block1, remove block2 - time1 = Time.now(); - blocks.add(block2, 2, 0, 3); - time2 = Time.now(); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() >= time1); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() <= time2); - - blocks.add(block1, 1, 0, 3); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() >= time1); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() <= time2); - - blocks.remove(block1, UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() >= time1); - assertTrue(blocks.getTimeOfTheOldestBlockToBeReplicated() <= time2); - - blocks.remove(block2, UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED); - assertEquals(blocks.getTimeOfTheOldestBlockToBeReplicated(), 0L); - } }