Repository: hadoop Updated Branches: refs/heads/branch-2 351fac25a -> 4e099c113
HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and even for failed transfers (Contributed by Vinayakumar B) (cherry picked from commit 9d8952f97f638ede27e4336b9601507d7bb1de7b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e099c11 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e099c11 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e099c11 Branch: refs/heads/branch-2 Commit: 4e099c113b5823f9bce92a014d469083a7863c1a Parents: 351fac2 Author: Vinayakumar B <vinayakum...@apache.org> Authored: Thu Apr 9 11:58:00 2015 +0530 Committer: Vinayakumar B <vinayakum...@apache.org> Committed: Thu Apr 9 11:59:05 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/datanode/BPOfferService.java | 1 - .../hadoop/hdfs/server/datanode/DataNode.java | 2 ++ .../datanode/metrics/DataNodeMetrics.java | 4 +-- .../server/datanode/TestDataNodeMetrics.java | 37 +++++++++++++++++--- 5 files changed, 40 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5a75f383..53b88dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -136,6 +136,9 @@ Release 2.8.0 - UNRELEASED HDFS-7725. Incorrect "nodes in service" metrics caused all writes to fail. (Ming Ma via wang) + HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and + even for failed transfers (vinayakumarb) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 8efad83..67979f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -657,7 +657,6 @@ class BPOfferService { // Send a copy of a block to another datanode dn.transferBlocks(bcmd.getBlockPoolId(), bcmd.getBlocks(), bcmd.getTargets(), bcmd.getTargetStorageTypes()); - dn.metrics.incrBlocksReplicated(bcmd.getBlocks().length); break; case DatanodeProtocol.DNA_INVALIDATE: // http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 6f70168..1285aaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -2181,6 +2181,8 @@ public class DataNode extends ReconfigurableBase + Arrays.asList(targets)); } } + } else { + metrics.incrBlocksReplicated(); } } catch (IOException ie) { LOG.warn(bpReg + ":Failed to transfer " + b + " to " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java index 2e8eb22..2e62b3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java @@ -210,8 +210,8 @@ public class DataNodeMetrics { cacheReports.add(latency); } - public void incrBlocksReplicated(int delta) { - blocksReplicated.incr(delta); + public void incrBlocksReplicated() { + blocksReplicated.incr(); } public void incrBlocksWritten() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java index 5d27fe6..0b47344 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java @@ -21,17 +21,15 @@ import static org.apache.hadoop.test.MetricsAsserts.assertCounter; import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.io.Closeable; import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.List; -import java.util.Map; import com.google.common.collect.Lists; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -45,6 +43,8 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.util.Time; @@ -293,4 +293,33 @@ public class TestDataNodeMetrics { } } + @Test + public void testDatanodeBlocksReplicatedMetric() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + try { + FileSystem fs = cluster.getFileSystem(); + List<DataNode> datanodes = cluster.getDataNodes(); + assertEquals(datanodes.size(), 1); + DataNode datanode = datanodes.get(0); + + MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name()); + long blocksReplicated = getLongCounter("BlocksReplicated", rb); + assertEquals("No blocks replicated yet", 0, blocksReplicated); + + Path path = new Path("/counter.txt"); + DFSTestUtil.createFile(fs, path, 1024, (short) 2, Time.monotonicNow()); + cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR, null); + ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fs, path); + DFSTestUtil.waitForReplication(cluster, firstBlock, 1, 2, 0); + + MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name()); + blocksReplicated = getLongCounter("BlocksReplicated", rbNew); + assertEquals("blocks replicated counter incremented", 1, blocksReplicated); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } }
