HDFS-9789. Correctly update DataNode's scheduled block size when writing small EC file. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19adb2bc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19adb2bc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19adb2bc Branch: refs/heads/yarn-2877 Commit: 19adb2bc641999b83e25ff0e107ba8c6edbad399 Parents: 39a71b6 Author: Jing Zhao <ji...@apache.org> Authored: Wed Feb 10 21:24:00 2016 -0800 Committer: Jing Zhao <ji...@apache.org> Committed: Wed Feb 10 21:24:00 2016 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/blockmanagement/BlockManager.java | 12 +++++-- .../BlockUnderConstructionFeature.java | 25 ++++++++++++++ .../server/namenode/TestAddStripedBlocks.java | 35 ++++++++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/19adb2bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e407ef3..17a05d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -421,6 +421,9 @@ Trunk (Unreleased) HDFS-9646. ErasureCodingWorker may fail when recovering data blocks with length less than the first internal block. (jing9) + HDFS-9789. Correctly update DataNode's scheduled block size when writing + small EC file. (jing9) + BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS HDFS-7347. Configurable erasure coding policy for individual files and http://git-wip-us.apache.org/repos/asf/hadoop/blob/19adb2bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index bee9372..9e8026b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -734,14 +734,20 @@ public class BlockManager implements BlockStatsMXBean { if(lastBlock.isComplete()) return false; // already completed (e.g. by syncBlock) - final boolean b = commitBlock(lastBlock, commitBlock); + final boolean committed = commitBlock(lastBlock, commitBlock); + if (committed && lastBlock.isStriped()) { + // update scheduled size for DatanodeStorages that do not store any + // internal blocks + lastBlock.getUnderConstructionFeature() + .updateStorageScheduledSize((BlockInfoStriped) lastBlock); + } if (hasMinStorage(lastBlock)) { - if (b) { + if (committed) { addExpectedReplicasToPending(lastBlock, bc); } completeBlock(lastBlock, false); } - return b; + return committed; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/19adb2bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java index 05ff809..b46b470 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java @@ -124,6 +124,31 @@ public class BlockUnderConstructionFeature { } /** + * when committing a striped block whose size is less than a stripe, we need + * to decrease the scheduled block size of the DataNodes that do not store + * any internal block. + */ + void updateStorageScheduledSize(BlockInfoStriped storedBlock) { + assert storedBlock.getUnderConstructionFeature() == this; + if (replicas == null) { + return; + } + final int dataBlockNum = storedBlock.getDataBlockNum(); + final int realDataBlockNum = storedBlock.getRealDataBlockNum(); + if (realDataBlockNum < dataBlockNum) { + for (ReplicaUnderConstruction replica : replicas) { + int index = BlockIdManager.getBlockIndex(replica); + if (index >= realDataBlockNum && index < dataBlockNum) { + final DatanodeStorageInfo storage = + replica.getExpectedStorageLocation(); + storage.getDatanodeDescriptor() + .decrementBlocksScheduled(storage.getStorageType()); + } + } + } + } + + /** * Return the state of the block under construction. * @see BlockUCState */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/19adb2bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index b31c580..7ca1a53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -37,9 +37,11 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten; import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; @@ -93,6 +95,39 @@ public class TestAddStripedBlocks { } /** + * Check if the scheduled block size on each DN storage is correctly updated + */ + @Test + public void testBlockScheduledUpdate() throws Exception { + final FSNamesystem fsn = cluster.getNamesystem(); + final Path foo = new Path("/foo"); + try (FSDataOutputStream out = dfs.create(foo, true)) { + DFSStripedOutputStream sout = (DFSStripedOutputStream) out.getWrappedStream(); + writeAndFlushStripedOutputStream(sout, DFS_BYTES_PER_CHECKSUM_DEFAULT); + + // make sure the scheduled block size has been updated for each DN storage + // in NN + final List<DatanodeDescriptor> dnList = new ArrayList<>(); + fsn.getBlockManager().getDatanodeManager().fetchDatanodes(dnList, null, false); + for (DatanodeDescriptor dn : dnList) { + Assert.assertEquals(1, dn.getBlocksScheduled()); + } + } + + // we have completed the file, force the DN to flush IBR + for (DataNode dn : cluster.getDataNodes()) { + DataNodeTestUtils.triggerBlockReport(dn); + } + + // check the scheduled block size again + final List<DatanodeDescriptor> dnList = new ArrayList<>(); + fsn.getBlockManager().getDatanodeManager().fetchDatanodes(dnList, null, false); + for (DatanodeDescriptor dn : dnList) { + Assert.assertEquals(0, dn.getBlocksScheduled()); + } + } + + /** * Make sure the IDs of striped blocks do not conflict */ @Test
