HDFS-8684. Erasure Coding: fix some block number calculation for striped block. (yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/904bd7d2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/904bd7d2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/904bd7d2 Branch: refs/heads/HDFS-7285-REBASE Commit: 904bd7d25b3c51265a6286f0a27031642194500b Parents: 185b173 Author: yliu <[email protected]> Authored: Mon Jul 6 19:14:18 2015 +0800 Committer: Vinayakumar B <[email protected]> Committed: Thu Aug 13 17:16:20 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 5 ++++- .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 11 ++++------- .../apache/hadoop/hdfs/server/namenode/INodeFile.java | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/904bd7d2/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 7e10753..8f720fc 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -325,4 +325,7 @@ multiple times. (Kai Sasaki via szetszwo) HDFS-8468. 2 RPC calls for every file read in DFSClient#open(..) resulting in - double Audit log entries (vinayakumarb) \ No newline at end of file + double Audit log entries (vinayakumarb) + + HDFS-8684. Erasure Coding: fix some block number calculation for striped + block. (yliu) http://git-wip-us.apache.org/repos/asf/hadoop/blob/904bd7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 66fca2f..d951d3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3155,12 +3155,12 @@ public class BlockManager implements BlockStatsMXBean { bc.getStoragePolicyID()); final List<StorageType> excessTypes = storagePolicy.chooseExcess( replication, DatanodeStorageInfo.toStorageTypes(nonExcess)); - if (!storedBlock.isStriped()) { - chooseExcessReplicasContiguous(bc, nonExcess, storedBlock, - replication, addedNode, delNodeHint, excessTypes); - } else { + if (storedBlock.isStriped()) { chooseExcessReplicasStriped(bc, nonExcess, storedBlock, delNodeHint, excessTypes); + } else { + chooseExcessReplicasContiguous(bc, nonExcess, storedBlock, + replication, addedNode, delNodeHint, excessTypes); } } @@ -3237,9 +3237,6 @@ public class BlockManager implements BlockStatsMXBean { assert storedBlock instanceof BlockInfoStriped; BlockInfoStriped sblk = (BlockInfoStriped) storedBlock; short groupSize = sblk.getTotalBlockNum(); - if (nonExcess.size() <= groupSize) { - return; - } BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true); List<DatanodeStorageInfo> empty = new ArrayList<>(0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/904bd7d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 05425bf..c8943ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -814,13 +814,9 @@ public class INodeFile extends INodeWithAdditionalFields if (!includesLastUcBlock) { size = 0; } else if (usePreferredBlockSize4LastUcBlock) { - // Striped blocks keeps block group which counts - // (data blocks num + parity blocks num). When you - // count actual used size by BlockInfoStripedUC must - // be multiplied by these blocks number. BlockInfoStripedUnderConstruction blockInfoStripedUC = (BlockInfoStripedUnderConstruction) blockInfos[last]; - size = getPreferredBlockSize() * blockInfoStripedUC.getTotalBlockNum(); + size = getPreferredBlockSize() * blockInfoStripedUC.getDataBlockNum(); } } //sum other blocks
