HDFS-7936. Erasure coding: resolving conflicts in the branch when merging trunk changes (this commit is for HDFS-8327 and HDFS-8357). Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10e2d80c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10e2d80c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10e2d80c Branch: refs/heads/HDFS-7285 Commit: 10e2d80c8bbcd9ec9380fc2f83a98debf37c1887 Parents: c31cb27 Author: Zhe Zhang <z...@apache.org> Authored: Mon May 11 12:22:12 2015 -0700 Committer: Zhe Zhang <z...@apache.org> Committed: Mon May 11 12:22:12 2015 -0700 ---------------------------------------------------------------------- .../hdfs/server/blockmanagement/BlockInfo.java | 12 +++++-- .../blockmanagement/BlockInfoContiguous.java | 38 -------------------- .../server/blockmanagement/BlockManager.java | 4 +-- .../erasurecode/ErasureCodingWorker.java | 3 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 10 ++---- .../server/namenode/TestStripedINodeFile.java | 8 ++--- .../namenode/TestTruncateQuotaUpdate.java | 3 +- 7 files changed, 23 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e2d80c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index aebfbb1..61068b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -88,13 +88,21 @@ public abstract class BlockInfo extends Block BlockInfo getPrevious(int index) { assert this.triplets != null : "BlockInfo is not initialized"; assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; - return (BlockInfo) triplets[index*3+1]; + BlockInfo info = (BlockInfo)triplets[index*3+1]; + assert info == null || + info.getClass().getName().startsWith(BlockInfo.class.getName()) : + "BlockInfo is expected at " + index*3; + return info; } BlockInfo getNext(int index) { assert this.triplets != null : "BlockInfo is not initialized"; assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; - return (BlockInfo) triplets[index*3+2]; + BlockInfo info = (BlockInfo)triplets[index*3+2]; + assert info == null || info.getClass().getName().startsWith( + BlockInfo.class.getName()) : + "BlockInfo is expected at " + index*3; + return info; } void setStorageInfo(int index, DatanodeStorageInfo storage) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e2d80c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java index a9ea625..be336b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java @@ -47,18 +47,6 @@ public class BlockInfoContiguous extends BlockInfo { this.setBlockCollection(from.getBlockCollection()); } - public BlockCollection getBlockCollection() { - return bc; - } - - public void setBlockCollection(BlockCollection bc) { - this.bc = bc; - } - - public boolean isDeleted() { - return (bc == null); - } - public DatanodeDescriptor getDatanode(int index) { DatanodeStorageInfo storage = getStorageInfo(index); return storage == null ? null : storage.getDatanodeDescriptor(); @@ -70,32 +58,6 @@ public class BlockInfoContiguous extends BlockInfo { return (DatanodeStorageInfo)triplets[index*3]; } - private BlockInfoContiguous getPrevious(int index) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; - BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1]; - assert info == null || - info.getClass().getName().startsWith(BlockInfoContiguous.class.getName()) : - "BlockInfo is expected at " + index*3; - return info; - } - - BlockInfoContiguous getNext(int index) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; - BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2]; - assert info == null || info.getClass().getName().startsWith( - BlockInfoContiguous.class.getName()) : - "BlockInfo is expected at " + index*3; - return info; - } - - private void setStorageInfo(int index, DatanodeStorageInfo storage) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; - triplets[index*3] = storage; - } - /** * Return the previous block on the block list for the datanode at * position index. Set the previous block on the list to "to". http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e2d80c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 0ad391a..6b98dcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2462,7 +2462,7 @@ public class BlockManager { if (rbi.getReportedState() == null) { // This is a DELETE_BLOCK request DatanodeStorageInfo storageInfo = rbi.getStorageInfo(); - removeStoredBlock(rbi.getBlock(), + removeStoredBlock(getStoredBlock(rbi.getBlock()), storageInfo.getDatanodeDescriptor()); } else { processAndHandleReportedBlock(rbi.getStorageInfo(), @@ -3207,7 +3207,7 @@ public class BlockManager { QUEUE_REASON_FUTURE_GENSTAMP); return; } - removeStoredBlock(block, node); + removeStoredBlock(getStoredBlock(block), node); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e2d80c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index c4e568f..5ede508 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -49,6 +49,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSPacket; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.RemoteBlockReader2; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; @@ -872,7 +873,7 @@ public final class ErasureCodingWorker { unbufIn = saslStreams.in; out = new DataOutputStream(new BufferedOutputStream(unbufOut, - HdfsServerConstants.SMALL_BUFFER_SIZE)); + DFSUtil.getSmallBufferSize(conf))); in = new DataInputStream(unbufIn); DatanodeInfo source = new DatanodeInfo(datanode.getDatanodeId()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e2d80c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 748c0be..79d4a29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -708,11 +708,7 @@ public class INodeFile extends INodeWithAdditionalFields */ public final QuotaCounts computeQuotaUsageWithStriped( BlockStoragePolicySuite bsps, QuotaCounts counts) { - long nsDelta = 1; - final long ssDelta = storagespaceConsumed(); - counts.addNameSpace(nsDelta); - counts.addStorageSpace(ssDelta); - return counts; + return null; } @Override @@ -981,11 +977,11 @@ public class INodeFile extends INodeWithAdditionalFields } long size = 0; - for (BlockInfoContiguous b : blocks) { + for (BlockInfo b : blocks) { size += b.getNumBytes(); } - BlockInfoContiguous[] sblocks = null; + BlockInfo[] sblocks = null; FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); if (sf != null) { FileDiff diff = sf.getDiffs().getLast(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e2d80c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index 7a330b9..a725e6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -109,8 +109,8 @@ public class TestStripedINodeFile { // a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0 // b. <Num Bytes> % <Num Bytes per Stripes> = 1 // c. <Last Stripe Length> * <Parity Block Num> = 1 * 3 - assertEquals(4, inf.storagespaceConsumedWithStriped()); - assertEquals(4, inf.storagespaceConsumed()); + assertEquals(4, inf.storagespaceConsumedWithStriped(null)); + assertEquals(4, inf.storagespaceConsumed(null)); } @Test @@ -134,8 +134,8 @@ public class TestStripedINodeFile { inf.addBlock(blockInfoStriped1); inf.addBlock(blockInfoStriped2); // This is the double size of one block in above case. - assertEquals(4 * 2, inf.storagespaceConsumedWithStriped()); - assertEquals(4 * 2, inf.storagespaceConsumed()); + assertEquals(4 * 2, inf.storagespaceConsumedWithStriped(null)); + assertEquals(4 * 2, inf.storagespaceConsumed(null)); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e2d80c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java index f6b18e6..57159db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; @@ -77,7 +78,7 @@ public class TestTruncateQuotaUpdate { @Test public void testTruncateWithSnapshotNoDivergence() { INodeFile file = createMockFile(BLOCKSIZE * 2 + BLOCKSIZE / 2, REPLICATION); - addSnapshotFeature(file, file.getBlocks()); + addSnapshotFeature(file, file.getContiguousBlocks()); // case 4: truncate to 1.5 blocks // all the blocks are in snapshot. truncate need to allocate a new block