Repository: hadoop Updated Branches: refs/heads/trunk d79c645b2 -> 355b907c0
HDFS-11121. Add assertions to BlockInfo#addStorage to protect from breaking reportedBlock-blockGroup mapping. Contributed by Takanobu Asanuma. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/355b907c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/355b907c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/355b907c Branch: refs/heads/trunk Commit: 355b907c09dca8cb813a1ae543dce6c1397a4026 Parents: d79c645 Author: Wei-Chiu Chuang <[email protected]> Authored: Sat Jan 21 05:56:22 2017 +0800 Committer: Wei-Chiu Chuang <[email protected]> Committed: Sat Jan 21 05:56:22 2017 +0800 ---------------------------------------------------------------------- .../blockmanagement/BlockInfoContiguous.java | 4 ++++ .../blockmanagement/BlockInfoStriped.java | 7 +++++++ .../server/blockmanagement/TestBlockInfo.java | 20 +++++++++++++++++--- .../blockmanagement/TestBlockInfoStriped.java | 17 +++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java index 088e068..149efc9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; @@ -55,6 +56,9 @@ public class BlockInfoContiguous extends BlockInfo { @Override boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) { + Preconditions.checkArgument(this.getBlockId() == reportedBlock.getBlockId(), + "reported blk_%s is different from stored blk_%s", + reportedBlock.getBlockId(), this.getBlockId()); // find the last null node int lastNode = ensureCapacity(1); setStorageInfo(lastNode, storage); http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index 10f531b..86e42a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; @@ -114,6 +115,12 @@ public class BlockInfoStriped extends BlockInfo { @Override boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) { + Preconditions.checkArgument(BlockIdManager.isStripedBlockID( + reportedBlock.getBlockId()), "reportedBlock is not striped"); + Preconditions.checkArgument(BlockIdManager.convertToStripedID( + reportedBlock.getBlockId()) == this.getBlockId(), + "reported blk_%s does not belong to the group of stored blk_%s", + reportedBlock.getBlockId(), this.getBlockId()); int blockIndex = BlockIdManager.getBlockIndex(reportedBlock); int index = blockIndex; DatanodeStorageInfo old = getStorageInfo(index); http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java index 4e7cf3d..2dfa900 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java @@ -23,6 +23,7 @@ import static org.hamcrest.core.Is.is; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.junit.Assert; @@ -54,7 +55,8 @@ public class TestBlockInfo { public void testAddStorage() throws Exception { BlockInfo blockInfo = new BlockInfoContiguous((short) 3); - final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1"); + final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo( + "storageID", "127.0.0.1"); boolean added = blockInfo.addStorage(storage, blockInfo); @@ -66,8 +68,10 @@ public class TestBlockInfo { public void testReplaceStorage() throws Exception { // Create two dummy storages. - final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1"); - final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2")); + final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo( + "storageID1", "127.0.0.1"); + final DatanodeStorageInfo storage2 = new DatanodeStorageInfo( + storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2")); final int NUM_BLOCKS = 10; BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS]; @@ -83,4 +87,14 @@ public class TestBlockInfo { Assert.assertThat(added, is(false)); Assert.assertThat(blockInfos[NUM_BLOCKS/2].getStorageInfo(0), is(storage2)); } + + @Test(expected=IllegalArgumentException.class) + public void testAddStorageWithDifferentBlock() throws Exception { + BlockInfo blockInfo1 = new BlockInfoContiguous(new Block(1000L), (short) 3); + BlockInfo blockInfo2 = new BlockInfoContiguous(new Block(1001L), (short) 3); + + final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo( + "storageID", "127.0.0.1"); + blockInfo1.addStorage(storage, blockInfo2); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java index 255127f..6a55d8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java @@ -220,4 +220,21 @@ public class TestBlockInfoStriped { assertEquals(byteBuffer.array().length, byteStream.toByteArray().length); assertArrayEquals(byteBuffer.array(), byteStream.toByteArray()); } + + @Test(expected=IllegalArgumentException.class) + public void testAddStorageWithReplicatedBlock() { + DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo( + "storageID", "127.0.0.1"); + BlockInfo replica = new BlockInfoContiguous(new Block(1000L), (short) 3); + info.addStorage(storage, replica); + } + + @Test(expected=IllegalArgumentException.class) + public void testAddStorageWithDifferentBlockGroup() { + DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo( + "storageID", "127.0.0.1"); + BlockInfo diffGroup = new BlockInfoStriped(new Block(BASE_ID + 100), + testECPolicy); + info.addStorage(storage, diffGroup); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
