HDFS-13542. TestBlockManager#testNeededReplicationWhileAppending fails due to improper cluster shutdown in TestBlockManager#testBlockManagerMachinesArray on Windows. Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d50c4d71 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d50c4d71 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d50c4d71 Branch: refs/heads/HDDS-4 Commit: d50c4d71dc42576f96ae5c268856fd1a7795f936 Parents: a922b9c Author: Inigo Goiri <inigo...@apache.org> Authored: Fri May 11 09:47:57 2018 -0700 Committer: Inigo Goiri <inigo...@apache.org> Committed: Fri May 11 09:47:57 2018 -0700 ---------------------------------------------------------------------- .../blockmanagement/TestBlockManager.java | 155 ++++++++++--------- 1 file changed, 85 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d50c4d71/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 5219a44..58ca2e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -452,8 +452,8 @@ public class TestBlockManager { String src = "/test-file"; Path file = new Path(src); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.waitActive(); try { + cluster.waitActive(); BlockManager bm = cluster.getNamesystem().getBlockManager(); FileSystem fs = cluster.getFileSystem(); NamenodeProtocols namenode = cluster.getNameNodeRpc(); @@ -492,7 +492,9 @@ public class TestBlockManager { IOUtils.closeStream(out); } } finally { - cluster.shutdown(); + if (cluster != null) { + cluster.shutdown(); + } } } @@ -1043,7 +1045,9 @@ public class TestBlockManager { assertTrue(fs.exists(file1)); fs.delete(file1, true); assertTrue(!fs.exists(file1)); - cluster.shutdown(); + if (cluster != null) { + cluster.shutdown(); + } } } @@ -1143,7 +1147,9 @@ public class TestBlockManager { assertEquals(0, bm.getBlockOpQueueLength()); assertTrue(doneLatch.await(1, TimeUnit.SECONDS)); } finally { - cluster.shutdown(); + if (cluster != null) { + cluster.shutdown(); + } } } @@ -1218,7 +1224,9 @@ public class TestBlockManager { long batched = MetricsAsserts.getLongCounter("BlockOpsBatched", rb); assertTrue(batched > 0); } finally { - cluster.shutdown(); + if (cluster != null) { + cluster.shutdown(); + } } } @@ -1227,76 +1235,83 @@ public class TestBlockManager { final Configuration conf = new HdfsConfiguration(); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); - cluster.waitActive(); - BlockManager blockManager = cluster.getNamesystem().getBlockManager(); - FileSystem fs = cluster.getFileSystem(); - final Path filePath = new Path("/tmp.txt"); - final long fileLen = 1L; - DFSTestUtil.createFile(fs, filePath, fileLen, (short) 3, 1L); - DFSTestUtil.waitForReplication((DistributedFileSystem)fs, - filePath, (short) 3, 60000); - ArrayList<DataNode> datanodes = cluster.getDataNodes(); - assertEquals(datanodes.size(), 4); - FSNamesystem ns = cluster.getNamesystem(); - // get the block - final String bpid = cluster.getNamesystem().getBlockPoolId(); - File storageDir = cluster.getInstanceStorageDir(0, 0); - File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); - assertTrue("Data directory does not exist", dataDir.exists()); - BlockInfo blockInfo = blockManager.blocksMap.getBlocks().iterator().next(); - ExtendedBlock blk = new ExtendedBlock(bpid, blockInfo.getBlockId(), - blockInfo.getNumBytes(), blockInfo.getGenerationStamp()); - DatanodeDescriptor failedStorageDataNode = - blockManager.getStoredBlock(blockInfo).getDatanode(0); - DatanodeDescriptor corruptStorageDataNode = - blockManager.getStoredBlock(blockInfo).getDatanode(1); - - ArrayList<StorageReport> reports = new ArrayList<StorageReport>(); - for(int i=0; i<failedStorageDataNode.getStorageInfos().length; i++) { - DatanodeStorageInfo storageInfo = failedStorageDataNode - .getStorageInfos()[i]; - DatanodeStorage dns = new DatanodeStorage( - failedStorageDataNode.getStorageInfos()[i].getStorageID(), - DatanodeStorage.State.FAILED, - failedStorageDataNode.getStorageInfos()[i].getStorageType()); - while(storageInfo.getBlockIterator().hasNext()) { - BlockInfo blockInfo1 = storageInfo.getBlockIterator().next(); - if(blockInfo1.equals(blockInfo)) { - StorageReport report = new StorageReport( - dns, true, storageInfo.getCapacity(), - storageInfo.getDfsUsed(), storageInfo.getRemaining(), - storageInfo.getBlockPoolUsed(), 0L); - reports.add(report); - break; + try { + cluster.waitActive(); + BlockManager blockManager = cluster.getNamesystem().getBlockManager(); + FileSystem fs = cluster.getFileSystem(); + final Path filePath = new Path("/tmp.txt"); + final long fileLen = 1L; + DFSTestUtil.createFile(fs, filePath, fileLen, (short) 3, 1L); + DFSTestUtil.waitForReplication((DistributedFileSystem)fs, + filePath, (short) 3, 60000); + ArrayList<DataNode> datanodes = cluster.getDataNodes(); + assertEquals(datanodes.size(), 4); + FSNamesystem ns = cluster.getNamesystem(); + // get the block + final String bpid = cluster.getNamesystem().getBlockPoolId(); + File storageDir = cluster.getInstanceStorageDir(0, 0); + File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); + assertTrue("Data directory does not exist", dataDir.exists()); + BlockInfo blockInfo = + blockManager.blocksMap.getBlocks().iterator().next(); + ExtendedBlock blk = new ExtendedBlock(bpid, blockInfo.getBlockId(), + blockInfo.getNumBytes(), blockInfo.getGenerationStamp()); + DatanodeDescriptor failedStorageDataNode = + blockManager.getStoredBlock(blockInfo).getDatanode(0); + DatanodeDescriptor corruptStorageDataNode = + blockManager.getStoredBlock(blockInfo).getDatanode(1); + + ArrayList<StorageReport> reports = new ArrayList<StorageReport>(); + for(int i=0; i<failedStorageDataNode.getStorageInfos().length; i++) { + DatanodeStorageInfo storageInfo = failedStorageDataNode + .getStorageInfos()[i]; + DatanodeStorage dns = new DatanodeStorage( + failedStorageDataNode.getStorageInfos()[i].getStorageID(), + DatanodeStorage.State.FAILED, + failedStorageDataNode.getStorageInfos()[i].getStorageType()); + while(storageInfo.getBlockIterator().hasNext()) { + BlockInfo blockInfo1 = storageInfo.getBlockIterator().next(); + if(blockInfo1.equals(blockInfo)) { + StorageReport report = new StorageReport( + dns, true, storageInfo.getCapacity(), + storageInfo.getDfsUsed(), storageInfo.getRemaining(), + storageInfo.getBlockPoolUsed(), 0L); + reports.add(report); + break; + } } } - } - failedStorageDataNode.updateHeartbeat(reports.toArray(StorageReport - .EMPTY_ARRAY), 0L, 0L, 0, 0, null); - ns.writeLock(); - DatanodeStorageInfo corruptStorageInfo= null; - for(int i=0; i<corruptStorageDataNode.getStorageInfos().length; i++) { - corruptStorageInfo = corruptStorageDataNode.getStorageInfos()[i]; - while(corruptStorageInfo.getBlockIterator().hasNext()) { - BlockInfo blockInfo1 = corruptStorageInfo.getBlockIterator().next(); - if (blockInfo1.equals(blockInfo)) { - break; + failedStorageDataNode.updateHeartbeat(reports.toArray(StorageReport + .EMPTY_ARRAY), 0L, 0L, 0, 0, null); + ns.writeLock(); + DatanodeStorageInfo corruptStorageInfo= null; + for(int i=0; i<corruptStorageDataNode.getStorageInfos().length; i++) { + corruptStorageInfo = corruptStorageDataNode.getStorageInfos()[i]; + while(corruptStorageInfo.getBlockIterator().hasNext()) { + BlockInfo blockInfo1 = corruptStorageInfo.getBlockIterator().next(); + if (blockInfo1.equals(blockInfo)) { + break; + } } } + blockManager.findAndMarkBlockAsCorrupt(blk, corruptStorageDataNode, + corruptStorageInfo.getStorageID(), + CorruptReplicasMap.Reason.ANY.toString()); + ns.writeUnlock(); + BlockInfo[] blockInfos = new BlockInfo[] {blockInfo}; + ns.readLock(); + LocatedBlocks locatedBlocks = + blockManager.createLocatedBlocks(blockInfos, 3L, false, 0L, 3L, + false, false, null, null); + assertTrue("Located Blocks should exclude corrupt" + + "replicas and failed storages", + locatedBlocks.getLocatedBlocks().size() == 1); + ns.readUnlock(); + } finally { + if (cluster != null) { + cluster.shutdown(); + } } - blockManager.findAndMarkBlockAsCorrupt(blk, corruptStorageDataNode, - corruptStorageInfo.getStorageID(), - CorruptReplicasMap.Reason.ANY.toString()); - ns.writeUnlock(); - BlockInfo[] blockInfos = new BlockInfo[] {blockInfo}; - ns.readLock(); - LocatedBlocks locatedBlocks = - blockManager.createLocatedBlocks(blockInfos, 3L, false, 0L, 3L, - false, false, null, null); - assertTrue("Located Blocks should exclude corrupt" + - "replicas and failed storages", - locatedBlocks.getLocatedBlocks().size() == 1); - ns.readUnlock(); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
