Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java Fri Jan 3 07:26:52 2014 @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.io.File; @@ -25,6 +27,7 @@ import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeoutException; @@ -68,7 +71,15 @@ import org.mockito.invocation.Invocation /** * This test simulates a variety of situations when blocks are being * intentionally corrupted, unexpectedly modified, and so on before a block - * report is happening + * report is happening. + * + * For each test case it runs two variations: + * #1 - For a given DN, the first variation sends block reports for all + * storages in a single call to the NN. + * #2 - For a given DN, the second variation sends block reports for each + * storage in a separate call. + * + * The behavior should be the same in either variation. */ public class TestBlockReport { public static final Log LOG = LogFactory.getLog(TestBlockReport.class); @@ -88,7 +99,7 @@ public class TestBlockReport { private MiniDFSCluster cluster; private DistributedFileSystem fs; - Random rand = new Random(RAND_LIMIT); + private static Random rand = new Random(RAND_LIMIT); private static Configuration conf; @@ -112,6 +123,155 @@ public class TestBlockReport { cluster.shutdown(); } + // Generate a block report, optionally corrupting the generation + // stamp and/or length of one block. + private static StorageBlockReport[] getBlockReports( + DataNode dn, String bpid, boolean corruptOneBlockGs, + boolean corruptOneBlockLen) { + Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpid); + + // Send block report + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + boolean corruptedGs = false; + boolean corruptedLen = false; + + int reportIndex = 0; + for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) { + DatanodeStorage dnStorage = kvPair.getKey(); + BlockListAsLongs blockList = kvPair.getValue(); + + // Walk the list of blocks until we find one each to corrupt the + // generation stamp and length, if so requested. + for (int i = 0; i < blockList.getNumberOfBlocks(); ++i) { + if (corruptOneBlockGs && !corruptedGs) { + blockList.corruptBlockGSForTesting(i, rand); + LOG.info("Corrupted the GS for block ID " + i); + corruptedGs = true; + } else if (corruptOneBlockLen && !corruptedLen) { + blockList.corruptBlockLengthForTesting(i, rand); + LOG.info("Corrupted the length for block ID " + i); + corruptedLen = true; + } else { + break; + } + } + + reports[reportIndex++] = + new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); + } + + return reports; + } + + /** + * Utility routine to send block reports to the NN, either in a single call + * or reporting one storage per call. + * + * @param dnR + * @param poolId + * @param reports + * @param needtoSplit + * @throws IOException + */ + private void sendBlockReports(DatanodeRegistration dnR, String poolId, + StorageBlockReport[] reports, boolean needtoSplit) throws IOException { + if (!needtoSplit) { + LOG.info("Sending combined block reports for " + dnR); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); + } else { + for (StorageBlockReport report : reports) { + LOG.info("Sending block report for storage " + report.getStorage().getStorageID()); + StorageBlockReport[] singletonReport = { report }; + cluster.getNameNodeRpc().blockReport(dnR, poolId, singletonReport); + } + } + } + + /** + * Test variations blockReport_01 through blockReport_09 with combined + * and split block reports. + */ + @Test + public void blockReportCombined_01() throws IOException { + blockReport_01(false); + } + + @Test + public void blockReportSplit_01() throws IOException { + blockReport_01(true); + } + + @Test + public void blockReportCombined_02() throws IOException { + blockReport_02(false); + } + + @Test + public void blockReportSplit_02() throws IOException { + blockReport_02(true); + } + + @Test + public void blockReportCombined_03() throws IOException { + blockReport_03(false); + } + + @Test + public void blockReportSplit_03() throws IOException { + blockReport_03(true); + } + + @Test + public void blockReportCombined_04() throws IOException { + blockReport_04(false); + } + + @Test + public void blockReportSplit_04() throws IOException { + blockReport_04(true); + } + + @Test + public void blockReportCombined_06() throws Exception { + blockReport_06(false); + } + + @Test + public void blockReportSplit_06() throws Exception { + blockReport_06(true); + } + + @Test + public void blockReportCombined_07() throws Exception { + blockReport_07(false); + } + + @Test + public void blockReportSplit_07() throws Exception { + blockReport_07(true); + } + + @Test + public void blockReportCombined_08() throws Exception { + blockReport_08(false); + } + + @Test + public void blockReportSplit_08() throws Exception { + blockReport_08(true); + } + + @Test + public void blockReportCombined_09() throws Exception { + blockReport_09(false); + } + + @Test + public void blockReportSplit_09() throws Exception { + blockReport_09(true); + } /** * Test write a file, verifies and closes it. Then the length of the blocks * are messed up and BlockReport is forced. @@ -119,8 +279,7 @@ public class TestBlockReport { * * @throws java.io.IOException on an error */ - @Test - public void blockReport_01() throws IOException { + private void blockReport_01(boolean splitBlockReports) throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); @@ -152,10 +311,8 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); + sendBlockReports(dnR, poolId, reports, splitBlockReports); List<LocatedBlock> blocksAfterReport = DFSTestUtil.getAllBlocks(fs.open(filePath)); @@ -181,8 +338,7 @@ public class TestBlockReport { * * @throws IOException in case of errors */ - @Test - public void blockReport_02() throws IOException { + private void blockReport_02(boolean splitBlockReports) throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); LOG.info("Running test " + METHOD_NAME); @@ -210,7 +366,6 @@ public class TestBlockReport { for (Integer aRemovedIndex : removedIndex) { blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock()); } - ArrayList<Block> blocks = locatedToBlocks(lBlocks, removedIndex); if(LOG.isDebugEnabled()) { LOG.debug("Number of blocks allocated " + lBlocks.size()); @@ -224,8 +379,11 @@ public class TestBlockReport { for (File f : findAllFiles(dataDir, new MyFileFilter(b.getBlockName(), true))) { DataNodeTestUtils.getFSDataset(dn0).unfinalizeBlock(b); - if (!f.delete()) + if (!f.delete()) { LOG.warn("Couldn't delete " + b.getBlockName()); + } else { + LOG.debug("Deleted file " + f.toString()); + } } } @@ -234,10 +392,8 @@ public class TestBlockReport { // all blocks belong to the same file, hence same BP String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn0, poolId, false, false); + sendBlockReports(dnR, poolId, reports, splitBlockReports); BlockManagerTestUtil.getComputedDatanodeWork(cluster.getNamesystem() .getBlockManager()); @@ -252,61 +408,64 @@ public class TestBlockReport { /** - * Test writes a file and closes it. Then test finds a block - * and changes its GS to be < of original one. - * New empty block is added to the list of blocks. + * Test writes a file and closes it. + * Block reported is generated with a bad GS for a single block. * Block report is forced and the check for # of corrupted blocks is performed. * * @throws IOException in case of an error */ - @Test - public void blockReport_03() throws IOException { + private void blockReport_03(boolean splitBlockReports) throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); - - ArrayList<Block> blocks = - prepareForRide(filePath, METHOD_NAME, FILE_SIZE); - - // The block with modified GS won't be found. Has to be deleted - blocks.get(0).setGenerationStamp(rand.nextLong()); - // This new block is unknown to NN and will be mark for deletion. - blocks.add(new Block()); + ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - DatanodeCommand dnCmd = - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); - if(LOG.isDebugEnabled()) { - LOG.debug("Got the command: " + dnCmd); - } + StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false); + sendBlockReports(dnR, poolId, reports, splitBlockReports); printStats(); - assertEquals("Wrong number of CorruptedReplica+PendingDeletion " + - "blocks is found", 2, - cluster.getNamesystem().getCorruptReplicaBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); } /** - * This test isn't a representative case for BlockReport - * The empty method is going to be left here to keep the naming - * of the test plan in synch with the actual implementation + * Test writes a file and closes it. + * Block reported is generated with an extra block. + * Block report is forced and the check for # of pendingdeletion + * blocks is performed. + * + * @throws IOException in case of an error */ - public void blockReport_04() { - } + private void blockReport_04(boolean splitBlockReports) throws IOException { + final String METHOD_NAME = GenericTestUtils.getMethodName(); + Path filePath = new Path("/" + METHOD_NAME + ".dat"); + DFSTestUtil.createFile(fs, filePath, + FILE_SIZE, REPL_FACTOR, rand.nextLong()); + + + DataNode dn = cluster.getDataNodes().get(DN_N0); + // all blocks belong to the same file, hence same BP + String poolId = cluster.getNamesystem().getBlockPoolId(); + + // Create a bogus new block which will not be present on the namenode. + ExtendedBlock b = new ExtendedBlock( + poolId, rand.nextLong(), 1024L, rand.nextLong()); + dn.getFSDataset().createRbw(b); + + DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); + sendBlockReports(dnR, poolId, reports, splitBlockReports); + printStats(); - // Client requests new block from NN. The test corrupts this very block - // and forces new block report. - // The test case isn't specific for BlockReport because it relies on - // BlockScanner which is out of scope of this test - // Keeping the name to be in synch with the test plan - // - public void blockReport_05() { + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); } /** @@ -317,23 +476,20 @@ public class TestBlockReport { * * @throws IOException in case of an error */ - @Test - public void blockReport_06() throws Exception { + private void blockReport_06(boolean splitBlockReports) throws Exception { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; - ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); + writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); - // all blocks belong to the same file, hence same BP + // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); + sendBlockReports(dnR, poolId, reports, splitBlockReports); printStats(); assertEquals("Wrong number of PendingReplication Blocks", 0, cluster.getNamesystem().getUnderReplicatedBlocks()); @@ -352,69 +508,40 @@ public class TestBlockReport { * * @throws IOException in case of an error */ - @Test - // Currently this test is failing as expected 'cause the correct behavior is - // not yet implemented (9/15/09) - public void blockReport_07() throws Exception { + private void blockReport_07(boolean splitBlockReports) throws Exception { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; // write file and start second node to be "older" than the original - ArrayList<Block> blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); + writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); - int randIndex = rand.nextInt(blocks.size()); - // Get a block and screw its GS - Block corruptedBlock = blocks.get(randIndex); - String secondNode = cluster.getDataNodes().get(DN_N1).getStorageId(); - if(LOG.isDebugEnabled()) { - LOG.debug("Working with " + secondNode); - LOG.debug("BlockGS before " + blocks.get(randIndex).getGenerationStamp()); - } - corruptBlockGS(corruptedBlock); - if(LOG.isDebugEnabled()) { - LOG.debug("BlockGS after " + blocks.get(randIndex).getGenerationStamp()); - LOG.debug("Done corrupting GS of " + corruptedBlock.getBlockName()); - } // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false); + sendBlockReports(dnR, poolId, reports, splitBlockReports); printStats(); - assertEquals("Wrong number of Corrupted blocks", - 1, cluster.getNamesystem().getCorruptReplicaBlocks() + -// the following might have to be added into the equation if -// the same block could be in two different states at the same time -// and then the expected number of has to be changed to '2' -// cluster.getNamesystem().getPendingReplicationBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); - - // Get another block and screw its length to be less than original - if (randIndex == 0) - randIndex++; - else - randIndex--; - corruptedBlock = blocks.get(randIndex); - corruptBlockLen(corruptedBlock); - if(LOG.isDebugEnabled()) { - LOG.debug("Done corrupting length of " + corruptedBlock.getBlockName()); - } - - report[0] = new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()); - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + assertThat("Wrong number of PendingReplication blocks", + cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); + + reports = getBlockReports(dn, poolId, true, true); + sendBlockReports(dnR, poolId, reports, splitBlockReports); printStats(); - assertEquals("Wrong number of Corrupted blocks", - 2, cluster.getNamesystem().getCorruptReplicaBlocks() + - cluster.getNamesystem().getPendingReplicationBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(2L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + assertThat("Wrong number of PendingReplication blocks", + cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); printStats(); @@ -432,8 +559,7 @@ public class TestBlockReport { * * @throws IOException in case of an error */ - @Test - public void blockReport_08() throws IOException { + private void blockReport_08(boolean splitBlockReports) throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; @@ -457,10 +583,8 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); + sendBlockReports(dnR, poolId, reports, splitBlockReports); printStats(); assertEquals("Wrong number of PendingReplication blocks", blocks.size(), cluster.getNamesystem().getPendingReplicationBlocks()); @@ -476,8 +600,7 @@ public class TestBlockReport { // Similar to BlockReport_08 but corrupts GS and len of the TEMPORARY's // replica block. Expect the same behaviour: NN should simply ignore this // block - @Test - public void blockReport_09() throws IOException { + private void blockReport_09(boolean splitBlockReports) throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; @@ -490,14 +613,11 @@ public class TestBlockReport { // write file and start second node to be "older" than the original try { - ArrayList<Block> blocks = - writeFile(METHOD_NAME, 12 * bytesChkSum, filePath); + writeFile(METHOD_NAME, 12 * bytesChkSum, filePath); Block bl = findBlock(filePath, 12 * bytesChkSum); BlockChecker bc = new BlockChecker(filePath); bc.start(); - corruptBlockGS(bl); - corruptBlockLen(bl); waitForTempReplica(bl, DN_N1); @@ -505,10 +625,8 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, true, true); + sendBlockReports(dnR, poolId, reports, splitBlockReports); printStats(); assertEquals("Wrong number of PendingReplication blocks", 2, cluster.getNamesystem().getPendingReplicationBlocks()); @@ -783,38 +901,6 @@ public class TestBlockReport { ((Log4JLogger) TestBlockReport.LOG).getLogger().setLevel(Level.ALL); } - private void corruptBlockLen(final Block block) - throws IOException { - if (block == null) { - throw new IOException("Block isn't suppose to be null"); - } - long oldLen = block.getNumBytes(); - long newLen = oldLen - rand.nextLong(); - assertTrue("Old and new length shouldn't be the same", - block.getNumBytes() != newLen); - block.setNumBytes(newLen); - if(LOG.isDebugEnabled()) { - LOG.debug("Length of " + block.getBlockName() + - " is changed to " + newLen + " from " + oldLen); - } - } - - private void corruptBlockGS(final Block block) - throws IOException { - if (block == null) { - throw new IOException("Block isn't suppose to be null"); - } - long oldGS = block.getGenerationStamp(); - long newGS = oldGS - rand.nextLong(); - assertTrue("Old and new GS shouldn't be the same", - block.getGenerationStamp() != newGS); - block.setGenerationStamp(newGS); - if(LOG.isDebugEnabled()) { - LOG.debug("Generation stamp of " + block.getBlockName() + - " is changed to " + block.getGenerationStamp() + " from " + oldGS); - } - } - private Block findBlock(Path path, long size) throws IOException { Block ret; List<LocatedBlock> lbs =
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java Fri Jan 3 07:26:52 2014 @@ -19,12 +19,14 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.*; -import java.net.URI; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; +import java.util.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.StorageType; import org.junit.Test; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; import static org.mockito.Mockito.*; @@ -34,19 +36,69 @@ import org.apache.hadoop.hdfs.server.dat public class TestDataDirs { - @Test (timeout = 10000) - public void testGetDataDirsFromURIs() throws Throwable { + @Test (timeout = 30000) + public void testDataDirParsing() throws Throwable { + Configuration conf = new Configuration(); + List<StorageLocation> locations; + File dir0 = new File("/dir0"); + File dir1 = new File("/dir1"); + File dir2 = new File("/dir2"); + File dir3 = new File("/dir3"); + + // Verify that a valid string is correctly parsed, and that storage + // type is not case-sensitive + String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1); + locations = DataNode.getStorageLocations(conf); + assertThat(locations.size(), is(4)); + assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(0).getUri(), is(dir0.toURI())); + assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(1).getUri(), is(dir1.toURI())); + assertThat(locations.get(2).getStorageType(), is(StorageType.SSD)); + assertThat(locations.get(2).getUri(), is(dir2.toURI())); + assertThat(locations.get(3).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(3).getUri(), is(dir3.toURI())); + + // Verify that an unrecognized storage type result in an exception. + String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations2); + try { + locations = DataNode.getStorageLocations(conf); + fail(); + } catch(IllegalArgumentException iae) { + DataNode.LOG.info("The exception is expected.", iae); + } + + // Assert that a string with no storage type specified is + // correctly parsed and the default storage type is picked up. + String locations3 = "/dir0,/dir1"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations3); + locations = DataNode.getStorageLocations(conf); + assertThat(locations.size(), is(2)); + assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(0).getUri(), is(dir0.toURI())); + assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(1).getUri(), is(dir1.toURI())); + } + + @Test (timeout = 30000) + public void testDataDirValidation() throws Throwable { DataNodeDiskChecker diskChecker = mock(DataNodeDiskChecker.class); doThrow(new IOException()).doThrow(new IOException()).doNothing() .when(diskChecker).checkDir(any(LocalFileSystem.class), any(Path.class)); LocalFileSystem fs = mock(LocalFileSystem.class); - Collection<URI> uris = Arrays.asList(new URI("file:/p1/"), - new URI("file:/p2/"), new URI("file:/p3/")); + AbstractList<StorageLocation> locations = new ArrayList<StorageLocation>(); - List<File> dirs = DataNode.getDataDirsFromURIs(uris, fs, diskChecker); - assertEquals("number of valid data dirs", 1, dirs.size()); - String validDir = dirs.iterator().next().getPath(); - assertEquals("p3 should be valid", new File("/p3").getPath(), validDir); + locations.add(StorageLocation.parse("file:/p1/")); + locations.add(StorageLocation.parse("file:/p2/")); + locations.add(StorageLocation.parse("file:/p3/")); + + List<StorageLocation> checkedLocations = + DataNode.checkStorageLocations(locations, fs, diskChecker); + assertEquals("number of valid data dirs", 1, checkedLocations.size()); + String validDir = checkedLocations.iterator().next().getFile().getPath(); + assertThat("p3 should be valid", new File("/p3/").getPath(), is(validDir)); } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java Fri Jan 3 07:26:52 2014 @@ -163,7 +163,7 @@ public class TestDataNodeMultipleRegistr for (BPOfferService bpos : dn.getAllBpOs()) { LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration + "; sid=" - + bpos.bpRegistration.getStorageID() + "; nna=" + + + bpos.bpRegistration.getDatanodeUuid() + "; nna=" + getNNSocketAddress(bpos)); } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Fri Jan 3 07:26:52 2014 @@ -42,11 +42,13 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; @@ -151,13 +153,23 @@ public class TestDataNodeVolumeFailure { DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3 String bpid = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid); - final StorageBlockReport[] report = { - new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), - DataNodeTestUtils.getFSDataset(dn).getBlockReport(bpid - ).getBlockListAsLongs()) - }; - cluster.getNameNodeRpc().blockReport(dnR, bpid, report); + + Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpid); + + // Send block report + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + + int reportIndex = 0; + for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair : perVolumeBlockLists.entrySet()) { + DatanodeStorage dnStorage = kvPair.getKey(); + BlockListAsLongs blockList = kvPair.getValue(); + reports[reportIndex++] = + new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); + } + + cluster.getNameNodeRpc().blockReport(dnR, bpid, reports); // verify number of blocks and files... verify(filename, filesize); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java Fri Jan 3 07:26:52 2014 @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; @@ -406,6 +407,16 @@ public class TestDirectoryScanner { public File getFinalizedDir(String bpid) throws IOException { return new File("/base/current/" + bpid + "/finalized"); } + + @Override + public StorageType getStorageType() { + return StorageType.DEFAULT; + } + + @Override + public String getStorageID() { + return ""; + } } private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi(); @@ -436,7 +447,7 @@ public class TestDirectoryScanner { void testScanInfoObject(long blockId) throws Exception { DirectoryScanner.ScanInfo scanInfo = - new DirectoryScanner.ScanInfo(blockId); + new DirectoryScanner.ScanInfo(blockId, null, null, null); assertEquals(blockId, scanInfo.getBlockId()); assertNull(scanInfo.getBlockFile()); assertNull(scanInfo.getMetaFile()); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java Fri Jan 3 07:26:52 2014 @@ -36,6 +36,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.HdfsBlockLocation; import org.apache.hadoop.fs.Path; @@ -82,7 +83,11 @@ public class TestFsDatasetCache { // Most Linux installs allow a default of 64KB locked memory private static final long CACHE_CAPACITY = 64 * 1024; - private static final long BLOCK_SIZE = 4096; + // mlock always locks the entire page. So we don't need to deal with this + // rounding, use the OS page size for the block size. + private static final long PAGE_SIZE = + NativeIO.POSIX.getCacheManipulator().getOperatingSystemPageSize(); + private static final long BLOCK_SIZE = PAGE_SIZE; private static Configuration conf; private static MiniDFSCluster cluster = null; @@ -104,14 +109,12 @@ public class TestFsDatasetCache { public void setUp() throws Exception { assumeTrue(!Path.WINDOWS); conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CACHING_ENABLED_KEY, true); conf.setLong(DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS, 500); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, CACHE_CAPACITY); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_CACHING_ENABLED_KEY, true); prevCacheManipulator = NativeIO.POSIX.getCacheManipulator(); NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); @@ -451,4 +454,27 @@ public class TestFsDatasetCache { } }, 100, 10000); } + + @Test(timeout=60000) + public void testPageRounder() throws Exception { + // Write a small file + Path fileName = new Path("/testPageRounder"); + final int smallBlocks = 512; // This should be smaller than the page size + assertTrue("Page size should be greater than smallBlocks!", + PAGE_SIZE > smallBlocks); + final int numBlocks = 5; + final int fileLen = smallBlocks * numBlocks; + FSDataOutputStream out = + fs.create(fileName, false, 4096, (short)1, smallBlocks); + out.write(new byte[fileLen]); + out.close(); + HdfsBlockLocation[] locs = (HdfsBlockLocation[])fs.getFileBlockLocations( + fileName, 0, fileLen); + // Cache the file and check the sizes match the page size + setHeartbeatResponse(cacheBlocks(locs)); + verifyExpectedCacheUsage(PAGE_SIZE * numBlocks, numBlocks); + // Uncache and check that it decrements by the page size too + setHeartbeatResponse(uncacheBlocks(locs)); + verifyExpectedCacheUsage(0, 0); + } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java Fri Jan 3 07:26:52 2014 @@ -311,7 +311,7 @@ public class TestSimulatedFSDataset { } private SimulatedFSDataset getSimulatedFSDataset() { - SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, null, conf); + SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf); fsdataset.addBlockPool(bpid, conf); return fsdataset; } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Fri Jan 3 07:26:52 2014 @@ -17,6 +17,14 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; @@ -25,28 +33,40 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.protocol.*; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataStorage; -import org.apache.hadoop.hdfs.server.protocol.*; +import org.apache.hadoop.hdfs.server.protocol.BlockCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.Groups; -import org.apache.hadoop.util.*; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.util.VersionInfo; import org.apache.log4j.Level; import org.apache.log4j.LogManager; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; - /** * Main class for a series of name-node benchmarks. * @@ -586,6 +606,98 @@ public class NNThroughputBenchmark imple } /** + * Directory creation statistics. + * + * Each thread creates the same (+ or -1) number of directories. + * Directory names are pre-generated during initialization. + */ + class MkdirsStats extends OperationStatsBase { + // Operation types + static final String OP_MKDIRS_NAME = "mkdirs"; + static final String OP_MKDIRS_USAGE = "-op mkdirs [-threads T] [-dirs N] " + + "[-dirsPerDir P]"; + + protected FileNameGenerator nameGenerator; + protected String[][] dirPaths; + + MkdirsStats(List<String> args) { + super(); + parseArguments(args); + } + + @Override + String getOpName() { + return OP_MKDIRS_NAME; + } + + @Override + void parseArguments(List<String> args) { + boolean ignoreUnrelatedOptions = verifyOpArgument(args); + int nrDirsPerDir = 2; + for (int i = 2; i < args.size(); i++) { // parse command line + if(args.get(i).equals("-dirs")) { + if(i+1 == args.size()) printUsage(); + numOpsRequired = Integer.parseInt(args.get(++i)); + } else if(args.get(i).equals("-threads")) { + if(i+1 == args.size()) printUsage(); + numThreads = Integer.parseInt(args.get(++i)); + } else if(args.get(i).equals("-dirsPerDir")) { + if(i+1 == args.size()) printUsage(); + nrDirsPerDir = Integer.parseInt(args.get(++i)); + } else if(!ignoreUnrelatedOptions) + printUsage(); + } + nameGenerator = new FileNameGenerator(getBaseDir(), nrDirsPerDir); + } + + @Override + void generateInputs(int[] opsPerThread) throws IOException { + assert opsPerThread.length == numThreads : "Error opsPerThread.length"; + nameNodeProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE, + false); + LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName()); + dirPaths = new String[numThreads][]; + for(int idx=0; idx < numThreads; idx++) { + int threadOps = opsPerThread[idx]; + dirPaths[idx] = new String[threadOps]; + for(int jdx=0; jdx < threadOps; jdx++) + dirPaths[idx][jdx] = nameGenerator. + getNextFileName("ThroughputBench"); + } + } + + /** + * returns client name + */ + @Override + String getExecutionArgument(int daemonId) { + return getClientName(daemonId); + } + + /** + * Do mkdirs operation. + */ + @Override + long executeOp(int daemonId, int inputIdx, String clientName) + throws IOException { + long start = Time.now(); + nameNodeProto.mkdirs(dirPaths[daemonId][inputIdx], + FsPermission.getDefault(), true); + long end = Time.now(); + return end-start; + } + + @Override + void printResults() { + LOG.info("--- " + getOpName() + " inputs ---"); + LOG.info("nrDirs = " + numOpsRequired); + LOG.info("nrThreads = " + numThreads); + LOG.info("nrDirsPerDir = " + nameGenerator.getFilesPerDirectory()); + printStats(); + } + } + + /** * Open file statistics. * * Measure how many open calls (getBlockLocations()) @@ -817,17 +929,16 @@ public class NNThroughputBenchmark imple dnRegistration = new DatanodeRegistration( new DatanodeID(DNS.getDefaultIP("default"), DNS.getDefaultHost("default", "default"), - "", getNodePort(dnIdx), + DataNode.generateUuid(), getNodePort(dnIdx), DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT), - new DataStorage(nsInfo, ""), + new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); - DataNode.setNewStorageID(dnRegistration); // register datanode dnRegistration = nameNodeProto.registerDatanode(dnRegistration); //first block reports - storage = new DatanodeStorage(dnRegistration.getStorageID()); + storage = new DatanodeStorage(dnRegistration.getDatanodeUuid()); final StorageBlockReport[] reports = { new StorageBlockReport(storage, new BlockListAsLongs(null, null).getBlockListAsLongs()) @@ -843,7 +954,7 @@ public class NNThroughputBenchmark imple void sendHeartbeat() throws IOException { // register datanode // TODO:FEDERATION currently a single block pool is supported - StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(), + StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0).getCommands(); @@ -890,7 +1001,7 @@ public class NNThroughputBenchmark imple @SuppressWarnings("unused") // keep it for future blockReceived benchmark int replicateBlocks() throws IOException { // register datanode - StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(), + StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0).getCommands(); @@ -920,14 +1031,14 @@ public class NNThroughputBenchmark imple DatanodeInfo dnInfo = blockTargets[t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo, - new DataStorage(nsInfo, dnInfo.getStorageID()), + new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo( blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - receivedDNReg.getStorageID(), rdBlocks) }; + receivedDNReg.getDatanodeUuid(), rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(receivedDNReg, nameNode .getNamesystem().getBlockPoolId(), report); } @@ -1050,7 +1161,7 @@ public class NNThroughputBenchmark imple loc.getBlock().getLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - datanodes[dnIdx].dnRegistration.getStorageID(), rdBlocks) }; + datanodes[dnIdx].dnRegistration.getDatanodeUuid(), rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, loc .getBlock().getBlockPoolId(), report); } @@ -1260,6 +1371,7 @@ public class NNThroughputBenchmark imple System.err.println("Usage: NNThroughputBenchmark" + "\n\t" + OperationStatsBase.OP_ALL_USAGE + " | \n\t" + CreateFileStats.OP_CREATE_USAGE + + " | \n\t" + MkdirsStats.OP_MKDIRS_USAGE + " | \n\t" + OpenFileStats.OP_OPEN_USAGE + " | \n\t" + DeleteFileStats.OP_DELETE_USAGE + " | \n\t" + FileStatusStats.OP_FILE_STATUS_USAGE @@ -1309,6 +1421,10 @@ public class NNThroughputBenchmark imple opStat = new CreateFileStats(args); ops.add(opStat); } + if(runAll || MkdirsStats.OP_MKDIRS_NAME.equals(type)) { + opStat = new MkdirsStats(args); + ops.add(opStat); + } if(runAll || OpenFileStats.OP_OPEN_NAME.equals(type)) { opStat = new OpenFileStats(args); ops.add(opStat); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Fri Jan 3 07:26:52 2014 @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp; @@ -111,8 +112,8 @@ public class NameNodeAdapter { public static HeartbeatResponse sendHeartBeat(DatanodeRegistration nodeReg, DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException { - return namesystem.handleHeartbeat(nodeReg, dd.getCapacity(), - dd.getDfsUsed(), dd.getRemaining(), dd.getBlockPoolUsed(), + return namesystem.handleHeartbeat(nodeReg, + BlockManagerTestUtil.getStorageReportsForDatanode(dd), dd.getCacheCapacity(), dd.getCacheRemaining(), 0, 0, 0); } @@ -238,3 +239,4 @@ public class NameNodeAdapter { return NNStorage.getInProgressEditsFile(sd, startTxId); } } + Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java Fri Jan 3 07:26:52 2014 @@ -239,7 +239,7 @@ public class OfflineEditsViewerHelper { .setOwnerName("carlton") .setGroupName("party") .setMode(new FsPermission((short)0700)) - .setWeight(1989)); + .setLimit(1989l)); // OP_ADD_PATH_BASED_CACHE_DIRECTIVE 33 long id = dfs.addCacheDirective( new CacheDirectiveInfo.Builder(). Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java?rev=1555021&r1=1555020&r2=1555021&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java Fri Jan 3 07:26:52 2014 @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.L import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.net.Node; @@ -99,13 +100,13 @@ public class TestAddBlockRetry { bmField.setAccessible(true); bmField.set(ns, spyBM); - doAnswer(new Answer<DatanodeDescriptor[]>() { + doAnswer(new Answer<DatanodeStorageInfo[]>() { @Override - public DatanodeDescriptor[] answer(InvocationOnMock invocation) + public DatanodeStorageInfo[] answer(InvocationOnMock invocation) throws Throwable { LOG.info("chooseTarget for " + src); - DatanodeDescriptor[] ret = - (DatanodeDescriptor[]) invocation.callRealMethod(); + DatanodeStorageInfo[] ret = + (DatanodeStorageInfo[]) invocation.callRealMethod(); count++; if(count == 1) { // run second addBlock() LOG.info("Starting second addBlock for " + src);
