Repository: hadoop Updated Branches: refs/heads/HDFS-9806 1fd812443 -> fc467d6bc
HDFS-11792. [READ] Test cases for ProvidedVolumeDF and ProviderBlockIteratorImpl Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc467d6b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc467d6b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc467d6b Branch: refs/heads/HDFS-9806 Commit: fc467d6bc1c0a8088b4c46114bd5201388c416dd Parents: 1fd8124 Author: Virajith Jalaparti <viraj...@apache.org> Authored: Wed May 31 15:17:12 2017 -0700 Committer: Virajith Jalaparti <viraj...@apache.org> Committed: Wed May 31 15:17:12 2017 -0700 ---------------------------------------------------------------------- .../fsdataset/impl/ProvidedVolumeImpl.java | 6 +- .../fsdataset/impl/TestProvidedImpl.java | 94 ++++++++++++++++++-- 2 files changed, 92 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc467d6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java index a48e117..421b9cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java @@ -191,7 +191,11 @@ public class ProvidedVolumeImpl extends FsVolumeImpl { @Override long getBlockPoolUsed(String bpid) throws IOException { - return df.getBlockPoolUsed(bpid); + if (bpSlices.containsKey(bpid)) { + return df.getBlockPoolUsed(bpid); + } else { + throw new IOException("block pool " + bpid + " is not found"); + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc467d6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java index 2c119fe..4753235 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java @@ -83,6 +83,7 @@ public class TestProvidedImpl { private static final String BASE_DIR = new FileSystemTestHelper().getTestRootDir(); private static final int NUM_LOCAL_INIT_VOLUMES = 1; + //only support one provided volume for now. private static final int NUM_PROVIDED_INIT_VOLUMES = 1; private static final String[] BLOCK_POOL_IDS = {"bpid-0", "bpid-1"}; private static final int NUM_PROVIDED_BLKS = 10; @@ -208,6 +209,39 @@ public class TestProvidedImpl { } } + public static class TestProvidedVolumeDF + implements ProvidedVolumeDF, Configurable { + + @Override + public void setConf(Configuration conf) { + } + + @Override + public Configuration getConf() { + return null; + } + + @Override + public long getCapacity() { + return Long.MAX_VALUE; + } + + @Override + public long getSpaceUsed() { + return -1; + } + + @Override + public long getBlockPoolUsed(String bpid) { + return -1; + } + + @Override + public long getAvailable() { + return Long.MAX_VALUE; + } + } + private static Storage.StorageDirectory createLocalStorageDirectory( File root, Configuration conf) throws SecurityException, IOException { @@ -299,8 +333,8 @@ public class TestProvidedImpl { public void setUp() throws IOException { datanode = mock(DataNode.class); storage = mock(DataStorage.class); - this.conf = new Configuration(); - this.conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 0); + conf = new Configuration(); + conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 0); when(datanode.getConf()).thenReturn(conf); final DNConf dnConf = new DNConf(datanode); @@ -312,8 +346,10 @@ public class TestProvidedImpl { new ShortCircuitRegistry(conf); when(datanode.getShortCircuitRegistry()).thenReturn(shortCircuitRegistry); - this.conf.setClass(DFSConfigKeys.DFS_PROVIDER_CLASS, + conf.setClass(DFSConfigKeys.DFS_PROVIDER_CLASS, TestFileRegionProvider.class, FileRegionProvider.class); + conf.setClass(DFSConfigKeys.DFS_PROVIDER_DF_CLASS, + TestProvidedVolumeDF.class, ProvidedVolumeDF.class); blkToPathMap = new HashMap<Long, String>(); providedVolumes = new LinkedList<FsVolumeImpl>(); @@ -333,17 +369,43 @@ public class TestProvidedImpl { for (String bpid : BLOCK_POOL_IDS) { dataset.addBlockPool(bpid, conf); } + } + + @Test + public void testProvidedVolumeImpl() throws IOException { assertEquals(NUM_LOCAL_INIT_VOLUMES + NUM_PROVIDED_INIT_VOLUMES, getNumVolumes()); + assertEquals(NUM_PROVIDED_INIT_VOLUMES, providedVolumes.size()); assertEquals(0, dataset.getNumFailedVolumes()); - } - @Test - public void testProvidedStorageID() throws IOException { + TestProvidedVolumeDF df = new TestProvidedVolumeDF(); + for (int i = 0; i < providedVolumes.size(); i++) { + //check basic information about provided volume assertEquals(DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT, providedVolumes.get(i).getStorageID()); + assertEquals(StorageType.PROVIDED, + providedVolumes.get(i).getStorageType()); + + //check the df stats of the volume + assertEquals(df.getAvailable(), providedVolumes.get(i).getAvailable()); + assertEquals(df.getBlockPoolUsed(BLOCK_POOL_IDS[CHOSEN_BP_ID]), + providedVolumes.get(i).getBlockPoolUsed( + BLOCK_POOL_IDS[CHOSEN_BP_ID])); + + providedVolumes.get(i).shutdownBlockPool( + BLOCK_POOL_IDS[1 - CHOSEN_BP_ID], null); + try { + assertEquals(df.getBlockPoolUsed(BLOCK_POOL_IDS[1 - CHOSEN_BP_ID]), + providedVolumes.get(i).getBlockPoolUsed( + BLOCK_POOL_IDS[1 - CHOSEN_BP_ID])); + //should not be triggered + assertTrue(false); + } catch (IOException e) { + LOG.info("Expected exception: " + e); + } + } } @@ -385,6 +447,8 @@ public class TestProvidedImpl { BlockIterator iter = vol.newBlockIterator(BLOCK_POOL_IDS[CHOSEN_BP_ID], "temp"); Set<Long> blockIdsUsed = new HashSet<Long>(); + + assertEquals(BLOCK_POOL_IDS[CHOSEN_BP_ID], iter.getBlockPoolId()); while(!iter.atEnd()) { ExtendedBlock eb = iter.nextBlock(); long blkId = eb.getBlockId(); @@ -394,10 +458,26 @@ public class TestProvidedImpl { blockIdsUsed.add(blkId); } assertEquals(NUM_PROVIDED_BLKS, blockIdsUsed.size()); + + // rewind the block iterator + iter.rewind(); + while(!iter.atEnd()) { + ExtendedBlock eb = iter.nextBlock(); + long blkId = eb.getBlockId(); + //the block should have already appeared in the first scan. + assertTrue(blockIdsUsed.contains(blkId)); + blockIdsUsed.remove(blkId); + } + //none of the blocks should remain in blockIdsUsed + assertEquals(0, blockIdsUsed.size()); + + //the other block pool should not contain any blocks! + BlockIterator nonProvidedBpIter = + vol.newBlockIterator(BLOCK_POOL_IDS[1 - CHOSEN_BP_ID], "temp"); + assertEquals(null, nonProvidedBpIter.nextBlock()); } } - @Test public void testRefresh() throws IOException { conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY, 1); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org