HDFS-10642. TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2cf8b54 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2cf8b54 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2cf8b54 Branch: refs/heads/HADOOP-12756 Commit: d2cf8b54c546fad098307b771ffe40173700f5a8 Parents: da6adf5 Author: Arpit Agarwal <a...@apache.org> Authored: Tue Jul 26 12:27:46 2016 -0700 Committer: Arpit Agarwal <a...@apache.org> Committed: Tue Jul 26 12:27:46 2016 -0700 ---------------------------------------------------------------------- .../fsdataset/impl/LazyPersistTestCase.java | 30 +++++++++++++++----- .../fsdataset/impl/TestLazyPersistFiles.java | 9 +++--- .../impl/TestLazyPersistLockedMemory.java | 3 +- .../impl/TestLazyPersistReplicaPlacement.java | 10 +++++-- .../impl/TestLazyPersistReplicaRecovery.java | 5 ++-- 5 files changed, 40 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java index 3593e2d..799d5d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java @@ -130,17 +130,33 @@ public abstract class LazyPersistTestCase { public Timeout timeout = new Timeout(300000); protected final LocatedBlocks ensureFileReplicasOnStorageType( - Path path, StorageType storageType) throws IOException { + Path path, StorageType storageType) + throws IOException, TimeoutException, InterruptedException { // Ensure that returned block locations returned are correct! LOG.info("Ensure path: " + path + " is on StorageType: " + storageType); assertThat(fs.exists(path), is(true)); long fileLength = client.getFileInfo(path.toString()).getLen(); - LocatedBlocks locatedBlocks = - client.getLocatedBlocks(path.toString(), 0, fileLength); - for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) { - assertThat(locatedBlock.getStorageTypes()[0], is(storageType)); - } - return locatedBlocks; + + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + try { + LocatedBlocks locatedBlocks = + client.getLocatedBlocks(path.toString(), 0, fileLength); + for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) { + if (locatedBlock.getStorageTypes()[0] != storageType) { + return false; + } + } + return true; + } catch (IOException ioe) { + LOG.warn("Exception got in ensureFileReplicasOnStorageType()", ioe); + return false; + } + } + }, 100, 30 * 1000); + + return client.getLocatedBlocks(path.toString(), 0, fileLength); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java index 950e9dc..8c43592 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java @@ -29,6 +29,7 @@ import java.util.EnumSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.fs.StorageType.RAM_DISK; @@ -89,7 +90,7 @@ public class TestLazyPersistFiles extends LazyPersistTestCase { */ @Test public void testCorruptFilesAreDiscarded() - throws IOException, InterruptedException { + throws IOException, InterruptedException, TimeoutException { getClusterBuilder().setRamDiskReplicaCapacity(2).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); @@ -123,7 +124,7 @@ public class TestLazyPersistFiles extends LazyPersistTestCase { @Test public void testDisableLazyPersistFileScrubber() - throws IOException, InterruptedException { + throws IOException, InterruptedException, TimeoutException { getClusterBuilder().setRamDiskReplicaCapacity(2).disableScrubber().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); @@ -151,8 +152,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase { * If NN restarted then lazyPersist files should not deleted */ @Test - public void testFileShouldNotDiscardedIfNNRestarted() throws IOException, - InterruptedException { + public void testFileShouldNotDiscardedIfNNRestarted() + throws IOException, InterruptedException, TimeoutException { getClusterBuilder().setRamDiskReplicaCapacity(2).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java index eef8f0b..d154b1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java @@ -53,7 +53,8 @@ public class TestLazyPersistLockedMemory extends LazyPersistTestCase { * fall back to disk. */ @Test - public void testWithNoLockedMemory() throws IOException { + public void testWithNoLockedMemory() + throws IOException, TimeoutException, InterruptedException { getClusterBuilder().setNumDatanodes(1) .setMaxLockedMemory(0).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java index c89475a..c16dbe5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java @@ -26,6 +26,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import java.io.IOException; +import java.util.concurrent.TimeoutException; import static org.apache.hadoop.fs.StorageType.DEFAULT; import static org.apache.hadoop.fs.StorageType.RAM_DISK; @@ -35,7 +36,8 @@ import static org.junit.Assert.fail; public class TestLazyPersistReplicaPlacement extends LazyPersistTestCase { @Test - public void testPlacementOnRamDisk() throws IOException { + public void testPlacementOnRamDisk() + throws IOException, TimeoutException, InterruptedException { getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -45,7 +47,8 @@ public class TestLazyPersistReplicaPlacement extends LazyPersistTestCase { } @Test - public void testPlacementOnSizeLimitedRamDisk() throws IOException { + public void testPlacementOnSizeLimitedRamDisk() + throws IOException, TimeoutException, InterruptedException { getClusterBuilder().setRamDiskReplicaCapacity(3).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); @@ -64,7 +67,8 @@ public class TestLazyPersistReplicaPlacement extends LazyPersistTestCase { * @throws IOException */ @Test - public void testFallbackToDisk() throws IOException { + public void testFallbackToDisk() + throws IOException, TimeoutException, InterruptedException { getClusterBuilder().setHasTransientStorage(false).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java index 231353a..537f9e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java @@ -23,6 +23,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import java.io.IOException; +import java.util.concurrent.TimeoutException; import static org.apache.hadoop.fs.StorageType.DEFAULT; import static org.apache.hadoop.fs.StorageType.RAM_DISK; @@ -30,7 +31,7 @@ import static org.apache.hadoop.fs.StorageType.RAM_DISK; public class TestLazyPersistReplicaRecovery extends LazyPersistTestCase { @Test public void testDnRestartWithSavedReplicas() - throws IOException, InterruptedException { + throws IOException, InterruptedException, TimeoutException { getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); @@ -55,7 +56,7 @@ public class TestLazyPersistReplicaRecovery extends LazyPersistTestCase { @Test public void testDnRestartWithUnsavedReplicas() - throws IOException, InterruptedException { + throws IOException, InterruptedException, TimeoutException { getClusterBuilder().build(); FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0)); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
