HDFS-6054. MiniQJMHACluster should not use static port to avoid binding failure in unit test. (Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57d0a943 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57d0a943 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57d0a943 Branch: refs/heads/YARN-1011 Commit: 57d0a94305b22a469062c7ce85b3159fee17a28b Parents: 1acc509 Author: Yongjun Zhang <yzh...@cloudera.com> Authored: Tue Jan 19 22:54:47 2016 -0800 Committer: Yongjun Zhang <yzh...@cloudera.com> Committed: Tue Jan 19 22:54:47 2016 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 15 +++++-- .../namenode/ha/TestFailureToReadEdits.java | 41 ++++++++++++++++---- 3 files changed, 47 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 327cf73..b051f2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2599,6 +2599,9 @@ Release 2.8.0 - UNRELEASED HDFS-9623. Update example configuration of block state change log in log4j.properties. (Masatake Iwasaki via aajisaka) + HDFS-6054. MiniQJMHACluster should not use static port to avoid binding + failure in unit test. (Yongjun Zhang) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java index 8e838c3..0764f12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -42,7 +42,6 @@ public class MiniQJMHACluster { public static final String NAMESERVICE = "ns1"; private static final Random RANDOM = new Random(); - private int basePort = 10000; public static class Builder { private final Configuration conf; @@ -92,9 +91,12 @@ public class MiniQJMHACluster { private MiniQJMHACluster(Builder builder) throws IOException { this.conf = builder.conf; int retryCount = 0; + int basePort = 10000; + while (true) { try { basePort = 10000 + RANDOM.nextInt(1000) * 4; + LOG.info("Set MiniQJMHACluster basePort to " + basePort); // start 3 journal nodes journalCluster = new MiniJournalCluster.Builder(conf).format(true) .build(); @@ -104,7 +106,7 @@ public class MiniQJMHACluster { // start cluster with specified NameNodes MiniDFSNNTopology topology = createDefaultTopology(builder.numNNs, basePort); - initHAConf(journalURI, builder.conf, builder.numNNs); + initHAConf(journalURI, builder.conf, builder.numNNs, basePort); // First start up the NNs just to format the namespace. The MinIDFSCluster // has no way to just format the NameNodes without also starting them. @@ -123,16 +125,21 @@ public class MiniQJMHACluster { // restart the cluster cluster.restartNameNodes(); - ++retryCount; break; } catch (BindException e) { + if (cluster != null) { + cluster.shutdown(true); + cluster = null; + } + ++retryCount; LOG.info("MiniQJMHACluster port conflicts, retried " + retryCount + " times"); } } } - private Configuration initHAConf(URI journalURI, Configuration conf, int numNNs) { + private Configuration initHAConf(URI journalURI, Configuration conf, + int numNNs, int basePort) { conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, journalURI.toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java index f82b616..5221ef9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java @@ -27,10 +27,14 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import java.io.IOException; +import java.net.BindException; import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; +import java.util.Random; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -62,11 +66,14 @@ import com.google.common.collect.ImmutableList; @RunWith(Parameterized.class) public class TestFailureToReadEdits { + private static final Log LOG = + LogFactory.getLog(TestFailureToReadEdits.class); private static final String TEST_DIR1 = "/test1"; private static final String TEST_DIR2 = "/test2"; private static final String TEST_DIR3 = "/test3"; - + private static final Random RANDOM = new Random(); + private final TestType clusterType; private Configuration conf; private MiniDFSCluster cluster; @@ -103,14 +110,32 @@ public class TestFailureToReadEdits { conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); HAUtil.setAllowStandbyReads(conf, true); - + if (clusterType == TestType.SHARED_DIR_HA) { - MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000); - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(0) - .checkExitOnShutdown(false) - .build(); + int basePort = 10000; + int retryCount = 0; + while (true) { + try { + basePort = 10000 + RANDOM.nextInt(1000) * 4; + LOG.info("Set SHARED_DIR_HA cluster's basePort to " + basePort); + MiniDFSNNTopology topology = + MiniQJMHACluster.createDefaultTopology(basePort); + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology) + .numDataNodes(0) + .checkExitOnShutdown(false) + .build(); + break; + } catch (BindException e) { + if (cluster != null) { + cluster.shutdown(true); + cluster = null; + } + ++retryCount; + LOG.info("SHARED_DIR_HA: MiniQJMHACluster port conflicts, retried " + + retryCount + " times " + e); + } + } } else { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder().numDataNodes(0).checkExitOnShutdown(false);
