hadoop git commit: HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. (iwasakims)

2016-10-18 Thread zhz 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 bd1676359 -> fd3df4bb0


HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. 
(iwasakims)

(cherry picked from commit 964e546ab1dba5f5d53b209ec6c9a70a85654765)

 Conflicts:
  
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java

(cherry picked from commit 1b91ebb7145860ecda394d764da5fc7063406fee)
(cherry picked from commit 1524e6320659df7d066ac96fd2c190d7a7c5b932)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd3df4bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd3df4bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd3df4bb

Branch: refs/heads/branch-2.7
Commit: fd3df4bb041ddc8ac3602c426c924254367f0952
Parents: bd16763
Author: Masatake Iwasaki 
Authored: Wed Sep 21 10:35:25 2016 +0900
Committer: Zhe Zhang 
Committed: Tue Oct 18 10:32:16 2016 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt   |  3 +++
 .../blockmanagement/TestBlockTokenWithDFS.java|  8 +++-
 .../namenode/ha/TestDFSZKFailoverController.java  | 18 +-
 3 files changed, 23 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd3df4bb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1776a49..03eca7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -160,6 +160,9 @@ Release 2.7.4 - UNRELEASED
 HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently.
 (xiaochen)
 
+HDFS-9333. Some tests using MiniDFSCluster errored complaining port in
+use. (iwasakims)
+
 Release 2.7.3 - 2016-08-25
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd3df4bb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index b15cb38..0de54ec 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.log4j.Level;
 import org.junit.Assert;
@@ -341,7 +342,12 @@ public class TestBlockTokenWithDFS {
 Configuration conf = getConf(numDataNodes);
 
 try {
-  cluster = new 
MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
+  // prefer non-ephemeral port to avoid port collision on restartNameNode
+  cluster = new MiniDFSCluster.Builder(conf)
+  .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+  .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+  .numDataNodes(numDataNodes)
+  .build();
   cluster.waitActive();
   assertEquals(numDataNodes, cluster.getDataNodes().size());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd3df4bb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java
index bcbd543..782f80c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java
@@ -39,6 +39,7 @@ import 
org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.tools.DFSHAAdmin;
 import

[26/50] [abbrv] hadoop git commit: HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. (iwasakims)

2016-09-26 Thread jianhe 
HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. 
(iwasakims)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/964e546a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/964e546a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/964e546a

Branch: refs/heads/MAPREDUCE-6608
Commit: 964e546ab1dba5f5d53b209ec6c9a70a85654765
Parents: 5a58bfe
Author: Masatake Iwasaki 
Authored: Wed Sep 21 10:35:25 2016 +0900
Committer: Masatake Iwasaki 
Committed: Wed Sep 21 10:35:25 2016 +0900

--
 .../blockmanagement/TestBlockTokenWithDFS.java  |  8 ++-
 .../TestBlockTokenWithDFSStriped.java   | 23 +++-
 .../hdfs/tools/TestDFSZKFailoverController.java | 18 ++-
 3 files changed, 42 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index e7e7739..9374ae8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
@@ -349,7 +350,12 @@ public class TestBlockTokenWithDFS {
 Configuration conf = getConf(numDataNodes);
 
 try {
-  cluster = new 
MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
+  // prefer non-ephemeral port to avoid port collision on restartNameNode
+  cluster = new MiniDFSCluster.Builder(conf)
+  .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+  .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+  .numDataNodes(numDataNodes)
+  .build();
   cluster.waitActive();
   assertEquals(numDataNodes, cluster.getDataNodes().size());
   doTestRead(conf, cluster, false);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index 64a48c2..1714561 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
 import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
 import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
@@ -59,7 +60,27 @@ public class TestBlockTokenWithDFSStriped extends 
TestBlockTokenWithDFS {
   @Override
   public void testRead() throws Exception {
 conf = getConf();
-cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+
+/*
+ * prefer non-ephemeral port to avoid conflict with tests using
+ * ephemeral ports on MiniDFSCluster#restartDataNode(true).
+ */
+Configuration[] overlays = new Configuration[numDNs];
+for (int i = 0; i < overlays.length; i++) {
+  int offset = i * 10;
+  Configuration c = new Configuration();
+  c.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:"
+  + ServerSocketUtil.getPort(19866 + offset, 100));
+  c.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:"
+  + ServerSocketUtil.getPort(19867 + offset, 100));
+  overlays[i] = c;
+}
+
+cluster = new

[10/13] hadoop git commit: HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. (iwasakims)

2016-09-21 Thread drankye 
HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. 
(iwasakims)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/964e546a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/964e546a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/964e546a

Branch: refs/heads/HADOOP-12756
Commit: 964e546ab1dba5f5d53b209ec6c9a70a85654765
Parents: 5a58bfe
Author: Masatake Iwasaki 
Authored: Wed Sep 21 10:35:25 2016 +0900
Committer: Masatake Iwasaki 
Committed: Wed Sep 21 10:35:25 2016 +0900

--
 .../blockmanagement/TestBlockTokenWithDFS.java  |  8 ++-
 .../TestBlockTokenWithDFSStriped.java   | 23 +++-
 .../hdfs/tools/TestDFSZKFailoverController.java | 18 ++-
 3 files changed, 42 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index e7e7739..9374ae8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
@@ -349,7 +350,12 @@ public class TestBlockTokenWithDFS {
 Configuration conf = getConf(numDataNodes);
 
 try {
-  cluster = new 
MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
+  // prefer non-ephemeral port to avoid port collision on restartNameNode
+  cluster = new MiniDFSCluster.Builder(conf)
+  .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+  .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+  .numDataNodes(numDataNodes)
+  .build();
   cluster.waitActive();
   assertEquals(numDataNodes, cluster.getDataNodes().size());
   doTestRead(conf, cluster, false);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index 64a48c2..1714561 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
 import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
 import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
@@ -59,7 +60,27 @@ public class TestBlockTokenWithDFSStriped extends 
TestBlockTokenWithDFS {
   @Override
   public void testRead() throws Exception {
 conf = getConf();
-cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+
+/*
+ * prefer non-ephemeral port to avoid conflict with tests using
+ * ephemeral ports on MiniDFSCluster#restartDataNode(true).
+ */
+Configuration[] overlays = new Configuration[numDNs];
+for (int i = 0; i < overlays.length; i++) {
+  int offset = i * 10;
+  Configuration c = new Configuration();
+  c.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:"
+  + ServerSocketUtil.getPort(19866 + offset, 100));
+  c.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:"
+  + ServerSocketUtil.getPort(19867 + offset, 100));
+  overlays[i] = c;
+}
+
+cluster = new

hadoop git commit: HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. (iwasakims)

2016-09-20 Thread iwasakims 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 6f6e62ad8 -> 1524e6320


HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. 
(iwasakims)

(cherry picked from commit 964e546ab1dba5f5d53b209ec6c9a70a85654765)

 Conflicts:
  
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java

(cherry picked from commit 1b91ebb7145860ecda394d764da5fc7063406fee)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1524e632
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1524e632
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1524e632

Branch: refs/heads/branch-2.8
Commit: 1524e6320659df7d066ac96fd2c190d7a7c5b932
Parents: 6f6e62a
Author: Masatake Iwasaki 
Authored: Wed Sep 21 10:35:25 2016 +0900
Committer: Masatake Iwasaki 
Committed: Wed Sep 21 10:52:00 2016 +0900

--
 .../blockmanagement/TestBlockTokenWithDFS.java|  8 +++-
 .../hdfs/tools/TestDFSZKFailoverController.java   | 18 +-
 2 files changed, 20 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1524e632/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index 4f74c7a..8ccbd48 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
@@ -345,7 +346,12 @@ public class TestBlockTokenWithDFS {
 Configuration conf = getConf(numDataNodes);
 
 try {
-  cluster = new 
MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
+  // prefer non-ephemeral port to avoid port collision on restartNameNode
+  cluster = new MiniDFSCluster.Builder(conf)
+  .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+  .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+  .numDataNodes(numDataNodes)
+  .build();
   cluster.waitActive();
   assertEquals(numDataNodes, cluster.getDataNodes().size());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1524e632/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
index dfdcf34..bbb787e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
@@ -39,6 +39,7 @@ import 
org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
 import org.apache.hadoop.hdfs.server.namenode.MockNameNodeResourceChecker;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
 import org.apache.hadoop.test.MultithreadedTestUtil.TestingThread;
@@ -75,14 +76,21 @@ public class TestDFSZKFailoverController extends 
ClientBaseWithFixes {
 conf.setInt(
 CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
 0);
-
-conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1", 10023);
-conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2", 10024);
 
+// Get random port numbers in advance. Because ZKFCs and DFSHAAdmin
+// needs rpc port numbers of all ZKFCs, Setting 0 does not work here.
+conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1",
+

hadoop git commit: HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. (iwasakims)

2016-09-20 Thread iwasakims 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b9b6396e0 -> 1b91ebb71


HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. 
(iwasakims)

(cherry picked from commit 964e546ab1dba5f5d53b209ec6c9a70a85654765)

 Conflicts:
  
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b91ebb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b91ebb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b91ebb7

Branch: refs/heads/branch-2
Commit: 1b91ebb7145860ecda394d764da5fc7063406fee
Parents: b9b6396
Author: Masatake Iwasaki 
Authored: Wed Sep 21 10:35:25 2016 +0900
Committer: Masatake Iwasaki 
Committed: Wed Sep 21 10:40:56 2016 +0900

--
 .../blockmanagement/TestBlockTokenWithDFS.java|  8 +++-
 .../hdfs/tools/TestDFSZKFailoverController.java   | 18 +-
 2 files changed, 20 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b91ebb7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index 4f74c7a..8ccbd48 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
@@ -345,7 +346,12 @@ public class TestBlockTokenWithDFS {
 Configuration conf = getConf(numDataNodes);
 
 try {
-  cluster = new 
MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
+  // prefer non-ephemeral port to avoid port collision on restartNameNode
+  cluster = new MiniDFSCluster.Builder(conf)
+  .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+  .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+  .numDataNodes(numDataNodes)
+  .build();
   cluster.waitActive();
   assertEquals(numDataNodes, cluster.getDataNodes().size());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b91ebb7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
index dfdcf34..bbb787e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
@@ -39,6 +39,7 @@ import 
org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
 import org.apache.hadoop.hdfs.server.namenode.MockNameNodeResourceChecker;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
 import org.apache.hadoop.test.MultithreadedTestUtil.TestingThread;
@@ -75,14 +76,21 @@ public class TestDFSZKFailoverController extends 
ClientBaseWithFixes {
 conf.setInt(
 CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
 0);
-
-conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1", 10023);
-conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2", 10024);
 
+// Get random port numbers in advance. Because ZKFCs and DFSHAAdmin
+// needs rpc port numbers of all ZKFCs, Setting 0 does not work here.
+conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1",
+ServerSocketUtil.getPort(10023, 100));
+conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY +

hadoop git commit: HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. (iwasakims)

2016-09-20 Thread iwasakims 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 5a58bfee3 -> 964e546ab


HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. 
(iwasakims)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/964e546a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/964e546a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/964e546a

Branch: refs/heads/trunk
Commit: 964e546ab1dba5f5d53b209ec6c9a70a85654765
Parents: 5a58bfe
Author: Masatake Iwasaki 
Authored: Wed Sep 21 10:35:25 2016 +0900
Committer: Masatake Iwasaki 
Committed: Wed Sep 21 10:35:25 2016 +0900

--
 .../blockmanagement/TestBlockTokenWithDFS.java  |  8 ++-
 .../TestBlockTokenWithDFSStriped.java   | 23 +++-
 .../hdfs/tools/TestDFSZKFailoverController.java | 18 ++-
 3 files changed, 42 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index e7e7739..9374ae8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
@@ -349,7 +350,12 @@ public class TestBlockTokenWithDFS {
 Configuration conf = getConf(numDataNodes);
 
 try {
-  cluster = new 
MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
+  // prefer non-ephemeral port to avoid port collision on restartNameNode
+  cluster = new MiniDFSCluster.Builder(conf)
+  .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+  .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+  .numDataNodes(numDataNodes)
+  .build();
   cluster.waitActive();
   assertEquals(numDataNodes, cluster.getDataNodes().size());
   doTestRead(conf, cluster, false);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index 64a48c2..1714561 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
 import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
 import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
@@ -59,7 +60,27 @@ public class TestBlockTokenWithDFSStriped extends 
TestBlockTokenWithDFS {
   @Override
   public void testRead() throws Exception {
 conf = getConf();
-cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+
+/*
+ * prefer non-ephemeral port to avoid conflict with tests using
+ * ephemeral ports on MiniDFSCluster#restartDataNode(true).
+ */
+Configuration[] overlays = new Configuration[numDNs];
+for (int i = 0; i < overlays.length; i++) {
+  int offset = i * 10;
+  Configuration c = new Configuration();
+  c.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:"
+  + ServerSocketUtil.getPort(19866 + offset, 100));
+  c.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:"
+  + ServerSocketUtil.getPort(19867 + offset,