HDFS-13462. Add BIND_HOST configuration for JournalNode's HTTP and RPC Servers. Contributed by Lukas Majercak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4b10786 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4b10786 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4b10786 Branch: refs/heads/YARN-8200 Commit: b4b10786c342b3a79344f4d5c6e06c78b60be619 Parents: 8d720da Author: Inigo Goiri <inigo...@apache.org> Authored: Tue Apr 17 15:18:01 2018 -0700 Committer: Inigo Goiri <inigo...@apache.org> Committed: Tue Apr 17 15:18:01 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +- .../hdfs/qjournal/server/JournalNode.java | 59 +++++- .../qjournal/server/JournalNodeHttpServer.java | 65 ++++-- .../qjournal/server/JournalNodeRpcServer.java | 31 ++- .../src/main/resources/hdfs-default.xml | 33 +++ .../TestJournalNodeRespectsBindHostKeys.java | 200 +++++++++++++++++++ 6 files changed, 364 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4b10786/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index c9fef06..ec50448 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -843,15 +843,19 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_JOURNALNODE_EDITS_DIR_DEFAULT = "/tmp/hadoop/dfs/journalnode/"; public static final String DFS_JOURNALNODE_RPC_ADDRESS_KEY = "dfs.journalnode.rpc-address"; public static final int DFS_JOURNALNODE_RPC_PORT_DEFAULT = 8485; + public static final String DFS_JOURNALNODE_RPC_BIND_HOST_KEY = "dfs.journalnode.rpc-bind-host"; public static final String DFS_JOURNALNODE_RPC_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_JOURNALNODE_RPC_PORT_DEFAULT; - + public static final String DFS_JOURNALNODE_HTTP_ADDRESS_KEY = "dfs.journalnode.http-address"; public static final int DFS_JOURNALNODE_HTTP_PORT_DEFAULT = 8480; + public static final String DFS_JOURNALNODE_HTTP_BIND_HOST_KEY = "dfs.journalnode.http-bind-host"; public static final String DFS_JOURNALNODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_JOURNALNODE_HTTP_PORT_DEFAULT; public static final String DFS_JOURNALNODE_HTTPS_ADDRESS_KEY = "dfs.journalnode.https-address"; public static final int DFS_JOURNALNODE_HTTPS_PORT_DEFAULT = 8481; + public static final String DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY = "dfs.journalnode.https-bind-host"; public static final String DFS_JOURNALNODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_JOURNALNODE_HTTPS_PORT_DEFAULT; + public static final String DFS_JOURNALNODE_KEYTAB_FILE_KEY = "dfs.journalnode.keytab.file"; public static final String DFS_JOURNALNODE_KERBEROS_PRINCIPAL_KEY = "dfs.journalnode.kerberos.principal"; public static final String DFS_JOURNALNODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY = "dfs.journalnode.kerberos.internal.spnego.principal"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4b10786/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java index 3eb3477..563c580 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.qjournal.server; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HTTP_BIND_HOST_KEY; import static org.apache.hadoop.util.ExitUtil.terminate; import java.io.File; @@ -44,6 +45,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.metrics2.util.MBeans; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.tracing.TraceUtils; import org.apache.hadoop.util.DiskChecker; @@ -159,7 +161,8 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean { registerJNMXBean(); - httpServer = new JournalNodeHttpServer(conf, this); + httpServer = new JournalNodeHttpServer(conf, this, + getHttpServerBindAddress(conf)); httpServer.start(); httpServerURI = httpServer.getServerURI().toString(); @@ -184,11 +187,6 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean { public InetSocketAddress getBoundIpcAddress() { return rpcServer.getAddress(); } - - @Deprecated - public InetSocketAddress getBoundHttpAddress() { - return httpServer.getAddress(); - } public String getHttpServerURI() { return httpServerURI; @@ -306,7 +304,7 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean { private void registerJNMXBean() { journalNodeInfoBeanName = MBeans.register("JournalNode", "JournalNodeInfo", this); } - + private class ErrorReporter implements StorageErrorReporter { @Override public void reportErrorOnFile(File f) { @@ -356,4 +354,51 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean { public Long getJournalCTime(String journalId) throws IOException { return getOrCreateJournal(journalId).getJournalCTime(); } + + public static InetSocketAddress getHttpAddress(Configuration conf) { + String addr = conf.get(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY, + DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_DEFAULT); + return NetUtils.createSocketAddr(addr, + DFSConfigKeys.DFS_JOURNALNODE_HTTP_PORT_DEFAULT, + DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY); + } + + protected InetSocketAddress getHttpServerBindAddress(Configuration conf) { + InetSocketAddress bindAddress = getHttpAddress(conf); + + // If DFS_JOURNALNODE_HTTP_BIND_HOST_KEY exists then it overrides the + // host name portion of DFS_JOURNALNODE_HTTP_ADDRESS_KEY. + final String bindHost = conf.getTrimmed(DFS_JOURNALNODE_HTTP_BIND_HOST_KEY); + if (bindHost != null && !bindHost.isEmpty()) { + bindAddress = new InetSocketAddress(bindHost, bindAddress.getPort()); + } + + return bindAddress; + } + + @VisibleForTesting + public JournalNodeRpcServer getRpcServer() { + return rpcServer; + } + + /** + * @return the actual JournalNode HTTP/HTTPS address. + */ + public InetSocketAddress getBoundHttpAddress() { + return httpServer.getAddress(); + } + + /** + * @return JournalNode HTTP address + */ + public InetSocketAddress getHttpAddress() { + return httpServer.getHttpAddress(); + } + + /** + * @return JournalNode HTTPS address + */ + public InetSocketAddress getHttpsAddress() { + return httpServer.getHttpsAddress(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4b10786/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java index 3adb93a..1d29c1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.common.JspHelper; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; @@ -41,21 +42,37 @@ public class JournalNodeHttpServer { private HttpServer2 httpServer; private final JournalNode localJournalNode; + private InetSocketAddress httpAddress; + private InetSocketAddress httpsAddress; + private final InetSocketAddress bindAddress; + private final Configuration conf; - JournalNodeHttpServer(Configuration conf, JournalNode jn) { + JournalNodeHttpServer(Configuration conf, JournalNode jn, + InetSocketAddress bindAddress) { this.conf = conf; this.localJournalNode = jn; + this.bindAddress = bindAddress; } void start() throws IOException { - final InetSocketAddress httpAddr = getAddress(conf); + final InetSocketAddress httpAddr = bindAddress; final String httpsAddrString = conf.get( DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_DEFAULT); InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString); + if (httpsAddr != null) { + // If DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY exists then it overrides the + // host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY. + final String bindHost = + conf.getTrimmed(DFSConfigKeys.DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY); + if (bindHost != null && !bindHost.isEmpty()) { + httpsAddr = new InetSocketAddress(bindHost, httpsAddr.getPort()); + } + } + HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf, httpAddr, httpsAddr, "journal", DFSConfigKeys.DFS_JOURNALNODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY, @@ -67,6 +84,20 @@ public class JournalNodeHttpServer { httpServer.addInternalServlet("getJournal", "/getJournal", GetJournalEditServlet.class, true); httpServer.start(); + + HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); + int connIdx = 0; + if (policy.isHttpEnabled()) { + httpAddress = httpServer.getConnectorAddress(connIdx++); + conf.set(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY, + NetUtils.getHostPortString(httpAddress)); + } + + if (policy.isHttpsEnabled()) { + httpsAddress = httpServer.getConnectorAddress(connIdx); + conf.set(DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, + NetUtils.getHostPortString(httpsAddress)); + } } void stop() throws IOException { @@ -78,15 +109,27 @@ public class JournalNodeHttpServer { } } } + + /** + * Return the actual HTTP/HTTPS address bound to by the running server. + */ + public InetSocketAddress getAddress() { + assert httpAddress != null || httpsAddress != null; + return httpAddress != null ? httpAddress : httpsAddress; + } /** * Return the actual address bound to by the running server. */ - @Deprecated - public InetSocketAddress getAddress() { - InetSocketAddress addr = httpServer.getConnectorAddress(0); - assert addr.getPort() != 0; - return addr; + public InetSocketAddress getHttpAddress() { + return httpAddress; + } + + /** + * Return the actual address bound to by the running server. + */ + public InetSocketAddress getHttpsAddress() { + return httpsAddress; } /** @@ -101,14 +144,6 @@ public class JournalNodeHttpServer { + NetUtils.getHostPortString(addr)); } - private static InetSocketAddress getAddress(Configuration conf) { - String addr = conf.get(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY, - DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_DEFAULT); - return NetUtils.createSocketAddr(addr, - DFSConfigKeys.DFS_JOURNALNODE_HTTP_PORT_DEFAULT, - DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY); - } - public static Journal getJournalFromContext(ServletContext context, String jid) throws IOException { JournalNode jn = (JournalNode)context.getAttribute(JN_ATTRIBUTE_KEY); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4b10786/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java index 42e6a4d..541f55b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.URL; +import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -49,10 +50,14 @@ import org.apache.hadoop.net.NetUtils; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_RPC_BIND_HOST_KEY; + @InterfaceAudience.Private @VisibleForTesting public class JournalNodeRpcServer implements QJournalProtocol { + private static final Log LOG = JournalNode.LOG; + private static final int HANDLER_COUNT = 5; private final JournalNode jn; private Server server; @@ -68,6 +73,12 @@ public class JournalNodeRpcServer implements QJournalProtocol { true); InetSocketAddress addr = getAddress(confCopy); + String bindHost = conf.getTrimmed(DFS_JOURNALNODE_RPC_BIND_HOST_KEY, null); + if (bindHost == null) { + bindHost = addr.getHostName(); + } + LOG.info("RPC server is binding to " + bindHost + ":" + addr.getPort()); + RPC.setProtocolEngine(confCopy, QJournalProtocolPB.class, ProtobufRpcEngine.class); QJournalProtocolServerSideTranslatorPB translator = @@ -76,13 +87,13 @@ public class JournalNodeRpcServer implements QJournalProtocol { .newReflectiveBlockingService(translator); this.server = new RPC.Builder(confCopy) - .setProtocol(QJournalProtocolPB.class) - .setInstance(service) - .setBindAddress(addr.getHostName()) - .setPort(addr.getPort()) - .setNumHandlers(HANDLER_COUNT) - .setVerbose(false) - .build(); + .setProtocol(QJournalProtocolPB.class) + .setInstance(service) + .setBindAddress(bindHost) + .setPort(addr.getPort()) + .setNumHandlers(HANDLER_COUNT) + .setVerbose(false) + .build(); // set service-level authorization security policy if (confCopy.getBoolean( @@ -248,4 +259,10 @@ public class JournalNodeRpcServer implements QJournalProtocol { throws IOException { jn.discardSegments(journalId, startTxId); } + + /** Allow access to the RPC server for testing. */ + @VisibleForTesting + Server getRpcServer() { + return server; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4b10786/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 66f9718..ba98257 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2115,6 +2115,17 @@ </property> <property> + <name>dfs.journalnode.rpc-bind-host</name> + <value></value> + <description> + The actual address the RPC server will bind to. If this optional address is + set, it overrides only the hostname portion of dfs.journalnode.rpc-address. + This is useful for making the JournalNode listen on all interfaces by + setting it to 0.0.0.0. + </description> +</property> + +<property> <name>dfs.journalnode.http-address</name> <value>0.0.0.0:8480</value> <description> @@ -2124,6 +2135,17 @@ </property> <property> + <name>dfs.journalnode.http-bind-host</name> + <value></value> + <description> + The actual address the HTTP server will bind to. If this optional address + is set, it overrides only the hostname portion of + dfs.journalnode.http-address. This is useful for making the JournalNode + HTTP server listen on allinterfaces by setting it to 0.0.0.0. + </description> +</property> + +<property> <name>dfs.journalnode.https-address</name> <value>0.0.0.0:8481</value> <description> @@ -2133,6 +2155,17 @@ </property> <property> + <name>dfs.journalnode.https-bind-host</name> + <value></value> + <description> + The actual address the HTTP server will bind to. If this optional address + is set, it overrides only the hostname portion of + dfs.journalnode.https-address. This is useful for making the JournalNode + HTTP server listen on all interfaces by setting it to 0.0.0.0. + </description> +</property> + +<property> <name>dfs.namenode.audit.loggers</name> <value>default</value> <description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4b10786/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeRespectsBindHostKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeRespectsBindHostKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeRespectsBindHostKeys.java new file mode 100644 index 0000000..79f3598 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeRespectsBindHostKeys.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.qjournal.server; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HTTP_BIND_HOST_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_RPC_BIND_HOST_KEY; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNot.not; + +import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hdfs.HdfsConfiguration; + +/** + * This test checks that the JournalNode respects the following keys. + * + * - DFS_JOURNALNODE_RPC_BIND_HOST_KEY + * - DFS_JOURNALNODE_HTTP_BIND_HOST_KEY + * - DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY + */ +public class TestJournalNodeRespectsBindHostKeys { + + public static final Log LOG = LogFactory.getLog( + TestJournalNodeRespectsBindHostKeys.class); + private static final String WILDCARD_ADDRESS = "0.0.0.0"; + private static final String LOCALHOST_SERVER_ADDRESS = "127.0.0.1:0"; + private static final int NUM_JN = 1; + + private HdfsConfiguration conf; + private MiniJournalCluster jCluster; + private JournalNode jn; + + @Before + public void setUp() { + conf = new HdfsConfiguration(); + } + + @After + public void tearDown() throws IOException { + if (jCluster != null) { + jCluster.shutdown(); + jCluster = null; + } + } + + private static String getRpcServerAddress(JournalNode jn) { + JournalNodeRpcServer rpcServer = jn.getRpcServer(); + return rpcServer.getRpcServer().getListenerAddress().getAddress(). + toString(); + } + + @Test (timeout=300000) + public void testRpcBindHostKey() throws IOException { + LOG.info("Testing without " + DFS_JOURNALNODE_RPC_BIND_HOST_KEY); + + // NN should not bind the wildcard address by default. + jCluster = new MiniJournalCluster.Builder(conf).format(true) + .numJournalNodes(NUM_JN).build(); + jn = jCluster.getJournalNode(0); + String address = getRpcServerAddress(jn); + assertThat("Bind address not expected to be wildcard by default.", + address, not("/" + WILDCARD_ADDRESS)); + + LOG.info("Testing with " + DFS_JOURNALNODE_RPC_BIND_HOST_KEY); + + // Tell NN to bind the wildcard address. + conf.set(DFS_JOURNALNODE_RPC_BIND_HOST_KEY, WILDCARD_ADDRESS); + + // Verify that NN binds wildcard address now. + jCluster = new MiniJournalCluster.Builder(conf).format(true) + .numJournalNodes(NUM_JN).build(); + jn = jCluster.getJournalNode(0); + address = getRpcServerAddress(jn); + assertThat("Bind address " + address + " is not wildcard.", + address, is("/" + WILDCARD_ADDRESS)); + } + + @Test(timeout=300000) + public void testHttpBindHostKey() throws IOException { + LOG.info("Testing without " + DFS_JOURNALNODE_HTTP_BIND_HOST_KEY); + + // NN should not bind the wildcard address by default. + conf.set(DFS_JOURNALNODE_HTTP_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS); + jCluster = new MiniJournalCluster.Builder(conf).format(true) + .numJournalNodes(NUM_JN).build(); + jn = jCluster.getJournalNode(0); + String address = jn.getHttpAddress().toString(); + assertFalse("HTTP Bind address not expected to be wildcard by default.", + address.startsWith(WILDCARD_ADDRESS)); + + LOG.info("Testing with " + DFS_JOURNALNODE_HTTP_BIND_HOST_KEY); + + // Tell NN to bind the wildcard address. + conf.set(DFS_JOURNALNODE_HTTP_BIND_HOST_KEY, WILDCARD_ADDRESS); + + // Verify that NN binds wildcard address now. + conf.set(DFS_JOURNALNODE_HTTP_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS); + jCluster = new MiniJournalCluster.Builder(conf).format(true) + .numJournalNodes(NUM_JN).build(); + jn = jCluster.getJournalNode(0); + address = jn.getHttpAddress().toString(); + assertTrue("HTTP Bind address " + address + " is not wildcard.", + address.startsWith(WILDCARD_ADDRESS)); + } + + private static final String BASEDIR = System.getProperty("test.build.dir", + "target/test-dir") + "/" + + TestJournalNodeRespectsBindHostKeys.class.getSimpleName(); + + private static void setupSsl() throws Exception { + Configuration conf = new Configuration(); + conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); + conf.set(DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, "localhost:0"); + conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); + + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + assertTrue(base.mkdirs()); + final String keystoresDir = new File(BASEDIR).getAbsolutePath(); + final String sslConfDir = KeyStoreTestUtil.getClasspathDir( + TestJournalNodeRespectsBindHostKeys.class); + + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + } + + /** + * HTTPS test is different since we need to setup SSL configuration. + * NN also binds the wildcard address for HTTPS port by default so we must + * pick a different host/port combination. + * @throws Exception + */ + @Test (timeout=300000) + public void testHttpsBindHostKey() throws Exception { + LOG.info("Testing behavior without " + DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY); + + setupSsl(); + + conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); + + // NN should not bind the wildcard address by default. + conf.set(DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS); + jCluster = new MiniJournalCluster.Builder(conf).format(true) + .numJournalNodes(NUM_JN).build(); + jn = jCluster.getJournalNode(0); + String address = jn.getHttpsAddress().toString(); + assertFalse("HTTP Bind address not expected to be wildcard by default.", + address.startsWith(WILDCARD_ADDRESS)); + + LOG.info("Testing behavior with " + DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY); + + // Tell NN to bind the wildcard address. + conf.set(DFS_JOURNALNODE_HTTPS_BIND_HOST_KEY, WILDCARD_ADDRESS); + + // Verify that NN binds wildcard address now. + conf.set(DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS); + jCluster = new MiniJournalCluster.Builder(conf).format(true) + .numJournalNodes(NUM_JN).build(); + jn = jCluster.getJournalNode(0); + address = jn.getHttpsAddress().toString(); + assertTrue("HTTP Bind address " + address + " is not wildcard.", + address.startsWith(WILDCARD_ADDRESS)); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
