HDFS-8572. DN always uses HTTP/localhost@REALM principals in SPNEGO. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6ff0e81 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6ff0e81 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6ff0e81 Branch: refs/heads/YARN-2928 Commit: d6ff0e81d6cf8ecd256be526efcd6538e60f17bf Parents: bc9d48a Author: Haohui Mai <whe...@apache.org> Authored: Thu Jun 11 18:53:29 2015 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Thu Jun 18 11:10:05 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/datanode/DataNode.java | 36 ++------------ .../server/datanode/web/DatanodeHttpServer.java | 52 ++++++++++++++++++-- 3 files changed, 55 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6ff0e81/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 033451e..51a0897 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -996,6 +996,9 @@ Release 2.7.1 - UNRELEASED HDFS-8583. Document that NFS gateway does not work with rpcbind on SLES 11. (Arpit Agarwal) + HDFS-8572. DN always uses HTTP/localhost@REALM principals in SPNEGO. + (wheat9) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6ff0e81/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index f73eb66..ed2925b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN; + import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; @@ -148,7 +148,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; -import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources; @@ -163,7 +162,6 @@ import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.nativeio.NativeIO; @@ -180,7 +178,6 @@ import org.apache.hadoop.security.SaslPropertiesResolver; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.tracing.SpanReceiverHost; @@ -299,7 +296,6 @@ public class DataNode extends ReconfigurableBase private volatile boolean heartbeatsDisabledForTests = false; private DataStorage storage = null; - private HttpServer2 infoServer = null; private DatanodeHttpServer httpServer = null; private int infoPort; private int infoSecurePort; @@ -761,29 +757,12 @@ public class DataNode extends ReconfigurableBase */ private void startInfoServer(Configuration conf) throws IOException { - Configuration confForInfoServer = new Configuration(conf); - confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10); - HttpServer2.Builder builder = new HttpServer2.Builder() - .setName("datanode") - .setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))) - .addEndpoint(URI.create("http://localhost:0")) - .setFindPort(true); - - this.infoServer = builder.build(); - - this.infoServer.setAttribute("datanode", this); - this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf); - this.infoServer.addServlet(null, "/blockScannerReport", - BlockScanner.Servlet.class); - - this.infoServer.start(); - InetSocketAddress jettyAddr = infoServer.getConnectorAddress(0); - // SecureDataNodeStarter will bind the privileged port to the channel if // the DN is started by JSVC, pass it along. ServerSocketChannel httpServerChannel = secureResources != null ? - secureResources.getHttpServerChannel() : null; - this.httpServer = new DatanodeHttpServer(conf, jettyAddr, httpServerChannel); + secureResources.getHttpServerChannel() : null; + + this.httpServer = new DatanodeHttpServer(conf, httpServerChannel); httpServer.start(); if (httpServer.getHttpAddress() != null) { infoPort = httpServer.getHttpAddress().getPort(); @@ -1715,13 +1694,6 @@ public class DataNode extends ReconfigurableBase shutdownPeriodicScanners(); // Stop the web server - if (infoServer != null) { - try { - infoServer.stop(); - } catch (Exception e) { - LOG.warn("Exception shutting down DataNode", e); - } - } if (httpServer != null) { try { httpServer.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6ff0e81/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java index 613d3ce..e9ad92f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode.web; -import io.netty.channel.ChannelFactory; import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.ChannelFactory; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelPipeline; @@ -35,9 +35,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.server.common.JspHelper; +import org.apache.hadoop.hdfs.server.datanode.BlockScanner; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.ssl.SSLFactory; import java.io.Closeable; @@ -46,13 +50,17 @@ import java.net.BindException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.SocketException; +import java.net.URI; import java.nio.channels.ServerSocketChannel; import java.security.GeneralSecurityException; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY; public class DatanodeHttpServer implements Closeable { + private final HttpServer2 infoServer; private final EventLoopGroup bossGroup; private final EventLoopGroup workerGroup; private final ServerSocketChannel externalHttpChannel; @@ -66,10 +74,31 @@ public class DatanodeHttpServer implements Closeable { static final Log LOG = LogFactory.getLog(DatanodeHttpServer.class); - public DatanodeHttpServer(final Configuration conf, final InetSocketAddress - jettyAddr, final ServerSocketChannel externalHttpChannel) + public DatanodeHttpServer(final Configuration conf, + final ServerSocketChannel externalHttpChannel) throws IOException { this.conf = conf; + + Configuration confForInfoServer = new Configuration(conf); + confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10); + HttpServer2.Builder builder = new HttpServer2.Builder() + .setName("datanode") + .setConf(confForInfoServer) + .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))) + .hostName(getHostnameForSpnegoPrincipal(confForInfoServer)) + .addEndpoint(URI.create("http://localhost:0")) + .setFindPort(true); + + this.infoServer = builder.build(); + + this.infoServer.setAttribute("datanode", this); + this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf); + this.infoServer.addServlet(null, "/blockScannerReport", + BlockScanner.Servlet.class); + + this.infoServer.start(); + final InetSocketAddress jettyAddr = infoServer.getConnectorAddress(0); + this.confForCreate = new Configuration(conf); confForCreate.set(FsPermission.UMASK_LABEL, "000"); @@ -191,5 +220,20 @@ public class DatanodeHttpServer implements Closeable { if (externalHttpChannel != null) { externalHttpChannel.close(); } + try { + infoServer.stop(); + } catch (Exception e) { + throw new IOException(e); + } + } + + private static String getHostnameForSpnegoPrincipal(Configuration conf) { + String addr = conf.getTrimmed(DFS_DATANODE_HTTP_ADDRESS_KEY, null); + if (addr == null) { + addr = conf.getTrimmed(DFS_DATANODE_HTTPS_ADDRESS_KEY, + DFS_DATANODE_HTTPS_ADDRESS_DEFAULT); + } + InetSocketAddress inetSocker = NetUtils.createSocketAddr(addr); + return inetSocker.getHostString(); } -} +} \ No newline at end of file