Repository: hive Updated Branches: refs/heads/master bdec08f9d -> 72c5d61f5
HIVE-12471: Secure HS2 web UI with SSL (Jimmy, reviewed by Mohit, Szehon) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/72c5d61f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/72c5d61f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/72c5d61f Branch: refs/heads/master Commit: 72c5d61f536e547c0c301855c221cc1ede6605f4 Parents: bdec08f Author: Jimmy Xiang <jxi...@apache.org> Authored: Fri Nov 20 13:39:30 2015 -0800 Committer: Jimmy Xiang <jxi...@apache.org> Committed: Thu Dec 3 13:37:25 2015 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 6 + .../java/org/apache/hive/http/HttpServer.java | 168 ++++++++++++++----- .../apache/hive/service/server/HiveServer2.java | 41 +++-- 3 files changed, 159 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/72c5d61f/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7f96071..c36d566 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1848,6 +1848,12 @@ public class HiveConf extends Configuration { HIVE_SERVER2_WEBUI_BIND_HOST("hive.server2.webui.host", "0.0.0.0", "The host address the HiveServer2 WebUI will listen on"), HIVE_SERVER2_WEBUI_PORT("hive.server2.webui.port", 10002, "The port the HiveServer2 WebUI will listen on"), HIVE_SERVER2_WEBUI_MAX_THREADS("hive.server2.webui.max.threads", 50, "The max HiveServer2 WebUI threads"), + HIVE_SERVER2_WEBUI_USE_SSL("hive.server2.webui.use.SSL", false, + "Set this to true for using SSL encryption for HiveServer2 WebUI."), + HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PATH("hive.server2.webui.keystore.path", "", + "SSL certificate keystore location for HiveServer2 WebUI."), + HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PASSWORD("hive.server2.webui.keystore.password", "", + "SSL certificate keystore password for HiveServer2 WebUI."), // Tez session settings HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "", http://git-wip-us.apache.org/repos/asf/hive/blob/72c5d61f/common/src/java/org/apache/hive/http/HttpServer.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index 1ff8d7c..4b0ed68 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -21,6 +21,9 @@ package org.apache.hive.http; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; import javax.servlet.ServletContext; import javax.servlet.http.HttpServlet; @@ -29,6 +32,8 @@ import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.Shell; @@ -44,12 +49,18 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ContextHandler.Context; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.nio.SelectChannelConnector; +import org.eclipse.jetty.server.ssl.SslSelectChannelConnector; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.webapp.WebAppContext; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Sets; + /** * A simple embedded Jetty server to serve as HS2/HMS web UI. */ @@ -57,32 +68,90 @@ public class HttpServer { public static final String CONF_CONTEXT_ATTRIBUTE = "hive.conf"; public static final String ADMINS_ACL = "admins.acl"; - private final AccessControlList adminsAcl; private final String appDir; - private final String name; - private final String host; private final int port; - private final int maxThreads; - private final Configuration conf; private final WebAppContext webAppContext; private final Server webServer; /** * Create a status server on the given port. */ - public HttpServer(String name, String host, int port, int maxThreads, - Configuration conf, AccessControlList adminsAcl) throws IOException { - this.name = name; - this.host = host; - this.port = port; - this.maxThreads = maxThreads; - this.conf = conf; - this.adminsAcl = adminsAcl; + private HttpServer(final Builder b) throws IOException { + this.port = b.port; webServer = new Server(); - appDir = getWebAppsPath(name); - webAppContext = createWebAppContext(); - initializeWebServer(); + appDir = getWebAppsPath(b.name); + webAppContext = createWebAppContext(b); + initializeWebServer(b); + } + + public static class Builder { + private String name; + private String host; + private int port; + private int maxThreads; + private HiveConf conf; + private Map<String, Object> contextAttrs = new HashMap<String, Object>(); + private String keyStorePassword; + private String keyStorePath; + private boolean useSSL; + + public HttpServer build() throws IOException { + return new HttpServer(this); + } + + public Builder setConf(HiveConf conf) { + setContextAttribute(CONF_CONTEXT_ATTRIBUTE, conf); + this.conf = conf; + return this; + } + + public Builder setName(String name) { + this.name = name; + return this; + } + + public Builder setHost(String host) { + this.host = host; + return this; + } + + public Builder setPort(int port) { + this.port = port; + return this; + } + + public Builder setMaxThreads(int maxThreads) { + this.maxThreads = maxThreads; + return this; + } + + public Builder setAdmins(String admins) { + if (admins != null) { + setContextAttribute(ADMINS_ACL, new AccessControlList(admins)); + } + return this; + } + + public Builder setKeyStorePassword(String keyStorePassword) { + this.keyStorePassword = keyStorePassword; + return this; + } + + public Builder setKeyStorePath(String keyStorePath) { + this.keyStorePath = keyStorePath; + return this; + } + + public Builder setUseSSL(boolean useSSL) { + this.useSSL = useSSL; + return this; + } + + public Builder setContextAttribute(String name, Object value) { + contextAttrs.put(name, value); + return this; + } } public void start() throws Exception { @@ -98,13 +167,6 @@ public class HttpServer { } /** - * Set servlet context attribute that can be used in jsp. - */ - public void setContextAttribute(String name, Object value) { - webAppContext.getServletContext().setAttribute(name, value); - } - - /** * Checks the user has privileges to access to instrumentation servlets. * <p/> * If <code>hadoop.security.instrumentation.requires.admin</code> is set to FALSE @@ -195,44 +257,64 @@ public class HttpServer { /** * Create the web context for the application of specified name */ - WebAppContext createWebAppContext() { + WebAppContext createWebAppContext(Builder b) { WebAppContext ctx = new WebAppContext(); - setContextAttributes(ctx.getServletContext()); - ctx.setDisplayName(name); + setContextAttributes(ctx.getServletContext(), b.contextAttrs); + ctx.setDisplayName(b.name); ctx.setContextPath("/"); - ctx.setWar(appDir + "/" + name); + ctx.setWar(appDir + "/" + b.name); return ctx; } /** - * Create a default regular channel connector for "http" requests + * Create a channel connector for "http/https" requests */ - Connector createDefaultChannelConnector() { - SelectChannelConnector connector = new SelectChannelConnector(); + Connector createChannelConnector(int queueSize, Builder b) { + SelectChannelConnector connector; + if (!b.useSSL) { + connector = new SelectChannelConnector(); + } else { + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePath(b.keyStorePath); + Set<String> excludedSSLProtocols = Sets.newHashSet( + Splitter.on(",").trimResults().omitEmptyStrings().split( + Strings.nullToEmpty(b.conf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST)))); + sslContextFactory.addExcludeProtocols(excludedSSLProtocols.toArray( + new String[excludedSSLProtocols.size()])); + sslContextFactory.setKeyStorePassword(b.keyStorePassword); + connector = new SslSelectChannelConnector(sslContextFactory); + } + connector.setLowResourcesMaxIdleTime(10000); - connector.setAcceptQueueSize(maxThreads); + connector.setAcceptQueueSize(queueSize); connector.setResolveNames(false); connector.setUseDirectBuffers(false); connector.setReuseAddress(!Shell.WINDOWS); return connector; } - void setContextAttributes(Context ctx) { - ctx.setAttribute(CONF_CONTEXT_ATTRIBUTE, conf); - ctx.setAttribute(ADMINS_ACL, adminsAcl); + /** + * Set servlet context attributes that can be used in jsp. + */ + void setContextAttributes(Context ctx, Map<String, Object> contextAttrs) { + for (Map.Entry<String, Object> e: contextAttrs.entrySet()) { + ctx.setAttribute(e.getKey(), e.getValue()); + } } - void initializeWebServer() { + void initializeWebServer(Builder b) { // Create the thread pool for the web server to handle HTTP requests - QueuedThreadPool threadPool = maxThreads <= 0 ? new QueuedThreadPool() - : new QueuedThreadPool(maxThreads); + QueuedThreadPool threadPool = new QueuedThreadPool(); + if (b.maxThreads > 0) { + threadPool.setMaxThreads(b.maxThreads); + } threadPool.setDaemon(true); - threadPool.setName(name + "-web"); + threadPool.setName(b.name + "-web"); webServer.setThreadPool(threadPool); // Create the channel connector for the web server - Connector connector = createDefaultChannelConnector(); - connector.setHost(host); + Connector connector = createChannelConnector(threadPool.getMaxThreads(), b); + connector.setHost(b.host); connector.setPort(port); webServer.addConnector(connector); @@ -250,18 +332,18 @@ public class HttpServer { staticCtx.addServlet(DefaultServlet.class, "/*"); staticCtx.setDisplayName("static"); - String logDir = getLogDir(); + String logDir = getLogDir(b.conf); if (logDir != null) { ServletContextHandler logCtx = new ServletContextHandler(contexts, "/logs"); - setContextAttributes(logCtx.getServletContext()); + setContextAttributes(logCtx.getServletContext(), b.contextAttrs); logCtx.addServlet(AdminAuthorizedServlet.class, "/*"); logCtx.setResourceBase(logDir); logCtx.setDisplayName("logs"); } } - String getLogDir() { + String getLogDir(Configuration conf) { String logDir = conf.get("hive.log.dir"); if (logDir == null) { logDir = System.getProperty("hive.log.dir"); http://git-wip-us.apache.org/repos/asf/hive/blob/72c5d61f/service/src/java/org/apache/hive/service/server/HiveServer2.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 204eb5a..cad541a 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -43,20 +43,18 @@ import org.apache.curator.framework.api.CuratorEventType; import org.apache.curator.framework.recipes.nodes.PersistentEphemeralNode; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.hive.common.JvmPauseMonitor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.common.LogUtils.LogInitializationException; -import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.common.ServerUtils; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.http.HttpServer; @@ -66,6 +64,7 @@ import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.hive.service.cli.thrift.ThriftHttpCLIService; +import org.apache.logging.log4j.util.Strings; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; @@ -73,6 +72,8 @@ import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.ZooDefs.Perms; import org.apache.zookeeper.data.ACL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Joiner; @@ -130,17 +131,31 @@ public class HiveServer2 extends CompositeService { if (webUIPort <= 0) { LOG.info("Web UI is disabled since port is set to " + webUIPort); } else { - AccessControlList adminsAcl = - new AccessControlList(hiveConf.getVar(ConfVars.USERS_IN_ADMIN_ROLE)); - hiveConf.set("startcode", String.valueOf(System.currentTimeMillis())); - webServer = new HttpServer("hiveserver2", - hiveConf.getVar(ConfVars.HIVE_SERVER2_WEBUI_BIND_HOST), - webUIPort, - hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_THREADS), - hiveConf, adminsAcl); + HttpServer.Builder builder = new HttpServer.Builder(); + builder.setName("hiveserver2").setPort(webUIPort).setConf(hiveConf); + builder.setHost(hiveConf.getVar(ConfVars.HIVE_SERVER2_WEBUI_BIND_HOST)); + builder.setMaxThreads( + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_THREADS)); + builder.setAdmins(hiveConf.getVar(ConfVars.USERS_IN_ADMIN_ROLE)); // SessionManager is initialized - webServer.setContextAttribute("hive.sm", + builder.setContextAttribute("hive.sm", cliService.getSessionManager()); + hiveConf.set("startcode", + String.valueOf(System.currentTimeMillis())); + if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_USE_SSL)) { + String keyStorePath = hiveConf.getVar( + ConfVars.HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PATH); + if (Strings.isBlank(keyStorePath)) { + throw new IllegalArgumentException( + ConfVars.HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PATH.varname + + " Not configured for SSL connection"); + } + builder.setKeyStorePassword(ShimLoader.getHadoopShims().getPassword( + hiveConf, ConfVars.HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PASSWORD.varname)); + builder.setKeyStorePath(keyStorePath); + builder.setUseSSL(true); + } + webServer = builder.build(); } } } catch (IOException ie) {