sourabh912 commented on code in PR #3105:
URL: https://github.com/apache/hive/pull/3105#discussion_r853417994
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java:
##########
@@ -343,21 +366,162 @@ public static void startMetaStore(int port,
HadoopThriftAuthBridge bridge,
startMetaStore(port, bridge, conf, false, null);
}
- /**
- * Start Metastore based on a passed {@link HadoopThriftAuthBridge}.
- *
- * @param port The port on which the Thrift server will start to serve
- * @param bridge
- * @param conf Configuration overrides
- * @param startMetaStoreThreads Start the background threads (initiator,
cleaner, statsupdater, etc.)
- * @param startedBackgroundThreads If startMetaStoreThreads is true, this
AtomicBoolean will be switched to true,
- * when all of the background threads are scheduled. Useful for testing
purposes to wait
- * until the MetaStore is fully initialized.
- * @throws Throwable
- */
- public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
- Configuration conf, boolean startMetaStoreThreads, AtomicBoolean
startedBackgroundThreads) throws Throwable {
- isMetaStoreRemote = true;
+ public static boolean isThriftServerRunning() {
+ return thriftServer != null && thriftServer.isRunning();
+ }
+
+ // TODO: Is it worth trying to use a server that supports HTTP/2?
+ // Does the Thrift http client support this?
+
+ public static ThriftServer startHttpMetastore(int port, Configuration conf)
+ throws Exception {
+ LOG.info("Attempting to start http metastore server on port: {}", port);
+
+ // This check is likely pointless, especially with the current state of
the http
+ // servlet which respects whatever comes in. Putting this in place for the
moment
+ // only to enable testing on an otherwise secure cluster.
+ LOG.info(" Checking if security is enabled");
+ if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.info("Logging in via keytab while starting HTTP metastore");
+ // Handle renewal
+ String kerberosName =
SecurityUtil.getServerPrincipal(MetastoreConf.getVar(conf,
ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0");
+ String keyTabFile = MetastoreConf.getVar(conf,
ConfVars.KERBEROS_KEYTAB_FILE);
+ UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile);
+ } else {
+ LOG.info("Security is not enabled. Not logging in via keytab");
+ }
+
+ // TODO Bunch of http specific variables need to be defined here.
+ long maxMessageSize = MetastoreConf.getLongVar(conf,
ConfVars.SERVER_MAX_MESSAGE_SIZE);
+ int minWorkerThreads = MetastoreConf.getIntVar(conf,
ConfVars.SERVER_MIN_THREADS);
+ int maxWorkerThreads = MetastoreConf.getIntVar(conf,
ConfVars.SERVER_MAX_THREADS);
+
+ boolean useCompactProtocol = MetastoreConf.getBoolVar(conf,
ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
+
+ // Server thread pool
+ // Start with minWorkerThreads, expand till maxWorkerThreads and reject
+ // subsequent requests
+ String threadPoolName = "HiveServer2-HttpHandler-Pool";
+ ExecutorService executorService = new ThreadPoolExecutor(
+ minWorkerThreads, maxWorkerThreads, 60, TimeUnit.SECONDS,
+ new SynchronousQueue<>());
+
+ ExecutorThreadPool threadPool = new
ExecutorThreadPool((ThreadPoolExecutor) executorService);
+
+ // HTTP Server
+ org.eclipse.jetty.server.Server server = new Server(threadPool);
+ server.setStopAtShutdown(true);
+
+ ServerConnector connector;
+
+ final HttpConfiguration httpServerConf = new HttpConfiguration();
+ // TODO: Read from Configuration
+ httpServerConf.setRequestHeaderSize(
+ MetastoreConf.getIntVar(conf,
ConfVars.METASTORE_THRIFT_HTTP_REQUEST_HEADER_SIZE));
+ httpServerConf.setResponseHeaderSize(
+ MetastoreConf.getIntVar(conf,
ConfVars.METASTORE_THRIFT_HTTP_RESPONSE_HEADER_SIZE));
+
+ final HttpConnectionFactory http = new
HttpConnectionFactory(httpServerConf);
+
+ boolean useSsl = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
+ String schemeName = useSsl ? "https" : "http";
+ if (useSsl) {
+ String keyStorePath = MetastoreConf.getVar(conf,
ConfVars.SSL_KEYSTORE_PATH).trim();
+ if (keyStorePath.isEmpty()) {
+ throw new
IllegalArgumentException(ConfVars.SSL_KEYSTORE_PATH.toString()
+ + " Not configured for SSL connection");
+ }
+ String keyStorePassword =
+ MetastoreConf.getPassword(conf,
MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD);
+ String keyStoreType =
+ MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_TYPE).trim();
+ String keyStoreAlgorithm =
+ MetastoreConf.getVar(conf,
ConfVars.SSL_KEYMANAGERFACTORY_ALGORITHM).trim();
+
+ SslContextFactory sslContextFactory = new SslContextFactory();
+ // TODO: Add support for excluding protocols?
+ String[] excludedProtocols = MetastoreConf.getVar(conf,
ConfVars.SSL_PROTOCOL_BLACKLIST).split(",");
+ LOG.info("HTTP Server SSL: adding excluded protocols: " +
Arrays.toString(excludedProtocols));
+ sslContextFactory.addExcludeProtocols(excludedProtocols);
+ LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = "
+ + Arrays.toString(sslContextFactory.getExcludeProtocols()));
+ sslContextFactory.setKeyStorePath(keyStorePath);
+ sslContextFactory.setKeyStorePassword(keyStorePassword);
+ sslContextFactory.setKeyStoreType(keyStoreType);
+ sslContextFactory.setKeyManagerFactoryAlgorithm(keyStoreAlgorithm);
+ connector = new ServerConnector(server, sslContextFactory, http);
+ } else {
+ connector = new ServerConnector(server, http);
+ }
+ connector.setPort(port);
+ connector.setReuseAddress(true);
+ // TODO: What should the idle timeout be for the metastore. 30 minutes
seems a little too long.
+ connector.setIdleTimeout(120 * 1000);
+ // TODO: AcceptQueueSize needs to be higher for HMS
+ connector.setAcceptQueueSize(maxWorkerThreads);
+ // TODO: Connection keepalive configuration?
+
+ server.addConnector(connector);
+
+ TProcessor processor;
+
+ // All of this code can be re-used.
+ // Eventually move the HTTP and Binary parts into separate
+ // classes.
+ final TProtocolFactory protocolFactory;
+ final TProtocolFactory inputProtoFactory;
+ if (useCompactProtocol) {
+ protocolFactory = new TCompactProtocol.Factory();
+ inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize,
maxMessageSize);
+ } else {
+ protocolFactory = new TBinaryProtocol.Factory();
+ inputProtoFactory = new TBinaryProtocol.Factory(true, true,
maxMessageSize, maxMessageSize);
+ }
+
+ // TODO ZZZ: HMS seems to have it's own set of handlers. Not sure if the
threadpool here is actually required.
+ HMSHandler baseHandler = new HMSHandler("new db based metaserver",
+ conf);
+ IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
+ processor = new ThriftHiveMetastore.Processor<>(handler);
+ LOG.info("Starting DB backed MetaStore Server with generic processor");
Review Comment:
Sure.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
