This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push: new b7316374cb3 HIVE-27888: Backport of HIVE-22429, HIVE-14898, HIVE-22231, HIVE-20507, HIVE-24786 to branch-3 (#4878) b7316374cb3 is described below commit b7316374cb35988ebb4bed3c96262b85bba22fc2 Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com> AuthorDate: Mon Nov 20 17:43:13 2023 +0530 HIVE-27888: Backport of HIVE-22429, HIVE-14898, HIVE-22231, HIVE-20507, HIVE-24786 to branch-3 (#4878) * HIVE-22429: Migrated clustered tables using bucketing_version 1 on hive 3 uses bucketing_version 2 for inserts (Ramesh Kumar Thangarajan, reviewed by Jesus Camacho Rodriguez) * HIVE-14898: HS2 shouldn't log callstack for an empty auth header error * HIVE-22231: Hive query with big size via knox fails with Broken pipe Write failed (Denys Kuzmenko via Peter Vary) * HIVE-20507: Beeline: Add a utility command to retrieve all uris from beeline-site.xml * HIVE-24786: JDBC HttpClient should retry for idempotent and unsent http methods (#1983) * HIVE-24786: JDBC HttpClient should retry for idempotent and unsent http methods --------- Co-authored-by: Ramesh Kumar Thangarajan <rameshku...@cloudera.com> Co-authored-by: Daniel Dai <dai...@gmail.com> Co-authored-by: denys kuzmenko <dkuzme...@cloudera.com> Co-authored-by: Vaibhav Gumashta <vgumas...@hortonworks.com> Co-authored-by: Prasanth Jayachandran <prasan...@apache.org> Co-authored-by: Prasanth Jayachandran <pjayachand...@cloudera.com> --------- Signed-off-by: Sankar Hariappan <sank...@apache.org> Closes (#4878) --- .../src/java/org/apache/hive/beeline/BeeLine.java | 57 ++++++- .../java/org/apache/hive/beeline/BeeLineOpts.java | 10 ++ beeline/src/main/resources/BeeLine.properties | 1 + .../java/org/apache/hive/jdbc/HiveConnection.java | 186 ++++++++++++++++++++- jdbc/src/java/org/apache/hive/jdbc/Utils.java | 2 +- .../apache/hadoop/hive/ql/parse/TezCompiler.java | 3 - .../ldap/HttpEmptyAuthenticationException.java | 23 +++ .../hive/service/cli/thrift/ThriftHttpServlet.java | 22 ++- 8 files changed, 291 insertions(+), 13 deletions(-) diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 73653d4217e..01adb1e1ff5 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -65,6 +65,7 @@ import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.ResourceBundle; import java.util.ServiceLoader; @@ -94,6 +95,7 @@ import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; import org.apache.hive.common.util.ShutdownHookManager; +import org.apache.hive.jdbc.HiveConnection; import org.apache.hive.jdbc.JdbcUriParseException; import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; @@ -389,6 +391,12 @@ public class BeeLine implements Closeable { .withLongOpt("help") .withDescription("Display this message") .create('h')); + + // -getUrlsFromBeelineSite + options.addOption(OptionBuilder + .withLongOpt("getUrlsFromBeelineSite") + .withDescription("Print all urls from beeline-site.xml, if it is present in the classpath") + .create()); // Substitution option --hivevar options.addOption(OptionBuilder @@ -712,7 +720,7 @@ public class BeeLine implements Closeable { private boolean isBeeLineOpt(String arg) { return arg.startsWith("--") && !(HIVE_VAR_PREFIX.equals(arg) || (HIVE_CONF_PREFIX.equals(arg)) - || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg)); + || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg) || "--getUrlsFromBeelineSite".equals(arg)); } } @@ -843,6 +851,12 @@ public class BeeLine implements Closeable { getOpts().setHelpAsked(true); return true; } + + if (cl.hasOption("getUrlsFromBeelineSite")) { + printBeelineSiteUrls(); + getOpts().setBeelineSiteUrlsAsked(true); + return true; + } Properties hiveVars = cl.getOptionProperties("hivevar"); for (String key : hiveVars.stringPropertyNames()) { @@ -919,6 +933,44 @@ public class BeeLine implements Closeable { return false; } + private void printBeelineSiteUrls() { + BeelineSiteParser beelineSiteParser = getUserBeelineSiteParser(); + if (!beelineSiteParser.configExists()) { + output("No beeline-site.xml in the path", true); + } + if (beelineSiteParser.configExists()) { + // Get the named url from user specific config file if present + try { + Properties userNamedConnectionURLs = beelineSiteParser.getConnectionProperties(); + userNamedConnectionURLs.remove(BeelineSiteParser.DEFAULT_NAMED_JDBC_URL_PROPERTY_KEY); + StringBuilder sb = new StringBuilder("urls: "); + for (Entry<Object, Object> entry : userNamedConnectionURLs.entrySet()) { + String urlFromBeelineSite = (String) entry.getValue(); + if (isZkBasedUrl(urlFromBeelineSite)) { + List<String> jdbcUrls = HiveConnection.getAllUrlStrings(urlFromBeelineSite); + for (String jdbcUrl : jdbcUrls) { + sb.append(jdbcUrl + ", "); + } + } else { + sb.append(urlFromBeelineSite + ", "); + } + } + output(sb.toString(), true); + } catch (Exception e) { + output(e.getMessage(), true); + return; + } + } + } + + private boolean isZkBasedUrl(String urlFromBeelineSite) { + String zkJdbcUriParam = ("serviceDiscoveryMode=zooKeeper").toLowerCase(); + if (urlFromBeelineSite.toLowerCase().contains(zkJdbcUriParam)) { + return true; + } + return false; + } + private void setHiveConfVar(String key, String val) { getOpts().getHiveConfVariables().put(key, val); if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) { @@ -1060,6 +1112,9 @@ public class BeeLine implements Closeable { if (getOpts().isHelpAsked()) { return 0; } + if (getOpts().isBeelineSiteUrlsAsked()) { + return 0; + } if (getOpts().getScriptFile() != null) { return executeFile(getOpts().getScriptFile()); } diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 5967b4d7bc0..de1cf092df2 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -115,6 +115,7 @@ public class BeeLineOpts implements Completer { private Map<String, String> hiveVariables = new HashMap<String, String>(); private Map<String, String> hiveConfVariables = new HashMap<String, String>(); private boolean helpAsked; + private boolean beelineSiteUrlsAsked; private String lastConnectedUrl = null; @@ -687,7 +688,16 @@ public class BeeLineOpts implements Completer { public boolean isHelpAsked() { return helpAsked; } + + public void setBeelineSiteUrlsAsked(boolean beelineSiteUrlsAsked) { + this.beelineSiteUrlsAsked = beelineSiteUrlsAsked; + } + + public boolean isBeelineSiteUrlsAsked() { + return beelineSiteUrlsAsked; + } + public String getLastConnectedUrl(){ return lastConnectedUrl; } diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index a4e342d089b..e3b0ba38303 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -212,6 +212,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --delimiter=DELIMITER set the query delimiter; multi-char delimiters are allowed, but quotation\n \ \ marks, slashes, and -- are not allowed; defaults to ;\n \ \ --convertBinaryArrayToString=[true/false] display binary column data as string or as byte array \n \ +\ --getUrlsFromBeelineSite Print all urls from beeline-site.xml, if it is present in the classpath\n \ \ --help display this message\n \ \n \ \ Example:\n \ diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index 9b364f9c90b..70696281b6a 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -43,11 +43,17 @@ import org.apache.hive.service.rpc.thrift.TProtocolVersion; import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenReq; import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenResp; import org.apache.hive.service.rpc.thrift.TSessionHandle; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; +import org.apache.http.NoHttpResponseException; import org.apache.http.client.CookieStore; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.socket.ConnectionSocketFactory; @@ -57,9 +63,11 @@ import org.apache.http.impl.client.BasicCookieStore; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.RequestWrapper; import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import org.apache.http.ssl.SSLContexts; +import org.apache.http.util.Args; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.transport.THttpClient; @@ -69,6 +77,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; import javax.net.ssl.TrustManagerFactory; import javax.security.auth.Subject; import javax.security.sasl.Sasl; @@ -76,12 +85,17 @@ import javax.security.sasl.SaslException; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.InterruptedIOException; import java.io.IOException; import java.io.InputStreamReader; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; +import java.net.ConnectException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import java.security.AccessControlContext; import java.security.AccessController; import java.security.KeyStore; @@ -105,6 +119,7 @@ import java.sql.Savepoint; import java.sql.Statement; import java.sql.Struct; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -144,6 +159,7 @@ public class HiveConnection implements java.sql.Connection { private String wmPool = null, wmApp = null; private Properties clientInfo; private Subject loggedInSubject; + private int maxRetries = 1; /** * Get all direct HiveServer2 URLs from a ZooKeeper based HiveServer2 URL @@ -162,6 +178,79 @@ public class HiveConnection implements java.sql.Connection { return ZooKeeperHiveClientHelper.getDirectParamsList(params); } + public static List<String> getAllUrlStrings(String zookeeperBasedHS2Url) throws Exception { + List<String> jdbcUrls = new ArrayList<>(); + List<JdbcConnectionParams> allConnectionParams = getAllUrls(zookeeperBasedHS2Url); + for (JdbcConnectionParams cp : allConnectionParams) { + String jdbcUrl = makeDirectJDBCUrlFromConnectionParams(cp); + if ((jdbcUrl != null) && (!jdbcUrl.isEmpty())) { + jdbcUrls.add(jdbcUrl); + } + } + return jdbcUrls; + } + + private static String makeDirectJDBCUrlFromConnectionParams(JdbcConnectionParams cp) { + // Direct JDBC Url format: + // jdbc:hive2://<host1>:<port1>/dbName;sess_var_list?hive_conf_list#hive_var_list + StringBuilder url = new StringBuilder(""); + if (cp != null) { + if (cp.getHost() != null) { + url.append(cp.getHost()); + url.append(":"); + url.append(cp.getPort()); + url.append("/"); + url.append(cp.getDbName()); + // Add session vars + if ((cp.getSessionVars() != null) && (!cp.getSessionVars().isEmpty())) { + for (Entry<String, String> sessVar : cp.getSessionVars().entrySet()) { + if ((sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.SERVICE_DISCOVERY_MODE)) + || (sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.ZOOKEEPER_NAMESPACE))) { + continue; + } + url.append(";"); + url.append(sessVar.getKey()); + url.append("="); + url.append(sessVar.getValue()); + } + } + // Add hive confs + if ((cp.getHiveConfs() != null) && (!cp.getHiveConfs().isEmpty())) { + url.append("?"); + boolean firstKV = true; + for (Entry<String, String> hiveConf : cp.getHiveConfs().entrySet()) { + if (!firstKV) { + url.append(";"); + } else { + firstKV = false; + } + url.append(hiveConf.getKey()); + url.append("="); + url.append(hiveConf.getValue()); + } + } + // Add hive vars + if ((cp.getHiveVars() != null) && (!cp.getHiveVars().isEmpty())) { + url.append("#"); + boolean firstKV = true; + for (Entry<String, String> hiveVar : cp.getHiveVars().entrySet()) { + if (!firstKV) { + url.append(";"); + } else { + firstKV = false; + } + url.append(hiveVar.getKey()); + url.append("="); + url.append(hiveVar.getValue()); + } + } + } else { + return url.toString(); + } + } + return url.toString(); + } + public HiveConnection(String uri, Properties info) throws SQLException { try { connParams = Utils.parseURL(uri, info); @@ -488,21 +577,99 @@ public class HiveConnection implements java.sql.Connection { } else { httpClientBuilder = HttpClientBuilder.create(); } - // In case the server's idletimeout is set to a lower value, it might close it's side of - // connection. However we retry one more time on NoHttpResponseException + + // Beeline <------> LB <------> Reverse Proxy <-----> Hiveserver2 + // In case of deployments like above, the LoadBalancer (LB) can be configured with Idle Timeout after which the LB + // will send TCP RST to Client (Beeline) and Backend (Reverse Proxy). If user is connected to beeline, idle for + // sometime and resubmits a query after the idle timeout there is a broken pipe between beeline and LB. When Beeline + // tries to submit the query one of two things happen, it either hangs or times out (if socketTimeout is defined in + // the jdbc param). The hang is because of the default infinite socket timeout for which there is no auto-recovery + // (user have to manually interrupt the query). If the socketTimeout jdbc param was specified, beeline will receive + // SocketTimeoutException (Read Timeout) or NoHttpResponseException both of which can be retried if maxRetries is + // also specified by the user (jdbc param). + // The following retry handler handles the above cases in addition to retries for idempotent and unsent requests. httpClientBuilder.setRetryHandler(new HttpRequestRetryHandler() { + // This handler is mostly a copy of DefaultHttpRequestRetryHandler except it also retries some exceptions + // which could be thrown in certain cases where idle timeout from intermediate proxy triggers a connection reset. + private final List<Class<? extends IOException>> nonRetriableClasses = Arrays.asList( + InterruptedIOException.class, + UnknownHostException.class, + ConnectException.class, + SSLException.class); + // socket exceptions could happen because of timeout, broken pipe or server not responding in which case it is + // better to reopen the connection and retry if user specified maxRetries + private final List<Class<? extends IOException>> retriableClasses = Arrays.asList( + SocketTimeoutException.class, + SocketException.class, + NoHttpResponseException.class + ); + @Override public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { - if (executionCount > 1) { - LOG.info("Retry attempts to connect to server exceeded."); + Args.notNull(exception, "Exception parameter"); + Args.notNull(context, "HTTP context"); + if (executionCount > maxRetries) { + // Do not retry if over max retry count + LOG.error("Max retries (" + maxRetries + ") exhausted.", exception); + return false; + } + if (this.retriableClasses.contains(exception.getClass())) { + LOG.info("Retrying " + exception.getClass() + " as it is in retriable classes list."); + return true; + } + if (this.nonRetriableClasses.contains(exception.getClass())) { + LOG.info("Not retrying as the class (" + exception.getClass() + ") is non-retriable class."); return false; + } else { + for (final Class<? extends IOException> rejectException : this.nonRetriableClasses) { + if (rejectException.isInstance(exception)) { + LOG.info("Not retrying as the class (" + exception.getClass() + ") is an instance of is non-retriable class.");; + return false; + } + } } - if (exception instanceof org.apache.http.NoHttpResponseException) { - LOG.info("Could not connect to the server. Retrying one more time."); + final HttpClientContext clientContext = HttpClientContext.adapt(context); + final HttpRequest request = clientContext.getRequest(); + + if(requestIsAborted(request)){ + LOG.info("Not retrying as request is aborted."); + return false; + } + + if (handleAsIdempotent(request)) { + LOG.info("Retrying idempotent request. Attempt " + executionCount + " of " + maxRetries); + // Retry if the request is considered idempotent return true; } + + if (!clientContext.isRequestSent()) { + LOG.info("Retrying unsent request. Attempt " + executionCount + " of " + maxRetries); + // Retry if the request has not been sent fully or + // if it's OK to retry methods that have been sent + return true; + } + + LOG.info("Not retrying as the request is not idempotent or is already sent."); + // otherwise do not retry return false; } + + // requests that handles "Expect continue" handshakes. If server received the header and is waiting for body + // then those requests can be retried. Most basic http method methods except DELETE are idempotent as long as they + // are not aborted. + protected boolean handleAsIdempotent(final HttpRequest request) { + return !(request instanceof HttpEntityEnclosingRequest); + } + + // checks if the request got aborted + protected boolean requestIsAborted(final HttpRequest request) { + HttpRequest req = request; + if (request instanceof RequestWrapper) { // does not forward request to original + req = ((RequestWrapper) request).getOriginal(); + } + return (req instanceof HttpUriRequest && ((HttpUriRequest)req).isAborted()); + } + }); // Add the request interceptor to the client builder @@ -511,6 +678,13 @@ public class HiveConnection implements java.sql.Connection { // Add an interceptor to add in an XSRF header httpClientBuilder.addInterceptorLast(new XsrfHttpRequestInterceptor()); + // set the specified timeout (socketTimeout jdbc param) for http connection as well + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(loginTimeout * 1000) + .setConnectionRequestTimeout(loginTimeout * 1000) + .setSocketTimeout(loginTimeout * 1000).build(); + httpClientBuilder.setDefaultRequestConfig(config); + // Configure http client for SSL if (useSsl) { String useTwoWaySSL = sessConfMap.get(JdbcConnectionParams.USE_TWO_WAY_SSL); diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java index 324776d3d5a..e54fdcebfaa 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -115,7 +115,7 @@ public class Utils { // Use ZooKeeper for indirection while using dynamic service discovery public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER = "zooKeeper"; public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER_HA = "zooKeeperHA"; - static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace"; + public static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace"; // Default namespace value on ZooKeeper. // This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri. static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2"; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 95ef33ffe20..c172c5d1baf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -1660,9 +1660,6 @@ public class TezCompiler extends TaskCompiler { for (FileSinkOperator fsOp : fsOpsAll) { - if (!fsOp.getConf().getTableInfo().isSetBucketingVersion()) { - continue; - } // Look for direct parent ReduceSinkOp // If there are more than 1 parent, bail out. Operator<?> parent = fsOp; diff --git a/service/src/java/org/apache/hive/service/auth/ldap/HttpEmptyAuthenticationException.java b/service/src/java/org/apache/hive/service/auth/ldap/HttpEmptyAuthenticationException.java new file mode 100644 index 00000000000..b6b71bcc7ad --- /dev/null +++ b/service/src/java/org/apache/hive/service/auth/ldap/HttpEmptyAuthenticationException.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hive.service.auth.ldap; + +import org.apache.hive.service.auth.HttpAuthenticationException; + +public class HttpEmptyAuthenticationException extends HttpAuthenticationException { + + public HttpEmptyAuthenticationException(String msg) { + super(msg); + } +} diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index 70ffa3c6a3a..a89fbc16183 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -18,6 +18,7 @@ package org.apache.hive.service.cli.thrift; +import java.io.EOFException; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.InetAddress; @@ -36,6 +37,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.NewCookie; +import com.google.common.io.ByteStreams; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -53,6 +55,7 @@ import org.apache.hive.service.auth.HiveAuthFactory; import org.apache.hive.service.auth.HttpAuthUtils; import org.apache.hive.service.auth.HttpAuthenticationException; import org.apache.hive.service.auth.PasswdAuthenticationProvider; +import org.apache.hive.service.auth.ldap.HttpEmptyAuthenticationException; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.session.SessionManager; import org.apache.thrift.TProcessor; @@ -207,7 +210,19 @@ public class ThriftHttpServlet extends TServlet { super.doPost(request, response); } catch (HttpAuthenticationException e) { - LOG.error("Error: ", e); + // Ignore HttpEmptyAuthenticationException, it is normal for knox + // to send a request with empty header + if (!(e instanceof HttpEmptyAuthenticationException)) { + LOG.error("Error: ", e); + } + // Wait until all the data is received and then respond with 401 + if (request.getContentLength() < 0) { + try { + ByteStreams.skipFully(request.getInputStream(), Integer.MAX_VALUE); + } catch (EOFException ex) { + LOG.info(ex.getMessage()); + } + } // Send a 401 to the client response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); if(isKerberosAuthMode(authType)) { @@ -404,6 +419,9 @@ public class ThriftHttpServlet extends TServlet { try { return serviceUGI.doAs(new HttpKerberosServerAction(request, serviceUGI)); } catch (Exception e) { + if (e.getCause() instanceof HttpEmptyAuthenticationException) { + throw (HttpEmptyAuthenticationException)e.getCause(); + } LOG.error("Failed to authenticate with hive/_HOST kerberos principal"); throw new HttpAuthenticationException(e); } @@ -546,7 +564,7 @@ public class ThriftHttpServlet extends TServlet { String authHeader = request.getHeader(HttpAuthUtils.AUTHORIZATION); // Each http request must have an Authorization header if (authHeader == null || authHeader.isEmpty()) { - throw new HttpAuthenticationException("Authorization header received " + + throw new HttpEmptyAuthenticationException("Authorization header received " + "from the client is empty."); }