Repository: knox Updated Branches: refs/heads/master 15ae0d781 -> 8cb451daf
[KNOX-695] - Expose configuration of HttpClient's connection and socket timeout settings Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/8cb451da Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/8cb451da Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/8cb451da Branch: refs/heads/master Commit: 8cb451daf2d2db841780eab78afe928de397c966 Parents: 15ae0d7 Author: Kevin Minder <kmin...@apache.org> Authored: Fri Mar 18 08:49:57 2016 -0400 Committer: Kevin Minder <kmin...@apache.org> Committed: Fri Mar 18 08:49:57 2016 -0400 ---------------------------------------------------------------------- CHANGES | 1 + .../gateway/config/impl/GatewayConfigImpl.java | 40 +++++++++++ gateway-spi/pom.xml | 4 ++ .../hadoop/gateway/config/GatewayConfig.java | 4 ++ .../gateway/dispatch/DefaultDispatch.java | 6 +- .../dispatch/DefaultHttpClientFactory.java | 74 ++++++++++++++++++-- .../hadoop/gateway/GatewayTestConfig.java | 10 +++ .../hadoop/gateway/GatewayTestConfig.java | 10 +++ 8 files changed, 142 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 5963fca..6d21512 100644 --- a/CHANGES +++ b/CHANGES @@ -11,6 +11,7 @@ Release Notes - Apache Knox - Version 0.9.0 * [KNOX-678] - Malformed UTF-8 characters in JSON Response * [KNOX-680] - Update Knox's HttpClient dependency to latest version * [KNOX-694] - Enhance LDAP user search configurability + * [KNOX-695] - Expose configuration of HttpClient's connection and socket timeout settings ** Bug * [KNOX-681] - A PUT with Content-Type application/xml but no body causes NullPointerException http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java index 711efc8..82a21ee 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java @@ -119,6 +119,8 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { private static final String XFORWARDED_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".xforwarded.enabled"; private static final String EPHEMERAL_DH_KEY_SIZE = GATEWAY_CONFIG_FILE_PREFIX + ".jdk.tls.ephemeralDHKeySize"; private static final String HTTP_CLIENT_MAX_CONNECTION = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.maxConnections"; + private static final String HTTP_CLIENT_CONNECTION_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.connectionTimeout"; + private static final String HTTP_CLIENT_SOCKET_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.socketTimeout"; private static final String THREAD_POOL_MAX = GATEWAY_CONFIG_FILE_PREFIX + ".threadpool.max"; public static final String HTTP_SERVER_REQUEST_BUFFER = GATEWAY_CONFIG_FILE_PREFIX + ".httpserver.requestBuffer"; public static final String HTTP_SERVER_REQUEST_HEADER_BUFFER = GATEWAY_CONFIG_FILE_PREFIX + ".httpserver.requestHeaderBuffer"; @@ -500,6 +502,34 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { return getInt( HTTP_CLIENT_MAX_CONNECTION, 32 ); } + @Override + public int getHttpClientConnectionTimeout() { + int t = -1; + String s = get( HTTP_CLIENT_CONNECTION_TIMEOUT, null ); + if ( s != null ) { + try { + t = (int)parseNetworkTimeout( s ); + } catch ( Exception e ) { + // Ignore it and use the default. + } + } + return t; + } + + @Override + public int getHttpClientSocketTimeout() { + int t = -1; + String s = get( HTTP_CLIENT_SOCKET_TIMEOUT, null ); + if ( s != null ) { + try { + t = (int)parseNetworkTimeout( s ); + } catch ( Exception e ) { + // Ignore it and use the default. + } + } + return t; + } + /* (non-Javadoc) * @see org.apache.hadoop.gateway.config.GatewayConfig#getThreadPoolMax() */ @@ -572,4 +602,14 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public String getSigningKeyAlias() { return get(SIGNING_KEY_ALIAS); } + + private static long parseNetworkTimeout( String s ) { + PeriodFormatter f = new PeriodFormatterBuilder() + .appendMinutes().appendSuffix("m"," min") + .appendSeconds().appendSuffix("s"," sec") + .appendMillis().toFormatter(); + Period p = Period.parse( s, f ); + return p.toStandardDuration().getMillis(); + } + } http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-spi/pom.xml b/gateway-spi/pom.xml index 196515e..1da99d9 100644 --- a/gateway-spi/pom.xml +++ b/gateway-spi/pom.xml @@ -110,6 +110,10 @@ <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + </dependency> <dependency> <groupId>junit</groupId> http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java index 5558578..d830887 100644 --- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java +++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java @@ -121,6 +121,10 @@ public interface GatewayConfig { int getHttpClientMaxConnections(); + int getHttpClientConnectionTimeout(); + + int getHttpClientSocketTimeout(); + int getThreadPoolMax(); int getHttpServerRequestBuffer(); http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java index d1cef55..6c72fde 100644 --- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java +++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java @@ -254,9 +254,11 @@ public class DefaultDispatch extends AbstractGatewayDispatch { @Override public void doGet(URI url, HttpServletRequest request, HttpServletResponse response) throws IOException, URISyntaxException { - HttpGet method = new HttpGet(url); + HttpGet method = new HttpGet(url); // https://issues.apache.org/jira/browse/KNOX-107 - Service URLs not rewritten for WebHDFS GET redirects - method.getParams().setBooleanParameter("http.protocol.handle-redirects", false); + // This is now taken care of in DefaultHttpClientFactory.createHttpClient + // and setting params here causes configuration setup there to be ignored there. + // method.getParams().setBooleanParameter("http.protocol.handle-redirects", false); copyRequestHeaderFields(method, request); executeRequest(method, request, response); } http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java index 57de2e4..92652e2 100644 --- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java +++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java @@ -30,17 +30,20 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.RedirectStrategy; import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.cookie.Cookie; import org.apache.http.impl.DefaultConnectionReuseStrategy; -import org.apache.http.impl.auth.SPNegoSchemeFactory; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.protocol.HttpContext; +import org.joda.time.Period; +import org.joda.time.format.PeriodFormatter; +import org.joda.time.format.PeriodFormatterBuilder; import javax.servlet.FilterConfig; import java.io.IOException; @@ -72,15 +75,31 @@ public class DefaultHttpClientFactory implements HttpClientFactory { builder.setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE ); builder.setConnectionReuseStrategy( DefaultConnectionReuseStrategy.INSTANCE ); + builder.setRedirectStrategy( new NeverRedirectStrategy() ); + builder.setRetryHandler( new NeverRetryHandler() ); int maxConnections = getMaxConnections( filterConfig ); builder.setMaxConnTotal( maxConnections ); builder.setMaxConnPerRoute( maxConnections ); - return builder - .setRedirectStrategy(new NeverRedirectStrategy()) - .setRetryHandler(new NeverRetryHandler()) - .build(); + builder.setDefaultRequestConfig( getRequestConfig( filterConfig ) ); + + HttpClient client = builder.build(); + return client; + } + + private static RequestConfig getRequestConfig( FilterConfig config ) { + RequestConfig.Builder builder = RequestConfig.custom(); + int connectionTimeout = getConnectionTimeout( config ); + if ( connectionTimeout != -1 ) { + builder.setConnectTimeout( connectionTimeout ); + builder.setConnectionRequestTimeout( connectionTimeout ); + } + int socketTimeout = getSocketTimeout( config ); + if( socketTimeout != -1 ) { + builder.setSocketTimeout( socketTimeout ); + } + return builder.build(); } private class NoCookieStore implements CookieStore { @@ -156,4 +175,49 @@ public class DefaultHttpClientFactory implements HttpClientFactory { return maxConnections; } + private static int getConnectionTimeout( FilterConfig filterConfig ) { + int timeout = -1; + GatewayConfig globalConfig = + (GatewayConfig)filterConfig.getServletContext().getAttribute( GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE ); + if( globalConfig != null ) { + timeout = globalConfig.getHttpClientConnectionTimeout(); + } + String str = filterConfig.getInitParameter( "httpclient.connectionTimeout" ); + if( str != null ) { + try { + timeout = (int)parseTimeout( str ); + } catch ( Exception e ) { + // Ignore it and use the default. + } + } + return timeout; + } + + private static int getSocketTimeout( FilterConfig filterConfig ) { + int timeout = -1; + GatewayConfig globalConfig = + (GatewayConfig)filterConfig.getServletContext().getAttribute( GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE ); + if( globalConfig != null ) { + timeout = globalConfig.getHttpClientSocketTimeout(); + } + String str = filterConfig.getInitParameter( "httpclient.socketTimeout" ); + if( str != null ) { + try { + timeout = (int)parseTimeout( str ); + } catch ( Exception e ) { + // Ignore it and use the default. + } + } + return timeout; + } + + private static long parseTimeout( String s ) { + PeriodFormatter f = new PeriodFormatterBuilder() + .appendMinutes().appendSuffix("m"," min") + .appendSeconds().appendSuffix("s"," sec") + .appendMillis().toFormatter(); + Period p = Period.parse( s, f ); + return p.toStandardDuration().getMillis(); + } + } http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java ---------------------------------------------------------------------- diff --git a/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java b/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java index e5be4be..3c53597 100644 --- a/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java +++ b/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java @@ -285,6 +285,16 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig { } @Override + public int getHttpClientConnectionTimeout() { + return -1; + } + + @Override + public int getHttpClientSocketTimeout() { + return -1; + } + + @Override public int getThreadPoolMax() { return 16; } http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java ---------------------------------------------------------------------- diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java index cd71e7f..4c75cb9 100644 --- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java +++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java @@ -326,6 +326,16 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig { } @Override + public int getHttpClientConnectionTimeout() { + return -1; + } + + @Override + public int getHttpClientSocketTimeout() { + return -1; + } + + @Override public int getThreadPoolMax() { return 16; }