Repository: zeppelin Updated Branches: refs/heads/master 88c5c3ccd -> e3ba23a9a
[ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl ### What is this PR for? We found that Livy doesn't work under kerberos and ssl, this is due to a bug when constructing KerberosRestTemplate. This PR fix this issue. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2584 ### How should this be tested? Unfornatelly it is difficult to add system test, so I manually tested it in a secure cluster. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: root <r...@ctr-e133-1493418528701-48413-01-000004.hwx.site> Closes #2367 from zjffdu/ZEPPELIN-2584 and squashes the following commits: 18b23dc [root] [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e3ba23a9 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e3ba23a9 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e3ba23a9 Branch: refs/heads/master Commit: e3ba23a9aa4909c7a8d3231e4d363771e0245b53 Parents: 88c5c3c Author: root <r...@ctr-e133-1493418528701-48413-01-000004.hwx.site> Authored: Fri May 26 08:44:09 2017 +0000 Committer: Jeff Zhang <zjf...@apache.org> Committed: Thu Jun 1 15:17:21 2017 +0800 ---------------------------------------------------------------------- livy/pom.xml | 18 ------- .../zeppelin/livy/BaseLivyInterpreter.java | 56 +++++++++++++++++--- pom.xml | 4 +- 3 files changed, 52 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/pom.xml ---------------------------------------------------------------------- diff --git a/livy/pom.xml b/livy/pom.xml index 3c121fc..5f9dec7 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -37,14 +37,10 @@ <properties> <!--library versions--> <commons.exec.version>1.3</commons.exec.version> - <httpcomponents.client.version>4.3.4</httpcomponents.client.version> <spring.web.version>4.3.0.RELEASE</spring.web.version> <spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient> <!--test library versions--> - <achilles.version>3.2.4-Zeppelin</achilles.version> - <assertj.version>1.7.0</assertj.version> - <mockito.version>1.9.5</mockito.version> <livy.version>0.3.0</livy.version> <spark.version>2.1.0</spark.version> <hadoop.version>2.6.0</hadoop.version> @@ -80,7 +76,6 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <version>${httpcomponents.client.version}</version> </dependency> <dependency> @@ -107,19 +102,6 @@ </dependency> <dependency> - <groupId>org.assertj</groupId> - <artifactId>assertj-core</artifactId> - <version>${assertj.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <version>${mockito.version}</version> - <scope>test</scope> - </dependency> - - <dependency> <groupId>com.cloudera.livy</groupId> <artifactId>livy-integration-test</artifactId> <version>${livy.version}</version> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index b52ba16..fcb7e9d 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -21,9 +21,20 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang.StringUtils; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.*; @@ -38,11 +49,11 @@ import org.springframework.security.kerberos.client.KerberosRestTemplate; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; - import javax.net.ssl.SSLContext; import java.io.FileInputStream; import java.io.IOException; import java.security.KeyStore; +import java.security.Principal; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -52,7 +63,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - /** * Base class for livy interpreters. */ @@ -407,6 +417,11 @@ public abstract class BaseLivyInterpreter extends Interpreter { private RestTemplate createRestTemplate() { + String keytabLocation = property.getProperty("zeppelin.livy.keytab"); + String principal = property.getProperty("zeppelin.livy.principal"); + boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) && + StringUtils.isNotEmpty(principal); + HttpClient httpClient = null; if (livyURL.startsWith("https:")) { String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore"); @@ -427,7 +442,37 @@ public abstract class BaseLivyInterpreter extends Interpreter { .loadTrustMaterial(trustStore) .build(); SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext); - httpClient = HttpClients.custom().setSSLSocketFactory(csf).build(); + HttpClientBuilder httpClientBuilder = HttpClients.custom().setSSLSocketFactory(csf); + RequestConfig reqConfig = new RequestConfig() { + @Override + public boolean isAuthenticationEnabled() { + return true; + } + }; + httpClientBuilder.setDefaultRequestConfig(reqConfig); + Credentials credentials = new Credentials() { + @Override + public String getPassword() { + return null; + } + + @Override + public Principal getUserPrincipal() { + return null; + } + }; + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(AuthScope.ANY, credentials); + httpClientBuilder.setDefaultCredentialsProvider(credsProvider); + if (isSpnegoEnabled) { + Registry<AuthSchemeProvider> authSchemeProviderRegistry = + RegistryBuilder.<AuthSchemeProvider>create() + .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()) + .build(); + httpClientBuilder.setDefaultAuthSchemeRegistry(authSchemeProviderRegistry); + } + + httpClient = httpClientBuilder.build(); } catch (Exception e) { throw new RuntimeException("Failed to create SSL HttpClient", e); } finally { @@ -441,9 +486,8 @@ public abstract class BaseLivyInterpreter extends Interpreter { } } - String keytabLocation = property.getProperty("zeppelin.livy.keytab"); - String principal = property.getProperty("zeppelin.livy.principal"); - if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) { + + if (isSpnegoEnabled) { if (httpClient == null) { return new KerberosRestTemplate(keytabLocation, principal); } else { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 87af323..9232b9e 100644 --- a/pom.xml +++ b/pom.xml @@ -96,8 +96,8 @@ <gson-extras.version>0.2.1</gson-extras.version> <guava.version>15.0</guava.version> <jetty.version>9.2.15.v20160210</jetty.version> - <httpcomponents.core.version>4.3.3</httpcomponents.core.version> - <httpcomponents.client.version>4.3.6</httpcomponents.client.version> + <httpcomponents.core.version>4.4.1</httpcomponents.core.version> + <httpcomponents.client.version>4.5.1</httpcomponents.client.version> <httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version> <commons.lang.version>2.5</commons.lang.version> <commons.configuration.version>1.9</commons.configuration.version>
