Repository: zeppelin Updated Branches: refs/heads/branch-0.7 a386253b7 -> 6a6fcc2ed
[ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl We found that Livy doesn't work under kerberos and ssl, this is due to a bug when constructing KerberosRestTemplate. This PR fix this issue. [Bug Fix] * [ ] - Task * https://issues.apache.org/jira/browse/ZEPPELIN-2584 Unfornatelly it is difficult to add system test, so I manually tested it in a secure cluster. * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: root <r...@ctr-e133-1493418528701-48413-01-000004.hwx.site> Closes #2367 from zjffdu/ZEPPELIN-2584 and squashes the following commits: 18b23dc [root] [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl (cherry picked from commit e3ba23a9aa4909c7a8d3231e4d363771e0245b53) Signed-off-by: Jeff Zhang <zjf...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/6a6fcc2e Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/6a6fcc2e Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/6a6fcc2e Branch: refs/heads/branch-0.7 Commit: 6a6fcc2ed6b0146e0a176232bedc0694fea43027 Parents: a386253 Author: root <r...@ctr-e133-1493418528701-48413-01-000004.hwx.site> Authored: Fri May 26 08:44:09 2017 +0000 Committer: Jeff Zhang <zjf...@apache.org> Committed: Thu Jun 1 15:47:05 2017 +0800 ---------------------------------------------------------------------- livy/pom.xml | 4 -- .../zeppelin/livy/BaseLivyInterpreter.java | 55 ++++++++++++++++++-- pom.xml | 4 +- 3 files changed, 52 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6a6fcc2e/livy/pom.xml ---------------------------------------------------------------------- diff --git a/livy/pom.xml b/livy/pom.xml index 5f41308..8a3d88d 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -37,14 +37,10 @@ <properties> <!--library versions--> <commons.exec.version>1.3</commons.exec.version> - <httpcomponents.client.version>4.3.4</httpcomponents.client.version> <spring.web.version>4.3.0.RELEASE</spring.web.version> <spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient> <!--test library versions--> - <achilles.version>3.2.4-Zeppelin</achilles.version> - <assertj.version>1.7.0</assertj.version> - <mockito.version>1.9.5</mockito.version> <livy.version>0.3.0</livy.version> <spark.version>2.1.0</spark.version> <hadoop.version>2.6.0</hadoop.version> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6a6fcc2e/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index dd27e7d..1322a4d 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -21,9 +21,20 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang.StringUtils; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.*; @@ -38,11 +49,11 @@ import org.springframework.security.kerberos.client.KerberosRestTemplate; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; - import javax.net.ssl.SSLContext; import java.io.FileInputStream; import java.io.IOException; import java.security.KeyStore; +import java.security.Principal; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -388,6 +399,11 @@ public abstract class BaseLivyInterpreter extends Interpreter { private RestTemplate createRestTemplate() { + String keytabLocation = property.getProperty("zeppelin.livy.keytab"); + String principal = property.getProperty("zeppelin.livy.principal"); + boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) && + StringUtils.isNotEmpty(principal); + HttpClient httpClient = null; if (livyURL.startsWith("https:")) { String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore"); @@ -408,7 +424,37 @@ public abstract class BaseLivyInterpreter extends Interpreter { .loadTrustMaterial(trustStore) .build(); SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext); - httpClient = HttpClients.custom().setSSLSocketFactory(csf).build(); + HttpClientBuilder httpClientBuilder = HttpClients.custom().setSSLSocketFactory(csf); + RequestConfig reqConfig = new RequestConfig() { + @Override + public boolean isAuthenticationEnabled() { + return true; + } + }; + httpClientBuilder.setDefaultRequestConfig(reqConfig); + Credentials credentials = new Credentials() { + @Override + public String getPassword() { + return null; + } + + @Override + public Principal getUserPrincipal() { + return null; + } + }; + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(AuthScope.ANY, credentials); + httpClientBuilder.setDefaultCredentialsProvider(credsProvider); + if (isSpnegoEnabled) { + Registry<AuthSchemeProvider> authSchemeProviderRegistry = + RegistryBuilder.<AuthSchemeProvider>create() + .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()) + .build(); + httpClientBuilder.setDefaultAuthSchemeRegistry(authSchemeProviderRegistry); + } + + httpClient = httpClientBuilder.build(); } catch (Exception e) { throw new RuntimeException("Failed to create SSL HttpClient", e); } finally { @@ -422,9 +468,8 @@ public abstract class BaseLivyInterpreter extends Interpreter { } } - String keytabLocation = property.getProperty("zeppelin.livy.keytab"); - String principal = property.getProperty("zeppelin.livy.principal"); - if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) { + + if (isSpnegoEnabled) { if (httpClient == null) { return new KerberosRestTemplate(keytabLocation, principal); } else { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6a6fcc2e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4df0094..1f67814 100644 --- a/pom.xml +++ b/pom.xml @@ -95,8 +95,8 @@ <gson.version>2.2</gson.version> <guava.version>15.0</guava.version> <jetty.version>9.2.15.v20160210</jetty.version> - <httpcomponents.core.version>4.3.3</httpcomponents.core.version> - <httpcomponents.client.version>4.3.6</httpcomponents.client.version> + <httpcomponents.core.version>4.4.1</httpcomponents.core.version> + <httpcomponents.client.version>4.5.1</httpcomponents.client.version> <httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version> <commons.lang.version>2.5</commons.lang.version> <commons.configuration.version>1.9</commons.configuration.version>
