zeppelin git commit: [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl

2017-06-01 Thread zjffdu 
Repository: zeppelin
Updated Branches:
  refs/heads/branch-0.7 a386253b7 -> 6a6fcc2ed


[ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl

We found that Livy doesn't work under kerberos and ssl, this is due to a bug 
when constructing KerberosRestTemplate. This PR fix this issue.

[Bug Fix]

* [ ] - Task

* https://issues.apache.org/jira/browse/ZEPPELIN-2584

Unfornatelly it is difficult to add system test, so I manually tested it in a 
secure cluster.

* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: root 

Closes #2367 from zjffdu/ZEPPELIN-2584 and squashes the following commits:

18b23dc [root] [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl

(cherry picked from commit e3ba23a9aa4909c7a8d3231e4d363771e0245b53)
Signed-off-by: Jeff Zhang 


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/6a6fcc2e
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/6a6fcc2e
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/6a6fcc2e

Branch: refs/heads/branch-0.7
Commit: 6a6fcc2ed6b0146e0a176232bedc0694fea43027
Parents: a386253
Author: root 
Authored: Fri May 26 08:44:09 2017 +
Committer: Jeff Zhang 
Committed: Thu Jun 1 15:47:05 2017 +0800

--
 livy/pom.xml|  4 --
 .../zeppelin/livy/BaseLivyInterpreter.java  | 55 ++--
 pom.xml |  4 +-
 3 files changed, 52 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6a6fcc2e/livy/pom.xml
--
diff --git a/livy/pom.xml b/livy/pom.xml
index 5f41308..8a3d88d 100644
--- a/livy/pom.xml
+++ b/livy/pom.xml
@@ -37,14 +37,10 @@
   
 
 1.3
-4.3.4
 4.3.0.RELEASE
 
1.0.1.RELEASE
 
 
-3.2.4-Zeppelin
-1.7.0
-1.9.5
 0.3.0
 2.1.0
 2.6.0

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6a6fcc2e/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
--
diff --git 
a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java 
b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
index dd27e7d..1322a4d 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
@@ -21,9 +21,20 @@ import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.annotations.SerializedName;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLContexts;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.zeppelin.interpreter.*;
@@ -38,11 +49,11 @@ import 
org.springframework.security.kerberos.client.KerberosRestTemplate;
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
-
 import javax.net.ssl.SSLContext;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.security.KeyStore;
+import java.security.Principal;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -388,6 +399,11 @@ public abstract class BaseLivyInterpreter extends 
Interpreter {
 
 
   private RestTemplate createRestTemplate() {
+String keytabLocation = property.getProperty("zeppelin.livy.keytab");
+String principal = property.getProperty("zeppelin.livy.principal");
+boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) &&
+StringUtils.isNotEmpty(principal);
+
 HttpClient httpClient = null;
 if (livyURL.startsWith("https:")) {
   String keystoreFile = 
property.getProperty("zeppelin.livy.ssl.trustStore");
@@ -408,7 +424,37 @@ public abstract class

zeppelin git commit: [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl

2017-06-01 Thread zjffdu 
Repository: zeppelin
Updated Branches:
  refs/heads/master 88c5c3ccd -> e3ba23a9a


[ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl

### What is this PR for?
We found that Livy doesn't work under kerberos and ssl, this is due to a bug 
when constructing KerberosRestTemplate. This PR fix this issue.

### What type of PR is it?
[Bug Fix]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-2584

### How should this be tested?
Unfornatelly it is difficult to add system test, so I manually tested it in a 
secure cluster.

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: root 

Closes #2367 from zjffdu/ZEPPELIN-2584 and squashes the following commits:

18b23dc [root] [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e3ba23a9
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e3ba23a9
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e3ba23a9

Branch: refs/heads/master
Commit: e3ba23a9aa4909c7a8d3231e4d363771e0245b53
Parents: 88c5c3c
Author: root 
Authored: Fri May 26 08:44:09 2017 +
Committer: Jeff Zhang 
Committed: Thu Jun 1 15:17:21 2017 +0800

--
 livy/pom.xml| 18 ---
 .../zeppelin/livy/BaseLivyInterpreter.java  | 56 +---
 pom.xml |  4 +-
 3 files changed, 52 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/pom.xml
--
diff --git a/livy/pom.xml b/livy/pom.xml
index 3c121fc..5f9dec7 100644
--- a/livy/pom.xml
+++ b/livy/pom.xml
@@ -37,14 +37,10 @@
   
 
 1.3
-4.3.4
 4.3.0.RELEASE
 
1.0.1.RELEASE
 
 
-3.2.4-Zeppelin
-1.7.0
-1.9.5
 0.3.0
 2.1.0
 2.6.0
@@ -80,7 +76,6 @@
 
   org.apache.httpcomponents
   httpclient
-  ${httpcomponents.client.version}
 
 
 
@@ -107,19 +102,6 @@
 
 
 
-  org.assertj
-  assertj-core
-  ${assertj.version}
-  test
-
-
-  org.mockito
-  mockito-all
-  ${mockito.version}
-  test
-
-
-
   com.cloudera.livy
   livy-integration-test
   ${livy.version}

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
--
diff --git 
a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java 
b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
index b52ba16..fcb7e9d 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
@@ -21,9 +21,20 @@ import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.annotations.SerializedName;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLContexts;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.zeppelin.interpreter.*;
@@ -38,11 +49,11 @@ import 
org.springframework.security.kerberos.client.KerberosRestTemplate;
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
-
 import javax.net.ssl.SSLContext;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.security.KeyStore;
+import java.security.Principal;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -52,7 +63,6 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 
-
 /**
  * Base class for livy interpreters.