This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 5584f0c [Fix-3262][common] When you request the URL through
applicationID to get the application status, you cannot get it if Kerberos
authentication is enabled (#3264)
5584f0c is described below
commit 5584f0cb4d27e3ae64ec28c65a8669b38c75d188
Author: felix.wang <59079269+felix-thinkingd...@users.noreply.github.com>
AuthorDate: Mon Aug 10 15:18:01 2020 +0800
[Fix-3262][common] When you request the URL through applicationID to get
the application status, you cannot get it if Kerberos authentication is enabled
(#3264)
* fix bug #3165 get resource.storage.type value toUpperCase
* fix bug #3176 optimize Gets the value of this property
“resource.storage.type”
* fix When you request the URL through applicationID to get the application
status, you cannot get it if Kerberos authentication is enabled
* fix When you request the URL through applicationID to get the application
status, you cannot get it if Kerberos authentication is enabled
* fix When you request the URL through applicationID to get the application
status, you cannot get it if Kerberos authentication is enabled
* fix miss
* add KerberosHttpClient fix kerberos bug
* fix map init
* add juint test
* Extraction of public methods
* Extraction of public methods
* Fix code quality
* Fix code quality
* Fix code quality
* Fix code quality
* Fix code quality
* Fix code quality
* Fix code quality
* rebuild test
* rebuild test
* check style
* check style
* check style
* revert UT POM
* Kerberos judgment goes up to hadoopUtils
* fix merge
* Remove connection pool shutdown
Co-authored-by: dailidong <dailidon...@gmail.com>
---
.../dolphinscheduler/common/utils/HadoopUtils.java | 7 +-
.../dolphinscheduler/common/utils/HttpUtils.java | 76 +++++-----
.../common/utils/KerberosHttpClient.java | 156 +++++++++++++++++++++
.../common/utils/HttpUtilsTest.java | 55 ++++++--
.../common/utils/KerberosHttpClientTest.java | 46 ++++++
pom.xml | 1 +
6 files changed, 296 insertions(+), 45 deletions(-)
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
index cf432a1..cc75624 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
@@ -417,7 +417,12 @@ public class HadoopUtils implements Closeable {
String applicationUrl = getApplicationUrl(applicationId);
logger.info("applicationUrl={}", applicationUrl);
- String responseContent = HttpUtils.get(applicationUrl);
+ String responseContent ;
+ if
(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,
false)) {
+ responseContent =
KerberosHttpClient.get(applicationUrl);
+ } else {
+ responseContent = HttpUtils.get(applicationUrl);
+ }
if (responseContent != null) {
ObjectNode jsonObject = JSONUtils.parseObject(responseContent);
result = jsonObject.path("app").path("finalStatus").asText();
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
index 8ea1531..7d1e0a5 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
@@ -128,40 +128,50 @@ public class HttpUtils {
CloseableHttpClient httpclient = HttpUtils.getInstance();
HttpGet httpget = new HttpGet(url);
- String responseContent = null;
- CloseableHttpResponse response = null;
-
- try {
- response = httpclient.execute(httpget);
- //check response status is 200
- if (response.getStatusLine().getStatusCode() == 200) {
- HttpEntity entity = response.getEntity();
- if (entity != null) {
- responseContent =
EntityUtils.toString(entity, Constants.UTF_8);
- }else{
- logger.warn("http entity is null");
- }
- }else{
- logger.error("http get:{} response status code
is not 200!", response.getStatusLine().getStatusCode());
- }
- }catch (Exception e){
- logger.error(e.getMessage(),e);
- }finally {
- try {
- if (response != null) {
-
EntityUtils.consume(response.getEntity());
- response.close();
- }
- } catch (IOException e) {
- logger.error(e.getMessage(),e);
- }
-
- if (!httpget.isAborted()) {
- httpget.releaseConnection();
- httpget.abort();
- }
+ return getResponseContentString(httpget,httpclient);
+ }
+
+ /**
+ * get http response content
+ *
+ * @param httpget httpget
+ * @param httpClient httpClient
+ * @return http get request response content
+ */
+ public static String getResponseContentString(HttpGet httpget,
CloseableHttpClient httpClient) {
+ String responseContent = null;
+ CloseableHttpResponse response = null;
+ try {
+ response = httpClient.execute(httpget);
+ // check response status is 200
+ if (response.getStatusLine().getStatusCode() == 200) {
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ responseContent = EntityUtils.toString(entity,
Constants.UTF_8);
+ } else {
+ logger.warn("http entity is null");
}
- return responseContent;
+ } else {
+ logger.error("http get:{} response status code is not 200!",
response.getStatusLine().getStatusCode());
+ }
+ } catch (IOException ioe) {
+ logger.error(ioe.getMessage(), ioe);
+ } finally {
+ try {
+ if (response != null) {
+ EntityUtils.consume(response.getEntity());
+ response.close();
+ }
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+ if (!httpget.isAborted()) {
+ httpget.releaseConnection();
+ httpget.abort();
+ }
+
}
+ return responseContent;
+ }
}
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java
new file mode 100644
index 0000000..5c1fd41
--- /dev/null
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.utils;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.Lookup;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.Subject;
+import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+import java.security.Principal;
+import java.security.PrivilegedAction;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * kerberos http client
+ */
+public class KerberosHttpClient {
+
+ public static final Logger logger =
LoggerFactory.getLogger(KerberosHttpClient.class);
+
+ private String principal;
+ private String keyTabLocation;
+
+ public KerberosHttpClient(String principal, String keyTabLocation) {
+ super();
+ this.principal = principal;
+ this.keyTabLocation = keyTabLocation;
+ }
+
+ public KerberosHttpClient(String principal, String keyTabLocation, boolean
isDebug) {
+ this(principal, keyTabLocation);
+ if (isDebug) {
+ System.setProperty("sun.security.spnego.debug", "true");
+ System.setProperty("sun.security.krb5.debug", "true");
+ }
+ }
+
+ public KerberosHttpClient(String principal, String keyTabLocation, String
krb5Location, boolean isDebug) {
+ this(principal, keyTabLocation, isDebug);
+ System.setProperty("java.security.krb5.conf", krb5Location);
+ }
+
+ private static CloseableHttpClient buildSpengoHttpClient() {
+ HttpClientBuilder builder = HttpClientBuilder.create();
+ Lookup<AuthSchemeProvider> authSchemeRegistry =
RegistryBuilder.<AuthSchemeProvider>create()
+ .register(AuthSchemes.SPNEGO, new
SPNegoSchemeFactory(true)).build();
+ builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+ BasicCredentialsProvider credentialsProvider = new
BasicCredentialsProvider();
+ credentialsProvider.setCredentials(new AuthScope(null, -1, null), new
Credentials() {
+ @Override
+ public Principal getUserPrincipal() {
+ return null;
+ }
+
+ @Override
+ public String getPassword() {
+ return null;
+ }
+ });
+ builder.setDefaultCredentialsProvider(credentialsProvider);
+ return builder.build();
+ }
+
+ public String get(final String url, final String userId) {
+ logger.info("Calling KerberosHttpClient {} {} {}", this.principal,
this.keyTabLocation, url);
+ Configuration config = new Configuration() {
+ @SuppressWarnings("serial")
+ @Override
+ public AppConfigurationEntry[] getAppConfigurationEntry(String
name) {
+ Map<String, Object> options = new HashMap<>(9);
+ options.put("useTicketCache", "false");
+ options.put("useKeyTab", "true");
+ options.put("keyTab", keyTabLocation);
+ options.put("refreshKrb5Config", "true");
+ options.put("principal", principal);
+ options.put("storeKey", "true");
+ options.put("doNotPrompt", "true");
+ options.put("isInitiator", "true");
+ options.put("debug", "true");
+ return new AppConfigurationEntry[] {
+ new
AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
+
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) };
+ }
+ };
+ Set<Principal> princ = new HashSet<>(1);
+ princ.add(new KerberosPrincipal(userId));
+ Subject sub = new Subject(false, princ, new HashSet<>(), new
HashSet<>());
+
+ LoginContext lc;
+ try {
+ lc = new LoginContext("", sub, null, config);
+ lc.login();
+ Subject serviceSubject = lc.getSubject();
+ return Subject.doAs(serviceSubject, (PrivilegedAction<String>) ()
-> {
+ CloseableHttpClient httpClient = buildSpengoHttpClient();
+ HttpGet httpget = new HttpGet(url);
+ return HttpUtils.getResponseContentString(httpget, httpClient);
+ });
+ } catch (LoginException le) {
+ logger.error("Kerberos authentication failed ", le);
+ }
+ return null;
+ }
+
+ /**
+ * get http request content by kerberosClient
+ *
+ * @param url url
+ * @return http get request response content
+ */
+ public static String get(String url) {
+
+ String responseContent;
+ KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(
+ PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
+ PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),
+
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
+ responseContent = kerberosHttpClient.get(url,
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
+ return responseContent;
+
+ }
+
+}
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
index aee7ac8..f9ce989 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
@@ -17,7 +17,13 @@
package org.apache.dolphinscheduler.common.utils;
import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -28,26 +34,53 @@ import org.slf4j.LoggerFactory;
*/
public class HttpUtilsTest {
+ public static final Logger logger =
LoggerFactory.getLogger(HttpUtilsTest.class);
+ private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
- public static final Logger logger =
LoggerFactory.getLogger(HttpUtilsTest.class);
+ @Test
+ public void testGetTest() {
+ // success
+ String result = HttpUtils.get("https://github.com/manifest.json";);
+ Assert.assertNotNull(result);
+ ObjectNode jsonObject = JSONUtils.parseObject(result);
+ Assert.assertEquals("GitHub", jsonObject.path("name").asText());
+ result = HttpUtils.get("https://123.333.111.33/ccc";);
+ Assert.assertNull(result);
+ }
+ @Test
+ public void testGetByKerberos() {
+ try {
+ String applicationUrl =
hadoopUtils.getApplicationUrl("application_1542010131334_0029");
+ String responseContent;
+ responseContent = HttpUtils.get(applicationUrl);
+ Assert.assertNull(responseContent);
- @Test
- public void testGetTest(){
- //success
- String result =
HttpUtils.get("https://github.com/manifest.json";);
- Assert.assertNotNull(result);
- ObjectNode jsonObject = JSONUtils.parseObject(result);
- Assert.assertEquals("GitHub", jsonObject.path("name").asText());
-
- result = HttpUtils.get("https://123.333.111.33/ccc";);
- Assert.assertNull(result);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
}
+ }
+
+ @Test
+ public void testGetResponseContentString() {
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+ HttpGet httpget = new HttpGet("https://github.com/manifest.json";);
+ /** set timeout、request time、socket timeout */
+ RequestConfig requestConfig =
RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
+
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
+
.setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
+ httpget.setConfig(requestConfig);
+ String responseContent = HttpUtils.getResponseContentString(httpget,
httpclient);
+ Assert.assertNotNull(responseContent);
+ }
+
+
@Test
public void testGetHttpClient() {
CloseableHttpClient httpClient1 = HttpUtils.getInstance();
CloseableHttpClient httpClient2 = HttpUtils.getInstance();
Assert.assertEquals(httpClient1, httpClient2);
}
+
}
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java
new file mode 100644
index 0000000..9911961
--- /dev/null
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.utils;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * KerberosHttpClient test
+ */
+public class KerberosHttpClientTest {
+ public static final Logger logger =
LoggerFactory.getLogger(KerberosHttpClientTest.class);
+ private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
+
+ @Test
+ public void get() {
+ try {
+ String applicationUrl =
hadoopUtils.getApplicationUrl("application_1542010131334_0029");
+ String responseContent;
+ KerberosHttpClient kerberosHttpClient = new
KerberosHttpClient(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
+
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
+ responseContent = kerberosHttpClient.get(applicationUrl,
+
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
+ Assert.assertNull(responseContent);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 173265a..1a869b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -779,6 +779,7 @@
<include>**/common/utils/TaskParametersUtilsTest.java</include>
<include>**/common/utils/HadoopUtilsTest.java</include>
<include>**/common/utils/HttpUtilsTest.java</include>
+
<include>**/common/utils/KerberosHttpClientTest.java</include>
<include>**/common/ConstantsTest.java</include>
<include>**/common/utils/HadoopUtils.java</include>
<include>**/common/utils/RetryerUtilsTest.java</include>
