Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 819a2a6f1 -> 97c193424
YARN-8474. Fixed ApiServiceClient kerberos negotiation.
Contributed by Billie Rinaldi
(cherry picked from commit 8990eaf5925afa533fbd9c3641859a146dc5a22c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97c19342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97c19342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97c19342
Branch: refs/heads/branch-3.1
Commit: 97c1934247b4863b60e6233c21cedeb6bc1e2a11
Parents: 819a2a6
Author: Eric Yang <ey...@apache.org>
Authored: Thu Aug 16 12:46:37 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Aug 16 12:50:52 2018 -0400
----------------------------------------------------------------------
.../hadoop-yarn-services-api/pom.xml | 57 +++++++++++++
.../yarn/service/client/ApiServiceClient.java | 85 ++++++++++++++++++--
.../client/TestSecureApiServiceClient.java | 83 +++++++++++++++++++
3 files changed, 218 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c19342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
index 2a2ee7f..646781f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
@@ -93,10 +93,18 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</dependency>
<dependency>
@@ -104,6 +112,14 @@
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
@@ -120,6 +136,42 @@
<artifactId>jsr311-api</artifactId>
</dependency>
<dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
@@ -155,6 +207,11 @@
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c19342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 18d45fa..e6dee79 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -20,19 +20,26 @@ import static
org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.List;
import java.util.Map;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.codec.binary.Base64;
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -53,6 +60,11 @@ import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.util.RMHAUtils;
import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.eclipse.jetty.util.UrlEncoded;
+import org.ietf.jgss.GSSContext;
+import org.ietf.jgss.GSSException;
+import org.ietf.jgss.GSSManager;
+import org.ietf.jgss.GSSName;
+import org.ietf.jgss.Oid;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -71,6 +83,7 @@ import static
org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
public class ApiServiceClient extends AppAdminClient {
private static final Logger LOG =
LoggerFactory.getLogger(ApiServiceClient.class);
+ private static final Base64 BASE_64_CODEC = new Base64(0);
protected YarnClient yarnClient;
@Override protected void serviceInit(Configuration configuration)
@@ -81,6 +94,54 @@ public class ApiServiceClient extends AppAdminClient {
}
/**
+ * Generate SPNEGO challenge request token.
+ *
+ * @param server - hostname to contact
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ String generateToken(String server) throws IOException, InterruptedException
{
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ LOG.debug("The user credential is {}", currentUser);
+ String challenge = currentUser
+ .doAs(new PrivilegedExceptionAction<String>() {
+ @Override
+ public String run() throws Exception {
+ try {
+ // This Oid for Kerberos GSS-API mechanism.
+ Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID");
+ GSSManager manager = GSSManager.getInstance();
+ // GSS name for server
+ GSSName serverName = manager.createName("HTTP@" + server,
+ GSSName.NT_HOSTBASED_SERVICE);
+ // Create a GSSContext for authentication with the service.
+ // We're passing client credentials as null since we want them to
+ // be read from the Subject.
+ GSSContext gssContext = manager.createContext(
+ serverName.canonicalize(mechOid), mechOid, null,
+ GSSContext.DEFAULT_LIFETIME);
+ gssContext.requestMutualAuth(true);
+ gssContext.requestCredDeleg(true);
+ // Establish context
+ byte[] inToken = new byte[0];
+ byte[] outToken = gssContext.initSecContext(inToken, 0,
+ inToken.length);
+ gssContext.dispose();
+ // Base64 encoded and stringified token for server
+ LOG.debug("Got valid challenge for host {}", serverName);
+ return new String(BASE_64_CODEC.encode(outToken),
+ StandardCharsets.US_ASCII);
+ } catch (GSSException | IllegalAccessException
+ | NoSuchFieldException | ClassNotFoundException e) {
+ LOG.error("Error: {}", e);
+ throw new AuthenticationException(e);
+ }
+ }
+ });
+ return challenge;
+ }
+
+ /**
* Calculate Resource Manager address base on working REST API.
*/
private String getRMWebAddress() {
@@ -100,6 +161,7 @@ public class ApiServiceClient extends AppAdminClient {
for (String host : rmServers) {
try {
Client client = Client.create();
+ client.setFollowRedirects(false);
StringBuilder sb = new StringBuilder();
sb.append(scheme);
sb.append(host);
@@ -116,8 +178,11 @@ public class ApiServiceClient extends AppAdminClient {
WebResource webResource = client
.resource(sb.toString());
if (useKerberos) {
- AuthenticatedURL.Token token = new AuthenticatedURL.Token();
- webResource.header("WWW-Authenticate", token);
+ String[] server = host.split(":");
+ String challenge = generateToken(server[0]);
+ webResource.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+ challenge);
+ LOG.debug("Authorization: Negotiate {}", challenge);
}
ClientResponse test = webResource.get(ClientResponse.class);
if (test.getStatus() == 200) {
@@ -125,7 +190,8 @@ public class ApiServiceClient extends AppAdminClient {
break;
}
} catch (Exception e) {
- LOG.debug("Fail to connect to: "+host, e);
+ LOG.info("Fail to connect to: "+host);
+ LOG.debug("Root cause: {}", e);
}
}
return scheme+rmAddress;
@@ -206,8 +272,13 @@ public class ApiServiceClient extends AppAdminClient {
Builder builder = client
.resource(requestPath).type(MediaType.APPLICATION_JSON);
if (conf.get("hadoop.http.authentication.type").equals("kerberos")) {
- AuthenticatedURL.Token token = new AuthenticatedURL.Token();
- builder.header("WWW-Authenticate", token);
+ try {
+ URI url = new URI(requestPath);
+ String challenge = generateToken(url.getHost());
+ builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
}
return builder
.accept("application/json;charset=utf-8");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c19342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
new file mode 100644
index 0000000..4f3b461
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+
+import javax.security.sasl.Sasl;
+
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test Spnego Client Login.
+ */
+public class TestSecureApiServiceClient extends KerberosSecurityTestcase {
+
+ private String clientPrincipal = "client";
+
+ private String server1Protocol = "HTTP";
+
+ private String server2Protocol = "server2";
+
+ private String host = "localhost";
+
+ private String server1Principal = server1Protocol + "/" + host;
+
+ private String server2Principal = server2Protocol + "/" + host;
+
+ private File keytabFile;
+
+ private Configuration conf = new Configuration();
+
+ private Map<String, String> props;
+
+ @Before
+ public void setUp() throws Exception {
+ keytabFile = new File(getWorkDir(), "keytab");
+ getKdc().createPrincipal(keytabFile, clientPrincipal, server1Principal,
+ server2Principal);
+ SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.setShouldRenewImmediatelyForTests(true);
+ props = new HashMap<String, String>();
+ props.put(Sasl.QOP, QualityOfProtection.AUTHENTICATION.saslQop);
+ }
+
+ @Test
+ public void testHttpSpnegoChallenge() throws Exception {
+ UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile
+ .getCanonicalPath());
+ ApiServiceClient asc = new ApiServiceClient();
+ String challenge = asc.generateToken("localhost");
+ assertNotNull(challenge);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
