YARN-3725. App submission via REST API is broken in secure mode due to Timeline DT service address is empty. (Zhijie Shen via wangda)
Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/732a70f1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/732a70f1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/732a70f1 Branch: refs/heads/YARN-2928 Commit: 732a70f12f5c9b84486503275bfe60a219aba44c Parents: a977bed Author: Wangda Tan <wan...@apache.org> Authored: Sun May 31 16:30:34 2015 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Tue Jun 2 16:44:04 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../client/api/impl/TimelineClientImpl.java | 30 ++++++++++++++------ .../TestTimelineAuthenticationFilter.java | 11 +++++++ 3 files changed, 36 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e770a5b..807165f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -697,6 +697,9 @@ Release 2.7.1 - UNRELEASED YARN-2900. Application (Attempt and Container) Not Found in AHS results in Internal Server Error (500). (Zhijie Shen and Mit Desai via xgong) + YARN-3725. App submission via REST API is broken in secure mode due to + Timeline DT service address is empty. (Zhijie Shen via wangda) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index e44a8a9..77e9af6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -522,9 +522,12 @@ public class TimelineClientImpl extends TimelineClient { public long renewDelegationToken( final Token<TimelineDelegationTokenIdentifier> timelineDT) throws IOException, YarnException { - boolean useHttps = YarnConfiguration.useHttps(this.getConfig()); - final String scheme = useHttps ? "https" : "http"; - final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT); + final boolean isTokenServiceAddrEmpty = + timelineDT.getService().toString().isEmpty(); + final String scheme = isTokenServiceAddrEmpty ? null + : (YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http"); + final InetSocketAddress address = isTokenServiceAddrEmpty ? null + : SecurityUtil.getTokenServiceAddr(timelineDT); PrivilegedExceptionAction<Long> renewDTAction = new PrivilegedExceptionAction<Long>() { @@ -539,7 +542,11 @@ public class TimelineClientImpl extends TimelineClient { DelegationTokenAuthenticatedURL authUrl = new DelegationTokenAuthenticatedURL(authenticator, connConfigurator); - final URI serviceURI = new URI(scheme, null, address.getHostName(), + // If the token service address is not available, fall back to use + // the configured service address. + final URI serviceURI = isTokenServiceAddrEmpty ? + constructResURI(getConfig(), getTimelineServiceAddress(), false) + : new URI(scheme, null, address.getHostName(), address.getPort(), RESOURCE_URI_STR_V1, null, null); return authUrl .renewDelegationToken(serviceURI.toURL(), token, doAsUser); @@ -553,9 +560,12 @@ public class TimelineClientImpl extends TimelineClient { public void cancelDelegationToken( final Token<TimelineDelegationTokenIdentifier> timelineDT) throws IOException, YarnException { - boolean useHttps = YarnConfiguration.useHttps(this.getConfig()); - final String scheme = useHttps ? "https" : "http"; - final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT); + final boolean isTokenServiceAddrEmpty = + timelineDT.getService().toString().isEmpty(); + final String scheme = isTokenServiceAddrEmpty ? null + : (YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http"); + final InetSocketAddress address = isTokenServiceAddrEmpty ? null + : SecurityUtil.getTokenServiceAddr(timelineDT); PrivilegedExceptionAction<Void> cancelDTAction = new PrivilegedExceptionAction<Void>() { @@ -570,7 +580,11 @@ public class TimelineClientImpl extends TimelineClient { DelegationTokenAuthenticatedURL authUrl = new DelegationTokenAuthenticatedURL(authenticator, connConfigurator); - final URI serviceURI = new URI(scheme, null, address.getHostName(), + // If the token service address is not available, fall back to use + // the configured service address. + final URI serviceURI = isTokenServiceAddrEmpty ? + constructResURI(getConfig(), getTimelineServiceAddress(), false) + : new URI(scheme, null, address.getHostName(), address.getPort(), RESOURCE_URI_STR_V1, null, null); authUrl.cancelDelegationToken(serviceURI.toURL(), token, doAsUser); return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java index c93e8f2..063f512 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java @@ -240,12 +240,21 @@ public class TestTimelineAuthenticationFilter { Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner()); // Renew token + Assert.assertFalse(token.getService().toString().isEmpty()); + // Renew the token from the token service address long renewTime1 = httpUserClient.renewDelegationToken(token); Thread.sleep(100); + token.setService(new Text()); + Assert.assertTrue(token.getService().toString().isEmpty()); + // If the token service address is not avaiable, it still can be renewed + // from the configured address long renewTime2 = httpUserClient.renewDelegationToken(token); Assert.assertTrue(renewTime1 < renewTime2); // Cancel token + Assert.assertTrue(token.getService().toString().isEmpty()); + // If the token service address is not avaiable, it still can be canceled + // from the configured address httpUserClient.cancelDelegationToken(token); // Renew should not be successful because the token is canceled try { @@ -280,6 +289,8 @@ public class TestTimelineAuthenticationFilter { Assert.assertTrue(renewTime1 < renewTime2); // Cancel token + Assert.assertFalse(tokenToRenew.getService().toString().isEmpty()); + // Cancel the token from the token service address fooUserClient.cancelDelegationToken(tokenToRenew); // Renew should not be successful because the token is canceled