Repository: hadoop Updated Branches: refs/heads/branch-2.8 959db3866 -> af65f8bb5
YARN-6137. Yarn client implicitly invoke ATS client which accesses HDFS. Contributed by Li Lu (cherry picked from commit 37b4acf7cee1f05599a84bbb1ebf07979a71f82f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af65f8bb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af65f8bb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af65f8bb Branch: refs/heads/branch-2.8 Commit: af65f8bb59863089d0b8f2accd44c27785652751 Parents: 959db38 Author: Jason Lowe <jl...@yahoo-inc.com> Authored: Wed Feb 8 14:59:09 2017 -0600 Committer: Jason Lowe <jl...@yahoo-inc.com> Committed: Wed Feb 8 15:01:10 2017 -0600 ---------------------------------------------------------------------- .../yarn/client/api/impl/YarnClientImpl.java | 54 +++++++++----------- .../yarn/client/api/impl/TestYarnClient.java | 24 ++++++--- 2 files changed, 40 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/af65f8bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 6d57994..93893f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -134,7 +134,7 @@ public class YarnClientImpl extends YarnClient { private long asyncApiPollTimeoutMillis; protected AHSClient historyClient; private boolean historyServiceEnabled; - protected TimelineClient timelineClient; + protected volatile TimelineClient timelineClient; @VisibleForTesting Text timelineService; @VisibleForTesting @@ -174,24 +174,9 @@ public class YarnClientImpl extends YarnClient { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { - try { - timelineServiceEnabled = true; - timelineClient = createTimelineClient(); - timelineClient.init(conf); - timelineDTRenewer = getTimelineDelegationTokenRenewer(conf); - timelineService = TimelineUtils.buildTimelineTokenService(conf); - } catch (NoClassDefFoundError error) { - // When attempt to initiate the timeline client with - // different set of dependencies, it may fail with - // NoClassDefFoundError. When some of them are not compatible - // with timeline server. This is not necessarily a fatal error - // to the client. - LOG.warn("Timeline client could not be initialized " - + "because dependency missing or incompatible," - + " disabling timeline client.", - error); - timelineServiceEnabled = false; - } + timelineServiceEnabled = true; + timelineDTRenewer = getTimelineDelegationTokenRenewer(conf); + timelineService = TimelineUtils.buildTimelineTokenService(conf); } timelineServiceBestEffort = conf.getBoolean( @@ -212,9 +197,6 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.start(); } - if (timelineServiceEnabled) { - timelineClient.start(); - } } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -229,7 +211,7 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.stop(); } - if (timelineServiceEnabled) { + if (timelineClient != null) { timelineClient.stop(); } super.serviceStop(); @@ -368,16 +350,26 @@ public class YarnClientImpl extends YarnClient { @VisibleForTesting org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier> getTimelineDelegationToken() throws IOException, YarnException { - try { - return timelineClient.getDelegationToken(timelineDTRenewer); - } catch (Exception e ) { - if (timelineServiceBestEffort) { - LOG.warn("Failed to get delegation token from the timeline server: " - + e.getMessage()); - return null; + try { + // Only reachable when both security and timeline service are enabled. + if (timelineClient == null) { + synchronized (this) { + if (timelineClient == null) { + timelineClient = createTimelineClient(); + timelineClient.init(getConfig()); + timelineClient.start(); } - throw e; } + } + return timelineClient.getDelegationToken(timelineDTRenewer); + } catch (Exception e) { + if (timelineServiceBestEffort) { + LOG.warn("Failed to get delegation token from the timeline server: " + + e.getMessage()); + return null; + } + throw e; + } } private static String getTimelineDelegationTokenRenewer(Configuration conf) http://git-wip-us.apache.org/repos/asf/hadoop/blob/af65f8bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 1e90a2a..8d199d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.client.api.impl; -import static org.junit.Assert.assertFalse; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -157,25 +156,36 @@ public class TestYarnClient { } @Test - public void testTimelineClientInitFailure() throws Exception{ + public void testStartWithTimelineV15Failure() throws Exception{ Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, + true); YarnClient client = YarnClient.createYarnClient(); if(client instanceof YarnClientImpl) { YarnClientImpl impl = (YarnClientImpl) client; YarnClientImpl spyClient = spy(impl); when(spyClient.createTimelineClient()).thenThrow( - new NoClassDefFoundError( - "Mock a failure when init timeline instance")); + new IOException("ATS v1.5 client initialization failed. ")); spyClient.init(conf); spyClient.start(); - assertFalse("Timeline client should be disabled when" - + "it is failed to init", - spyClient.timelineServiceEnabled); + spyClient.getTimelineDelegationToken(); spyClient.stop(); } } + @Test + public void testStartWithTimelineV15() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f); + YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient(); + client.init(conf); + client.start(); + client.stop(); + } + @SuppressWarnings("deprecation") @Test (timeout = 30000) public void testSubmitApplication() throws Exception { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
