Repository: hadoop Updated Branches: refs/heads/trunk 285b31e75 -> 265ed1fe8
YARN-2890. MiniYarnCluster should turn on timeline service if configured to do so. Contributed by Mit Desai. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/265ed1fe Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/265ed1fe Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/265ed1fe Branch: refs/heads/trunk Commit: 265ed1fe804743601a8b62cabc1e4dc2ec8e502f Parents: 285b31e Author: Hitesh Shah <hit...@apache.org> Authored: Wed Apr 8 14:13:10 2015 -0700 Committer: Hitesh Shah <hit...@apache.org> Committed: Wed Apr 8 14:13:10 2015 -0700 ---------------------------------------------------------------------- .../jobhistory/TestJobHistoryEventHandler.java | 2 +- .../mapred/TestMRTimelineEventHandling.java | 52 ++++++++- hadoop-yarn-project/CHANGES.txt | 3 + .../distributedshell/TestDistributedShell.java | 2 +- .../hadoop/yarn/server/MiniYARNCluster.java | 6 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 115 +++++++++++++++++++ 6 files changed, 172 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/265ed1fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index 43e3dbe..de260c9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -453,7 +453,7 @@ public class TestJobHistoryEventHandler { long currentTime = System.currentTimeMillis(); try { yarnCluster = new MiniYARNCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); jheh.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/265ed1fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index c2ef128..eab9026 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -35,6 +35,52 @@ import org.junit.Test; public class TestMRTimelineEventHandling { @Test + public void testTimelineServiceStartInMiniCluster() throws Exception { + Configuration conf = new YarnConfiguration(); + + /* + * Timeline service should not start if the config is set to false + * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true); + MiniMRYarnCluster cluster = null; + try { + cluster = new MiniMRYarnCluster( + TestJobHistoryEventHandler.class.getSimpleName(), 1); + cluster.init(conf); + cluster.start(); + + //verify that the timeline service is not started. + Assert.assertNull("Timeline Service should not have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false); + cluster = null; + try { + cluster = new MiniMRYarnCluster( + TestJobHistoryEventHandler.class.getSimpleName(), 1); + cluster.init(conf); + cluster.start(); + + //verify that the timeline service is not started. + Assert.assertNull("Timeline Service should not have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + } + + @Test public void testMRTimelineEventHandling() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); @@ -42,7 +88,7 @@ public class TestMRTimelineEventHandling { MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() @@ -96,7 +142,7 @@ public class TestMRTimelineEventHandling { MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() @@ -133,7 +179,7 @@ public class TestMRTimelineEventHandling { cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, true); + TestJobHistoryEventHandler.class.getSimpleName(), 1); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() http://git-wip-us.apache.org/repos/asf/hadoop/blob/265ed1fe/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7f0ddd8..4714f4e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -159,6 +159,9 @@ Release 2.8.0 - UNRELEASED YARN-3459. Fix failiure of TestLog4jWarningErrorMetricsAppender. (Varun Vasudev via wangda) + YARN-2890. MiniYarnCluster should turn on timeline service if + configured to do so. (Mit Desai via hitesh) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/265ed1fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 5e6fa46..967d172 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -85,7 +85,7 @@ public class TestDistributedShell { if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1, - numNodeManager, 1, 1, true); + numNodeManager, 1, 1); yarnCluster.init(conf); yarnCluster.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/265ed1fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 365e0bb..f8b27b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -262,8 +261,9 @@ public class MiniYARNCluster extends CompositeService { addService(new NodeManagerWrapper(index)); } - if (enableAHS) { - addService(new ApplicationHistoryServerWrapper()); + if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) { + addService(new ApplicationHistoryServerWrapper()); } super.serviceInit( http://git-wip-us.apache.org/repos/asf/hadoop/blob/265ed1fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java new file mode 100644 index 0000000..8a3c9e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; +import org.junit.Test; + +public class TestMiniYarnCluster { + @Test + public void testTimelineServiceStartInMiniCluster() throws Exception { + Configuration conf = new YarnConfiguration(); + int numNodeManagers = 1; + int numLocalDirs = 1; + int numLogDirs = 1; + boolean enableAHS; + + /* + * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false + * and enableAHS flag == false + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + enableAHS = false; + MiniYARNCluster cluster = null; + try { + cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + cluster.init(conf); + cluster.start(); + + //verify that the timeline service is not started. + Assert.assertNull("Timeline Service should not have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + + /* + * Timeline service should start if TIMELINE_SERVICE_ENABLED == true + * and enableAHS == false + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + enableAHS = false; + cluster = null; + try { + cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + cluster.init(conf); + cluster.start(); + + //Timeline service may sometime take a while to get started + int wait = 0; + while(cluster.getApplicationHistoryServer() == null && wait < 20) { + Thread.sleep(500); + wait++; + } + //verify that the timeline service is started. + Assert.assertNotNull("Timeline Service should have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + /* + * Timeline service should start if TIMELINE_SERVICE_ENABLED == false + * and enableAHS == true + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + enableAHS = true; + cluster = null; + try { + cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS); + cluster.init(conf); + cluster.start(); + + //Timeline service may sometime take a while to get started + int wait = 0; + while(cluster.getApplicationHistoryServer() == null && wait < 20) { + Thread.sleep(500); + wait++; + } + //verify that the timeline service is started. + Assert.assertNotNull("Timeline Service should have been started", + cluster.getApplicationHistoryServer()); + } + finally { + if(cluster != null) { + cluster.stop(); + } + } + } +}
