YARN-2301. Improved yarn container command. Contributed by Naganarasimha G R (cherry picked from commit 258623ff8bb1a1057ae3501d4f20982d5a59ea34)
(cherry picked from commit 1d1e7682c9cad6a2f819b390ca3368dfa29c7097) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92742b44 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92742b44 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92742b44 Branch: refs/heads/branch-2.6 Commit: 92742b4402906264b519f08c4c9355ed2f46a8cc Parents: c3928b3 Author: Jian He <jia...@apache.org> Authored: Thu Dec 4 12:51:15 2014 -0800 Committer: Vinod Kumar Vavilapalli <vino...@apache.org> Committed: Tue Sep 8 22:57:28 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 8 +++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 41 +++++++++++++------- .../yarn/server/resourcemanager/RMContext.java | 3 ++ .../server/resourcemanager/RMContextImpl.java | 11 ++++++ .../server/resourcemanager/ResourceManager.java | 2 + .../rmcontainer/RMContainerImpl.java | 9 ++++- .../resourcemanager/TestClientRMService.java | 1 + .../rmcontainer/TestRMContainerImpl.java | 6 ++- 9 files changed, 63 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1ad0199..f2b8c07 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -22,6 +22,8 @@ Release 2.6.1 - UNRELEASED YARN-3098. Created common QueueCapacities class in Capacity Scheduler to track capacities-by-labels of queues. (Wangda Tan via jianhe) + YARN-2301. Improved yarn container command. (Naganarasimha G R via jianhe) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index a847cd5..83d212d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import com.google.common.annotations.VisibleForTesting; @@ -536,8 +537,11 @@ public class ApplicationCLI extends YarnCLI { writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time", "Finish Time", "State", "Host", "LOG-URL"); for (ContainerReport containerReport : appsReport) { - writer.printf(CONTAINER_PATTERN, containerReport.getContainerId(), - containerReport.getCreationTime(), containerReport.getFinishTime(), + writer.printf( + CONTAINER_PATTERN, + containerReport.getContainerId(), + Times.format(containerReport.getCreationTime()), + Times.format(containerReport.getFinishTime()), containerReport.getContainerState(), containerReport .getAssignedNode(), containerReport.getLogUrl()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 9d9a86a..b4df551 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -32,19 +32,17 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.commons.cli.Options; import org.apache.commons.lang.time.DateFormatUtils; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -65,9 +63,8 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.util.Records; -import org.jboss.netty.logging.CommonsLoggerFactory; +import org.apache.hadoop.yarn.util.Times; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -257,25 +254,30 @@ public class TestYarnCLI { applicationId, 1); ContainerId containerId = ContainerId.newContainerId(attemptId, 1); ContainerId containerId1 = ContainerId.newContainerId(attemptId, 2); + ContainerId containerId2 = ContainerId.newContainerId(attemptId, 3); + long time1=1234,time2=5678; ContainerReport container = ContainerReport.newInstance(containerId, null, - NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, + NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); ContainerReport container1 = ContainerReport.newInstance(containerId1, null, - NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, + NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + ContainerReport container2 = ContainerReport.newInstance(containerId2, null, + NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0, + "diagnosticInfo", "", 0, ContainerState.RUNNING); List<ContainerReport> reports = new ArrayList<ContainerReport>(); reports.add(container); reports.add(container1); + reports.add(container2); when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn( reports); int result = cli.run(new String[] { "container", "-list", attemptId.toString() }); assertEquals(0, result); verify(client).getContainers(attemptId); - Log.info(sysOutStream.toString()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); - pw.println("Total number of containers :2"); + pw.println("Total number of containers :3"); pw.print(" Container-Id"); pw.print("\t Start Time"); pw.print("\t Finish Time"); @@ -283,19 +285,30 @@ public class TestYarnCLI { pw.print("\t Host"); pw.println("\t LOG-URL"); pw.print(" container_1234_0005_01_000001"); - pw.print("\t 1234"); - pw.print("\t 5678"); + pw.print("\t"+Times.format(time1)); + pw.print("\t"+Times.format(time2)); pw.print("\t COMPLETE"); pw.print("\t host:1234"); pw.println("\t logURL"); pw.print(" container_1234_0005_01_000002"); - pw.print("\t 1234"); - pw.print("\t 5678"); + pw.print("\t"+Times.format(time1)); + pw.print("\t"+Times.format(time2)); pw.print("\t COMPLETE"); pw.print("\t host:1234"); pw.println("\t logURL"); + pw.print(" container_1234_0005_01_000003"); + pw.print("\t"+Times.format(time1)); + pw.print("\t N/A"); + pw.print("\t RUNNING"); + pw.print("\t host:1234"); + pw.println("\t "); pw.close(); String appReportStr = baos.toString("UTF-8"); + Log.info("ExpectedOutput"); + Log.info("["+appReportStr+"]"); + Log.info("OutputFrom command"); + String actualOutput = sysOutStream.toString(); + Log.info("["+actualOutput+"]"); Assert.assertEquals(appReportStr, sysOutStream.toString()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 56984e6..ecf6166 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentMap; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -121,4 +122,6 @@ public interface RMContext { ReservationSystem getReservationSystem(); boolean isSchedulerReadyForAllocatingContainers(); + + Configuration getYarnConfiguration(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 32216e5..1d0d6c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; @@ -65,6 +66,8 @@ public class RMContextImpl implements RMContext { private RMActiveServiceContext activeServiceContext; + private Configuration yarnConfiguration; + /** * Default constructor. To be used in conjunction with setter methods for * individual fields. @@ -427,4 +430,12 @@ public class RMContextImpl implements RMContext { this.activeServiceContext = activeServiceContext; } + @Override + public Configuration getYarnConfiguration() { + return this.yarnConfiguration; + } + + public void setYarnConfiguration(Configuration yarnConfiguration) { + this.yarnConfiguration=yarnConfiguration; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 353851c..711acf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -252,6 +252,8 @@ public class ResourceManager extends CompositeService implements Recoverable { addService(adminService); rmContext.setRMAdminService(adminService); + rmContext.setYarnConfiguration(conf); + createAndInitActiveServices(); webAppAddress = WebAppUtils.getWebAppBindURL(this.conf, http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 479734a..e37d8fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -285,8 +285,13 @@ public class RMContainerImpl implements RMContainer { public String getLogURL() { try { readLock.lock(); - return WebAppUtils.getRunningLogURL("//" + container.getNodeHttpAddress(), - ConverterUtils.toString(containerId), user); + StringBuilder logURL = new StringBuilder(); + logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext + .getYarnConfiguration())); + logURL.append(WebAppUtils.getRunningLogURL( + container.getNodeHttpAddress(), ConverterUtils.toString(containerId), + user)); + return logURL.toString(); } finally { readLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 15aca42..a344e9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -443,6 +443,7 @@ public class TestClientRMService { ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext, yarnScheduler); when(rmContext.getRMApps()).thenReturn(apps); + when(rmContext.getYarnConfiguration()).thenReturn(new Configuration()); RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration()); when(rmContext.getDispatcher().getEventHandler()).thenReturn( http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 76cdcae..21aba3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -107,6 +107,7 @@ public class TestRMContainerImpl { when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getRMApps()).thenReturn(rmApps); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); + when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); @@ -130,7 +131,7 @@ public class TestRMContainerImpl { RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); - assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user", + assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. @@ -196,6 +197,7 @@ public class TestRMContainerImpl { when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); + when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); @@ -220,7 +222,7 @@ public class TestRMContainerImpl { RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); - assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user", + assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions.