Repository: hadoop Updated Branches: refs/heads/branch-2 b68e6b1d6 -> 3895058a6
YARN-4747. AHS error 500 due to NPE when container start event is missing. Contributed by Varun Saxena (cherry picked from commit b2ed6ae73197990a950ce71ece80c0f23221c384) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3895058a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3895058a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3895058a Branch: refs/heads/branch-2 Commit: 3895058a675cfcda8d6fa7c7a969251fa2db2a66 Parents: b68e6b1 Author: Jason Lowe <jl...@apache.org> Authored: Fri May 6 22:58:17 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Fri May 6 23:00:25 2016 +0000 ---------------------------------------------------------------------- ...pplicationHistoryManagerOnTimelineStore.java | 21 ++++++++------ .../metrics/ContainerFinishedEvent.java | 9 +++++- .../metrics/SystemMetricsPublisher.java | 8 +++++- .../metrics/TestSystemMetricsPublisher.java | 30 ++++++++++++++++++++ 4 files changed, 57 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index fd11aea..19afb25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -587,19 +587,22 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService } } } - NodeId allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort); ContainerId containerId = ConverterUtils.toContainerId(entity.getEntityId()); - String logUrl = WebAppUtils.getAggregatedLogURL( - serverHttpAddress, - allocatedNode.toString(), - containerId.toString(), - containerId.toString(), - user); + String logUrl = null; + NodeId allocatedNode = null; + if (allocatedHost != null) { + allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort); + logUrl = WebAppUtils.getAggregatedLogURL( + serverHttpAddress, + allocatedNode.toString(), + containerId.toString(), + containerId.toString(), + user); + } return ContainerReport.newInstance( ConverterUtils.toContainerId(entity.getEntityId()), - Resource.newInstance(allocatedMem, allocatedVcore), - NodeId.newInstance(allocatedHost, allocatedPort), + Resource.newInstance(allocatedMem, allocatedVcore), allocatedNode, Priority.newInstance(allocatedPriority), createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java index aafd760..ca4d311 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerFinishedEvent.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.NodeId; public class ContainerFinishedEvent extends SystemMetricsEvent { @@ -27,17 +28,20 @@ public class ContainerFinishedEvent extends SystemMetricsEvent { private String diagnosticsInfo; private int containerExitStatus; private ContainerState state; + private NodeId allocatedNode; public ContainerFinishedEvent( ContainerId containerId, String diagnosticsInfo, int containerExitStatus, ContainerState state, - long finishedTime) { + long finishedTime, + NodeId allocatedNode) { super(SystemMetricsEventType.CONTAINER_FINISHED, finishedTime); this.containerId = containerId; this.diagnosticsInfo = diagnosticsInfo; this.containerExitStatus = containerExitStatus; + this.allocatedNode = allocatedNode; this.state = state; } @@ -62,4 +66,7 @@ public class ContainerFinishedEvent extends SystemMetricsEvent { return state; } + public NodeId getAllocatedNode() { + return allocatedNode; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 84a3b19..cba8790 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -218,7 +218,7 @@ public class SystemMetricsPublisher extends CompositeService { container.getDiagnosticsInfo(), container.getContainerExitStatus(), container.getContainerState(), - finishedTime)); + finishedTime, container.getAllocatedNode())); } } @@ -479,6 +479,12 @@ public class SystemMetricsPublisher extends CompositeService { event.getContainerExitStatus()); eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, event.getContainerState().toString()); + Map<String, Object> entityInfo = new HashMap<String, Object>(); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, + event.getAllocatedNode().getHost()); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, + event.getAllocatedNode().getPort()); + entity.setOtherInfo(entityInfo); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3895058a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index f2b02bc..0738a2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -345,6 +345,36 @@ public class TestSystemMetricsPublisher { } @Test(timeout = 10000) + public void testPublishHostPortInfoOnContainerFinished() throws Exception { + ContainerId containerId = + ContainerId.newContainerId(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(0, 1), 1), 1); + RMContainer container = createRMContainer(containerId); + metricsPublisher.containerFinished(container, container.getFinishTime()); + TimelineEntity entity = null; + do { + entity = + store.getEntity(containerId.toString(), + ContainerMetricsConstants.ENTITY_TYPE, + EnumSet.allOf(Field.class)); + } while (entity == null || entity.getEvents().size() < 1); + Assert.assertNotNull(entity.getOtherInfo()); + Assert.assertEquals(2, entity.getOtherInfo().size()); + Assert.assertNotNull(entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO)); + Assert.assertNotNull(entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO)); + Assert.assertEquals( + container.getAllocatedNode().getHost(), + entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO)); + Assert.assertEquals( + container.getAllocatedNode().getPort(), + entity.getOtherInfo().get( + ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO)); + } + + @Test(timeout = 10000) public void testPublishContainerMetrics() throws Exception { ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance( --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
