Repository: hadoop Updated Branches: refs/heads/branch-2.8 f42222a75 -> c992cf638
YARN-6598. History server getApplicationReport NPE when fetching report for pre-2.8 job (Jason Lowe via jeagles) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c992cf63 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c992cf63 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c992cf63 Branch: refs/heads/branch-2.8 Commit: c992cf638e9b391dd8ad5dbb87df59c187a98f18 Parents: f42222a Author: Jonathan Eagles <jeag...@yahoo-inc.com> Authored: Mon May 15 12:10:12 2017 -0500 Committer: Jonathan Eagles <jeag...@yahoo-inc.com> Committed: Mon May 15 12:10:12 2017 -0500 ---------------------------------------------------------------------- ...pplicationHistoryManagerOnTimelineStore.java | 35 ++++++++++++-------- ...pplicationHistoryManagerOnTimelineStore.java | 31 +++++++++++++---- 2 files changed, 47 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c992cf63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 3a11c92..7434178 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -330,20 +330,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { - long vcoreSeconds=Long.parseLong(entityInfo.get( - ApplicationMetricsConstants.APP_CPU_METRICS).toString()); - long memorySeconds=Long.parseLong(entityInfo.get( - ApplicationMetricsConstants.APP_MEM_METRICS).toString()); - long preemptedMemorySeconds = Long.parseLong(entityInfo.get( - ApplicationMetricsConstants - .APP_MEM_PREEMPT_METRICS).toString()); - long preemptedVcoreSeconds = Long.parseLong(entityInfo.get( - ApplicationMetricsConstants - .APP_CPU_PREEMPT_METRICS).toString()); - appResources = ApplicationResourceUsageReport - .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0, - 0, preemptedMemorySeconds, preemptedVcoreSeconds); + long vcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_CPU_METRICS); + long memorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_MEM_METRICS); + long preemptedMemorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS); + long preemptedVcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS); + appResources = ApplicationResourceUsageReport.newInstance(0, 0, null, + null, null, memorySeconds, vcoreSeconds, 0, 0, + preemptedMemorySeconds, preemptedVcoreSeconds); } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { appTags = new HashSet<String>(); Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO); @@ -445,6 +444,16 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService amNodeLabelExpression), appViewACLs); } + private static long parseLong(Map<String, Object> entityInfo, + String infoKey) { + long result = 0; + Object infoValue = entityInfo.get(infoKey); + if (infoValue != null) { + result = Long.parseLong(infoValue.toString()); + } + return result; + } + private static boolean isFinalState(YarnApplicationState state) { return state == YarnApplicationState.FINISHED || state == YarnApplicationState.FAILED http://git-wip-us.apache.org/repos/asf/hadoop/blob/c992cf63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 2f9f7aa..8fece99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -141,6 +141,9 @@ public class TestApplicationHistoryManagerOnTimelineStore { if (i == 2) { entities.addEntity(createApplicationTimelineEntity( appId, true, false, false, true)); + } else if (i == 3) { + entities.addEntity(createApplicationTimelineEntity( + appId, false, false, false, false, true)); } else { entities.addEntity(createApplicationTimelineEntity( appId, false, false, false, false)); @@ -164,7 +167,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { @Test public void testGetApplicationReport() throws Exception { - for (int i = 1; i <= 2; ++i) { + for (int i = 1; i <= 3; ++i) { final ApplicationId appId = ApplicationId.newInstance(0, i); ApplicationReport app; if (callerUGI == null) { @@ -202,7 +205,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2")); // App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // Nobody except admin and owner has access to the details of the app. - if ((i == 1 && callerUGI != null && + if ((i != 2 && callerUGI != null && callerUGI.getShortUserName().equals("user3")) || (i == 2 && callerUGI != null && (callerUGI.getShortUserName().equals("user2") || @@ -233,10 +236,16 @@ public class TestApplicationHistoryManagerOnTimelineStore { applicationResourceUsageReport.getMemorySeconds()); Assert .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); - Assert.assertEquals(456, + long expectedPreemptMemSecs = 456; + long expectedPreemptVcoreSecs = 789; + if (i == 3) { + expectedPreemptMemSecs = 0; + expectedPreemptVcoreSecs = 0; + } + Assert.assertEquals(expectedPreemptMemSecs, applicationResourceUsageReport.getPreemptedMemorySeconds()); Assert - .assertEquals(789, applicationResourceUsageReport + .assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport .getPreemptedVcoreSeconds()); Assert.assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus()); @@ -473,6 +482,14 @@ public class TestApplicationHistoryManagerOnTimelineStore { private static TimelineEntity createApplicationTimelineEntity( ApplicationId appId, boolean emptyACLs, boolean noAttemptId, boolean wrongAppId, boolean enableUpdateEvent) { + return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId, + wrongAppId, enableUpdateEvent, false); + } + + private static TimelineEntity createApplicationTimelineEntity( + ApplicationId appId, boolean emptyACLs, boolean noAttemptId, + boolean wrongAppId, boolean enableUpdateEvent, + boolean missingPreemptMetrics) { TimelineEntity entity = new TimelineEntity(); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); if (wrongAppId) { @@ -497,8 +514,10 @@ public class TestApplicationHistoryManagerOnTimelineStore { Integer.MAX_VALUE + 1L); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); - entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,456); - entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,789); + if (!missingPreemptMetrics) { + entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456); + entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789); + } if (emptyACLs) { entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); } else { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org