YARN-5473. Expose per-application over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc40e3ae Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc40e3ae Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc40e3ae Branch: refs/heads/YARN-1011 Commit: cc40e3ae5c61849c3b394018d4cadba6c6ef8e75 Parents: b04006f Author: Miklos Szegedi <szege...@apache.org> Authored: Tue Jan 23 22:34:49 2018 -0800 Committer: Haibo Chen <haiboc...@apache.org> Committed: Tue Jun 5 10:46:28 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/mapreduce/TypeConverter.java | 4 +- .../hadoop/mapreduce/TestTypeConverter.java | 4 +- .../hadoop/mapred/TestResourceMgrDelegate.java | 2 +- .../records/ApplicationResourceUsageReport.java | 158 +++++++++++--- .../src/main/proto/yarn_protos.proto | 2 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 18 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 23 ++- .../ApplicationResourceUsageReportPBImpl.java | 205 +++++++++++++++---- .../hadoop/yarn/util/resource/Resources.java | 20 ++ ...pplicationHistoryManagerOnTimelineStore.java | 83 +++++--- .../TestApplicationHistoryClientService.java | 8 +- ...pplicationHistoryManagerOnTimelineStore.java | 18 +- .../metrics/ApplicationMetricsConstants.java | 10 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 15 +- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 39 +++- .../server/resourcemanager/RMAppManager.java | 18 +- .../server/resourcemanager/RMServerUtils.java | 3 +- .../metrics/TimelineServiceV1Publisher.java | 14 +- .../metrics/TimelineServiceV2Publisher.java | 14 +- .../resourcemanager/recovery/RMStateStore.java | 5 +- .../records/ApplicationAttemptStateData.java | 144 +++++++++++-- .../pb/ApplicationAttemptStateDataPBImpl.java | 110 ++++++++-- .../server/resourcemanager/rmapp/RMAppImpl.java | 38 ++-- .../resourcemanager/rmapp/RMAppMetrics.java | 38 ++-- .../attempt/AggregateAppResourceUsage.java | 51 +++-- .../rmapp/attempt/RMAppAttemptImpl.java | 21 +- .../rmapp/attempt/RMAppAttemptMetrics.java | 49 +++-- .../ContainerResourceUsageReport.java | 46 +++++ .../rmcontainer/RMContainer.java | 4 +- .../rmcontainer/RMContainerImpl.java | 118 ++++++++--- .../scheduler/AbstractYarnScheduler.java | 2 +- .../scheduler/SchedulerApplicationAttempt.java | 63 +++--- .../scheduler/YarnScheduler.java | 2 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 9 +- .../webapp/FairSchedulerAppsBlock.java | 8 + .../resourcemanager/webapp/RMAppBlock.java | 9 +- .../resourcemanager/webapp/RMAppsBlock.java | 10 + .../resourcemanager/webapp/dao/AppInfo.java | 78 +++++-- .../yarn_server_resourcemanager_recovery.proto | 1 + .../server/resourcemanager/TestAppManager.java | 39 ++-- .../resourcemanager/TestApplicationACLs.java | 4 +- .../resourcemanager/TestClientRMService.java | 45 ++-- .../TestContainerResourceUsage.java | 184 ++++++++++------- .../applicationsmanager/MockAsm.java | 4 +- .../TestCombinedSystemMetricsPublisher.java | 2 +- .../metrics/TestSystemMetricsPublisher.java | 23 ++- .../TestSystemMetricsPublisherForV2.java | 12 +- .../recovery/RMStateStoreTestBase.java | 12 +- .../recovery/TestZKRMStateStore.java | 40 ++-- .../rmapp/TestRMAppTransitions.java | 6 +- .../attempt/TestRMAppAttemptTransitions.java | 32 +-- .../capacity/TestCapacityScheduler.java | 4 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 20 +- .../resourcemanager/webapp/TestAppPage.java | 2 +- .../resourcemanager/webapp/TestRMWebApp.java | 3 +- .../webapp/TestRMWebAppFairScheduler.java | 2 +- .../webapp/TestRMWebServicesApps.java | 2 +- .../router/webapp/RouterWebServiceUtil.java | 14 +- .../router/webapp/TestRouterWebServiceUtil.java | 14 +- 61 files changed, 1428 insertions(+), 513 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java index aea931a..23409fc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java @@ -527,8 +527,8 @@ public class TypeConverter { jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers()); jobStatus.setReservedMem( (int)resourceUsageReport.getReservedResources().getMemorySize()); - jobStatus.setUsedMem( - (int) resourceUsageReport.getUsedResources().getMemorySize()); + jobStatus.setUsedMem((int) resourceUsageReport + .getGuaranteedResourcesUsed().getMemorySize()); } return jobStatus; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index 629a246..acd9c35 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -94,7 +94,7 @@ public class TestTypeConverter { appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); appUsageRpt.setReservedResources(r); - appUsageRpt.setUsedResources(r); + appUsageRpt.setGuaranteedResourcesUsed(r); applicationReport.setApplicationResourceUsageReport(appUsageRpt); JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile"); Assert.assertEquals(appStartTime, jobStatus.getStartTime()); @@ -133,7 +133,7 @@ public class TestTypeConverter { appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); appUsageRpt.setReservedResources(r); - appUsageRpt.setUsedResources(r); + appUsageRpt.setGuaranteedResourcesUsed(r); when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt); JobStatus status = TypeConverter.fromYarn(mockReport, jobFile); Assert.assertNotNull("fromYarn returned null status", status); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java index d6754f0..0506b27 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java @@ -137,7 +137,7 @@ public class TestResourceMgrDelegate { Records.newRecord(Resource.class)); Mockito.when(appResources.getReservedResources()).thenReturn( Records.newRecord(Resource.class)); - Mockito.when(appResources.getUsedResources()).thenReturn( + Mockito.when(appResources.getGuaranteedResourcesUsed()).thenReturn( Records.newRecord(Resource.class)); Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn( appResources); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index d2e33ff..4517db9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -36,21 +36,27 @@ public abstract class ApplicationResourceUsageReport { @Private @Unstable public static ApplicationResourceUsageReport newInstance( - int numUsedContainers, int numReservedContainers, Resource usedResources, + int numUsedContainers, int numReservedContainers, + Resource guaranteedResourcesUsed, Resource reservedResources, Resource neededResources, - Map<String, Long> resourceSecondsMap, float queueUsagePerc, - float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap) { + Map<String, Long> guaranteedResourceSecondsMap, float queueUsagePerc, + float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap, + Resource opportunisticResourcesUsed, + Map<String, Long> opportunisticResourcesSecondsMap) { + ApplicationResourceUsageReport report = Records.newRecord(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); report.setNumReservedContainers(numReservedContainers); - report.setUsedResources(usedResources); + report.setGuaranteedResourcesUsed(guaranteedResourcesUsed); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setResourceSecondsMap(resourceSecondsMap); + report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap); report.setQueueUsagePercentage(queueUsagePerc); report.setClusterUsagePercentage(clusterUsagePerc); report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap); + report.setOpportunisticResourcesUsed(opportunisticResourcesUsed); + report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap); return report; } @@ -87,16 +93,40 @@ public abstract class ApplicationResourceUsageReport { public abstract void setNumReservedContainers(int num_reserved_containers); /** - * Get the used <code>Resource</code>. -1 for invalid/inaccessible reports. - * @return the used <code>Resource</code> + * Get the guaranteed <code>Resource</code> used. + * -1 for invalid/inaccessible reports. + * @return the guaranteed <code>Resource</code> used */ @Public @Stable + @Deprecated public abstract Resource getUsedResources(); + /** + * Get the guaranteed <code>Resource</code> used. + * -1 for invalid/inaccessible reports. + * @return the guaranteed <code>Resource</code> used + */ + @Public + @Unstable + public abstract Resource getGuaranteedResourcesUsed(); + + @Private + @Unstable + public abstract void setGuaranteedResourcesUsed(Resource resources); + + /** + * Get the opportunistic <code>Resource</code> used. + * -1 for invalid/inaccessible reports. + * @return the opportunistic <code>Resource</code> used + */ + @Public + @Unstable + public abstract Resource getOpportunisticResourcesUsed(); + @Private @Unstable - public abstract void setUsedResources(Resource resources); + public abstract void setOpportunisticResourcesUsed(Resource resources); /** * Get the reserved <code>Resource</code>. -1 for invalid/inaccessible reports. @@ -123,42 +153,84 @@ public abstract class ApplicationResourceUsageReport { public abstract void setNeededResources(Resource needed_resources); /** - * Set the aggregated amount of memory (in megabytes) the application has - * allocated times the number of seconds the application has been running. - * @param memory_seconds the aggregated amount of memory seconds + * Set the aggregated amount of guaranteed memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @param memorySeconds the aggregated amount of guaranteed memory seconds */ @Private @Unstable - public abstract void setMemorySeconds(long memory_seconds); + public abstract void setGuaranteedMemorySeconds(long memorySeconds); /** - * Get the aggregated amount of memory (in megabytes) the application has - * allocated times the number of seconds the application has been running. - * @return the aggregated amount of memory seconds + * Get the aggregated amount of guaranteed memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @return the aggregated amount of guaranteed memory seconds + */ + @Public + @Unstable + public abstract long getGuaranteedMemorySeconds(); + + /** + * Get the aggregated amount of guaranteed memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @return the aggregated amount of guaranteed memory seconds */ @Public @Unstable + @Deprecated public abstract long getMemorySeconds(); /** - * Set the aggregated number of vcores that the application has allocated - * times the number of seconds the application has been running. - * @param vcore_seconds the aggregated number of vcore seconds + * Set the aggregated number of guaranteed vcores that the application has + * allocated times the number of seconds the application has been running. + * @param vcoreSeconds the aggregated number of guaranteed vcore seconds */ @Private @Unstable - public abstract void setVcoreSeconds(long vcore_seconds); + public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds); /** - * Get the aggregated number of vcores that the application has allocated - * times the number of seconds the application has been running. - * @return the aggregated number of vcore seconds + * Get the aggregated number of guaranteed vcores that the application has + * allocated times the number of seconds the application has been running. + * @return the aggregated number of guaranteed vcore seconds + */ + @Public + @Unstable + public abstract long getGuaranteedVcoreSeconds(); + + /** + * Get the aggregated number of guaranteed vcores that the application has + * allocated times the number of seconds the application has been running. + * @return the aggregated number of guaranteed vcore seconds */ @Public @Unstable + @Deprecated public abstract long getVcoreSeconds(); /** + * Get the aggregated amount of opportunistic memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @return the aggregated amount of opportunistic memory seconds + */ + @Public + @Unstable + public abstract long getOpportunisticMemorySeconds(); + + /** + * Get the aggregated number of opportunistic vcores that the application + * has allocated times the number of seconds the application has been running. + * @return the aggregated number of opportunistic vcore seconds + */ + @Public + @Unstable + public abstract long getOpportunisticVcoreSeconds(); + + /** * Get the percentage of resources of the queue that the app is using. * @return the percentage of resources of the queue that the app is using. */ @@ -231,23 +303,35 @@ public abstract class ApplicationResourceUsageReport { public abstract long getPreemptedVcoreSeconds(); /** - * Get the aggregated number of resources that the application has + * Get the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. - * @return map containing the resource name and aggregated resource-seconds + * @return map containing the resource name and aggregated guaranteed + * resource-seconds */ @Public @Unstable + @Deprecated public abstract Map<String, Long> getResourceSecondsMap(); /** - * Set the aggregated number of resources that the application has + * Get the aggregated number of guaranteed resources that the application has + * allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated guaranteed + * resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getGuaranteedResourceSecondsMap(); + + /** + * Set the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. * @param resourceSecondsMap map containing the resource name and aggregated - * resource-seconds + * guaranteed resource-seconds */ @Private @Unstable - public abstract void setResourceSecondsMap( + public abstract void setGuaranteedResourceSecondsMap( Map<String, Long> resourceSecondsMap); @@ -272,4 +356,24 @@ public abstract class ApplicationResourceUsageReport { public abstract void setPreemptedResourceSecondsMap( Map<String, Long> preemptedResourceSecondsMap); + /** + * Get the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated opportunistic + * resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getOpportunisticResourceSecondsMap(); + + /** + * Set the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * @param opportunisticResourceSecondsMap map containing the resource name + * and aggregated opportunistic resource-seconds + */ + @Private + @Unstable + public abstract void setOpportunisticResourceSecondsMap( + Map<String, Long> opportunisticResourceSecondsMap); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 31d090e..c671819 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -252,6 +252,8 @@ message ApplicationResourceUsageReportProto { optional int64 preempted_vcore_seconds = 11; repeated StringLongMapProto application_resource_usage_map = 12; repeated StringLongMapProto application_preempted_resource_usage_map = 13; + optional ResourceProto used_opportunistic_resources = 14; + repeated StringLongMapProto application_opportunistic_resource_usage_map = 15; } message ApplicationReportProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 1d26a96..2a89307 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -1070,15 +1070,20 @@ public class ApplicationCLI extends YarnCLI { private void printResourceUsage(PrintWriter appReportStr, ApplicationResourceUsageReport usageReport) { - appReportStr.print("\tAggregate Resource Allocation : "); + appReportStr.print("\tAggregate Guaranteed Resource Allocation : "); if (usageReport != null) { - appReportStr.println( - getResourceSecondsString(usageReport.getResourceSecondsMap())); + appReportStr.println(getResourceSecondsString( + usageReport.getGuaranteedResourceSecondsMap())); + appReportStr.print("\tAggregate Opportunistic Resource Allocation : "); + appReportStr.println(getResourceSecondsString( + usageReport.getOpportunisticResourceSecondsMap())); appReportStr.print("\tAggregate Resource Preempted : "); appReportStr.println(getResourceSecondsString( usageReport.getPreemptedResourceSecondsMap())); } else { appReportStr.println("N/A"); + appReportStr.print("\tAggregate Opportunistic Resource Allocation : "); + appReportStr.println("N/A"); appReportStr.print("\tAggregate Resource Preempted : "); appReportStr.println("N/A"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java index d6c33f4..ba6eb3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java @@ -170,13 +170,11 @@ public class TopCLI extends YarnCLI { appReport.getApplicationResourceUsageReport() .getNumReservedContainers(); displayStringsMap.put(Columns.RCONT, String.valueOf(reservedContainers)); - usedVirtualCores = - appReport.getApplicationResourceUsageReport().getUsedResources() - .getVirtualCores(); + usedVirtualCores = appReport.getApplicationResourceUsageReport() + .getGuaranteedResourcesUsed().getVirtualCores(); displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores)); - usedMemory = - appReport.getApplicationResourceUsageReport().getUsedResources() - .getMemorySize() / 1024; + usedMemory = appReport.getApplicationResourceUsageReport() + .getGuaranteedResourcesUsed().getMemorySize() / 1024; displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G"); reservedVirtualCores = appReport.getApplicationResourceUsageReport().getReservedResources() @@ -195,11 +193,11 @@ public class TopCLI extends YarnCLI { progress = appReport.getProgress() * 100; displayStringsMap.put(Columns.PROGRESS, String.format("%.2f", progress)); // store in GBSeconds - memorySeconds = - appReport.getApplicationResourceUsageReport().getMemorySeconds() / 1024; + memorySeconds = appReport.getApplicationResourceUsageReport() + .getGuaranteedMemorySeconds() / 1024; displayStringsMap.put(Columns.MEMSECS, String.valueOf(memorySeconds)); - vcoreSeconds = - appReport.getApplicationResourceUsageReport().getVcoreSeconds(); + vcoreSeconds = appReport.getApplicationResourceUsageReport() + .getGuaranteedVcoreSeconds(); displayStringsMap.put(Columns.VCORESECS, String.valueOf(vcoreSeconds)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 0f8ef7b..9d12ab5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -121,18 +121,27 @@ public class TestYarnCLI { for (int i = 0; i < 2; ++i) { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); - Map<String, Long> resourceSecondsMap = new HashMap<>(); + Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(); + Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>(); Map<String, Long> preemptedResoureSecondsMap = new HashMap<>(); - resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L); - resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L); + guaranteedResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 123456L); + guaranteedResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 4567L); + opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 123456L); + opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 4567L); preemptedResoureSecondsMap .put(ResourceInformation.MEMORY_MB.getName(), 1111L); preemptedResoureSecondsMap .put(ResourceInformation.VCORES.getName(), 2222L); ApplicationResourceUsageReport usageReport = i == 0 ? null : ApplicationResourceUsageReport - .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0, - preemptedResoureSecondsMap); + .newInstance(2, 0, null, null, null, + guaranteedResourceSecondsMap, 0, 0, + preemptedResoureSecondsMap, null, + opportunisticResourceSecondsMap); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, @@ -168,7 +177,9 @@ public class TestYarnCLI { pw.println("\tTracking-URL : N/A"); pw.println("\tRPC Port : 124"); pw.println("\tAM Host : host"); - pw.println("\tAggregate Resource Allocation : " + + pw.println("\tAggregate Guaranteed Resource Allocation : " + + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); + pw.println("\tAggregate Opportunistic Resource Allocation : " + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); pw.println("\tAggregate Resource Preempted : " + (i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java index 14ede5d..8ff1631 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java @@ -40,12 +40,14 @@ extends ApplicationResourceUsageReport { ApplicationResourceUsageReportProto.Builder builder = null; boolean viaProto = false; - Resource usedResources; - Resource reservedResources; - Resource neededResources; + private Resource guaranteedResourceUsed; + private Resource reservedResources; + private Resource neededResources; + private Resource opportunisticResourceUsed; - private Map<String, Long> resourceSecondsMap; + private Map<String, Long> guaranteedResourceSecondsMap; private Map<String, Long> preemptedResourceSecondsMap; + private Map<String, Long> opportunisticResourceSecondsMap; public ApplicationResourceUsageReportPBImpl() { builder = ApplicationResourceUsageReportProto.newBuilder(); @@ -55,8 +57,9 @@ extends ApplicationResourceUsageReport { ApplicationResourceUsageReportProto proto) { this.proto = proto; viaProto = true; - getResourceSecondsMap(); + getGuaranteedResourceSecondsMap(); getPreemptedResourceSecondsMap(); + getOpportunisticResourceSecondsMap(); } public synchronized ApplicationResourceUsageReportProto getProto() { @@ -87,8 +90,13 @@ extends ApplicationResourceUsageReport { } private void mergeLocalToBuilder() { - if (this.usedResources != null) { - builder.setUsedResources(convertToProtoFormat(this.usedResources)); + if (this.guaranteedResourceUsed != null) { + builder.setUsedResources( + convertToProtoFormat(this.guaranteedResourceUsed)); + } + if (this.opportunisticResourceUsed != null) { + builder.setUsedOpportunisticResources( + convertToProtoFormat(this.opportunisticResourceUsed)); } if (this.reservedResources != null) { builder.setReservedResources( @@ -99,19 +107,28 @@ extends ApplicationResourceUsageReport { } builder.clearApplicationResourceUsageMap(); builder.clearApplicationPreemptedResourceUsageMap(); + builder.clearApplicationOpportunisticResourceUsageMap(); if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap .isEmpty()) { builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); } - if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) { + if (guaranteedResourceSecondsMap != null && + !guaranteedResourceSecondsMap.isEmpty()) { builder.addAllApplicationResourceUsageMap( - ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + ProtoUtils.convertMapToStringLongMapProtoList( + guaranteedResourceSecondsMap)); + } + if (opportunisticResourceSecondsMap != null && + !opportunisticResourceSecondsMap.isEmpty()) { + builder.addAllApplicationOpportunisticResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList( + opportunisticResourceSecondsMap)); } - builder.setMemorySeconds(this.getMemorySeconds()); - builder.setVcoreSeconds(this.getVcoreSeconds()); + builder.setMemorySeconds(this.getGuaranteedMemorySeconds()); + builder.setVcoreSeconds(this.getGuaranteedVcoreSeconds()); builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds()); builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds()); } @@ -157,24 +174,53 @@ extends ApplicationResourceUsageReport { } @Override + @Deprecated public synchronized Resource getUsedResources() { + return getGuaranteedResourcesUsed(); + } + + @Override + public synchronized Resource getGuaranteedResourcesUsed() { ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - if (this.usedResources != null) { - return this.usedResources; + if (this.guaranteedResourceUsed != null) { + return this.guaranteedResourceUsed; } if (!p.hasUsedResources()) { return null; } - this.usedResources = convertFromProtoFormat(p.getUsedResources()); - return this.usedResources; + this.guaranteedResourceUsed = convertFromProtoFormat(p.getUsedResources()); + return this.guaranteedResourceUsed; } @Override - public synchronized void setUsedResources(Resource resources) { + public synchronized void setGuaranteedResourcesUsed(Resource resources) { maybeInitBuilder(); if (resources == null) builder.clearUsedResources(); - this.usedResources = resources; + this.guaranteedResourceUsed = resources; + } + + @Override + public synchronized Resource getOpportunisticResourcesUsed() { + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + if (this.opportunisticResourceUsed != null) { + return this.opportunisticResourceUsed; + } + if (!p.hasUsedOpportunisticResources()) { + return null; + } + this.opportunisticResourceUsed = + convertFromProtoFormat(p.getUsedOpportunisticResources()); + return this.opportunisticResourceUsed; + } + + @Override + public synchronized void setOpportunisticResourcesUsed(Resource resources) { + maybeInitBuilder(); + if (resources == null) { + builder.clearUsedOpportunisticResources(); + } + this.opportunisticResourceUsed = resources; } @Override @@ -220,14 +266,14 @@ extends ApplicationResourceUsageReport { } @Override - public synchronized void setMemorySeconds(long memory_seconds) { - getResourceSecondsMap() - .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds); + public synchronized void setGuaranteedMemorySeconds(long memorySeconds) { + getGuaranteedResourceSecondsMap() + .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); } @Override - public synchronized long getMemorySeconds() { - Map<String, Long> tmp = getResourceSecondsMap(); + public synchronized long getGuaranteedMemorySeconds() { + Map<String, Long> tmp = getGuaranteedResourceSecondsMap(); if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { return tmp.get(ResourceInformation.MEMORY_MB.getName()); } @@ -235,20 +281,50 @@ extends ApplicationResourceUsageReport { } @Override - public synchronized void setVcoreSeconds(long vcore_seconds) { - getResourceSecondsMap() - .put(ResourceInformation.VCORES.getName(), vcore_seconds); + @Deprecated + public synchronized long getMemorySeconds() { + return getGuaranteedMemorySeconds(); + } + + @Override + public synchronized void setGuaranteedVcoreSeconds(long vcoreSeconds) { + getGuaranteedResourceSecondsMap() + .put(ResourceInformation.VCORES.getName(), vcoreSeconds); } @Override + public synchronized long getGuaranteedVcoreSeconds() { + Map<String, Long> tmp = getGuaranteedResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; + } + + @Override + @Deprecated public synchronized long getVcoreSeconds() { - Map<String, Long> tmp = getResourceSecondsMap(); + return getGuaranteedVcoreSeconds(); + } + + @Override + public synchronized long getOpportunisticMemorySeconds() { + Map<String, Long> tmp = getOpportunisticResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; + } + + @Override + public synchronized long getOpportunisticVcoreSeconds() { + Map<String, Long> tmp = getOpportunisticResourceSecondsMap(); if (tmp.containsKey(ResourceInformation.VCORES.getName())) { return tmp.get(ResourceInformation.VCORES.getName()); } return 0; } - + @Override public synchronized void setPreemptedMemorySeconds( long preemptedMemorySeconds) { @@ -314,41 +390,47 @@ extends ApplicationResourceUsageReport { } @Override - public synchronized void setResourceSecondsMap( + public synchronized void setGuaranteedResourceSecondsMap( Map<String, Long> resourceSecondsMap) { - this.resourceSecondsMap = resourceSecondsMap; + this.guaranteedResourceSecondsMap = resourceSecondsMap; if (resourceSecondsMap == null) { return; } if (!resourceSecondsMap .containsKey(ResourceInformation.MEMORY_MB.getName())) { - this.setMemorySeconds(0L); + this.setGuaranteedMemorySeconds(0L); } if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) { - this.setVcoreSeconds(0L); + this.setGuaranteedVcoreSeconds(0L); } } @Override + @Deprecated public synchronized Map<String, Long> getResourceSecondsMap() { - if (this.resourceSecondsMap != null) { - return this.resourceSecondsMap; + return getGuaranteedResourceSecondsMap(); + } + + @Override + public synchronized Map<String, Long> getGuaranteedResourceSecondsMap() { + if (this.guaranteedResourceSecondsMap != null) { + return this.guaranteedResourceSecondsMap; } ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - this.resourceSecondsMap = ProtoUtils + this.guaranteedResourceSecondsMap = ProtoUtils .convertStringLongMapProtoListToMap( p.getApplicationResourceUsageMapList()); - if (!this.resourceSecondsMap + if (!this.guaranteedResourceSecondsMap .containsKey(ResourceInformation.MEMORY_MB.getName())) { - this.setMemorySeconds(p.getMemorySeconds()); + this.setGuaranteedMemorySeconds(p.getMemorySeconds()); } - if (!this.resourceSecondsMap + if (!this.guaranteedResourceSecondsMap .containsKey(ResourceInformation.VCORES.getName())) { - this.setVcoreSeconds(p.getVcoreSeconds()); + this.setGuaranteedVcoreSeconds(p.getVcoreSeconds()); } - this.setMemorySeconds(p.getMemorySeconds()); - this.setVcoreSeconds(p.getVcoreSeconds()); - return this.resourceSecondsMap; + this.setGuaranteedMemorySeconds(p.getMemorySeconds()); + this.setGuaranteedVcoreSeconds(p.getVcoreSeconds()); + return this.guaranteedResourceSecondsMap; } @Override @@ -389,4 +471,45 @@ extends ApplicationResourceUsageReport { this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds()); return this.preemptedResourceSecondsMap; } + + @Override + public synchronized Map<String, Long> getOpportunisticResourceSecondsMap() { + if (this.opportunisticResourceSecondsMap != null) { + return this.opportunisticResourceSecondsMap; + } + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + this.opportunisticResourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationOpportunisticResourceUsageMapList()); + if (!opportunisticResourceSecondsMap.containsKey( + ResourceInformation.MEMORY_MB.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 0L); + } + if (!opportunisticResourceSecondsMap.containsKey( + ResourceInformation.VCORES.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 0L); + } + return this.opportunisticResourceSecondsMap; + } + + @Override + public synchronized void setOpportunisticResourceSecondsMap( + Map<String, Long> opportunisticResourceSecondsMap) { + this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap; + if (opportunisticResourceSecondsMap == null) { + return; + } + if (!opportunisticResourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 0L); + } + if (!opportunisticResourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 0L); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 7826f51..96aff88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import java.util.Map; + /** * Resources is a computation class which provides a set of apis to do * mathematical operations on Resource object. @@ -551,4 +553,22 @@ public class Resources { Resource resource, Resource factor) { return calculator.normalizeDown(resource, factor); } + + /** + * Merge resource usage entries from a map to another map. + * @param mergeFrom the map to merge from + * @param mergeTo the map to merge to + */ + public static void mergeResourceSecondsMap(Map<String, Long> mergeFrom, + Map<String, Long> mergeTo) { + for (Map.Entry<String, Long> entry : mergeFrom.entrySet()) { + Long value = mergeTo.get(entry.getKey()); + if (value != null) { + value += entry.getValue(); + } else { + value = entry.getValue(); + } + mergeTo.put(entry.getKey(), value); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 0b57717..85cc006 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -330,29 +329,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService .toString(); } - if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { - long vcoreSeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_CPU_METRICS); - long memorySeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_MEM_METRICS); - long preemptedMemorySeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS); - long preemptedVcoreSeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS); - Map<String, Long> resourceSecondsMap = new HashMap<>(); - Map<String, Long> preemptedResoureSecondsMap = new HashMap<>(); - resourceSecondsMap - .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); - resourceSecondsMap - .put(ResourceInformation.VCORES.getName(), vcoreSeconds); - preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), - preemptedMemorySeconds); - preemptedResoureSecondsMap - .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds); - + if (entityInfo.containsKey( + ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)) { + Map<String, Long> guaranteedResourceSecondsMap = + extractGuaranteedResourceSecondsMap(entityInfo); + Map<String, Long> preemptedResoureSecondsMap = + extractPreemptedResourceSecondsMap(entityInfo); + Map<String, Long> opportunisticResourceSecondsMap = + extractOpportunisticResourceSecondsMap(entityInfo); appResources = ApplicationResourceUsageReport - .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0, - preemptedResoureSecondsMap); + .newInstance(0, 0, null, null, null, + guaranteedResourceSecondsMap, 0, 0, + preemptedResoureSecondsMap, null, + opportunisticResourceSecondsMap); } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { @@ -456,6 +445,54 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService amNodeLabelExpression), appViewACLs); } + private static Map<String, Long> extractGuaranteedResourceSecondsMap( + Map<String, Object> entityInfo) { + Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(); + + long guaranteedVcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS); + long guaranteedMemorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS); + guaranteedResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), guaranteedMemorySeconds); + guaranteedResourceSecondsMap.put(ResourceInformation.VCORES.getName(), + guaranteedVcoreSeconds); + + return guaranteedResourceSecondsMap; + } + + private static Map<String, Long> extractOpportunisticResourceSecondsMap( + Map<String, Object> entityInfo) { + Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>(); + + long opportunisticVcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS); + long opportunisticMemorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS); + opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), + opportunisticMemorySeconds); + opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), opportunisticVcoreSeconds); + + return opportunisticResourceSecondsMap; + } + + private static Map<String, Long> extractPreemptedResourceSecondsMap( + Map<String, Object> entityInfo) { + Map<String, Long> preemptedResoureSecondsMap = new HashMap<>(); + + long preemptedMemorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS); + long preemptedVcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS); + preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), + preemptedMemorySeconds); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds); + + return preemptedResoureSecondsMap; + } private static long parseLong(Map<String, Object> entityInfo, String infoKey) { long result = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index 7ef6eca..6cb21ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -152,9 +152,13 @@ public class TestApplicationHistoryClientService { ApplicationReport appReport = response.getApplicationReport(); Assert.assertNotNull(appReport); Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() - .getMemorySeconds()); + .getGuaranteedMemorySeconds()); Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() - .getVcoreSeconds()); + .getGuaranteedVcoreSeconds()); + Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() + .getOpportunisticMemorySeconds()); + Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() + .getOpportunisticVcoreSeconds()); Assert.assertEquals("application_0_0001", appReport.getApplicationId() .toString()); Assert.assertEquals("test app type", http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index ecaaf1e..35996aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -250,9 +250,13 @@ public class TestApplicationHistoryManagerOnTimelineStore { ApplicationResourceUsageReport applicationResourceUsageReport = app.getApplicationResourceUsageReport(); Assert.assertEquals(123, - applicationResourceUsageReport.getMemorySeconds()); - Assert - .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); + applicationResourceUsageReport.getGuaranteedMemorySeconds()); + Assert.assertEquals(345, + applicationResourceUsageReport.getGuaranteedVcoreSeconds()); + Assert.assertEquals(123, + applicationResourceUsageReport.getOpportunisticMemorySeconds()); + Assert.assertEquals(345, + applicationResourceUsageReport.getOpportunisticVcoreSeconds()); long expectedPreemptMemSecs = 456; long expectedPreemptVcoreSecs = 789; if (i == 3) { @@ -534,8 +538,12 @@ public class TestApplicationHistoryManagerOnTimelineStore { Priority.newInstance(0)); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); - entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); - entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); + entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, 123); + entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, 345); + entityInfo.put( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, 123); + entityInfo.put( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, 345); if (!missingPreemptMetrics) { entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456); entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index 4cec409..b4123ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -70,12 +70,18 @@ public class ApplicationMetricsConstants { public static final String STATE_EVENT_INFO = "YARN_APPLICATION_STATE"; - public static final String APP_CPU_METRICS = + public static final String APP_GUARANTEED_CPU_METRICS = "YARN_APPLICATION_CPU"; - public static final String APP_MEM_METRICS = + public static final String APP_GUARANTEED_MEM_METRICS = "YARN_APPLICATION_MEMORY"; + public static final String APP_OPPORTUNISTIC_CPU_METRICS = + "YARN_APPLICATION_OPPORTUNISTIC_CPU"; + + public static final String APP_OPPORTUNISTIC_MEM_METRICS = + "YARN_APPLICATION_OPPORTUNISTIC_MEMORY"; + public static final String APP_RESOURCE_PREEMPTED_CPU = "YARN_APPLICATION_RESOURCE_PREEMPTED_CPU"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index fd9a9fb..708ffaf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -466,19 +466,24 @@ public class BuilderUtils { } public static ApplicationResourceUsageReport newApplicationResourceUsageReport( - int numUsedContainers, int numReservedContainers, Resource usedResources, + int numUsedContainers, int numReservedContainers, + Resource guaranteedResourcesUsed, Resource reservedResources, Resource neededResources, - Map<String, Long> resourceSecondsMap, - Map<String, Long> preemptedResourceSecondsMap) { + Map<String, Long> guaranteedResourceSecondsMap, + Map<String, Long> preemptedResourceSecondsMap, + Resource opportunisticResourcesUsed, + Map<String, Long> opportunisticResourcesSecondsMap) { ApplicationResourceUsageReport report = recordFactory.newRecordInstance(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); report.setNumReservedContainers(numReservedContainers); - report.setUsedResources(usedResources); + report.setGuaranteedResourcesUsed(guaranteedResourcesUsed); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setResourceSecondsMap(resourceSecondsMap); + report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap); report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); + report.setOpportunisticResourcesUsed(opportunisticResourcesUsed); + report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap); return report; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index b4687a4..be6c17c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Times; @@ -62,6 +63,8 @@ public class AppInfo { protected int priority; private long allocatedCpuVcores; private long allocatedMemoryMB; + private long allocatedOpportunisticCpuVcores; + private long allocatedOpportunisticMemoryMB; private long reservedCpuVcores; private long reservedMemoryMB; protected boolean unmanagedApplication; @@ -100,15 +103,25 @@ public class AppInfo { if (app.getApplicationResourceUsageReport() != null) { runningContainers = app.getApplicationResourceUsageReport() .getNumUsedContainers(); - if (app.getApplicationResourceUsageReport().getUsedResources() != null) { - allocatedCpuVcores = app.getApplicationResourceUsageReport() - .getUsedResources().getVirtualCores(); - allocatedMemoryMB = app.getApplicationResourceUsageReport() - .getUsedResources().getMemorySize(); - reservedCpuVcores = app.getApplicationResourceUsageReport() - .getReservedResources().getVirtualCores(); - reservedMemoryMB = app.getApplicationResourceUsageReport() - .getReservedResources().getMemorySize(); + Resource guaranteedResourceUsed = + app.getApplicationResourceUsageReport().getGuaranteedResourcesUsed(); + if (guaranteedResourceUsed != null) { + allocatedCpuVcores = guaranteedResourceUsed.getVirtualCores(); + allocatedMemoryMB = guaranteedResourceUsed.getMemorySize(); + } + Resource opportunisticResourceUsed = app + .getApplicationResourceUsageReport().getOpportunisticResourcesUsed(); + if (opportunisticResourceUsed != null) { + allocatedOpportunisticCpuVcores = + opportunisticResourceUsed.getVirtualCores(); + allocatedOpportunisticMemoryMB = + opportunisticResourceUsed.getMemorySize(); + } + Resource resourceReserved = + app.getApplicationResourceUsageReport().getReservedResources(); + if (resourceReserved != null) { + reservedCpuVcores = resourceReserved.getVirtualCores(); + reservedMemoryMB = resourceReserved.getMemorySize(); } } progress = app.getProgress() * 100; // in percent @@ -168,6 +181,14 @@ public class AppInfo { return allocatedMemoryMB; } + public long getAllocatedOpportunisticCpuVcores() { + return allocatedOpportunisticCpuVcores; + } + + public long getAllocatedOpportunisticMemoryMB() { + return allocatedOpportunisticMemoryMB; + } + public long getReservedCpuVcores() { return reservedCpuVcores; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 357f1c1..04c1298 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -186,19 +186,27 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, .add("startTime", app.getStartTime()) .add("finishTime", app.getFinishTime()) .add("finalStatus", app.getFinalApplicationStatus()) - .add("memorySeconds", metrics.getMemorySeconds()) - .add("vcoreSeconds", metrics.getVcoreSeconds()) + .add("guaranteedMemorySeconds", metrics.getGuaranteedMemorySeconds()) + .add("guaranteedVcoreSeconds", metrics.getGuaranteedVcoreSeconds()) + .add("opportunisticMemorySeconds", + metrics.getOpportunisticMemorySeconds()) + .add("opportunisticVcoreSeconds", + metrics.getOpportunisticVcoreSeconds()) .add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds()) .add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds()) .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) .add("applicationType", app.getApplicationType()) - .add("resourceSeconds", StringHelper - .getResourceSecondsString(metrics.getResourceSecondsMap())) + .add("guaranteedResourceSeconds", StringHelper + .getResourceSecondsString( + metrics.getGuaranteedResourceSecondsMap())) .add("preemptedResourceSeconds", StringHelper .getResourceSecondsString( - metrics.getPreemptedResourceSecondsMap())); + metrics.getPreemptedResourceSecondsMap())) + .add("opportunisticResourceSeconds", StringHelper + .getResourceSecondsString( + metrics.getOpportunisticResourceSecondsMap())); return summary; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index ab6bbcf..bf93df2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -481,7 +481,8 @@ public class RMServerUtils { DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = BuilderUtils.newApplicationResourceUsageReport(-1, -1, Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>()); + Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>(), + Resources.createResource(-1, -1), new HashMap<>()); /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index 31b07f1..f14c112 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -142,10 +142,16 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { latestApplicationAttemptId); } RMAppMetrics appMetrics = app.getRMAppMetrics(); - entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS, - appMetrics.getVcoreSeconds()); - entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, - appMetrics.getMemorySeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, + appMetrics.getGuaranteedVcoreSeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, + appMetrics.getGuaranteedMemorySeconds()); + entity.addOtherInfo( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, + appMetrics.getOpportunisticVcoreSeconds()); + entity.addOtherInfo( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, + appMetrics.getOpportunisticMemorySeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, appMetrics.getPreemptedMemorySeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index ea286a0..4e00595 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -192,11 +192,17 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>(); entityMetrics.add(getTimelineMetric( - ApplicationMetricsConstants.APP_CPU_METRICS, timestamp, - appMetrics.getVcoreSeconds())); + ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, timestamp, + appMetrics.getGuaranteedVcoreSeconds())); entityMetrics.add(getTimelineMetric( - ApplicationMetricsConstants.APP_MEM_METRICS, timestamp, - appMetrics.getMemorySeconds())); + ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, timestamp, + appMetrics.getGuaranteedMemorySeconds())); + entityMetrics.add(getTimelineMetric( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, + timestamp, appMetrics.getOpportunisticVcoreSeconds())); + entityMetrics.add(getTimelineMetric( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, + timestamp, appMetrics.getOpportunisticMemorySeconds())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp, appMetrics.getPreemptedMemorySeconds())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 242b5d0..4a7696f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -904,8 +904,9 @@ public abstract class RMStateStore extends AbstractService { appAttempt.getAppAttemptId(), appAttempt.getMasterContainer(), credentials, appAttempt.getStartTime(), - resUsage.getResourceUsageSecondsMap(), - attempMetrics.getPreemptedResourceSecondsMap()); + resUsage.getGuaranteedResourceUsageSecondsMap(), + attempMetrics.getPreemptedResourceSecondsMap(), + resUsage.getOpportunisticResourceSecondsMap()); getRMStateStoreEventHandler().handle( new RMStateStoreAppAttemptEvent(attemptState)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc40e3ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java index 2de071a..b08d2eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery.records; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -45,8 +46,9 @@ public abstract class ApplicationAttemptStateData { Credentials attemptTokens, long startTime, RMAppAttemptState finalState, String finalTrackingUrl, String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, - long finishTime, Map<String, Long> resourceSecondsMap, - Map<String, Long> preemptedResourceSecondsMap) { + long finishTime, Map<String, Long> guaranteedResourceSecondsMap, + Map<String, Long> preemptedResourceSecondsMap, + Map<String, Long> opportunisticResourcesSecondsMap) { ApplicationAttemptStateData attemptStateData = Records.newRecord(ApplicationAttemptStateData.class); attemptStateData.setAttemptId(attemptId); @@ -59,19 +61,22 @@ public abstract class ApplicationAttemptStateData { attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus); attemptStateData.setAMContainerExitStatus(exitStatus); attemptStateData.setFinishTime(finishTime); - attemptStateData.setMemorySeconds(RMServerUtils - .getOrDefault(resourceSecondsMap, + attemptStateData.setGuaranteedMemorySeconds(RMServerUtils + .getOrDefault(guaranteedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L)); - attemptStateData.setVcoreSeconds(RMServerUtils - .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), - 0L)); + attemptStateData.setGuaranteedVcoreSeconds(RMServerUtils + .getOrDefault(guaranteedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L)); attemptStateData.setPreemptedMemorySeconds(RMServerUtils .getOrDefault(preemptedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L)); attemptStateData.setPreemptedVcoreSeconds(RMServerUtils .getOrDefault(preemptedResourceSecondsMap, ResourceInformation.VCORES.getName(), 0L)); - attemptStateData.setResourceSecondsMap(resourceSecondsMap); + attemptStateData.setGuaranteedResourceSecondsMap( + guaranteedResourceSecondsMap); + attemptStateData.setOpportunisticResourceSecondsMap( + opportunisticResourcesSecondsMap); attemptStateData .setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); return attemptStateData; @@ -80,11 +85,13 @@ public abstract class ApplicationAttemptStateData { public static ApplicationAttemptStateData newInstance( ApplicationAttemptId attemptId, Container masterContainer, Credentials attemptTokens, long startTime, - Map<String, Long> resourceSeondsMap, - Map<String, Long> preemptedResourceSecondsMap) { + Map<String, Long> guaranteedResourceSecondsMap, + Map<String, Long> preemptedResourceSecondsMap, + Map<String, Long> opportunisticResourcesSecondsMap) { return newInstance(attemptId, masterContainer, attemptTokens, startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0, - resourceSeondsMap, preemptedResourceSecondsMap); + guaranteedResourceSecondsMap, preemptedResourceSecondsMap, + opportunisticResourcesSecondsMap); } @@ -180,30 +187,78 @@ public abstract class ApplicationAttemptStateData { public abstract void setFinishTime(long finishTime); /** - * Get the <em>memory seconds</em> (in MB seconds) of the application. - * @return <em>memory seconds</em> (in MB seconds) of the application + * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the + * application. + * @return <em>guaranteed memory seconds</em> (in MB seconds) of the + * application */ @Public @Unstable + @Deprecated public abstract long getMemorySeconds(); @Public @Unstable + @Deprecated public abstract void setMemorySeconds(long memorySeconds); /** - * Get the <em>vcore seconds</em> of the application. - * @return <em>vcore seconds</em> of the application + * Get the <em>guaranteed vcore seconds</em> of the application. + * @return <em>guaranteed vcore seconds</em> of the application */ @Public @Unstable + @Deprecated public abstract long getVcoreSeconds(); @Public @Unstable + @Deprecated public abstract void setVcoreSeconds(long vcoreSeconds); /** + * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the + * application. + * @return <em>guaranteed memory seconds</em> (in MB seconds) of the + * application + */ + @Public + @Unstable + public abstract long getGuaranteedMemorySeconds(); + + @Private + public abstract void setGuaranteedMemorySeconds(long memorySeconds); + + /** + * Get the <em>guaranteed vcore seconds</em> of the application. + * @return <em>guaranteed vcore seconds</em> of the application + */ + @Public + @Unstable + public abstract long getGuaranteedVcoreSeconds(); + + @Private + public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds); + + /** + * Get the <em>opportunistic memory seconds</em> (in MB seconds) of the + * application. + * @return <em>opportunistic memory seconds</em> (in MB seconds) of the + * application + */ + @Public + @Unstable + public abstract long getOpportunisticMemorySeconds(); + + /** + * Get the <em>opportunistic vcore seconds</em> of the application. + * @return <em>opportunistic vcore seconds</em> of the application + */ + @Public + @Unstable + public abstract long getOpportunisticVcoreSeconds(); + + /** * Get the <em>preempted memory seconds</em> * (in MB seconds) of the application. * @return <em>preempted memory seconds</em> @@ -232,28 +287,54 @@ public abstract class ApplicationAttemptStateData { public abstract void setPreemptedVcoreSeconds(long vcoreSeconds); /** - * Get the aggregated number of resources preempted that the application has + * Get the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. * - * @return map containing the resource name and aggregated preempted - * resource-seconds + * @return map containing the resource name and aggregated guaranteed + * resource-seconds */ @Public @Unstable + @Deprecated public abstract Map<String, Long> getResourceSecondsMap(); /** - * Set the aggregated number of resources that the application has + * Set the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. * * @param resourceSecondsMap map containing the resource name and aggregated - * resource-seconds + * guaranteed resource-seconds */ @Public @Unstable + @Deprecated public abstract void setResourceSecondsMap( Map<String, Long> resourceSecondsMap); + + /** + * Get the aggregated number of guaranteed resources that the application has + * allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated guaranteed + * resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getGuaranteedResourceSecondsMap(); + + /** + * Set the aggregated number of guaranteed resources that the application has + * allocated times the number of seconds the application has been running. + * + * @param resourceSecondsMap map containing the resource name and aggregated + * guaranteed resource-seconds + */ + @Public + @Unstable + public abstract void setGuaranteedResourceSecondsMap( + Map<String, Long> resourceSecondsMap); + /** * Get the aggregated number of resources preempted that the application has * allocated times the number of seconds the application has been running. @@ -276,4 +357,27 @@ public abstract class ApplicationAttemptStateData { @Unstable public abstract void setPreemptedResourceSecondsMap( Map<String, Long> preemptedResourceSecondsMap); + + /** + * Get the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated opportunistic + * resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getOpportunisticResourceSecondsMap(); + + /** + * Set the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * + * @param resourceSecondsMap map containing the resource name and aggregated + * opportunistic resource-seconds + */ + @Public + @Unstable + public abstract void setOpportunisticResourceSecondsMap( + Map<String, Long> resourceSecondsMap); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org