YARN-6232. Update resource usage and preempted resource calculations to take into account all resource types. Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fb5a99d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fb5a99d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fb5a99d Branch: refs/heads/YARN-3926 Commit: 5fb5a99dc59c451575ca43a7880c7dfe9916ff62 Parents: b680025 Author: Sunil G <sun...@apache.org> Authored: Mon Mar 6 11:34:20 2017 +0530 Committer: Wangda Tan <wan...@apache.org> Committed: Mon Aug 21 16:56:01 2017 -0700 ---------------------------------------------------------------------- .../records/ApplicationResourceUsageReport.java | 58 ++++++- .../src/main/proto/yarn_protos.proto | 7 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 35 +++-- .../hadoop/yarn/client/cli/TestYarnCLI.java | 16 +- .../ApplicationResourceUsageReportPBImpl.java | 151 ++++++++++++++++--- .../yarn/api/records/impl/pb/ProtoUtils.java | 34 +++++ .../apache/hadoop/yarn/util/StringHelper.java | 36 +++++ .../hadoop/yarn/api/BasePBImplRecordsTest.java | 12 ++ .../hadoop/yarn/api/TestPBImplRecords.java | 4 + ...pplicationHistoryManagerOnTimelineStore.java | 18 ++- .../hadoop/yarn/server/utils/BuilderUtils.java | 16 +- .../server/resourcemanager/RMAppManager.java | 8 +- .../server/resourcemanager/RMServerUtils.java | 10 +- .../resourcemanager/recovery/RMStateStore.java | 7 +- .../records/ApplicationAttemptStateData.java | 89 +++++++++-- .../pb/ApplicationAttemptStateDataPBImpl.java | 50 ++++++ .../server/resourcemanager/rmapp/RMAppImpl.java | 43 +++--- .../resourcemanager/rmapp/RMAppMetrics.java | 41 +++-- .../attempt/AggregateAppResourceUsage.java | 34 ++--- .../rmapp/attempt/RMAppAttemptImpl.java | 32 ++-- .../rmapp/attempt/RMAppAttemptMetrics.java | 106 ++++++++++--- .../rmcontainer/RMContainerImpl.java | 16 +- .../scheduler/SchedulerApplicationAttempt.java | 47 +++--- .../resourcemanager/webapp/RMAppBlock.java | 14 +- .../resourcemanager/webapp/dao/AppInfo.java | 15 +- .../webapp/dao/ResourceInfo.java | 36 ++++- .../webapp/dao/SchedulerInfo.java | 2 +- .../yarn_server_resourcemanager_recovery.proto | 2 + .../server/resourcemanager/TestAppManager.java | 7 +- .../TestContainerResourceUsage.java | 7 +- .../applicationsmanager/MockAsm.java | 9 +- .../metrics/TestSystemMetricsPublisher.java | 15 +- .../TestSystemMetricsPublisherForV2.java | 22 ++- .../recovery/RMStateStoreTestBase.java | 8 +- .../recovery/TestZKRMStateStore.java | 23 +-- .../resourcemanager/webapp/TestAppPage.java | 8 +- .../webapp/TestRMWebAppFairScheduler.java | 5 +- .../DefaultClientRequestInterceptor.java | 16 ++ .../clientrm/FederationClientInterceptor.java | 15 ++ .../router/clientrm/RouterClientRMService.java | 18 +++ .../PassThroughClientRequestInterceptor.java | 16 ++ 41 files changed, 854 insertions(+), 254 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index 3cf8f3d..f9c8975 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -24,6 +24,9 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; +import java.util.HashMap; +import java.util.Map; + /** * Contains various scheduling metrics to be reported by UI and CLI. */ @@ -35,9 +38,9 @@ public abstract class ApplicationResourceUsageReport { @Unstable public static ApplicationResourceUsageReport newInstance( int numUsedContainers, int numReservedContainers, Resource usedResources, - Resource reservedResources, Resource neededResources, long memorySeconds, - long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc, - long preemptedMemorySeconds, long preemptedVcoresSeconds) { + Resource reservedResources, Resource neededResources, + Map<String, Long> resourceSecondsMap, float queueUsagePerc, + float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap) { ApplicationResourceUsageReport report = Records.newRecord(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -45,12 +48,10 @@ public abstract class ApplicationResourceUsageReport { report.setUsedResources(usedResources); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setMemorySeconds(memorySeconds); - report.setVcoreSeconds(vcoreSeconds); + report.setResourceSecondsMap(resourceSecondsMap); report.setQueueUsagePercentage(queueUsagePerc); report.setClusterUsagePercentage(clusterUsagePerc); - report.setPreemptedMemorySeconds(preemptedMemorySeconds); - report.setPreemptedVcoreSeconds(preemptedVcoresSeconds); + report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap); return report; } @@ -229,4 +230,47 @@ public abstract class ApplicationResourceUsageReport { @Public @Unstable public abstract long getPreemptedVcoreSeconds(); + + /** + * Get the aggregated number of resources that the application has + * allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getResourceSecondsMap(); + + /** + * Set the aggregated number of resources that the application has + * allocated times the number of seconds the application has been running. + * @param resourceSecondsMap map containing the resource name and aggregated + * resource-seconds + */ + @Private + @Unstable + public abstract void setResourceSecondsMap( + Map<String, Long> resourceSecondsMap); + + + /** + * Get the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated preempted + * resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getPreemptedResourceSecondsMap(); + + /** + * Set the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * @param preemptedResourceSecondsMap map containing the resource name and + * aggregated preempted resource-seconds + */ + @Private + @Unstable + public abstract void setPreemptedResourceSecondsMap( + Map<String, Long> preemptedResourceSecondsMap); + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 472a758..292cdc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -195,6 +195,11 @@ message LocalResourceProto { optional bool should_be_uploaded_to_shared_cache = 7; } +message StringLongMapProto { + required string key = 1; + required int64 value = 2; +} + message ApplicationResourceUsageReportProto { optional int32 num_used_containers = 1; optional int32 num_reserved_containers = 2; @@ -207,6 +212,8 @@ message ApplicationResourceUsageReportProto { optional float cluster_usage_percentage = 9; optional int64 preempted_memory_seconds = 10; optional int64 preempted_vcore_seconds = 11; + repeated StringLongMapProto application_resource_usage_map = 12; + repeated StringLongMapProto application_preempted_resource_usage_map = 13; } message ApplicationReportProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 893348a..e9a1d3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -61,6 +61,8 @@ import org.apache.hadoop.yarn.util.Times; import com.google.common.annotations.VisibleForTesting; +import static org.apache.hadoop.yarn.util.StringHelper.getResourceSecondsString; + @Private @Unstable public class ApplicationCLI extends YarnCLI { @@ -699,24 +701,9 @@ public class ApplicationCLI extends YarnCLI { appReportStr.println(appReport.getRpcPort()); appReportStr.print("\tAM Host : "); appReportStr.println(appReport.getHost()); - appReportStr.print("\tAggregate Resource Allocation : "); - ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport(); - if (usageReport != null) { - //completed app report in the timeline server doesn't have usage report - appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, "); - appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds"); - appReportStr.print("\tAggregate Resource Preempted : "); - appReportStr.print(usageReport.getPreemptedMemorySeconds() + - " MB-seconds, "); - appReportStr.println(usageReport.getPreemptedVcoreSeconds() + - " vcore-seconds"); - } else { - appReportStr.println("N/A"); - appReportStr.print("\tAggregate Resource Preempted : "); - appReportStr.println("N/A"); - } + printResourceUsage(appReportStr, usageReport); appReportStr.print("\tLog Aggregation Status : "); appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" : appReport.getLogAggregationStatus()); @@ -747,6 +734,22 @@ public class ApplicationCLI extends YarnCLI { return 0; } + private void printResourceUsage(PrintWriter appReportStr, + ApplicationResourceUsageReport usageReport) { + appReportStr.print("\tAggregate Resource Allocation : "); + if (usageReport != null) { + appReportStr.println( + getResourceSecondsString(usageReport.getResourceSecondsMap())); + appReportStr.print("\tAggregate Resource Preempted : "); + appReportStr.println(getResourceSecondsString( + usageReport.getPreemptedResourceSecondsMap())); + } else { + appReportStr.println("N/A"); + appReportStr.print("\tAggregate Resource Preempted : "); + appReportStr.println("N/A"); + } + } + private String getAllValidApplicationStates() { StringBuilder sb = new StringBuilder(); sb.append("The valid application state can be" + " one of the following: "); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 3c35b9c..9870877 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -39,8 +39,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -68,6 +70,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; @@ -117,9 +120,18 @@ public class TestYarnCLI { for (int i = 0; i < 2; ++i) { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + Map<String, Long> resourceSecondsMap = new HashMap<>(); + Map<String, Long> preemptedResoureSecondsMap = new HashMap<>(); + resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L); + resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L); + preemptedResoureSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), 1111L); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), 2222L); ApplicationResourceUsageReport usageReport = i == 0 ? null : - ApplicationResourceUsageReport.newInstance( - 2, 0, null, null, null, 123456, 4567, 0, 0, 1111, 2222); + ApplicationResourceUsageReport + .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0, + preemptedResoureSecondsMap); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java index 1c85e28..2f94a24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java @@ -22,12 +22,16 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import com.google.protobuf.TextFormat; +import java.util.HashMap; +import java.util.Map; + @Private @Unstable public class ApplicationResourceUsageReportPBImpl @@ -41,6 +45,9 @@ extends ApplicationResourceUsageReport { Resource reservedResources; Resource neededResources; + private Map<String, Long> resourceSecondsMap; + private Map<String, Long> preemptedResourceSecondsMap; + public ApplicationResourceUsageReportPBImpl() { builder = ApplicationResourceUsageReportProto.newBuilder(); } @@ -49,6 +56,8 @@ extends ApplicationResourceUsageReport { ApplicationResourceUsageReportProto proto) { this.proto = proto; viaProto = true; + getResourceSecondsMap(); + getPreemptedResourceSecondsMap(); } public synchronized ApplicationResourceUsageReportProto getProto() { @@ -89,6 +98,23 @@ extends ApplicationResourceUsageReport { if (this.neededResources != null) { builder.setNeededResources(convertToProtoFormat(this.neededResources)); } + builder.clearApplicationResourceUsageMap(); + builder.clearApplicationPreemptedResourceUsageMap(); + + if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap + .isEmpty()) { + builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils + .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); + } + if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) { + builder.addAllApplicationResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + } + + builder.setMemorySeconds(this.getMemorySeconds()); + builder.setVcoreSeconds(this.getVcoreSeconds()); + builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds()); + builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds()); } private void mergeLocalToProto() { @@ -196,54 +222,64 @@ extends ApplicationResourceUsageReport { @Override public synchronized void setMemorySeconds(long memory_seconds) { - maybeInitBuilder(); - builder.setMemorySeconds(memory_seconds); + getResourceSecondsMap() + .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds); } - + @Override public synchronized long getMemorySeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - return p.getMemorySeconds(); + Map<String, Long> tmp = getResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; } @Override public synchronized void setVcoreSeconds(long vcore_seconds) { - maybeInitBuilder(); - builder.setVcoreSeconds(vcore_seconds); + getResourceSecondsMap() + .put(ResourceInformation.VCORES.getName(), vcore_seconds); } @Override public synchronized long getVcoreSeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - return (p.getVcoreSeconds()); + Map<String, Long> tmp = getResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; } @Override public synchronized void setPreemptedMemorySeconds( long preemptedMemorySeconds) { - maybeInitBuilder(); - builder.setPreemptedMemorySeconds(preemptedMemorySeconds); + getPreemptedResourceSecondsMap() + .put(ResourceInformation.MEMORY_MB.getName(), preemptedMemorySeconds); } @Override public synchronized long getPreemptedMemorySeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = - viaProto ? proto : builder; - return p.getPreemptedMemorySeconds(); + Map<String, Long> tmp = getPreemptedResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; } @Override public synchronized void setPreemptedVcoreSeconds( long vcoreSeconds) { - maybeInitBuilder(); - builder.setPreemptedVcoreSeconds(vcoreSeconds); + getPreemptedResourceSecondsMap() + .put(ResourceInformation.VCORES.getName(), vcoreSeconds); } @Override public synchronized long getPreemptedVcoreSeconds() { - ApplicationResourceUsageReportProtoOrBuilder p = - viaProto ? proto : builder; - return (p.getPreemptedVcoreSeconds()); + Map<String, Long> tmp = getPreemptedResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; } private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { @@ -277,4 +313,81 @@ extends ApplicationResourceUsageReport { maybeInitBuilder(); builder.setClusterUsagePercentage((clusterUsagePerc)); } + + @Override + public synchronized void setResourceSecondsMap( + Map<String, Long> resourceSecondsMap) { + this.resourceSecondsMap = resourceSecondsMap; + if (resourceSecondsMap == null) { + return; + } + if (!resourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setMemorySeconds(0L); + } + if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) { + this.setVcoreSeconds(0L); + } + } + + @Override + public synchronized Map<String, Long> getResourceSecondsMap() { + if (this.resourceSecondsMap != null) { + return this.resourceSecondsMap; + } + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + this.resourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + if (!this.resourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setMemorySeconds(p.getMemorySeconds()); + } + if (!this.resourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.setVcoreSeconds(p.getVcoreSeconds()); + } + this.setMemorySeconds(p.getMemorySeconds()); + this.setVcoreSeconds(p.getVcoreSeconds()); + return this.resourceSecondsMap; + } + + @Override + public synchronized void setPreemptedResourceSecondsMap( + Map<String, Long> preemptedResourceSecondsMap) { + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + if (preemptedResourceSecondsMap == null) { + return; + } + if (!preemptedResourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setPreemptedMemorySeconds(0L); + } + if (!preemptedResourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.setPreemptedVcoreSeconds(0L); + } + } + + @Override + public synchronized Map<String, Long> getPreemptedResourceSecondsMap() { + if (this.preemptedResourceSecondsMap != null) { + return this.preemptedResourceSecondsMap; + } + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + this.preemptedResourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationPreemptedResourceUsageMapList()); + if (!this.preemptedResourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.setPreemptedMemorySeconds(p.getPreemptedMemorySeconds()); + } + if (!this.preemptedResourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds()); + } + this.setPreemptedMemorySeconds(p.getPreemptedMemorySeconds()); + this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds()); + return this.preemptedResourceSecondsMap; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index a80ff82..626ff9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -19,6 +19,10 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -45,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; @@ -446,6 +451,35 @@ public class ProtoUtils { public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { return ResourceTypes.valueOf(e.name()); } + + public static Map<String, Long> convertStringLongMapProtoListToMap( + List<YarnProtos.StringLongMapProto> pList) { + Resource tmp = Resource.newInstance(0, 0); + Map<String, Long> ret = new HashMap<>(); + for (Map.Entry<String, ResourceInformation> entry : tmp.getResources() + .entrySet()) { + ret.put(entry.getKey(), 0L); + } + if (pList != null) { + for (YarnProtos.StringLongMapProto p : pList) { + ret.put(p.getKey(), p.getValue()); + } + } + return ret; + } + + public static List<YarnProtos.StringLongMapProto> convertMapToStringLongMapProtoList( + Map<String, Long> map) { + List<YarnProtos.StringLongMapProto> ret = new ArrayList<>(); + for (Map.Entry<String, Long> entry : map.entrySet()) { + YarnProtos.StringLongMapProto.Builder tmp = + YarnProtos.StringLongMapProto.newBuilder(); + tmp.setKey(entry.getKey()); + tmp.setValue(entry.getValue()); + ret.add(tmp.build()); + } + return ret; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java index 0240fbc..331be30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java @@ -20,9 +20,15 @@ package org.apache.hadoop.yarn.util; import com.google.common.base.Joiner; import com.google.common.base.Splitter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.regex.Pattern; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** * Common string manipulation helpers @@ -174,4 +180,34 @@ public final class StringHelper { } sb.append(part); } + + public static String getResourceSecondsString(Map<String, Long> targetMap) { + List<String> strings = new ArrayList<>(targetMap.size()); + //completed app report in the timeline server doesn't have usage report + Long memorySeconds = 0L; + Long vcoreSeconds = 0L; + if (targetMap.containsKey(ResourceInformation.MEMORY_MB.getName())) { + memorySeconds = targetMap.get(ResourceInformation.MEMORY_MB.getName()); + } + if (targetMap.containsKey(ResourceInformation.VCORES.getName())) { + vcoreSeconds = targetMap.get(ResourceInformation.VCORES.getName()); + } + strings.add(memorySeconds + " MB-seconds"); + strings.add(vcoreSeconds + " vcore-seconds"); + Map<String, ResourceInformation> tmp = ResourceUtils.getResourceTypes(); + if (targetMap.size() > 2) { + for (Map.Entry<String, Long> entry : targetMap.entrySet()) { + if (!entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) + && !entry.getKey().equals(ResourceInformation.VCORES.getName())) { + String units = ""; + if (tmp.containsKey(entry.getKey())) { + units = tmp.get(entry.getKey()).getUnits(); + } + strings.add(entry.getValue() + " " + entry.getKey() + "-" + units + + "seconds"); + } + } + } + return String.join(", ", strings); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java index 82170b3..8694651 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -37,6 +37,9 @@ public class BasePBImplRecordsTest { @SuppressWarnings("checkstyle:visibilitymodifier") protected static HashMap<Type, Object> typeValueCache = new HashMap<Type, Object>(); + @SuppressWarnings("checkstyle:visibilitymodifier") + protected static HashMap<Type, List<String>> excludedPropertiesMap = + new HashMap<>(); private static Random rand = new Random(); private static byte [] bytes = new byte[] {'1', '2', '3', '4'}; @@ -167,6 +170,10 @@ public class BasePBImplRecordsTest { private <R> Map<String, GetSetPair> getGetSetPairs(Class<R> recordClass) throws Exception { Map<String, GetSetPair> ret = new HashMap<String, GetSetPair>(); + List<String> excluded = null; + if (excludedPropertiesMap.containsKey(recordClass.getClass())) { + excluded = excludedPropertiesMap.get(recordClass.getClass()); + } Method [] methods = recordClass.getDeclaredMethods(); // get all get methods for (int i = 0; i < methods.length; i++) { @@ -224,6 +231,11 @@ public class BasePBImplRecordsTest { (gsp.setMethod == null)) { LOG.info(String.format("Exclude potential property: %s\n", gsp.propertyName)); itr.remove(); + } else if ((excluded != null && excluded.contains(gsp.propertyName))) { + LOG.info(String.format( + "Excluding potential property(present in exclusion list): %s\n", + gsp.propertyName)); + itr.remove(); } else { LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type)); gsp.testValue = genTypeValue(gsp.type); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 74701ab..77824c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -342,6 +342,8 @@ import org.junit.Ignore; import org.junit.Test; import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.Arrays; /** * Test class for YARN API protocol records. @@ -743,6 +745,8 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { @Test public void testApplicationResourceUsageReportPBImpl() throws Exception { + excludedPropertiesMap.put(ApplicationResourceUsageReportPBImpl.class.getClass(), + Arrays.asList("PreemptedResourceSecondsMap", "ResourceSecondsMap")); validatePBImplRecord(ApplicationResourceUsageReportPBImpl.class, ApplicationResourceUsageReportProto.class); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 9240ed8..0b57717 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -338,9 +339,20 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS); long preemptedVcoreSeconds = parseLong(entityInfo, ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS); - appResources = ApplicationResourceUsageReport.newInstance(0, 0, null, - null, null, memorySeconds, vcoreSeconds, 0, 0, - preemptedMemorySeconds, preemptedVcoreSeconds); + Map<String, Long> resourceSecondsMap = new HashMap<>(); + Map<String, Long> preemptedResoureSecondsMap = new HashMap<>(); + resourceSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); + resourceSecondsMap + .put(ResourceInformation.VCORES.getName(), vcoreSeconds); + preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), + preemptedMemorySeconds); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds); + + appResources = ApplicationResourceUsageReport + .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0, + preemptedResoureSecondsMap); } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index e7f47af..3b37abd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -65,8 +65,6 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; @@ -447,12 +445,12 @@ public class BuilderUtils { queue, priority, amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, resource, null); } - + public static ApplicationResourceUsageReport newApplicationResourceUsageReport( int numUsedContainers, int numReservedContainers, Resource usedResources, - Resource reservedResources, Resource neededResources, long memorySeconds, - long vcoreSeconds, long preemptedMemorySeconds, - long preemptedVcoreSeconds) { + Resource reservedResources, Resource neededResources, + Map<String, Long> resourceSecondsMap, + Map<String, Long> preemptedResourceSecondsMap) { ApplicationResourceUsageReport report = recordFactory.newRecordInstance(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); @@ -460,10 +458,8 @@ public class BuilderUtils { report.setUsedResources(usedResources); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setMemorySeconds(memorySeconds); - report.setVcoreSeconds(vcoreSeconds); - report.setPreemptedMemorySeconds(preemptedMemorySeconds); - report.setPreemptedVcoreSeconds(preemptedVcoreSeconds); + report.setResourceSecondsMap(resourceSecondsMap); + report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); return report; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index bcd1a9c..2dc8c62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.SettableFuture; +import org.apache.hadoop.yarn.util.StringHelper; /** * This class manages the list of applications for the resource manager. @@ -188,7 +189,12 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) - .add("applicationType", app.getApplicationType()); + .add("applicationType", app.getApplicationType()) + .add("resourceSeconds", StringHelper + .getResourceSecondsString(metrics.getResourceSecondsMap())) + .add("preemptedResourceSeconds", StringHelper + .getResourceSecondsString( + metrics.getPreemptedResourceSecondsMap())); return summary; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 5f590f3..d3e8345 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -483,7 +483,7 @@ public class RMServerUtils { DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = BuilderUtils.newApplicationResourceUsageReport(-1, -1, Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1), 0, 0, 0, 0); + Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>()); /** @@ -672,4 +672,12 @@ public class RMServerUtils { .debug("Converted profile to resource capability for ask " + ask); } } + + public static Long getOrDefault(Map<String, Long> map, String key, + Long defaultValue) { + if (map.containsKey(key)) { + return map.get(key); + } + return defaultValue; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 35340e6..00ef39f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -851,11 +851,8 @@ public abstract class RMStateStore extends AbstractService { appAttempt.getAppAttemptId(), appAttempt.getMasterContainer(), credentials, appAttempt.getStartTime(), - resUsage.getMemorySeconds(), - resUsage.getVcoreSeconds(), - attempMetrics.getPreemptedMemory(), - attempMetrics.getPreemptedVcore() - ); + resUsage.getResourceUsageSecondsMap(), + attempMetrics.getPreemptedResourceSecondsMap()); getRMStateStoreEventHandler().handle( new RMStateStoreAppAttemptEvent(attemptState)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java index 67aaf94..2de071a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java @@ -25,23 +25,28 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.Records; +import java.util.Map; + /* * Contains the state data that needs to be persisted for an ApplicationAttempt */ @Public @Unstable public abstract class ApplicationAttemptStateData { + public static ApplicationAttemptStateData newInstance( ApplicationAttemptId attemptId, Container container, Credentials attemptTokens, long startTime, RMAppAttemptState finalState, String finalTrackingUrl, String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, - long finishTime, long memorySeconds, long vcoreSeconds, - long preemptedMemorySeconds, long preemptedVcoreSeconds) { + long finishTime, Map<String, Long> resourceSecondsMap, + Map<String, Long> preemptedResourceSecondsMap) { ApplicationAttemptStateData attemptStateData = Records.newRecord(ApplicationAttemptStateData.class); attemptStateData.setAttemptId(attemptId); @@ -54,23 +59,33 @@ public abstract class ApplicationAttemptStateData { attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus); attemptStateData.setAMContainerExitStatus(exitStatus); attemptStateData.setFinishTime(finishTime); - attemptStateData.setMemorySeconds(memorySeconds); - attemptStateData.setVcoreSeconds(vcoreSeconds); - attemptStateData.setPreemptedMemorySeconds(preemptedMemorySeconds); - attemptStateData.setPreemptedVcoreSeconds(preemptedVcoreSeconds); + attemptStateData.setMemorySeconds(RMServerUtils + .getOrDefault(resourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L)); + attemptStateData.setVcoreSeconds(RMServerUtils + .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), + 0L)); + attemptStateData.setPreemptedMemorySeconds(RMServerUtils + .getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L)); + attemptStateData.setPreemptedVcoreSeconds(RMServerUtils + .getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L)); + attemptStateData.setResourceSecondsMap(resourceSecondsMap); + attemptStateData + .setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); return attemptStateData; } public static ApplicationAttemptStateData newInstance( ApplicationAttemptId attemptId, Container masterContainer, - Credentials attemptTokens, long startTime, long memorySeconds, - long vcoreSeconds, long preemptedMemorySeconds, - long preemptedVcoreSeconds) { - return newInstance(attemptId, masterContainer, attemptTokens, - startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0, - memorySeconds, vcoreSeconds, - preemptedMemorySeconds, preemptedVcoreSeconds); - } + Credentials attemptTokens, long startTime, + Map<String, Long> resourceSeondsMap, + Map<String, Long> preemptedResourceSecondsMap) { + return newInstance(attemptId, masterContainer, attemptTokens, startTime, + null, "N/A", "", null, ContainerExitStatus.INVALID, 0, + resourceSeondsMap, preemptedResourceSecondsMap); + } public abstract ApplicationAttemptStateDataProto getProto(); @@ -215,4 +230,50 @@ public abstract class ApplicationAttemptStateData { @Public @Unstable public abstract void setPreemptedVcoreSeconds(long vcoreSeconds); + + /** + * Get the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated preempted + * resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getResourceSecondsMap(); + + /** + * Set the aggregated number of resources that the application has + * allocated times the number of seconds the application has been running. + * + * @param resourceSecondsMap map containing the resource name and aggregated + * resource-seconds + */ + @Public + @Unstable + public abstract void setResourceSecondsMap( + Map<String, Long> resourceSecondsMap); + + /** + * Get the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated preempted + * resource-seconds + */ + @Public + @Unstable + public abstract Map<String, Long> getPreemptedResourceSecondsMap(); + + /** + * Set the aggregated number of resources preempted that the application has + * allocated times the number of seconds the application has been running. + * + * @param preemptedResourceSecondsMap map containing the resource name and + * aggregated preempted resource-seconds + */ + @Public + @Unstable + public abstract void setPreemptedResourceSecondsMap( + Map<String, Long> preemptedResourceSecondsMap); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java index e89726f..ed71ea2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -55,6 +56,9 @@ public class ApplicationAttemptStateDataPBImpl extends private Container masterContainer = null; private ByteBuffer appAttemptTokens = null; + private Map<String, Long> resourceSecondsMap; + private Map<String, Long> preemptedResourceSecondsMap; + public ApplicationAttemptStateDataPBImpl() { builder = ApplicationAttemptStateDataProto.newBuilder(); } @@ -404,4 +408,50 @@ public class ApplicationAttemptStateDataPBImpl extends IOUtils.closeStream(dibb); } } + + @Override + public Map<String, Long> getResourceSecondsMap() { + if (this.resourceSecondsMap != null) { + return this.resourceSecondsMap; + } + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + return this.resourceSecondsMap; + } + + @Override + public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) { + maybeInitBuilder(); + builder.clearApplicationResourceUsageMap(); + this.resourceSecondsMap = resourceSecondsMap; + if (resourceSecondsMap != null) { + builder.addAllApplicationResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + } + } + + @Override + public Map<String, Long> getPreemptedResourceSecondsMap() { + if (this.preemptedResourceSecondsMap != null) { + return this.preemptedResourceSecondsMap; + } + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + this.preemptedResourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + return this.preemptedResourceSecondsMap; + } + + @Override + public void setPreemptedResourceSecondsMap( + Map<String, Long> preemptedResourceSecondsMap) { + maybeInitBuilder(); + builder.clearPreemptedResourceUsageMap(); + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + if (preemptedResourceSecondsMap != null) { + builder.addAllPreemptedResourceUsageMap(ProtoUtils + .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 03be793..1e14409 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -741,14 +741,10 @@ public class RMAppImpl implements RMApp, Recoverable { } RMAppMetrics rmAppMetrics = getRMAppMetrics(); - appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds()); - appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds()); - appUsageReport. - setPreemptedMemorySeconds(rmAppMetrics. - getPreemptedMemorySeconds()); - appUsageReport. - setPreemptedVcoreSeconds(rmAppMetrics. - getPreemptedVcoreSeconds()); + appUsageReport + .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap()); + appUsageReport.setPreemptedResourceSecondsMap( + rmAppMetrics.getPreemptedResourceSecondsMap()); } if (currentApplicationAttemptId == null) { @@ -1625,10 +1621,9 @@ public class RMAppImpl implements RMApp, Recoverable { Resource resourcePreempted = Resource.newInstance(0, 0); int numAMContainerPreempted = 0; int numNonAMContainerPreempted = 0; - long memorySeconds = 0; - long vcoreSeconds = 0; - long preemptedMemorySeconds = 0; - long preemptedVcoreSeconds = 0; + Map<String, Long> resourceSecondsMap = new HashMap<>(); + Map<String, Long> preemptedSecondsMap = new HashMap<>(); + for (RMAppAttempt attempt : attempts.values()) { if (null != attempt) { RMAppAttemptMetrics attemptMetrics = @@ -1642,17 +1637,25 @@ public class RMAppImpl implements RMApp, Recoverable { // for both running and finished containers. AggregateAppResourceUsage resUsage = attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); - memorySeconds += resUsage.getMemorySeconds(); - vcoreSeconds += resUsage.getVcoreSeconds(); - preemptedMemorySeconds += attemptMetrics.getPreemptedMemory(); - preemptedVcoreSeconds += attemptMetrics.getPreemptedVcore(); + for (Map.Entry<String, Long> entry : resUsage + .getResourceUsageSecondsMap().entrySet()) { + long value = RMServerUtils + .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); + value += entry.getValue(); + resourceSecondsMap.put(entry.getKey(), value); + } + for (Map.Entry<String, Long> entry : attemptMetrics + .getPreemptedResourceSecondsMap().entrySet()) { + long value = RMServerUtils + .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L); + value += entry.getValue(); + preemptedSecondsMap.put(entry.getKey(), value); + } } } - return new RMAppMetrics(resourcePreempted, - numNonAMContainerPreempted, numAMContainerPreempted, - memorySeconds, vcoreSeconds, - preemptedMemorySeconds, preemptedVcoreSeconds); + return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted, + numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap); } @Private http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index fa068ea..2bb7fd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -19,27 +19,27 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; + +import java.util.Map; public class RMAppMetrics { final Resource resourcePreempted; final int numNonAMContainersPreempted; final int numAMContainersPreempted; - final long memorySeconds; - final long vcoreSeconds; - private final long preemptedMemorySeconds; - private final long preemptedVcoreSeconds; + private final Map<String, Long> resourceSecondsMap; + private final Map<String, Long> preemptedResourceSecondsMap; public RMAppMetrics(Resource resourcePreempted, int numNonAMContainersPreempted, int numAMContainersPreempted, - long memorySeconds, long vcoreSeconds, long preemptedMemorySeconds, - long preemptedVcoreSeconds) { + Map<String, Long> resourceSecondsMap, + Map<String, Long> preemptedResourceSecondsMap) { this.resourcePreempted = resourcePreempted; this.numNonAMContainersPreempted = numNonAMContainersPreempted; this.numAMContainersPreempted = numAMContainersPreempted; - this.memorySeconds = memorySeconds; - this.vcoreSeconds = vcoreSeconds; - this.preemptedMemorySeconds = preemptedMemorySeconds; - this.preemptedVcoreSeconds = preemptedVcoreSeconds; + this.resourceSecondsMap = resourceSecondsMap; + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; } public Resource getResourcePreempted() { @@ -55,19 +55,32 @@ public class RMAppMetrics { } public long getMemorySeconds() { - return memorySeconds; + return RMServerUtils.getOrDefault(resourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } public long getVcoreSeconds() { - return vcoreSeconds; + return RMServerUtils + .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), + 0L); } public long getPreemptedMemorySeconds() { - return preemptedMemorySeconds; + return RMServerUtils.getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } public long getPreemptedVcoreSeconds() { - return preemptedVcoreSeconds; + return RMServerUtils.getOrDefault(preemptedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } + + public Map<String, Long> getResourceSecondsMap() { + return resourceSecondsMap; + } + + public Map<String, Long> getPreemptedResourceSecondsMap() { + return preemptedResourceSecondsMap; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java index f0c2b34..b858712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java @@ -19,42 +19,38 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; + +import java.util.HashMap; +import java.util.Map; @Private public class AggregateAppResourceUsage { - long memorySeconds; - long vcoreSeconds; + private Map<String, Long> resourceSecondsMap = new HashMap<>(); - public AggregateAppResourceUsage(long memorySeconds, long vcoreSeconds) { - this.memorySeconds = memorySeconds; - this.vcoreSeconds = vcoreSeconds; + public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) { + this.resourceSecondsMap.putAll(resourceSecondsMap); } /** * @return the memorySeconds */ public long getMemorySeconds() { - return memorySeconds; - } - - /** - * @param memorySeconds the memorySeconds to set - */ - public void setMemorySeconds(long memorySeconds) { - this.memorySeconds = memorySeconds; + return RMServerUtils.getOrDefault(resourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } /** * @return the vcoreSeconds */ public long getVcoreSeconds() { - return vcoreSeconds; + return RMServerUtils + .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), + 0L); } - /** - * @param vcoreSeconds the vcoreSeconds to set - */ - public void setVcoreSeconds(long vcoreSeconds) { - this.vcoreSeconds = vcoreSeconds; + public Map<String, Long> getResourceUsageSecondsMap() { + return resourceSecondsMap; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 7d453bd..5d0bd44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -943,12 +943,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage(); - report.setMemorySeconds(resUsage.getMemorySeconds()); - report.setVcoreSeconds(resUsage.getVcoreSeconds()); - report.setPreemptedMemorySeconds( - this.attemptMetrics.getPreemptedMemory()); - report.setPreemptedVcoreSeconds( - this.attemptMetrics.getPreemptedVcore()); + report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap()); + report.setPreemptedResourceSecondsMap( + this.attemptMetrics.getPreemptedResourceSecondsMap()); return report; } finally { this.readLock.unlock(); @@ -985,11 +982,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.finalStatus = attemptState.getFinalApplicationStatus(); this.startTime = attemptState.getStartTime(); this.finishTime = attemptState.getFinishTime(); - this.attemptMetrics.updateAggregateAppResourceUsage( - attemptState.getMemorySeconds(), attemptState.getVcoreSeconds()); + this.attemptMetrics + .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap()); this.attemptMetrics.updateAggregatePreemptedAppResourceUsage( - attemptState.getPreemptedMemorySeconds(), - attemptState.getPreemptedVcoreSeconds()); + attemptState.getPreemptedResourceSecondsMap()); } public void transferStateFromAttempt(RMAppAttempt attempt) { @@ -1365,16 +1361,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMStateStore rmStore = rmContext.getStateStore(); setFinishTime(System.currentTimeMillis()); - ApplicationAttemptStateData attemptState = - ApplicationAttemptStateData.newInstance( - applicationAttemptId, getMasterContainer(), - rmStore.getCredentialsFromAppAttempt(this), - startTime, stateToBeStored, finalTrackingUrl, diags.toString(), - finalStatus, exitStatus, - getFinishTime(), resUsage.getMemorySeconds(), - resUsage.getVcoreSeconds(), - this.attemptMetrics.getPreemptedMemory(), - this.attemptMetrics.getPreemptedVcore()); + ApplicationAttemptStateData attemptState = ApplicationAttemptStateData + .newInstance(applicationAttemptId, getMasterContainer(), + rmStore.getCredentialsFromAppAttempt(this), startTime, + stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, + getFinishTime(), resUsage.getResourceUsageSecondsMap(), + this.attemptMetrics.getPreemptedResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index 0655609..8ef6d99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -25,11 +27,13 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -49,10 +53,8 @@ public class RMAppAttemptMetrics { private ReadLock readLock; private WriteLock writeLock; - private AtomicLong finishedMemorySeconds = new AtomicLong(0); - private AtomicLong finishedVcoreSeconds = new AtomicLong(0); - private AtomicLong preemptedMemorySeconds = new AtomicLong(0); - private AtomicLong preemptedVcoreSeconds = new AtomicLong(0); + private Map<String, AtomicLong> resourceUsageMap = new HashMap<>(); + private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>(); private RMContext rmContext; private int[][] localityStatistics = @@ -102,11 +104,16 @@ public class RMAppAttemptMetrics { } public long getPreemptedMemory() { - return preemptedMemorySeconds.get(); + return preemptedResourceMap.get(ResourceInformation.MEMORY_MB.getName()) + .get(); } public long getPreemptedVcore() { - return preemptedVcoreSeconds.get(); + return preemptedResourceMap.get(ResourceInformation.VCORES.getName()).get(); + } + + public Map<String, Long> getPreemptedResourceSecondsMap() { + return convertAtomicLongMaptoLongMap(preemptedResourceMap); } public int getNumNonAMContainersPreempted() { @@ -122,35 +129,90 @@ public class RMAppAttemptMetrics { } public AggregateAppResourceUsage getAggregateAppResourceUsage() { - long memorySeconds = finishedMemorySeconds.get(); - long vcoreSeconds = finishedVcoreSeconds.get(); + Map<String, Long> resourcesUsed = + convertAtomicLongMaptoLongMap(resourceUsageMap); // Only add in the running containers if this is the active attempt. RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId()); - if (null != rmApp) { - RMAppAttempt currentAttempt = rmApp.getCurrentAppAttempt(); + if (rmApp != null) { + RMAppAttempt currentAttempt = rmContext.getRMApps().get(attemptId.getApplicationId()).getCurrentAppAttempt(); if (currentAttempt.getAppAttemptId().equals(attemptId)) { - ApplicationResourceUsageReport appResUsageReport = rmContext - .getScheduler().getAppResourceUsageReport(attemptId); + ApplicationResourceUsageReport appResUsageReport = + rmContext.getScheduler().getAppResourceUsageReport(attemptId); if (appResUsageReport != null) { - memorySeconds += appResUsageReport.getMemorySeconds(); - vcoreSeconds += appResUsageReport.getVcoreSeconds(); + Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap(); + for (Map.Entry<String, Long> entry : tmp.entrySet()) { + if (resourcesUsed.containsKey(entry.getKey())) { + Long value = resourcesUsed.get(entry.getKey()); + value += entry.getValue(); + resourcesUsed.put(entry.getKey(), value); + } else{ + resourcesUsed.put(entry.getKey(), entry.getValue()); + } + } } } } - return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds); + return new AggregateAppResourceUsage(resourcesUsed); + } + + public void updateAggregateAppResourceUsage(Resource allocated, + long deltaUsedMillis) { + updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap); + } + + public void updateAggregatePreemptedAppResourceUsage(Resource allocated, + long deltaUsedMillis) { + updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap); } - public void updateAggregateAppResourceUsage(long finishedMemorySeconds, - long finishedVcoreSeconds) { - this.finishedMemorySeconds.addAndGet(finishedMemorySeconds); - this.finishedVcoreSeconds.addAndGet(finishedVcoreSeconds); + public void updateAggregateAppResourceUsage( + Map<String, Long> resourceSecondsMap) { + updateUsageMap(resourceSecondsMap, resourceUsageMap); } public void updateAggregatePreemptedAppResourceUsage( - long preemptedMemorySeconds, long preemptedVcoreSeconds) { - this.preemptedMemorySeconds.addAndGet(preemptedMemorySeconds); - this.preemptedVcoreSeconds.addAndGet(preemptedVcoreSeconds); + Map<String, Long> preemptedResourceSecondsMap) { + updateUsageMap(preemptedResourceSecondsMap, preemptedResourceMap); + } + + private void updateUsageMap(Resource allocated, long deltaUsedMillis, + Map<String, AtomicLong> targetMap) { + for (Map.Entry<String, ResourceInformation> entry : allocated.getResources() + .entrySet()) { + AtomicLong resourceUsed; + if (!targetMap.containsKey(entry.getKey())) { + resourceUsed = new AtomicLong(0); + targetMap.put(entry.getKey(), resourceUsed); + + } + resourceUsed = targetMap.get(entry.getKey()); + resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis) + / DateUtils.MILLIS_PER_SECOND); + } + } + + private void updateUsageMap(Map<String, Long> sourceMap, + Map<String, AtomicLong> targetMap) { + for (Map.Entry<String, Long> entry : sourceMap.entrySet()) { + AtomicLong resourceUsed; + if (!targetMap.containsKey(entry.getKey())) { + resourceUsed = new AtomicLong(0); + targetMap.put(entry.getKey(), resourceUsed); + + } + resourceUsed = targetMap.get(entry.getKey()); + resourceUsed.set(entry.getValue()); + } + } + + private Map<String, Long> convertAtomicLongMaptoLongMap( + Map<String, AtomicLong> source) { + Map<String, Long> ret = new HashMap<>(); + for (Map.Entry<String, AtomicLong> entry : source.entrySet()) { + ret.put(entry.getKey(), entry.getValue().get()); + } + return ret; } public void incNumAllocatedContainers(NodeType containerType, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index f49db7e..2e0f202 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -25,7 +25,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; -import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -713,20 +712,15 @@ public class RMContainerImpl implements RMContainer { if (rmAttempt != null) { long usedMillis = container.finishTime - container.creationTime; - long memorySeconds = resource.getMemorySize() - * usedMillis / DateUtils.MILLIS_PER_SECOND; - long vcoreSeconds = resource.getVirtualCores() - * usedMillis / DateUtils.MILLIS_PER_SECOND; rmAttempt.getRMAppAttemptMetrics() - .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds); + .updateAggregateAppResourceUsage(resource, usedMillis); // If this is a preempted container, update preemption metrics if (ContainerExitStatus.PREEMPTED == container.finishedStatus - .getExitStatus()) { - rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource, - container); + .getExitStatus()) { rmAttempt.getRMAppAttemptMetrics() - .updateAggregatePreemptedAppResourceUsage(memorySeconds, - vcoreSeconds); + .updatePreemptionInfo(resource, container); + rmAttempt.getRMAppAttemptMetrics() + .updateAggregatePreemptedAppResourceUsage(resource, usedMillis); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb5a99d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index cc14a1e..77c5e52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -55,11 +55,13 @@ import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.ContainerType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; @@ -106,9 +108,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000; protected long lastMemoryAggregateAllocationUpdateTime = 0; - private long lastMemorySeconds = 0; - private long lastVcoreSeconds = 0; - + private Map<String, Long> lastResourceSecondsMap = new HashMap<>(); protected final AppSchedulingInfo appSchedulingInfo; protected ApplicationAttemptId attemptId; protected Map<ContainerId, RMContainer> liveContainers = @@ -997,22 +997,24 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { // recently. if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) { - long memorySeconds = 0; - long vcoreSeconds = 0; + Map<String, Long> resourceSecondsMap = new HashMap<>(); for (RMContainer rmContainer : this.liveContainers.values()) { long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); Resource resource = rmContainer.getContainer().getResource(); - memorySeconds += resource.getMemorySize() * usedMillis / - DateUtils.MILLIS_PER_SECOND; - vcoreSeconds += resource.getVirtualCores() * usedMillis - / DateUtils.MILLIS_PER_SECOND; + for (Map.Entry<String, ResourceInformation> entry : resource + .getResources().entrySet()) { + long value = RMServerUtils + .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); + value += entry.getValue().getValue() * usedMillis + / DateUtils.MILLIS_PER_SECOND; + resourceSecondsMap.put(entry.getKey(), value); + } } lastMemoryAggregateAllocationUpdateTime = currentTimeMillis; - lastMemorySeconds = memorySeconds; - lastVcoreSeconds = vcoreSeconds; + lastResourceSecondsMap = resourceSecondsMap; } - return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds); + return new AggregateAppResourceUsage(lastResourceSecondsMap); } public ApplicationResourceUsageReport getResourceUsageReport() { @@ -1027,6 +1029,11 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { Resource cluster = rmContext.getScheduler().getClusterResource(); ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); + Map<String, Long> preemptedResourceSecondsMaps = new HashMap<>(); + preemptedResourceSecondsMaps + .put(ResourceInformation.MEMORY_MB.getName(), 0L); + preemptedResourceSecondsMaps + .put(ResourceInformation.VCORES.getName(), 0L); float queueUsagePerc = 0.0f; float clusterUsagePerc = 0.0f; if (!calc.isInvalidDivisor(cluster)) { @@ -1036,15 +1043,15 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { queueUsagePerc = calc.divide(cluster, usedResourceClone, Resources.multiply(cluster, queueCapacityPerc)) * 100; } - clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster) - * 100; + clusterUsagePerc = + calc.divide(cluster, usedResourceClone, cluster) * 100; } - return ApplicationResourceUsageReport.newInstance(liveContainers.size(), - reservedContainers.size(), usedResourceClone, reservedResourceClone, - Resources.add(usedResourceClone, reservedResourceClone), - runningResourceUsage.getMemorySeconds(), - runningResourceUsage.getVcoreSeconds(), queueUsagePerc, - clusterUsagePerc, 0, 0); + return ApplicationResourceUsageReport + .newInstance(liveContainers.size(), reservedContainers.size(), + usedResourceClone, reservedResourceClone, + Resources.add(usedResourceClone, reservedResourceClone), + runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc, + clusterUsagePerc, preemptedResourceSecondsMaps); } finally { writeLock.unlock(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
