This is an automated email from the ASF dual-hosted git repository. epayne pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 6f2226a YARN-9756: Create metric that sums total memory/vcores preempted per round. Contributed by Manikandan R (manirajv06). 6f2226a is described below commit 6f2226a013daf30b00c6676a5f12160ac0b197b7 Author: Eric E Payne <er...@verizonmedia.com> AuthorDate: Wed Aug 28 20:40:34 2019 +0000 YARN-9756: Create metric that sums total memory/vcores preempted per round. Contributed by Manikandan R (manirajv06). --- .../resourcemanager/scheduler/QueueMetrics.java | 31 ++++++++++++++++++++++ .../scheduler/QueueMetricsForCustomResources.java | 6 +++++ .../scheduler/capacity/LeafQueue.java | 2 ++ .../TestCapacitySchedulerSurgicalPreemption.java | 7 +++++ 4 files changed, 46 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index c126338..ed5f546 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -77,6 +77,10 @@ public class QueueMetrics implements MetricsSource { @Metric("# of active applications") MutableGaugeInt activeApplications; @Metric("App Attempt First Container Allocation Delay") MutableRate appAttemptFirstContainerAllocationDelay; + @Metric("Aggregate total of preempted memory MB") + MutableCounterLong aggregateMemoryMBPreempted; + @Metric("Aggregate total of preempted vcores") + MutableCounterLong aggregateVcoresPreempted; //Metrics updated only for "default" partition @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB; @@ -586,6 +590,23 @@ public class QueueMetrics implements MetricsSource { } } + public void updatePreemptedResources(Resource res) { + aggregateMemoryMBPreempted.incr(res.getMemorySize()); + aggregateVcoresPreempted.incr(res.getVirtualCores()); + if (parent != null) { + parent.updatePreemptedResources(res); + } + } + + public void updatePreemptedForCustomResources(Resource res) { + if (queueMetricsForCustomResources != null) { + queueMetricsForCustomResources.increaseAggregatedPreempted(res); + } + if (parent != null) { + parent.updatePreemptedForCustomResources(res); + } + } + public void updatePreemptedSecondsForCustomResources(Resource res, long seconds) { if (queueMetricsForCustomResources != null) { @@ -753,6 +774,16 @@ public class QueueMetrics implements MetricsSource { return aggregateVcoreSecondsPreempted; } + @VisibleForTesting + public long getAggregateMemoryMBPreempted() { + return aggregateMemoryMBPreempted.value(); + } + + @VisibleForTesting + public long getAggregateVcoresPreempted() { + return aggregateVcoresPreempted.value(); + } + public long getAllocatedMB() { return allocatedMB.value(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java index 3470858..ddfffc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java @@ -28,6 +28,8 @@ import java.util.Map; public class QueueMetricsForCustomResources { private final QueueMetricsCustomResource aggregatePreemptedSeconds = new QueueMetricsCustomResource(); + private final QueueMetricsCustomResource aggregatePreempted = + new QueueMetricsCustomResource(); private final QueueMetricsCustomResource allocated = new QueueMetricsCustomResource(); private final QueueMetricsCustomResource available = @@ -82,6 +84,10 @@ public class QueueMetricsForCustomResources { aggregatePreemptedSeconds.increaseWithMultiplier(res, seconds); } + public void increaseAggregatedPreempted(Resource res) { + aggregatePreempted.increase(res); + } + Map<String, Long> getAllocatedValues() { return allocated.getValues(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index a178f9e..4d2b8e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -2222,7 +2222,9 @@ public class LeafQueue extends AbstractCSQueue { / DateUtils.MILLIS_PER_SECOND; metrics.updatePreemptedMemoryMBSeconds(mbSeconds); metrics.updatePreemptedVcoreSeconds(vcSeconds); + metrics.updatePreemptedResources(containerResource); metrics.updatePreemptedSecondsForCustomResources(containerResource, usedSeconds); + metrics.updatePreemptedForCustomResources(containerResource); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index 800789a..c2f4511 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -185,6 +185,13 @@ public class TestCapacitySchedulerSurgicalPreemption Assert.assertEquals("Number of preempted containers incorrectly recorded:", 4, cs.getQueue("root").getMetrics().getAggregatePreemptedContainers()); + Assert.assertEquals("Amount of preempted memory incorrectly recorded:", + 4 * GB, + cs.getQueue("root").getMetrics().getAggregateMemoryMBPreempted()); + + Assert.assertEquals("Number of preempted vcores incorrectly recorded:", 4, + cs.getQueue("root").getMetrics().getAggregateVcoresPreempted()); + rm1.close(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org