Github user squito commented on a diff in the pull request:
https://github.com/apache/spark/pull/22612#discussion_r226522582
--- Diff:
core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala ---
@@ -48,17 +56,47 @@ private[spark] abstract class
MBeanExecutorMetricType(mBeanName: String)
}
case object JVMHeapMemory extends ExecutorMetricType {
+ override private[spark] def getMetricSet(memoryManager: MemoryManager):
Map[String, Long] = {
+ var metricAsSet = Map.empty[String, Long]
+ metricAsSet += (name ->
ManagementFactory.getMemoryMXBean.getHeapMemoryUsage().getUsed())
+ metricAsSet
+ }
override private[spark] def getMetricValue(memoryManager:
MemoryManager): Long = {
ManagementFactory.getMemoryMXBean.getHeapMemoryUsage().getUsed()
}
}
case object JVMOffHeapMemory extends ExecutorMetricType {
+ override private[spark] def getMetricSet(memoryManager: MemoryManager):
Map[String, Long] = {
+ var metricAsSet = Map.empty[String, Long]
+ metricAsSet += (name ->
ManagementFactory.getMemoryMXBean.getNonHeapMemoryUsage().getUsed())
+ metricAsSet
+ }
override private[spark] def getMetricValue(memoryManager:
MemoryManager): Long = {
ManagementFactory.getMemoryMXBean.getNonHeapMemoryUsage().getUsed()
}
}
+case object ProcessTreeMetrics extends ExecutorMetricType {
+ override private[spark] def getMetricSet(memoryManager: MemoryManager):
Map[String, Long] = {
+ ExecutorMetricType.pTreeInfo.computeAllMetrics()
+ var processTreeMetrics = Map.empty[String, Long]
+ processTreeMetrics += ("ProcessTreeJVMVMemory" ->
+ ExecutorMetricType.pTreeInfo.allMetrics.jvmVmemTotal )
+ processTreeMetrics += ("ProcessTreeJVMRSSMemory" ->
+ ExecutorMetricType.pTreeInfo.allMetrics.jvmRSSTotal )
+ processTreeMetrics += ("ProcessTreePythonVMemory" ->
+ ExecutorMetricType.pTreeInfo.allMetrics.pythonVmemTotal )
+ processTreeMetrics += ("ProcessTreePythonRSSMemory" ->
+ ExecutorMetricType.pTreeInfo.allMetrics.pythonRSSTotal )
+ processTreeMetrics += ("ProcessTreeOtherVMemory" ->
+ ExecutorMetricType.pTreeInfo.allMetrics.otherVmemTotal )
+ processTreeMetrics += ("ProcessTreeOtherRSSMemory" ->
+ ExecutorMetricType.pTreeInfo.allMetrics.otherRSSTotal )
+ processTreeMetrics
--- End diff --
I'd make `computeAllMetrics` just return the latest metrics, and then this
can just be
```scala
val allMetrics = ExecutorMetricType.pTreeInfo.computeAllMetrics()
Map(
"ProcessTreeJVMMemory" -> allMetrics.jvmVMemTotal,
"ProcessTreeJVMRSSMemory" -> allMetrics.jvmRssTotal,
...
)
```
(also some of this munging might actually be easier if the values were
stored in arrays as I mentioned below)
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
