This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c2df895 [SPARK-37493][CORE] show gc time and duration time of driver
in executors page
c2df895 is described below
commit c2df895a8cbdfcbffc320e837ad1826933685912
Author: zhoubin11 <zhoubi...@baidu.com>
AuthorDate: Mon Dec 13 08:47:01 2021 -0600
[SPARK-37493][CORE] show gc time and duration time of driver in executors
page
### What changes were proposed in this pull request?
show driver's gc time & duration time(equivalent to application time) of
driver in both driver side and history side UI
### Why are the changes needed?
help user to config driver's resource more appropriately
### Does this PR introduce _any_ user-facing change?
yes,user will see driver's gc time & duration time in executors page .
when `spark.eventLog.logStageExecutorMetrics` is enabled driver's gc time
can be logged.
before this change,user always get zero
### How was this patch tested?
unit tests
Closes #34749 from summaryzb/SPARK-37493.
Authored-by: zhoubin11 <zhoubi...@baidu.com>
Signed-off-by: Sean Owen <sro...@gmail.com>
---
.../apache/spark/metrics/ExecutorMetricType.scala | 9 ++--
.../org/apache/spark/status/AppStatusStore.scala | 54 +++++++++++++++++++-
.../complete_stage_list_json_expectation.json | 9 ++--
.../excludeOnFailure_for_stage_expectation.json | 9 ++--
...xcludeOnFailure_node_for_stage_expectation.json | 18 ++++---
.../executor_list_json_expectation.json | 7 +--
...ist_with_executor_metrics_json_expectation.json | 14 ++++--
.../executor_memory_usage_expectation.json | 32 ++++++------
...executor_node_excludeOnFailure_expectation.json | 40 ++++++++-------
...e_excludeOnFailure_unexcluding_expectation.json | 14 ++++--
.../executor_resource_information_expectation.json | 2 +-
.../failed_stage_list_json_expectation.json | 3 +-
..._json_details_with_failed_task_expectation.json | 6 ++-
.../one_stage_attempt_json_expectation.json | 6 ++-
.../one_stage_json_expectation.json | 6 ++-
.../one_stage_json_with_details_expectation.json | 6 ++-
.../stage_list_json_expectation.json | 12 +++--
...age_list_with_accumulable_json_expectation.json | 3 +-
.../stage_list_with_peak_metrics_expectation.json | 9 ++--
.../stage_with_accumulable_json_expectation.json | 6 ++-
.../stage_with_peak_metrics_expectation.json | 9 ++--
...stage_with_speculation_summary_expectation.json | 17 ++++---
.../stage_with_summaries_expectation.json | 12 +++--
.../scheduler/EventLoggingListenerSuite.scala | 58 +++++++++++-----------
.../org/apache/spark/util/JsonProtocolSuite.scala | 36 +++++++++-----
25 files changed, 260 insertions(+), 137 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
index 76e2813..a536919 100644
--- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
@@ -109,7 +109,8 @@ case object GarbageCollectionMetrics extends
ExecutorMetricType with Logging {
"MinorGCCount",
"MinorGCTime",
"MajorGCCount",
- "MajorGCTime"
+ "MajorGCTime",
+ "TotalGCTime"
)
/* We builtin some common GC collectors which categorized as young
generation and old */
@@ -136,8 +137,10 @@ case object GarbageCollectionMetrics extends
ExecutorMetricType with Logging {
}
override private[spark] def getMetricValues(memoryManager: MemoryManager):
Array[Long] = {
- val gcMetrics = new Array[Long](names.length) // minorCount, minorTime,
majorCount, majorTime
- ManagementFactory.getGarbageCollectorMXBeans.asScala.foreach { mxBean =>
+ val gcMetrics = new Array[Long](names.length)
+ val mxBeans = ManagementFactory.getGarbageCollectorMXBeans.asScala
+ gcMetrics(4) = mxBeans.map(_.getCollectionTime).sum
+ mxBeans.foreach { mxBean =>
if (youngGenerationGarbageCollector.contains(mxBean.getName)) {
gcMetrics(0) = mxBean.getCollectionCount
gcMetrics(1) = mxBean.getCollectionTime
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index ef9f55f..398cd45 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -22,7 +22,7 @@ import java.util.{List => JList}
import scala.collection.JavaConverters._
import scala.collection.mutable.HashMap
-import org.apache.spark.{JobExecutionStatus, SparkConf}
+import org.apache.spark.{JobExecutionStatus, SparkConf, SparkContext}
import org.apache.spark.status.api.v1
import org.apache.spark.storage.FallbackStorage.FALLBACK_BLOCK_MANAGER_ID
import org.apache.spark.ui.scope._
@@ -89,7 +89,57 @@ private[spark] class AppStatusStore(
} else {
base
}
- filtered.asScala.map(_.info).filter(_.id !=
FALLBACK_BLOCK_MANAGER_ID.executorId).toSeq
+ filtered.asScala.map(_.info)
+ .filter(_.id != FALLBACK_BLOCK_MANAGER_ID.executorId)
+ .map(replaceExec).toSeq
+ }
+
+ private def replaceExec(origin: v1.ExecutorSummary): v1.ExecutorSummary = {
+ if (origin.id == SparkContext.DRIVER_IDENTIFIER) {
+ replaceDriverGcTime(origin, extractGcTime(origin), extractAppTime)
+ } else {
+ origin
+ }
+ }
+
+ private def replaceDriverGcTime(source: v1.ExecutorSummary,
+ totalGcTime: Option[Long], totalAppTime: Option[Long]): v1.ExecutorSummary
= {
+ new v1.ExecutorSummary(source.id, source.hostPort, source.isActive,
source.rddBlocks,
+ source.memoryUsed, source.diskUsed, source.totalCores, source.maxTasks,
source.activeTasks,
+ source.failedTasks, source.completedTasks, source.totalTasks,
+ totalAppTime.getOrElse(source.totalDuration),
+ totalGcTime.getOrElse(source.totalGCTime),
+ source.totalInputBytes, source.totalShuffleRead,
+ source.totalShuffleWrite, source.isBlacklisted, source.maxMemory,
source.addTime,
+ source.removeTime, source.removeReason, source.executorLogs,
source.memoryMetrics,
+ source.blacklistedInStages, source.peakMemoryMetrics, source.attributes,
source.resources,
+ source.resourceProfileId, source.isExcluded, source.excludedInStages)
+ }
+
+ private def extractGcTime(source: v1.ExecutorSummary): Option[Long] = {
+ source.peakMemoryMetrics.map(_.getMetricValue("TotalGCTime"))
+ }
+
+ private def extractAppTime: Option[Long] = {
+ var startTime = 0L
+ // -1 when SparkListenerApplicationStart event written to kvStore
+ // event time when SparkListenerApplicationStart event written to kvStore
+ var endTime = 0L
+ try {
+ val appInfo = applicationInfo()
+ startTime = appInfo.attempts.head.startTime.getTime()
+ endTime = appInfo.attempts.head.endTime.getTime()
+ } catch {
+ // too early to get appInfo, should wait a while
+ case _: NoSuchElementException =>
+ }
+ if (endTime == 0) {
+ None
+ } else if (endTime < 0) {
+ Option(System.currentTimeMillis() - startTime)
+ } else {
+ Option(endTime - startTime)
+ }
}
def miscellaneousProcessList(activeOnly: Boolean): Seq[v1.ProcessSummary] = {
diff --git
a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
index f04543e..67b4d63 100644
---
a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
@@ -63,7 +63,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -130,7 +131,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -197,6 +199,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
index dcad8a6..9be6b6e 100644
---
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
@@ -718,7 +718,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : true
},
@@ -758,7 +759,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -785,6 +787,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
index 2ab1546..1661f7c 100644
---
a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
@@ -826,7 +826,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : true
},
@@ -866,7 +867,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : true
},
@@ -906,7 +908,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -946,7 +949,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -986,7 +990,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : true
}
@@ -1013,6 +1018,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index be12507..ec3fc28 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -11,8 +11,8 @@
"failedTasks" : 1,
"completedTasks" : 31,
"totalTasks" : 32,
- "totalDuration" : 8820,
- "totalGCTime" : 352,
+ "totalDuration" : 9011,
+ "totalGCTime" : 0,
"totalInputBytes" : 28000288,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 13180,
@@ -41,7 +41,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
index bf3e93f..9b7498d 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
@@ -11,7 +11,7 @@
"failedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
- "totalDuration" : 0,
+ "totalDuration" : 62168,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalShuffleRead" : 0,
@@ -47,7 +47,8 @@
"MinorGCCount" : 7,
"MinorGCTime" : 55,
"MajorGCCount" : 3,
- "MajorGCTime" : 144
+ "MajorGCTime" : 144,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -106,7 +107,8 @@
"MinorGCCount" : 24,
"MinorGCTime" : 145,
"MajorGCCount" : 2,
- "MajorGCTime" : 63
+ "MajorGCTime" : 63,
+ "TotalGCTime" : 0
},
"attributes" : {
"NM_HTTP_ADDRESS" : "test-3.vpc.company.com:8042",
@@ -175,7 +177,8 @@
"MinorGCCount" : 15,
"MinorGCTime" : 106,
"MajorGCCount" : 2,
- "MajorGCTime" : 75
+ "MajorGCTime" : 75,
+ "TotalGCTime" : 0
},
"attributes" : {
"NM_HTTP_ADDRESS" : "test-4.vpc.company.com:8042",
@@ -244,7 +247,8 @@
"MinorGCCount" : 27,
"MinorGCTime" : 140,
"MajorGCCount" : 2,
- "MajorGCTime" : 60
+ "MajorGCTime" : 60,
+ "TotalGCTime" : 0
},
"attributes" : {
"NM_HTTP_ADDRESS" : "test-2.vpc.company.com:8042",
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index 0a3eb81..fbb7b66 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -11,7 +11,7 @@
"failedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
- "totalDuration" : 0,
+ "totalDuration" : 10671,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalShuffleRead" : 0,
@@ -57,7 +57,7 @@
"stdout" :
"http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout";,
"stderr" :
"http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr";
},
- "memoryMetrics": {
+ "memoryMetrics" : {
"usedOnHeapStorageMemory" : 0,
"usedOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
@@ -84,14 +84,15 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
"resourceProfileId" : 0,
"isExcluded" : true,
"excludedInStages" : [ ]
-} ,{
+}, {
"id" : "2",
"hostPort" : "172.22.0.167:51487",
"isActive" : true,
@@ -116,7 +117,7 @@
"stdout" :
"http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout";,
"stderr" :
"http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr";
},
- "memoryMetrics": {
+ "memoryMetrics" : {
"usedOnHeapStorageMemory" : 0,
"usedOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
@@ -143,7 +144,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -175,11 +177,11 @@
"stdout" :
"http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout";,
"stderr" :
"http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr";
},
- "memoryMetrics": {
- "usedOnHeapStorageMemory": 0,
- "usedOffHeapStorageMemory": 0,
- "totalOnHeapStorageMemory": 384093388,
- "totalOffHeapStorageMemory": 524288000
+ "memoryMetrics" : {
+ "usedOnHeapStorageMemory" : 0,
+ "usedOffHeapStorageMemory" : 0,
+ "totalOnHeapStorageMemory" : 384093388,
+ "totalOffHeapStorageMemory" : 524288000
},
"blacklistedInStages" : [ ],
"peakMemoryMetrics" : {
@@ -202,7 +204,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -234,7 +237,7 @@
"stdout" :
"http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout";,
"stderr" :
"http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr";
},
- "memoryMetrics": {
+ "memoryMetrics" : {
"usedOnHeapStorageMemory" : 0,
"usedOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
@@ -261,7 +264,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
index 8869fb4..fbb7b66 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json
@@ -11,7 +11,7 @@
"failedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
- "totalDuration" : 0,
+ "totalDuration" : 10671,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalShuffleRead" : 0,
@@ -20,7 +20,7 @@
"maxMemory" : 908381388,
"addTime" : "2016-11-16T22:33:31.477GMT",
"executorLogs" : { },
- "memoryMetrics": {
+ "memoryMetrics" : {
"usedOnHeapStorageMemory" : 0,
"usedOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
@@ -57,7 +57,7 @@
"stdout" :
"http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout";,
"stderr" :
"http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr";
},
- "memoryMetrics": {
+ "memoryMetrics" : {
"usedOnHeapStorageMemory" : 0,
"usedOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
@@ -84,7 +84,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -116,7 +117,7 @@
"stdout" :
"http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout";,
"stderr" :
"http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr";
},
- "memoryMetrics": {
+ "memoryMetrics" : {
"usedOnHeapStorageMemory" : 0,
"usedOffHeapStorageMemory" : 0,
"totalOnHeapStorageMemory" : 384093388,
@@ -143,7 +144,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -175,11 +177,11 @@
"stdout" :
"http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout";,
"stderr" :
"http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr";
},
- "memoryMetrics": {
- "usedOnHeapStorageMemory": 0,
- "usedOffHeapStorageMemory": 0,
- "totalOnHeapStorageMemory": 384093388,
- "totalOffHeapStorageMemory": 524288000
+ "memoryMetrics" : {
+ "usedOnHeapStorageMemory" : 0,
+ "usedOffHeapStorageMemory" : 0,
+ "totalOnHeapStorageMemory" : 384093388,
+ "totalOffHeapStorageMemory" : 524288000
},
"blacklistedInStages" : [ ],
"peakMemoryMetrics" : {
@@ -202,7 +204,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -234,11 +237,11 @@
"stdout" :
"http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout";,
"stderr" :
"http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr";
},
- "memoryMetrics": {
- "usedOnHeapStorageMemory": 0,
- "usedOffHeapStorageMemory": 0,
- "totalOnHeapStorageMemory": 384093388,
- "totalOffHeapStorageMemory": 524288000
+ "memoryMetrics" : {
+ "usedOnHeapStorageMemory" : 0,
+ "usedOffHeapStorageMemory" : 0,
+ "totalOnHeapStorageMemory" : 384093388,
+ "totalOffHeapStorageMemory" : 524288000
},
"blacklistedInStages" : [ ],
"peakMemoryMetrics" : {
@@ -261,7 +264,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
index 21cc9d0..b72ed0a 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json
@@ -11,7 +11,7 @@
"failedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
- "totalDuration" : 0,
+ "totalDuration" : 101795,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalShuffleRead" : 0,
@@ -72,7 +72,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -125,7 +126,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -178,7 +180,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
@@ -231,7 +234,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"attributes" : { },
"resources" : { },
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json
index 53ae9a0..2caa11e 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json
@@ -11,7 +11,7 @@
"failedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
- "totalDuration" : 0,
+ "totalDuration" : 18794,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalShuffleRead" : 0,
diff --git
a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
index 5573cf9..e1fd06b 100644
---
a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
@@ -64,6 +64,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
index 4579d3b..18dac90 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
@@ -79,7 +79,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -106,6 +107,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 9edb518..94d343d 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -480,7 +480,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -507,6 +508,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 9e661bd..6713486 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -480,7 +480,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -507,6 +508,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
index 9e661bd..6713486 100644
---
a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
@@ -480,7 +480,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -507,6 +508,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
index d109c73..05e47ff 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
@@ -63,7 +63,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}, {
"status" : "FAILED",
@@ -131,7 +132,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -198,7 +200,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -265,6 +268,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
index 7901c4f..04d7493 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
@@ -67,6 +67,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
index d455b97..f709724 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
@@ -63,7 +63,8 @@
"MinorGCCount" : 13,
"MinorGCTime" : 115,
"MajorGCCount" : 4,
- "MajorGCTime" : 339
+ "MajorGCTime" : 339,
+ "TotalGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -131,7 +132,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}, {
"status" : "COMPLETE",
@@ -199,6 +201,7 @@
"MinorGCCount" : 7,
"MinorGCTime" : 33,
"MajorGCCount" : 3,
- "MajorGCTime" : 110
+ "MajorGCTime" : 110,
+ "TotalGCTime" : 0
}
} ]
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index a5958e0..281f7b3 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -524,7 +524,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -551,6 +552,7 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
index 20a9580..5b2c205 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
@@ -928,7 +928,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -968,7 +969,8 @@
"MinorGCCount" : 13,
"MinorGCTime" : 115,
"MajorGCCount" : 4,
- "MajorGCTime" : 339
+ "MajorGCTime" : 339,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -995,6 +997,7 @@
"MinorGCCount" : 13,
"MinorGCTime" : 115,
"MajorGCCount" : 4,
- "MajorGCTime" : 339
+ "MajorGCTime" : 339,
+ "TotalGCTime" : 0
}
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
index 5f6090d..32cf571 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
@@ -346,7 +346,8 @@
"MinorGCCount" : 2,
"MinorGCTime" : 280,
"MajorGCCount" : 2,
- "MajorGCTime" : 1116
+ "MajorGCTime" : 1116,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -386,7 +387,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -426,7 +428,8 @@
"MinorGCCount" : 2,
"MinorGCTime" : 587,
"MajorGCCount" : 2,
- "MajorGCTime" : 906
+ "MajorGCTime" : 906,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -466,7 +469,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -502,6 +506,7 @@
"MinorGCCount" : 2,
"MinorGCTime" : 587,
"MajorGCCount" : 2,
- "MajorGCTime" : 1116
+ "MajorGCTime" : 1116,
+ "TotalGCTime" : 0
}
-}
\ No newline at end of file
+}
diff --git
a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
index f200da3..3be20df 100644
---
a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
@@ -928,7 +928,8 @@
"MinorGCCount" : 0,
"MinorGCTime" : 0,
"MajorGCCount" : 0,
- "MajorGCTime" : 0
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
},
@@ -968,7 +969,8 @@
"MinorGCCount" : 13,
"MinorGCTime" : 115,
"MajorGCCount" : 4,
- "MajorGCTime" : 339
+ "MajorGCTime" : 339,
+ "TotalGCTime" : 0
},
"isExcludedForStage" : false
}
@@ -995,7 +997,8 @@
"MinorGCCount" : 13,
"MinorGCTime" : 115,
"MajorGCCount" : 4,
- "MajorGCTime" : 339
+ "MajorGCTime" : 339,
+ "TotalGCTime" : 0
},
"taskMetricsDistributions" : {
"quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ],
@@ -1072,7 +1075,8 @@
"MinorGCCount" : [ 0.0, 0.0, 13.0, 13.0, 13.0 ],
"MinorGCTime" : [ 0.0, 0.0, 115.0, 115.0, 115.0 ],
"MajorGCCount" : [ 0.0, 0.0, 4.0, 4.0, 4.0 ],
- "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ]
+ "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ],
+ "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
}
}
}
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index 09ad223..b06e83e 100644
---
a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -337,55 +337,55 @@ class EventLoggingListenerSuite extends SparkFunSuite
with LocalSparkContext wit
// Executor metrics
// driver
val md_1 = Array(4000L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, 70L, 0L, 7500L,
3500L,
- 0L, 0L, 0L, 0L, 10L, 90L, 2L, 20L)
+ 0L, 0L, 0L, 0L, 10L, 90L, 2L, 20L, 110L)
val md_2 = Array(4500L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, 70L, 0L, 8000L,
3500L,
- 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L)
+ 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L, 110L)
val md_3 = Array(4200L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, 70L, 0L, 7800L,
3500L,
- 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L)
+ 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L, 120L)
// executors 1 and 2
val m1_1 = Array(4000L, 50L, 20L, 0L, 40L, 0L, 60L, 0L, 70L, 20L, 7500L,
3500L,
- 6500L, 2500L, 5500L, 1500L, 10L, 90L, 2L, 20L)
+ 6500L, 2500L, 5500L, 1500L, 10L, 90L, 2L, 20L, 110L)
val m2_1 = Array(1500L, 50L, 20L, 0L, 0L, 0L, 20L, 0L, 70L, 0L, 8500L,
3500L,
- 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L)
+ 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L, 110L)
val m1_2 = Array(4000L, 50L, 50L, 0L, 50L, 0L, 100L, 0L, 70L, 20L, 8000L,
4000L,
- 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L)
+ 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L)
val m2_2 = Array(2000L, 50L, 10L, 0L, 10L, 0L, 30L, 0L, 70L, 0L, 9000L,
4000L,
- 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L)
+ 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L, 110L)
val m1_3 = Array(2000L, 40L, 50L, 0L, 40L, 10L, 90L, 10L, 50L, 0L, 8000L,
3500L,
- 7000L, 2500L, 6000L, 1500L, 10L, 90L, 2L, 20L)
+ 7000L, 2500L, 6000L, 1500L, 10L, 90L, 2L, 20L, 110L)
val m2_3 = Array(3500L, 50L, 15L, 0L, 10L, 10L, 35L, 10L, 80L, 0L, 8500L,
3500L,
- 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L)
+ 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L, 110L)
val m1_4 = Array(5000L, 30L, 50L, 20L, 30L, 10L, 80L, 30L, 50L,
- 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L)
+ 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L, 110L)
val m2_4 = Array(7000L, 70L, 50L, 20L, 0L, 10L, 50L, 30L, 10L,
- 40L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L)
+ 40L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L)
val m1_5 = Array(6000L, 70L, 20L, 30L, 10L, 0L, 30L, 30L, 30L, 0L, 5000L,
3000L,
- 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L)
+ 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L, 110L)
val m2_5 = Array(5500L, 30L, 20L, 40L, 10L, 0L, 30L, 40L, 40L,
- 20L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L)
+ 20L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L, 110L)
val m1_6 = Array(7000L, 70L, 5L, 25L, 60L, 30L, 65L, 55L, 30L, 0L, 3000L,
2500L,
- 2000L, 1500L, 1000L, 500L, 10L, 90L, 2L, 20L)
+ 2000L, 1500L, 1000L, 500L, 10L, 90L, 2L, 20L, 110L)
val m2_6 = Array(5500L, 40L, 25L, 30L, 10L, 30L, 35L, 60L, 0L,
- 20L, 7000L, 3000L, 6000L, 2000L, 5000L, 1000L, 10L, 90L, 2L, 20L)
+ 20L, 7000L, 3000L, 6000L, 2000L, 5000L, 1000L, 10L, 90L, 2L, 20L, 110L)
val m1_7 = Array(5500L, 70L, 15L, 20L, 55L, 20L, 70L, 40L, 20L,
- 0L, 4000L, 2500L, 3000L, 1500L, 2000L, 500L, 10L, 90L, 2L, 20L)
+ 0L, 4000L, 2500L, 3000L, 1500L, 2000L, 500L, 10L, 90L, 2L, 20L, 110L)
val m2_7 = Array(4000L, 20L, 25L, 30L, 10L, 30L, 35L, 60L, 0L, 0L, 7000L,
- 4000L, 6000L, 3000L, 5000L, 2000L, 10L, 90L, 2L, 20L)
+ 4000L, 6000L, 3000L, 5000L, 2000L, 10L, 90L, 2L, 20L, 110L)
// tasks
val t1 = Array(4500L, 60L, 50L, 0L, 50L, 10L, 100L, 10L, 70L, 20L,
- 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L)
+ 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L)
val t2 = Array(3500L, 50L, 20L, 0L, 10L, 10L, 35L, 10L, 80L, 0L,
- 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L)
+ 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L, 110L)
val t3 = Array(5000L, 60L, 50L, 20L, 50L, 10L, 100L, 30L, 70L, 20L,
- 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L)
+ 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L)
val t4 = Array(7000L, 70L, 50L, 20L, 10L, 10L, 50L, 30L, 80L, 40L,
- 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L)
+ 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L, 110L)
val t5 = Array(7000L, 100L, 50L, 30L, 60L, 30L, 80L, 55L, 50L, 0L,
- 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L)
+ 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L, 110L)
val t6 = Array(7200L, 70L, 50L, 40L, 10L, 30L, 50L, 60L, 40L, 40L,
- 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L)
+ 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L, 110L)
def max(a: Array[Long], b: Array[Long]): Array[Long] =
(a, b).zipped.map(Math.max).toArray
@@ -402,17 +402,17 @@ class EventLoggingListenerSuite extends SparkFunSuite
with LocalSparkContext wit
// expected metric peaks per stage per executor
val p0_1 = Array(5000L, 60L, 50L, 20L, 50L, 10L, 100L, 30L,
- 70L, 20L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L)
+ 70L, 20L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L,
110L)
val p0_2 = Array(7000L, 70L, 50L, 20L, 10L, 10L, 50L, 30L,
- 80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L)
+ 80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L,
110L)
val p0_d = Array(4500L, 50L, 0L, 0L, 40L, 0L, 40L, 0L,
- 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L)
+ 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L, 110L)
val p1_1 = Array(7000L, 100L, 50L, 30L, 60L, 30L, 80L, 55L,
- 50L, 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L)
+ 50L, 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L,
110L)
val p1_2 = Array(7200L, 70L, 50L, 40L, 10L, 30L, 50L, 60L,
- 40L, 40L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L)
+ 40L, 40L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L,
110L)
val p1_d = Array(4500L, 50L, 0L, 0L, 40L, 0L, 40L, 0L,
- 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L)
+ 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L, 120L)
assert(Arrays.equals(p0_1, cp0_1))
assert(Arrays.equals(p0_2, cp0_2))
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 8dbfe53..4eea225 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -51,17 +51,20 @@ class JsonProtocolSuite extends SparkFunSuite {
val taskEnd = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success,
makeTaskInfo(123L, 234, 67, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
- 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L)),
+ 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
+ 0, 0, 0, 0, 80001L)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput =
false, hasOutput = false))
val taskEndWithHadoopInput = SparkListenerTaskEnd(1, 0, "ShuffleMapTask",
Success,
makeTaskInfo(123L, 234, 67, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
- 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L)),
+ 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
+ 0, 0, 0, 0, 80001L)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput = true,
hasOutput = false))
val taskEndWithOutput = SparkListenerTaskEnd(1, 0, "ResultTask", Success,
makeTaskInfo(123L, 234, 67, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
- 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L)),
+ 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
+ 0, 0, 0, 0, 80001L)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput = true,
hasOutput = true))
val jobStart = {
val stageIds = Seq[Int](1, 2, 3, 4)
@@ -114,7 +117,8 @@ class JsonProtocolSuite extends SparkFunSuite {
.zipWithIndex.map { case (a, i) => a.copy(id = i) }
val executorUpdates = new ExecutorMetrics(
Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
- 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
10L, 90L, 2L, 20L))
+ 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L,
+ 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L))
SparkListenerExecutorMetricsUpdate("exec3", Seq((1L, 2, 3,
accumUpdates)),
Map((0, 0) -> executorUpdates))
}
@@ -124,7 +128,8 @@ class JsonProtocolSuite extends SparkFunSuite {
val stageExecutorMetrics =
SparkListenerStageExecutorMetrics("1", 2, 3,
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L,
432L,
- 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
10L, 90L, 2L, 20L)))
+ 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L,
+ 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L)))
val rprofBuilder = new ResourceProfileBuilder()
val taskReq = new TaskResourceRequests().cpus(1).resource("gpu", 1)
val execReq =
@@ -502,12 +507,12 @@ class JsonProtocolSuite extends SparkFunSuite {
test("executorMetricsFromJson backward compatibility: handle missing
metrics") {
// any missing metrics should be set to 0
val executorMetrics = new ExecutorMetrics(Array(12L, 23L, 45L, 67L, 78L,
89L,
- 90L, 123L, 456L, 789L, 40L, 20L, 20L, 10L, 20L, 10L))
+ 90L, 123L, 456L, 789L, 40L, 20L, 20L, 10L, 20L, 10L, 301L))
val oldExecutorMetricsJson =
JsonProtocol.executorMetricsToJson(executorMetrics)
.removeField( _._1 == "MappedPoolMemory")
val expectedExecutorMetrics = new ExecutorMetrics(Array(12L, 23L, 45L, 67L,
- 78L, 89L, 90L, 123L, 456L, 0L, 40L, 20L, 20L, 10L, 20L, 10L))
+ 78L, 89L, 90L, 123L, 456L, 0L, 40L, 20L, 20L, 10L, 20L, 10L, 301L))
assertEquals(expectedExecutorMetrics,
JsonProtocol.executorMetricsFromJson(oldExecutorMetricsJson))
}
@@ -1047,7 +1052,7 @@ private[spark] object JsonProtocolSuite extends
Assertions {
val executorMetricsUpdate: Map[(Int, Int), ExecutorMetrics] =
if (includeExecutorMetrics) {
Map((0, 0) -> new ExecutorMetrics(Array(123456L, 543L, 0L, 0L, 0L, 0L,
0L,
- 0L, 0L, 0L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, 10L,
90L, 2L, 20L)))
+ 0L, 0L, 0L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, 10L,
90L, 2L, 20L, 301L)))
} else {
Map.empty
}
@@ -1377,7 +1382,8 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCCount" : 0,
| "MinorGCTime" : 0,
| "MajorGCCount" : 0,
- | "MajorGCTime" : 0
+ | "MajorGCTime" : 0,
+ | "TotalGCTime" : 80001
| },
| "Task Metrics": {
| "Executor Deserialize Time": 300,
@@ -1501,7 +1507,8 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCCount" : 0,
| "MinorGCTime" : 0,
| "MajorGCCount" : 0,
- | "MajorGCTime" : 0
+ | "MajorGCTime" : 0,
+ | "TotalGCTime" : 80001
| },
| "Task Metrics": {
| "Executor Deserialize Time": 300,
@@ -1625,7 +1632,8 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCCount" : 0,
| "MinorGCTime" : 0,
| "MajorGCCount" : 0,
- | "MajorGCTime" : 0
+ | "MajorGCTime" : 0,
+ | "TotalGCTime" : 80001
| },
| "Task Metrics": {
| "Executor Deserialize Time": 300,
@@ -2378,7 +2386,8 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCCount": 10,
| "MinorGCTime": 90,
| "MajorGCCount": 2,
- | "MajorGCTime": 20
+ | "MajorGCTime": 20,
+ | "TotalGCTime" : 80001
| }
| }
| ]
@@ -2412,7 +2421,8 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "MinorGCCount": 10,
| "MinorGCTime": 90,
| "MajorGCCount": 2,
- | "MajorGCTime": 20
+ | "MajorGCTime": 20,
+ | "TotalGCTime" : 80001
| }
|}
""".stripMargin
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
