This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new c2df895 [SPARK-37493][CORE] show gc time and duration time of driver in executors page c2df895 is described below commit c2df895a8cbdfcbffc320e837ad1826933685912 Author: zhoubin11 <zhoubi...@baidu.com> AuthorDate: Mon Dec 13 08:47:01 2021 -0600 [SPARK-37493][CORE] show gc time and duration time of driver in executors page ### What changes were proposed in this pull request? show driver's gc time & duration time(equivalent to application time) of driver in both driver side and history side UI ### Why are the changes needed? help user to config driver's resource more appropriately ### Does this PR introduce _any_ user-facing change? yes,user will see driver's gc time & duration time in executors page . when `spark.eventLog.logStageExecutorMetrics` is enabled driver's gc time can be logged. before this change,user always get zero ![image](https://user-images.githubusercontent.com/37905939/144010082-5ebc1f80-b9f9-4286-ba6a-109700168124.png) ![image](https://user-images.githubusercontent.com/37905939/144010127-389c7b74-f5df-49c2-b600-626825af194e.png) ![image](https://user-images.githubusercontent.com/37905939/144012944-527852b9-681b-4a97-8dad-e4b029408c21.png) ### How was this patch tested? unit tests Closes #34749 from summaryzb/SPARK-37493. Authored-by: zhoubin11 <zhoubi...@baidu.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- .../apache/spark/metrics/ExecutorMetricType.scala | 9 ++-- .../org/apache/spark/status/AppStatusStore.scala | 54 +++++++++++++++++++- .../complete_stage_list_json_expectation.json | 9 ++-- .../excludeOnFailure_for_stage_expectation.json | 9 ++-- ...xcludeOnFailure_node_for_stage_expectation.json | 18 ++++--- .../executor_list_json_expectation.json | 7 +-- ...ist_with_executor_metrics_json_expectation.json | 14 ++++-- .../executor_memory_usage_expectation.json | 32 ++++++------ ...executor_node_excludeOnFailure_expectation.json | 40 ++++++++------- ...e_excludeOnFailure_unexcluding_expectation.json | 14 ++++-- .../executor_resource_information_expectation.json | 2 +- .../failed_stage_list_json_expectation.json | 3 +- ..._json_details_with_failed_task_expectation.json | 6 ++- .../one_stage_attempt_json_expectation.json | 6 ++- .../one_stage_json_expectation.json | 6 ++- .../one_stage_json_with_details_expectation.json | 6 ++- .../stage_list_json_expectation.json | 12 +++-- ...age_list_with_accumulable_json_expectation.json | 3 +- .../stage_list_with_peak_metrics_expectation.json | 9 ++-- .../stage_with_accumulable_json_expectation.json | 6 ++- .../stage_with_peak_metrics_expectation.json | 9 ++-- ...stage_with_speculation_summary_expectation.json | 17 ++++--- .../stage_with_summaries_expectation.json | 12 +++-- .../scheduler/EventLoggingListenerSuite.scala | 58 +++++++++++----------- .../org/apache/spark/util/JsonProtocolSuite.scala | 36 +++++++++----- 25 files changed, 260 insertions(+), 137 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala index 76e2813..a536919 100644 --- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala +++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala @@ -109,7 +109,8 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { "MinorGCCount", "MinorGCTime", "MajorGCCount", - "MajorGCTime" + "MajorGCTime", + "TotalGCTime" ) /* We builtin some common GC collectors which categorized as young generation and old */ @@ -136,8 +137,10 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { } override private[spark] def getMetricValues(memoryManager: MemoryManager): Array[Long] = { - val gcMetrics = new Array[Long](names.length) // minorCount, minorTime, majorCount, majorTime - ManagementFactory.getGarbageCollectorMXBeans.asScala.foreach { mxBean => + val gcMetrics = new Array[Long](names.length) + val mxBeans = ManagementFactory.getGarbageCollectorMXBeans.asScala + gcMetrics(4) = mxBeans.map(_.getCollectionTime).sum + mxBeans.foreach { mxBean => if (youngGenerationGarbageCollector.contains(mxBean.getName)) { gcMetrics(0) = mxBean.getCollectionCount gcMetrics(1) = mxBean.getCollectionTime diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala index ef9f55f..398cd45 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala @@ -22,7 +22,7 @@ import java.util.{List => JList} import scala.collection.JavaConverters._ import scala.collection.mutable.HashMap -import org.apache.spark.{JobExecutionStatus, SparkConf} +import org.apache.spark.{JobExecutionStatus, SparkConf, SparkContext} import org.apache.spark.status.api.v1 import org.apache.spark.storage.FallbackStorage.FALLBACK_BLOCK_MANAGER_ID import org.apache.spark.ui.scope._ @@ -89,7 +89,57 @@ private[spark] class AppStatusStore( } else { base } - filtered.asScala.map(_.info).filter(_.id != FALLBACK_BLOCK_MANAGER_ID.executorId).toSeq + filtered.asScala.map(_.info) + .filter(_.id != FALLBACK_BLOCK_MANAGER_ID.executorId) + .map(replaceExec).toSeq + } + + private def replaceExec(origin: v1.ExecutorSummary): v1.ExecutorSummary = { + if (origin.id == SparkContext.DRIVER_IDENTIFIER) { + replaceDriverGcTime(origin, extractGcTime(origin), extractAppTime) + } else { + origin + } + } + + private def replaceDriverGcTime(source: v1.ExecutorSummary, + totalGcTime: Option[Long], totalAppTime: Option[Long]): v1.ExecutorSummary = { + new v1.ExecutorSummary(source.id, source.hostPort, source.isActive, source.rddBlocks, + source.memoryUsed, source.diskUsed, source.totalCores, source.maxTasks, source.activeTasks, + source.failedTasks, source.completedTasks, source.totalTasks, + totalAppTime.getOrElse(source.totalDuration), + totalGcTime.getOrElse(source.totalGCTime), + source.totalInputBytes, source.totalShuffleRead, + source.totalShuffleWrite, source.isBlacklisted, source.maxMemory, source.addTime, + source.removeTime, source.removeReason, source.executorLogs, source.memoryMetrics, + source.blacklistedInStages, source.peakMemoryMetrics, source.attributes, source.resources, + source.resourceProfileId, source.isExcluded, source.excludedInStages) + } + + private def extractGcTime(source: v1.ExecutorSummary): Option[Long] = { + source.peakMemoryMetrics.map(_.getMetricValue("TotalGCTime")) + } + + private def extractAppTime: Option[Long] = { + var startTime = 0L + // -1 when SparkListenerApplicationStart event written to kvStore + // event time when SparkListenerApplicationStart event written to kvStore + var endTime = 0L + try { + val appInfo = applicationInfo() + startTime = appInfo.attempts.head.startTime.getTime() + endTime = appInfo.attempts.head.endTime.getTime() + } catch { + // too early to get appInfo, should wait a while + case _: NoSuchElementException => + } + if (endTime == 0) { + None + } else if (endTime < 0) { + Option(System.currentTimeMillis() - startTime) + } else { + Option(endTime - startTime) + } } def miscellaneousProcessList(activeOnly: Boolean): Seq[v1.ProcessSummary] = { diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json index f04543e..67b4d63 100644 --- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json @@ -63,7 +63,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } }, { "status" : "COMPLETE", @@ -130,7 +131,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } }, { "status" : "COMPLETE", @@ -197,6 +199,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json index dcad8a6..9be6b6e 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json @@ -718,7 +718,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : true }, @@ -758,7 +759,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -785,6 +787,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json index 2ab1546..1661f7c 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json @@ -826,7 +826,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : true }, @@ -866,7 +867,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : true }, @@ -906,7 +908,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false }, @@ -946,7 +949,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false }, @@ -986,7 +990,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : true } @@ -1013,6 +1018,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json index be12507..ec3fc28 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json @@ -11,8 +11,8 @@ "failedTasks" : 1, "completedTasks" : 31, "totalTasks" : 32, - "totalDuration" : 8820, - "totalGCTime" : 352, + "totalDuration" : 9011, + "totalGCTime" : 0, "totalInputBytes" : 28000288, "totalShuffleRead" : 0, "totalShuffleWrite" : 13180, @@ -41,7 +41,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json index bf3e93f..9b7498d 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json @@ -11,7 +11,7 @@ "failedTasks" : 0, "completedTasks" : 0, "totalTasks" : 0, - "totalDuration" : 0, + "totalDuration" : 62168, "totalGCTime" : 0, "totalInputBytes" : 0, "totalShuffleRead" : 0, @@ -47,7 +47,8 @@ "MinorGCCount" : 7, "MinorGCTime" : 55, "MajorGCCount" : 3, - "MajorGCTime" : 144 + "MajorGCTime" : 144, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -106,7 +107,8 @@ "MinorGCCount" : 24, "MinorGCTime" : 145, "MajorGCCount" : 2, - "MajorGCTime" : 63 + "MajorGCTime" : 63, + "TotalGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-3.vpc.company.com:8042", @@ -175,7 +177,8 @@ "MinorGCCount" : 15, "MinorGCTime" : 106, "MajorGCCount" : 2, - "MajorGCTime" : 75 + "MajorGCTime" : 75, + "TotalGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-4.vpc.company.com:8042", @@ -244,7 +247,8 @@ "MinorGCCount" : 27, "MinorGCTime" : 140, "MajorGCCount" : 2, - "MajorGCTime" : 60 + "MajorGCTime" : 60, + "TotalGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-2.vpc.company.com:8042", diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json index 0a3eb81..fbb7b66 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json @@ -11,7 +11,7 @@ "failedTasks" : 0, "completedTasks" : 0, "totalTasks" : 0, - "totalDuration" : 0, + "totalDuration" : 10671, "totalGCTime" : 0, "totalInputBytes" : 0, "totalShuffleRead" : 0, @@ -57,7 +57,7 @@ "stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout", "stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr" }, - "memoryMetrics": { + "memoryMetrics" : { "usedOnHeapStorageMemory" : 0, "usedOffHeapStorageMemory" : 0, "totalOnHeapStorageMemory" : 384093388, @@ -84,14 +84,15 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, "resourceProfileId" : 0, "isExcluded" : true, "excludedInStages" : [ ] -} ,{ +}, { "id" : "2", "hostPort" : "172.22.0.167:51487", "isActive" : true, @@ -116,7 +117,7 @@ "stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout", "stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr" }, - "memoryMetrics": { + "memoryMetrics" : { "usedOnHeapStorageMemory" : 0, "usedOffHeapStorageMemory" : 0, "totalOnHeapStorageMemory" : 384093388, @@ -143,7 +144,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -175,11 +177,11 @@ "stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout", "stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr" }, - "memoryMetrics": { - "usedOnHeapStorageMemory": 0, - "usedOffHeapStorageMemory": 0, - "totalOnHeapStorageMemory": 384093388, - "totalOffHeapStorageMemory": 524288000 + "memoryMetrics" : { + "usedOnHeapStorageMemory" : 0, + "usedOffHeapStorageMemory" : 0, + "totalOnHeapStorageMemory" : 384093388, + "totalOffHeapStorageMemory" : 524288000 }, "blacklistedInStages" : [ ], "peakMemoryMetrics" : { @@ -202,7 +204,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -234,7 +237,7 @@ "stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout", "stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr" }, - "memoryMetrics": { + "memoryMetrics" : { "usedOnHeapStorageMemory" : 0, "usedOffHeapStorageMemory" : 0, "totalOnHeapStorageMemory" : 384093388, @@ -261,7 +264,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json index 8869fb4..fbb7b66 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json @@ -11,7 +11,7 @@ "failedTasks" : 0, "completedTasks" : 0, "totalTasks" : 0, - "totalDuration" : 0, + "totalDuration" : 10671, "totalGCTime" : 0, "totalInputBytes" : 0, "totalShuffleRead" : 0, @@ -20,7 +20,7 @@ "maxMemory" : 908381388, "addTime" : "2016-11-16T22:33:31.477GMT", "executorLogs" : { }, - "memoryMetrics": { + "memoryMetrics" : { "usedOnHeapStorageMemory" : 0, "usedOffHeapStorageMemory" : 0, "totalOnHeapStorageMemory" : 384093388, @@ -57,7 +57,7 @@ "stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout", "stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr" }, - "memoryMetrics": { + "memoryMetrics" : { "usedOnHeapStorageMemory" : 0, "usedOffHeapStorageMemory" : 0, "totalOnHeapStorageMemory" : 384093388, @@ -84,7 +84,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -116,7 +117,7 @@ "stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout", "stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr" }, - "memoryMetrics": { + "memoryMetrics" : { "usedOnHeapStorageMemory" : 0, "usedOffHeapStorageMemory" : 0, "totalOnHeapStorageMemory" : 384093388, @@ -143,7 +144,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -175,11 +177,11 @@ "stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout", "stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr" }, - "memoryMetrics": { - "usedOnHeapStorageMemory": 0, - "usedOffHeapStorageMemory": 0, - "totalOnHeapStorageMemory": 384093388, - "totalOffHeapStorageMemory": 524288000 + "memoryMetrics" : { + "usedOnHeapStorageMemory" : 0, + "usedOffHeapStorageMemory" : 0, + "totalOnHeapStorageMemory" : 384093388, + "totalOffHeapStorageMemory" : 524288000 }, "blacklistedInStages" : [ ], "peakMemoryMetrics" : { @@ -202,7 +204,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -234,11 +237,11 @@ "stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout", "stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr" }, - "memoryMetrics": { - "usedOnHeapStorageMemory": 0, - "usedOffHeapStorageMemory": 0, - "totalOnHeapStorageMemory": 384093388, - "totalOffHeapStorageMemory": 524288000 + "memoryMetrics" : { + "usedOnHeapStorageMemory" : 0, + "usedOffHeapStorageMemory" : 0, + "totalOnHeapStorageMemory" : 384093388, + "totalOffHeapStorageMemory" : 524288000 }, "blacklistedInStages" : [ ], "peakMemoryMetrics" : { @@ -261,7 +264,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json index 21cc9d0..b72ed0a 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json @@ -11,7 +11,7 @@ "failedTasks" : 0, "completedTasks" : 0, "totalTasks" : 0, - "totalDuration" : 0, + "totalDuration" : 101795, "totalGCTime" : 0, "totalInputBytes" : 0, "totalShuffleRead" : 0, @@ -72,7 +72,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -125,7 +126,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -178,7 +180,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -231,7 +234,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json index 53ae9a0..2caa11e 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_resource_information_expectation.json @@ -11,7 +11,7 @@ "failedTasks" : 0, "completedTasks" : 0, "totalTasks" : 0, - "totalDuration" : 0, + "totalDuration" : 18794, "totalGCTime" : 0, "totalInputBytes" : 0, "totalShuffleRead" : 0, diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json index 5573cf9..e1fd06b 100644 --- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json @@ -64,6 +64,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json index 4579d3b..18dac90 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json @@ -79,7 +79,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -106,6 +107,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index 9edb518..94d343d 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -480,7 +480,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -507,6 +508,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 9e661bd..6713486 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -480,7 +480,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -507,6 +508,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json index 9e661bd..6713486 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json @@ -480,7 +480,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -507,6 +508,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json index d109c73..05e47ff 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json @@ -63,7 +63,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } }, { "status" : "FAILED", @@ -131,7 +132,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } }, { "status" : "COMPLETE", @@ -198,7 +200,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } }, { "status" : "COMPLETE", @@ -265,6 +268,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json index 7901c4f..04d7493 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json @@ -67,6 +67,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json index d455b97..f709724 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json @@ -63,7 +63,8 @@ "MinorGCCount" : 13, "MinorGCTime" : 115, "MajorGCCount" : 4, - "MajorGCTime" : 339 + "MajorGCTime" : 339, + "TotalGCTime" : 0 } }, { "status" : "COMPLETE", @@ -131,7 +132,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } }, { "status" : "COMPLETE", @@ -199,6 +201,7 @@ "MinorGCCount" : 7, "MinorGCTime" : 33, "MajorGCCount" : 3, - "MajorGCTime" : 110 + "MajorGCTime" : 110, + "TotalGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index a5958e0..281f7b3 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -524,7 +524,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -551,6 +552,7 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json index 20a9580..5b2c205 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json @@ -928,7 +928,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false }, @@ -968,7 +969,8 @@ "MinorGCCount" : 13, "MinorGCTime" : 115, "MajorGCCount" : 4, - "MajorGCTime" : 339 + "MajorGCTime" : 339, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -995,6 +997,7 @@ "MinorGCCount" : 13, "MinorGCTime" : 115, "MajorGCCount" : 4, - "MajorGCTime" : 339 + "MajorGCTime" : 339, + "TotalGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json index 5f6090d..32cf571 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json @@ -346,7 +346,8 @@ "MinorGCCount" : 2, "MinorGCTime" : 280, "MajorGCCount" : 2, - "MajorGCTime" : 1116 + "MajorGCTime" : 1116, + "TotalGCTime" : 0 }, "isExcludedForStage" : false }, @@ -386,7 +387,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false }, @@ -426,7 +428,8 @@ "MinorGCCount" : 2, "MinorGCTime" : 587, "MajorGCCount" : 2, - "MajorGCTime" : 906 + "MajorGCTime" : 906, + "TotalGCTime" : 0 }, "isExcludedForStage" : false }, @@ -466,7 +469,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -502,6 +506,7 @@ "MinorGCCount" : 2, "MinorGCTime" : 587, "MajorGCCount" : 2, - "MajorGCTime" : 1116 + "MajorGCTime" : 1116, + "TotalGCTime" : 0 } -} \ No newline at end of file +} diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json index f200da3..3be20df 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json @@ -928,7 +928,8 @@ "MinorGCCount" : 0, "MinorGCTime" : 0, "MajorGCCount" : 0, - "MajorGCTime" : 0 + "MajorGCTime" : 0, + "TotalGCTime" : 0 }, "isExcludedForStage" : false }, @@ -968,7 +969,8 @@ "MinorGCCount" : 13, "MinorGCTime" : 115, "MajorGCCount" : 4, - "MajorGCTime" : 339 + "MajorGCTime" : 339, + "TotalGCTime" : 0 }, "isExcludedForStage" : false } @@ -995,7 +997,8 @@ "MinorGCCount" : 13, "MinorGCTime" : 115, "MajorGCCount" : 4, - "MajorGCTime" : 339 + "MajorGCTime" : 339, + "TotalGCTime" : 0 }, "taskMetricsDistributions" : { "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ], @@ -1072,7 +1075,8 @@ "MinorGCCount" : [ 0.0, 0.0, 13.0, 13.0, 13.0 ], "MinorGCTime" : [ 0.0, 0.0, 115.0, 115.0, 115.0 ], "MajorGCCount" : [ 0.0, 0.0, 4.0, 4.0, 4.0 ], - "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ] + "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ], + "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] } } } diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala index 09ad223..b06e83e 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala @@ -337,55 +337,55 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit // Executor metrics // driver val md_1 = Array(4000L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, 70L, 0L, 7500L, 3500L, - 0L, 0L, 0L, 0L, 10L, 90L, 2L, 20L) + 0L, 0L, 0L, 0L, 10L, 90L, 2L, 20L, 110L) val md_2 = Array(4500L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, 70L, 0L, 8000L, 3500L, - 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L) + 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L, 110L) val md_3 = Array(4200L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, 70L, 0L, 7800L, 3500L, - 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L) + 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L, 120L) // executors 1 and 2 val m1_1 = Array(4000L, 50L, 20L, 0L, 40L, 0L, 60L, 0L, 70L, 20L, 7500L, 3500L, - 6500L, 2500L, 5500L, 1500L, 10L, 90L, 2L, 20L) + 6500L, 2500L, 5500L, 1500L, 10L, 90L, 2L, 20L, 110L) val m2_1 = Array(1500L, 50L, 20L, 0L, 0L, 0L, 20L, 0L, 70L, 0L, 8500L, 3500L, - 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L) + 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L, 110L) val m1_2 = Array(4000L, 50L, 50L, 0L, 50L, 0L, 100L, 0L, 70L, 20L, 8000L, 4000L, - 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L) + 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L) val m2_2 = Array(2000L, 50L, 10L, 0L, 10L, 0L, 30L, 0L, 70L, 0L, 9000L, 4000L, - 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L) + 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L, 110L) val m1_3 = Array(2000L, 40L, 50L, 0L, 40L, 10L, 90L, 10L, 50L, 0L, 8000L, 3500L, - 7000L, 2500L, 6000L, 1500L, 10L, 90L, 2L, 20L) + 7000L, 2500L, 6000L, 1500L, 10L, 90L, 2L, 20L, 110L) val m2_3 = Array(3500L, 50L, 15L, 0L, 10L, 10L, 35L, 10L, 80L, 0L, 8500L, 3500L, - 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L) + 7500L, 2500L, 6500L, 1500L, 10L, 90L, 2L, 20L, 110L) val m1_4 = Array(5000L, 30L, 50L, 20L, 30L, 10L, 80L, 30L, 50L, - 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L) + 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L, 110L) val m2_4 = Array(7000L, 70L, 50L, 20L, 0L, 10L, 50L, 30L, 10L, - 40L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L) + 40L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L) val m1_5 = Array(6000L, 70L, 20L, 30L, 10L, 0L, 30L, 30L, 30L, 0L, 5000L, 3000L, - 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L) + 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L, 110L) val m2_5 = Array(5500L, 30L, 20L, 40L, 10L, 0L, 30L, 40L, 40L, - 20L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L) + 20L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L, 110L) val m1_6 = Array(7000L, 70L, 5L, 25L, 60L, 30L, 65L, 55L, 30L, 0L, 3000L, 2500L, - 2000L, 1500L, 1000L, 500L, 10L, 90L, 2L, 20L) + 2000L, 1500L, 1000L, 500L, 10L, 90L, 2L, 20L, 110L) val m2_6 = Array(5500L, 40L, 25L, 30L, 10L, 30L, 35L, 60L, 0L, - 20L, 7000L, 3000L, 6000L, 2000L, 5000L, 1000L, 10L, 90L, 2L, 20L) + 20L, 7000L, 3000L, 6000L, 2000L, 5000L, 1000L, 10L, 90L, 2L, 20L, 110L) val m1_7 = Array(5500L, 70L, 15L, 20L, 55L, 20L, 70L, 40L, 20L, - 0L, 4000L, 2500L, 3000L, 1500L, 2000L, 500L, 10L, 90L, 2L, 20L) + 0L, 4000L, 2500L, 3000L, 1500L, 2000L, 500L, 10L, 90L, 2L, 20L, 110L) val m2_7 = Array(4000L, 20L, 25L, 30L, 10L, 30L, 35L, 60L, 0L, 0L, 7000L, - 4000L, 6000L, 3000L, 5000L, 2000L, 10L, 90L, 2L, 20L) + 4000L, 6000L, 3000L, 5000L, 2000L, 10L, 90L, 2L, 20L, 110L) // tasks val t1 = Array(4500L, 60L, 50L, 0L, 50L, 10L, 100L, 10L, 70L, 20L, - 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L) + 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L) val t2 = Array(3500L, 50L, 20L, 0L, 10L, 10L, 35L, 10L, 80L, 0L, - 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L) + 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L, 110L) val t3 = Array(5000L, 60L, 50L, 20L, 50L, 10L, 100L, 30L, 70L, 20L, - 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L) + 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L) val t4 = Array(7000L, 70L, 50L, 20L, 10L, 10L, 50L, 30L, 80L, 40L, - 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L) + 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L, 110L) val t5 = Array(7000L, 100L, 50L, 30L, 60L, 30L, 80L, 55L, 50L, 0L, - 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L) + 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L, 110L) val t6 = Array(7200L, 70L, 50L, 40L, 10L, 30L, 50L, 60L, 40L, 40L, - 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L) + 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L, 110L) def max(a: Array[Long], b: Array[Long]): Array[Long] = (a, b).zipped.map(Math.max).toArray @@ -402,17 +402,17 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit // expected metric peaks per stage per executor val p0_1 = Array(5000L, 60L, 50L, 20L, 50L, 10L, 100L, 30L, - 70L, 20L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L) + 70L, 20L, 8000L, 4000L, 7000L, 3000L, 6000L, 2000L, 10L, 90L, 2L, 20L, 110L) val p0_2 = Array(7000L, 70L, 50L, 20L, 10L, 10L, 50L, 30L, - 80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L) + 80L, 40L, 9000L, 4000L, 8000L, 3000L, 7000L, 2000L, 10L, 90L, 2L, 20L, 110L) val p0_d = Array(4500L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, - 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L) + 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 10L, 90L, 3L, 20L, 110L) val p1_1 = Array(7000L, 100L, 50L, 30L, 60L, 30L, 80L, 55L, - 50L, 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L) + 50L, 0L, 5000L, 3000L, 4000L, 2000L, 3000L, 1000L, 10L, 90L, 2L, 20L, 110L) val p1_2 = Array(7200L, 70L, 50L, 40L, 10L, 30L, 50L, 60L, - 40L, 40L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L) + 40L, 40L, 8000L, 5000L, 7000L, 4000L, 6000L, 3000L, 10L, 90L, 2L, 20L, 110L) val p1_d = Array(4500L, 50L, 0L, 0L, 40L, 0L, 40L, 0L, - 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L) + 70L, 0L, 8000L, 3500L, 0L, 0L, 0L, 0L, 15L, 100L, 5L, 20L, 120L) assert(Arrays.equals(p0_1, cp0_1)) assert(Arrays.equals(p0_2, cp0_2)) diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 8dbfe53..4eea225 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -51,17 +51,20 @@ class JsonProtocolSuite extends SparkFunSuite { val taskEnd = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success, makeTaskInfo(123L, 234, 67, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, - 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L)), + 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, + 0, 0, 0, 0, 80001L)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput = false, hasOutput = false)) val taskEndWithHadoopInput = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success, makeTaskInfo(123L, 234, 67, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, - 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L)), + 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, + 0, 0, 0, 0, 80001L)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput = true, hasOutput = false)) val taskEndWithOutput = SparkListenerTaskEnd(1, 0, "ResultTask", Success, makeTaskInfo(123L, 234, 67, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, - 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L)), + 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, + 0, 0, 0, 0, 80001L)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput = true, hasOutput = true)) val jobStart = { val stageIds = Seq[Int](1, 2, 3, 4) @@ -114,7 +117,8 @@ class JsonProtocolSuite extends SparkFunSuite { .zipWithIndex.map { case (a, i) => a.copy(id = i) } val executorUpdates = new ExecutorMetrics( Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, - 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, 10L, 90L, 2L, 20L)) + 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L)) SparkListenerExecutorMetricsUpdate("exec3", Seq((1L, 2, 3, accumUpdates)), Map((0, 0) -> executorUpdates)) } @@ -124,7 +128,8 @@ class JsonProtocolSuite extends SparkFunSuite { val stageExecutorMetrics = SparkListenerStageExecutorMetrics("1", 2, 3, new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, - 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, 10L, 90L, 2L, 20L))) + 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L))) val rprofBuilder = new ResourceProfileBuilder() val taskReq = new TaskResourceRequests().cpus(1).resource("gpu", 1) val execReq = @@ -502,12 +507,12 @@ class JsonProtocolSuite extends SparkFunSuite { test("executorMetricsFromJson backward compatibility: handle missing metrics") { // any missing metrics should be set to 0 val executorMetrics = new ExecutorMetrics(Array(12L, 23L, 45L, 67L, 78L, 89L, - 90L, 123L, 456L, 789L, 40L, 20L, 20L, 10L, 20L, 10L)) + 90L, 123L, 456L, 789L, 40L, 20L, 20L, 10L, 20L, 10L, 301L)) val oldExecutorMetricsJson = JsonProtocol.executorMetricsToJson(executorMetrics) .removeField( _._1 == "MappedPoolMemory") val expectedExecutorMetrics = new ExecutorMetrics(Array(12L, 23L, 45L, 67L, - 78L, 89L, 90L, 123L, 456L, 0L, 40L, 20L, 20L, 10L, 20L, 10L)) + 78L, 89L, 90L, 123L, 456L, 0L, 40L, 20L, 20L, 10L, 20L, 10L, 301L)) assertEquals(expectedExecutorMetrics, JsonProtocol.executorMetricsFromJson(oldExecutorMetricsJson)) } @@ -1047,7 +1052,7 @@ private[spark] object JsonProtocolSuite extends Assertions { val executorMetricsUpdate: Map[(Int, Int), ExecutorMetrics] = if (includeExecutorMetrics) { Map((0, 0) -> new ExecutorMetrics(Array(123456L, 543L, 0L, 0L, 0L, 0L, 0L, - 0L, 0L, 0L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, 10L, 90L, 2L, 20L))) + 0L, 0L, 0L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, 10L, 90L, 2L, 20L, 301L))) } else { Map.empty } @@ -1377,7 +1382,8 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCCount" : 0, | "MinorGCTime" : 0, | "MajorGCCount" : 0, - | "MajorGCTime" : 0 + | "MajorGCTime" : 0, + | "TotalGCTime" : 80001 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -1501,7 +1507,8 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCCount" : 0, | "MinorGCTime" : 0, | "MajorGCCount" : 0, - | "MajorGCTime" : 0 + | "MajorGCTime" : 0, + | "TotalGCTime" : 80001 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -1625,7 +1632,8 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCCount" : 0, | "MinorGCTime" : 0, | "MajorGCCount" : 0, - | "MajorGCTime" : 0 + | "MajorGCTime" : 0, + | "TotalGCTime" : 80001 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -2378,7 +2386,8 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCCount": 10, | "MinorGCTime": 90, | "MajorGCCount": 2, - | "MajorGCTime": 20 + | "MajorGCTime": 20, + | "TotalGCTime" : 80001 | } | } | ] @@ -2412,7 +2421,8 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCCount": 10, | "MinorGCTime": 90, | "MajorGCCount": 2, - | "MajorGCTime": 20 + | "MajorGCTime": 20, + | "TotalGCTime" : 80001 | } |} """.stripMargin --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org