This is an automated email from the ASF dual-hosted git repository. mridulm80 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 74447ab5817 [SPARK-44162][CORE] Support G1GC in spark metrics 74447ab5817 is described below commit 74447ab5817bcb76f8979ae21bf38a6d193a3e46 Author: Jia Fan <fanjiaemi...@qq.com> AuthorDate: Thu Aug 31 23:16:24 2023 -0500 [SPARK-44162][CORE] Support G1GC in spark metrics ### What changes were proposed in this pull request? As a part of support JDK21, add support `G1 Concurrent GC` GarbageCollectorMXBean in Spark metrics. Refer https://github.com/openjdk/jdk/pull/11341 and https://bugs.openjdk.org/browse/JDK-8297247 , the `G1 Concurrent GC` not a part of YoungGC or FullGC. So we follow the JDK definition, bring two new metrics: `ConcurrentGCCount` and `ConcurrentGCTime`. ### Why are the changes needed? add new builtin garbage collectors for metrics. ### Does this PR introduce _any_ user-facing change? Yes, will receive new metrics. ### How was this patch tested? Test in local, I will add test if necessary. Closes #41808 from Hisoka-X/SPARK-44162_G1GC_on_JDK21. Authored-by: Jia Fan <fanjiaemi...@qq.com> Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com> --- .../apache/spark/metrics/ExecutorMetricType.scala | 11 ++++++-- .../spark/status/api/v1/PrometheusResource.scala | 4 +-- .../complete_stage_list_json_expectation.json | 12 ++++++--- .../excludeOnFailure_for_stage_expectation.json | 12 ++++++--- ...xcludeOnFailure_node_for_stage_expectation.json | 24 ++++++++++++----- .../executor_list_json_expectation.json | 4 ++- ...ist_with_executor_metrics_json_expectation.json | 16 +++++++++--- .../executor_memory_usage_expectation.json | 16 +++++++++--- ...executor_node_excludeOnFailure_expectation.json | 16 +++++++++--- ...e_excludeOnFailure_unexcluding_expectation.json | 16 +++++++++--- .../failed_stage_list_json_expectation.json | 4 ++- ..._json_details_with_failed_task_expectation.json | 8 ++++-- .../one_stage_attempt_json_expectation.json | 8 ++++-- .../one_stage_json_expectation.json | 8 ++++-- .../one_stage_json_with_details_expectation.json | 8 ++++-- ...ne_stage_json_with_partitionId_expectation.json | 8 ++++-- .../stage_list_json_expectation.json | 16 +++++++++--- ...age_list_with_accumulable_json_expectation.json | 4 ++- .../stage_list_with_peak_metrics_expectation.json | 12 ++++++--- .../stage_with_accumulable_json_expectation.json | 8 ++++-- .../stage_with_peak_metrics_expectation.json | 12 ++++++--- ...stage_with_speculation_summary_expectation.json | 20 +++++++++++---- .../stage_with_summaries_expectation.json | 16 +++++++++--- .../org/apache/spark/util/JsonProtocolSuite.scala | 30 ++++++++++++++-------- 24 files changed, 217 insertions(+), 76 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala index 648532faa3a..1e80eb66dc5 100644 --- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala +++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala @@ -110,10 +110,12 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { "MinorGCTime", "MajorGCCount", "MajorGCTime", - "TotalGCTime" + "TotalGCTime", + "ConcurrentGCCount", + "ConcurrentGCTime" ) - /* We builtin some common GC collectors which categorized as young generation and old */ + /* We builtin some common GC collectors */ private[spark] val YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS = Seq( "Copy", "PS Scavenge", @@ -128,6 +130,8 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { "G1 Old Generation" ) + private[spark] val BUILTIN_CONCURRENT_GARBAGE_COLLECTOR = "G1 Concurrent GC" + private lazy val youngGenerationGarbageCollector: Seq[String] = { SparkEnv.get.conf.get(config.EVENT_LOG_GC_METRICS_YOUNG_GENERATION_GARBAGE_COLLECTORS) } @@ -147,6 +151,9 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { } else if (oldGenerationGarbageCollector.contains(mxBean.getName)) { gcMetrics(2) = mxBean.getCollectionCount gcMetrics(3) = mxBean.getCollectionTime + } else if (BUILTIN_CONCURRENT_GARBAGE_COLLECTOR.equals(mxBean.getName)) { + gcMetrics(5) = mxBean.getCollectionCount + gcMetrics(6) = mxBean.getCollectionTime } else if (!nonBuiltInCollectors.contains(mxBean.getName)) { nonBuiltInCollectors = mxBean.getName +: nonBuiltInCollectors // log it when first seen diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala index 9658e5e6277..ca088dc8055 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala @@ -97,10 +97,10 @@ private[v1] class PrometheusResource extends ApiRequestContext { names.foreach { name => sb.append(s"$prefix${name}_bytes$labels ${m.getMetricValue(name)}\n") } - Seq("MinorGCCount", "MajorGCCount").foreach { name => + Seq("MinorGCCount", "MajorGCCount", "ConcurrentGCCount").foreach { name => sb.append(s"$prefix${name}_total$labels ${m.getMetricValue(name)}\n") } - Seq("MinorGCTime", "MajorGCTime").foreach { name => + Seq("MinorGCTime", "MajorGCTime", "ConcurrentGCTime").foreach { name => sb.append(s"$prefix${name}_seconds_total$labels ${m.getMetricValue(name) * 0.001}\n") } } diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json index 850c3777ec4..ac0f2ce2605 100644 --- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json @@ -76,7 +76,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } }, { "status" : "COMPLETE", @@ -156,7 +158,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } }, { "status" : "COMPLETE", @@ -236,6 +240,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json index ed4ed9ad871..d614bb000e4 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json @@ -887,7 +887,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -928,7 +930,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -956,6 +960,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json index f96a59fae53..475dee00a26 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json @@ -1021,7 +1021,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -1062,7 +1064,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -1103,7 +1107,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1144,7 +1150,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1185,7 +1193,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true } @@ -1213,6 +1223,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json index ec3fc280b0a..a860682ca2e 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json @@ -42,7 +42,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json index 9b7498d9e91..2833cdcfde5 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json @@ -48,7 +48,9 @@ "MinorGCTime" : 55, "MajorGCCount" : 3, "MajorGCTime" : 144, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -108,7 +110,9 @@ "MinorGCTime" : 145, "MajorGCCount" : 2, "MajorGCTime" : 63, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-3.vpc.company.com:8042", @@ -178,7 +182,9 @@ "MinorGCTime" : 106, "MajorGCCount" : 2, "MajorGCTime" : 75, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-4.vpc.company.com:8042", @@ -248,7 +254,9 @@ "MinorGCTime" : 140, "MajorGCCount" : 2, "MajorGCTime" : 60, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-2.vpc.company.com:8042", diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json index fbb7b6631f0..8a96858a201 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json @@ -85,7 +85,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -145,7 +147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -205,7 +209,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -265,7 +271,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json index fbb7b6631f0..8a96858a201 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json @@ -85,7 +85,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -145,7 +147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -205,7 +209,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -265,7 +271,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json index b72ed0a6254..0e5e73f36fa 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json @@ -73,7 +73,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -127,7 +129,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -181,7 +185,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -235,7 +241,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json index fee7377f181..dc1bcd6a396 100644 --- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json @@ -75,7 +75,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json index 9e390a995c3..e24ac4f82b8 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json @@ -92,7 +92,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -120,6 +122,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index 887d2678e61..659e3c41d92 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -595,7 +595,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 3bb59aaf5b5..f84cf26fcf1 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -595,7 +595,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json index b688b72b04d..564f3eadd1c 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json @@ -597,7 +597,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json index 83ffb7da8e7..2bf7f348037 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json @@ -721,7 +721,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -749,7 +751,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json index e3cd9809434..8df41bfcc8d 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json @@ -74,7 +74,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -155,7 +157,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -235,7 +239,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -315,7 +321,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json index e4caffcf107..730df3fbd53 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json @@ -78,7 +78,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json index d3459be777d..16d92244ee0 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json @@ -74,7 +74,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -155,7 +157,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -236,7 +240,9 @@ "MinorGCTime" : 33, "MajorGCCount" : 3, "MajorGCTime" : 110, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index 3880818a7b5..e38741b7bf6 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -639,7 +639,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -667,7 +669,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json index d3eb7d55e0e..630b0512e8f 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json @@ -1147,7 +1147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1188,7 +1190,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -1216,7 +1220,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json index 3ad18f816fb..23770480ad6 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json @@ -424,7 +424,9 @@ "MinorGCTime" : 280, "MajorGCCount" : 2, "MajorGCTime" : 1116, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -465,7 +467,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -506,7 +510,9 @@ "MinorGCTime" : 587, "MajorGCCount" : 2, "MajorGCTime" : 906, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -547,7 +553,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -584,6 +592,8 @@ "MinorGCTime" : 587, "MajorGCCount" : 2, "MajorGCTime" : 1116, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json index c89b82caf38..c8458a40958 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json @@ -1147,7 +1147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1188,7 +1190,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -1216,7 +1220,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "taskMetricsDistributions" : { "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ], @@ -1306,7 +1312,9 @@ "MinorGCTime" : [ 0.0, 0.0, 115.0, 115.0, 115.0 ], "MajorGCCount" : [ 0.0, 0.0, 4.0, 4.0, 4.0 ], "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ], - "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] + "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "ConcurrentGCCount" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "ConcurrentGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] } }, "isShufflePushEnabled" : false, diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 8105df64705..e8d41c4d46e 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -58,21 +58,21 @@ class JsonProtocolSuite extends SparkFunSuite { makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = false, hasOutput = false)) val taskEndWithHadoopInput = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success, makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = true, hasOutput = false)) val taskEndWithOutput = SparkListenerTaskEnd(1, 0, "ResultTask", Success, makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = true, hasOutput = true)) val jobStart = { @@ -136,7 +136,7 @@ class JsonProtocolSuite extends SparkFunSuite { val executorUpdates = new ExecutorMetrics( Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, - 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L)) + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3)) SparkListenerExecutorMetricsUpdate("exec3", Seq((1L, 2, 3, accumUpdates)), Map((0, 0) -> executorUpdates)) } @@ -147,7 +147,7 @@ class JsonProtocolSuite extends SparkFunSuite { SparkListenerStageExecutorMetrics("1", 2, 3, new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, - 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L))) + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3))) val rprofBuilder = new ResourceProfileBuilder() val taskReq = new TaskResourceRequests() .cpus(1) @@ -1754,7 +1754,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -1893,7 +1895,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -2032,7 +2036,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -2933,7 +2939,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime": 90, | "MajorGCCount": 2, | "MajorGCTime": 20, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | } | } | ] @@ -2968,7 +2976,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime": 90, | "MajorGCCount": 2, | "MajorGCTime": 20, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | } |} """.stripMargin --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org