[
https://issues.apache.org/jira/browse/SPARK-25064?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16726281#comment-16726281
]
ASF GitHub Bot commented on SPARK-25064:
----------------------------------------
vanzin closed pull request #22051: [SPARK-25064][WEBUI] Add killed tasks count
info to WebUI
URL: https://github.com/apache/spark/pull/22051
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
index 5c91304e49fd7..547f70515316a 100644
---
a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
+++
b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
@@ -38,6 +38,7 @@ <h4 style="clear: left; display: inline-block;">Summary</h4>
<th>Cores</th>
<th>Active Tasks</th>
<th>Failed Tasks</th>
+ <th>Killed Tasks</th>
<th>Complete Tasks</th>
<th>Total Tasks</th>
<th><span data-toggle="tooltip"
@@ -93,6 +94,7 @@ <h4 style="clear: left; display: inline-block;">Executors</h4>
<th><span data-toggle="tooltip" data-placement="top"
title="Cores">Cores</span></th>
<th><span data-toggle="tooltip" data-placement="top" title="Active
Tasks">Active Tasks</span></th>
<th><span data-toggle="tooltip" data-placement="top" title="Failed
Tasks">Failed Tasks</span></th>
+ <th><span data-toggle="tooltip" data-placement="top" title="Killed
Tasks">Killed Tasks</span></th>
<th><span data-toggle="tooltip" data-placement="top" title="Complete
Tasks">Complete Tasks</span></th>
<th><span data-toggle="tooltip" data-placement="top" title="Total
Tasks">Total Tasks</span></th>
<th>
diff --git
a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index 6717af3ac4daf..f8fc21cfaef6d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -204,6 +204,7 @@ $(document).ready(function () {
var allMaxTasks = 0;
var allActiveTasks = 0;
var allFailedTasks = 0;
+ var allKilledTasks = 0;
var allCompletedTasks = 0;
var allTotalTasks = 0;
var allTotalDuration = 0;
@@ -226,6 +227,7 @@ $(document).ready(function () {
var activeMaxTasks = 0;
var activeActiveTasks = 0;
var activeFailedTasks = 0;
+ var activeKilledTasks = 0;
var activeCompletedTasks = 0;
var activeTotalTasks = 0;
var activeTotalDuration = 0;
@@ -248,6 +250,7 @@ $(document).ready(function () {
var deadMaxTasks = 0;
var deadActiveTasks = 0;
var deadFailedTasks = 0;
+ var deadKilledTasks = 0;
var deadCompletedTasks = 0;
var deadTotalTasks = 0;
var deadTotalDuration = 0;
@@ -282,6 +285,7 @@ $(document).ready(function () {
allMaxTasks += exec.maxTasks;
allActiveTasks += exec.activeTasks;
allFailedTasks += exec.failedTasks;
+ allKilledTasks += exec.killedTasks;
allCompletedTasks += exec.completedTasks;
allTotalTasks += exec.totalTasks;
allTotalDuration += exec.totalDuration;
@@ -304,6 +308,7 @@ $(document).ready(function () {
activeMaxTasks += exec.maxTasks;
activeActiveTasks += exec.activeTasks;
activeFailedTasks += exec.failedTasks;
+ activeKilledTasks += exec.killedTasks;
activeCompletedTasks += exec.completedTasks;
activeTotalTasks += exec.totalTasks;
activeTotalDuration += exec.totalDuration;
@@ -326,6 +331,7 @@ $(document).ready(function () {
deadMaxTasks += exec.maxTasks;
deadActiveTasks += exec.activeTasks;
deadFailedTasks += exec.failedTasks;
+ deadKilledTasks += exec.killedTasks;
deadCompletedTasks += exec.completedTasks;
deadTotalTasks += exec.totalTasks;
deadTotalDuration += exec.totalDuration;
@@ -351,6 +357,7 @@ $(document).ready(function () {
"allMaxTasks": allMaxTasks,
"allActiveTasks": allActiveTasks,
"allFailedTasks": allFailedTasks,
+ "allKilledTasks": allKilledTasks,
"allCompletedTasks": allCompletedTasks,
"allTotalTasks": allTotalTasks,
"allTotalDuration": allTotalDuration,
@@ -374,6 +381,7 @@ $(document).ready(function () {
"allMaxTasks": activeMaxTasks,
"allActiveTasks": activeActiveTasks,
"allFailedTasks": activeFailedTasks,
+ "allKilledTasks": activeKilledTasks,
"allCompletedTasks": activeCompletedTasks,
"allTotalTasks": activeTotalTasks,
"allTotalDuration": activeTotalDuration,
@@ -397,6 +405,7 @@ $(document).ready(function () {
"allMaxTasks": deadMaxTasks,
"allActiveTasks": deadActiveTasks,
"allFailedTasks": deadFailedTasks,
+ "allKilledTasks": deadKilledTasks,
"allCompletedTasks": deadCompletedTasks,
"allTotalTasks": deadTotalTasks,
"allTotalDuration": deadTotalDuration,
@@ -481,6 +490,7 @@ $(document).ready(function () {
}
}
},
+ {data: 'killedTasks'},
{data: 'completedTasks'},
{data: 'totalTasks'},
{
@@ -577,6 +587,7 @@ $(document).ready(function () {
}
}
},
+ {data: 'allKilledTasks'},
{data: 'allCompletedTasks'},
{data: 'allTotalTasks'},
{
diff --git
a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 5ea161cd0d151..b874ed47ba632 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -546,6 +546,7 @@ private[spark] class AppStatusListener(
exec.activeTasks -= 1
exec.completedTasks += completedDelta
exec.failedTasks += failedDelta
+ exec.killedTasks += killedDelta
exec.totalDuration += event.taskInfo.duration
// Note: For resubmitted tasks, we continue to use the metrics that
belong to the
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index 79e3f13b826ce..cb945e47a7e29 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -249,6 +249,7 @@ private class LiveExecutor(val executorId: String,
_addTime: Long) extends LiveE
var activeTasks = 0
var completedTasks = 0
var failedTasks = 0
+ var killedTasks = 0
var totalDuration = 0L
var totalGcTime = 0L
var totalInputBytes = 0L
@@ -288,6 +289,7 @@ private class LiveExecutor(val executorId: String,
_addTime: Long) extends LiveE
maxTasks,
activeTasks,
failedTasks,
+ killedTasks,
completedTasks,
totalTasks,
totalDuration,
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 971d7e90fa7b8..4ccc83e56b4fb 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -84,6 +84,7 @@ class ExecutorSummary private[spark](
val maxTasks: Int,
val activeTasks: Int,
val failedTasks: Int,
+ val killedTasks: Int,
val completedTasks: Int,
val totalTasks: Int,
val totalDuration: Long,
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index 7bb8fe8fd8f98..d0f6f1a8aabc9 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -9,6 +9,7 @@
"maxTasks" : 0,
"activeTasks" : 0,
"failedTasks" : 1,
+ "killedTasks" : 0,
"completedTasks" : 31,
"totalTasks" : 32,
"totalDuration" : 8820,
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index dd5b1dcb7372b..c7497236c8dec 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -9,6 +9,7 @@
"maxTasks" : 0,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
"totalDuration" : 0,
@@ -38,6 +39,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 12,
"totalTasks" : 12,
"totalDuration" : 2453,
@@ -70,6 +72,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 4,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 4,
"totalDuration" : 2537,
@@ -102,6 +105,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 4,
"totalTasks" : 4,
"totalDuration" : 3152,
@@ -134,6 +138,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 4,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 4,
"totalDuration" : 2551,
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
index 3e55d3d9d7eb9..855c251d65dcb 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
@@ -9,6 +9,7 @@
"maxTasks" : 0,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
"totalDuration" : 0,
@@ -38,6 +39,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 12,
"totalTasks" : 12,
"totalDuration" : 2453,
@@ -70,6 +72,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 4,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 4,
"totalDuration" : 2537,
@@ -102,6 +105,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 4,
"totalTasks" : 4,
"totalDuration" : 3152,
@@ -134,6 +138,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 4,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 4,
"totalDuration" : 2551,
diff --git
a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
index e87f3e78f2dc8..10b25a3ca8269 100644
---
a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
@@ -9,6 +9,7 @@
"maxTasks" : 0,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 0,
"totalDuration" : 0,
@@ -32,6 +33,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 4,
"totalTasks" : 4,
"totalDuration" : 3457,
@@ -58,6 +60,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 6,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 6,
"totalDuration" : 2792,
@@ -84,6 +87,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 0,
+ "killedTasks" : 0,
"completedTasks" : 12,
"totalTasks" : 12,
"totalDuration" : 2613,
@@ -110,6 +114,7 @@
"maxTasks" : 4,
"activeTasks" : 0,
"failedTasks" : 4,
+ "killedTasks" : 0,
"completedTasks" : 0,
"totalTasks" : 4,
"totalDuration" : 2741,
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Total Tasks in WebUI does not match Active+Failed+Complete Tasks
> ----------------------------------------------------------------
>
> Key: SPARK-25064
> URL: https://issues.apache.org/jira/browse/SPARK-25064
> Project: Spark
> Issue Type: Bug
> Components: Web UI
> Affects Versions: 2.2.2, 2.3.0, 2.3.1
> Reporter: StanZhai
> Priority: Minor
> Attachments: 1533128402933_3.png
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
