Repository: spark Updated Branches: refs/heads/master be846db48 -> 190d8b0b6
[SPARK-20591][WEB UI] Succeeded tasks num not equal in all jobs page and job detail page on spark web ui when speculative task(s) exist. ## What changes were proposed in this pull request? Modified succeeded num in job detail page from "completed = stageData.completedIndices.size" to "completed = stageData.numCompleteTasks",which making succeeded tasks num in all jobs page and job detail page look more consistent, and more easily to find which stages the speculative task(s) were in. ## How was this patch tested? manual tests Author: fjh100456 <fu.jinh...@zte.com.cn> Closes #17923 from fjh100456/master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/190d8b0b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/190d8b0b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/190d8b0b Branch: refs/heads/master Commit: 190d8b0b6393290349ab834457f9e916fdaef530 Parents: be846db Author: fjh100456 <fu.jinh...@zte.com.cn> Authored: Mon May 22 13:58:42 2017 +0100 Committer: Sean Owen <so...@cloudera.com> Committed: Mon May 22 13:58:42 2017 +0100 ---------------------------------------------------------------------- .../src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 3 ++- .../scala/org/apache/spark/ui/jobs/JobProgressListener.scala | 1 + core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala | 1 + core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala | 7 +------ 4 files changed, 5 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index a0fd29c..cce7a76 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -631,7 +631,8 @@ private[ui] class JobPagedTable( {if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"} </td> <td class="progress-cell"> - {UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks, + {UIUtils.makeProgressBar(started = job.numActiveTasks, + completed = job.completedIndices.size, failed = job.numFailedTasks, skipped = job.numSkippedTasks, reasonToNumKilled = job.reasonToNumKilled, total = job.numTasks - job.numSkippedTasks)} </td> http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index 7370f9f..1b10feb 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -423,6 +423,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { jobData.numActiveTasks -= 1 taskEnd.reason match { case Success => + jobData.completedIndices.add((taskEnd.stageId, info.index)) jobData.numCompletedTasks += 1 case kill: TaskKilled => jobData.reasonToNumKilled = jobData.reasonToNumKilled.updated( http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala index 8d280bc..048c4ad 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala @@ -62,6 +62,7 @@ private[spark] object UIData { var numTasks: Int = 0, var numActiveTasks: Int = 0, var numCompletedTasks: Int = 0, + var completedIndices: OpenHashSet[(Int, Int)] = new OpenHashSet[(Int, Int)](), var numSkippedTasks: Int = 0, var numFailedTasks: Int = 0, var reasonToNumKilled: Map[String, Int] = Map.empty, http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index bdd1488..267c8dc 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -320,12 +320,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B eventually(timeout(5 seconds), interval(50 milliseconds)) { goToUi(sc, "/jobs") find(cssSelector(".stage-progress-cell")).get.text should be ("2/2 (1 failed)") - // Ideally, the following test would pass, but currently we overcount completed tasks - // if task recomputations occur: - // find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)") - // Instead, we guarantee that the total number of tasks is always correct, while the number - // of completed tasks may be higher: - find(cssSelector(".progress-cell .progress")).get.text should be ("3/2 (1 failed)") + find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)") } val jobJson = getJson(sc.ui.get, "jobs") (jobJson \ "numTasks").extract[Int]should be (2) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
