Github user vanzin commented on a diff in the pull request: https://github.com/apache/spark/pull/23038#discussion_r234382254 --- Diff: core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala --- @@ -1275,6 +1275,49 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter { assert(allJobs.head.numFailedStages == 1) } + test("SPARK-25451: total tasks in the executor summary should match total stage tasks") { + val testConf = conf.clone() + .set("spark.ui.liveUpdate.period", s"${Int.MaxValue}s") + + val listener = new AppStatusListener(store, testConf, true) + + val stage = new StageInfo(1, 0, "stage", 4, Nil, Nil, "details") + listener.onJobStart(SparkListenerJobStart(1, time, Seq(stage), null)) + listener.onStageSubmitted(SparkListenerStageSubmitted(stage, new Properties())) + + val tasks = createTasks(4, Array("1", "2")) + tasks.foreach { task => + listener.onTaskStart(SparkListenerTaskStart(stage.stageId, stage.attemptNumber, task)) + } + + tasks.filter(_.index < 2).foreach { task => + time += 1 + var execId = (task.index % 2 + 1).toString + tasks(task.index).markFinished(TaskState.FAILED, time) + listener.onTaskEnd( + SparkListenerTaskEnd(stage.stageId, stage.attemptId, "taskType", + ExecutorLostFailure(execId, true, Some("Lost executor")), tasks(task.index), null)) + } + + stage.failureReason = Some("Failed") + listener.onStageCompleted(SparkListenerStageCompleted(stage)) + time += 1 + listener.onJobEnd(SparkListenerJobEnd(1, time, JobFailed(new RuntimeException("Bad Executor")))) + + tasks.filter(_.index >= 2).foreach { task => + time += 1 + var execId = (task.index % 2 + 1).toString + tasks(task.index).markFinished(TaskState.FAILED, time) + listener.onTaskEnd(SparkListenerTaskEnd(stage.stageId, stage.attemptId, "taskType", + ExecutorLostFailure(execId, true, Some("Lost executor")), tasks(task.index), null)) + } + + val esummary = store.view(classOf[ExecutorStageSummaryWrapper]).asScala.map(_.info) + esummary.foreach { + execSummary => assert(execSummary.failedTasks == 2) --- End diff -- keep `execSummary =>` in the previous line.
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org