Repository: spark Updated Branches: refs/heads/branch-2.0 f895d6d85 -> c1390ccbb
Revert [SPARK-14485][CORE] ignore task finished for executor lost This reverts commit 695dbc816a6d70289abeb145cb62ff4e62b3f49b. This change is being reverted because it hurts performance of some jobs, and only helps in a narrow set of cases. For more discussion, refer to the JIRA. Author: Kay Ousterhout <kayousterh...@gmail.com> Closes #13580 from kayousterhout/revert-SPARK-14485. (cherry picked from commit 5c16ad0d522e5124a6977533077afb7b38fc42a1) Signed-off-by: Kay Ousterhout <kayousterh...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c1390ccb Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c1390ccb Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c1390ccb Branch: refs/heads/branch-2.0 Commit: c1390ccbb2968156245e267e6c5cd2a27f7d0121 Parents: f895d6d Author: Kay Ousterhout <kayousterh...@gmail.com> Authored: Fri Jun 10 12:50:27 2016 -0700 Committer: Kay Ousterhout <kayousterh...@gmail.com> Committed: Fri Jun 10 12:51:29 2016 -0700 ---------------------------------------------------------------------- .../apache/spark/scheduler/TaskSchedulerImpl.scala | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/c1390ccb/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index d6f58e4..01e85ca 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -346,11 +346,9 @@ private[spark] class TaskSchedulerImpl( } taskIdToTaskSetManager.get(tid) match { case Some(taskSet) => - var executorId: String = null if (TaskState.isFinished(state)) { taskIdToTaskSetManager.remove(tid) taskIdToExecutorId.remove(tid).foreach { execId => - executorId = execId if (executorIdToTaskCount.contains(execId)) { executorIdToTaskCount(execId) -= 1 } @@ -358,17 +356,7 @@ private[spark] class TaskSchedulerImpl( } if (state == TaskState.FINISHED) { taskSet.removeRunningTask(tid) - // In some case, executor has already been removed by driver for heartbeats timeout, - // but at sometime, before executor killed by cluster, the task of running on this - // executor is finished and return task success state to driver. However, this kinds - // of task should be ignored, because the task on this executor is already re-queued - // by driver. For more details, can check in SPARK-14485. - if (executorId != null && !executorIdToTaskCount.contains(executorId)) { - logInfo(s"Ignoring update with state $state for TID $tid because its executor " + - s"has already been removed by driver") - } else { - taskResultGetter.enqueueSuccessfulTask(taskSet, tid, serializedData) - } + taskResultGetter.enqueueSuccessfulTask(taskSet, tid, serializedData) } else if (Set(TaskState.FAILED, TaskState.KILLED, TaskState.LOST).contains(state)) { taskSet.removeRunningTask(tid) taskResultGetter.enqueueFailedTask(taskSet, tid, state, serializedData) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
