cloud-fan commented on a change in pull request #24892: [SPARK-25341][Core]
Support rolling back a shuffle map stage and re-generate the shuffle files
URL: https://github.com/apache/spark/pull/24892#discussion_r299352482
##########
File path:
core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
##########
@@ -2710,37 +2710,32 @@ class DAGSchedulerSuite extends SparkFunSuite with
LocalSparkContext with TimeLi
assert(countSubmittedMapStageAttempts() === 2)
}
- test("SPARK-23207: retry all the succeeding stages when the map stage is
indeterminate") {
+ private def constructIndeterminateStageRetryScenario(): (Int, Int) = {
val shuffleMapRdd1 = new MyRDD(sc, 2, Nil, indeterminate = true)
-
val shuffleDep1 = new ShuffleDependency(shuffleMapRdd1, new
HashPartitioner(2))
val shuffleId1 = shuffleDep1.shuffleId
val shuffleMapRdd2 = new MyRDD(sc, 2, List(shuffleDep1), tracker =
mapOutputTracker)
-
val shuffleDep2 = new ShuffleDependency(shuffleMapRdd2, new
HashPartitioner(2))
val shuffleId2 = shuffleDep2.shuffleId
val finalRdd = new MyRDD(sc, 2, List(shuffleDep2), tracker =
mapOutputTracker)
- submit(finalRdd, Array(0, 1))
+ submit(finalRdd, Array(0, 1), properties = new Properties())
// Finish the first shuffle map stage.
complete(taskSets(0), Seq(
(Success, makeMapStatus("hostA", 2)),
(Success, makeMapStatus("hostB", 2))))
assert(mapOutputTracker.findMissingPartitions(shuffleId1) ===
Some(Seq.empty))
-
// Finish the second shuffle map stage.
complete(taskSets(1), Seq(
(Success, makeMapStatus("hostC", 2)),
(Success, makeMapStatus("hostD", 2))))
assert(mapOutputTracker.findMissingPartitions(shuffleId2) ===
Some(Seq.empty))
-
// The first task of the final stage failed with fetch failure
runEvent(makeCompletionEvent(
taskSets(2).tasks(0),
FetchFailed(makeBlockManagerId("hostC"), shuffleId2, 0, 0, "ignored"),
null))
-
val failedStages = scheduler.failedStages.toSeq
assert(failedStages.length == 2)
Review comment:
It's `Seq(0, 1)` or `Seq(1, 2)`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
