Github user vanzin commented on a diff in the pull request:

    https://github.com/apache/spark/pull/23088#discussion_r236405732
  
    --- Diff: core/src/main/scala/org/apache/spark/status/AppStatusStore.scala 
---
    @@ -222,29 +223,20 @@ private[spark] class AppStatusStore(
         val indices = quantiles.map { q => math.min((q * count).toLong, count 
- 1) }
     
         def scanTasks(index: String)(fn: TaskDataWrapper => Long): 
IndexedSeq[Double] = {
    -      Utils.tryWithResource(
    -        store.view(classOf[TaskDataWrapper])
    -          .parent(stageKey)
    -          .index(index)
    -          .first(0L)
    -          .closeableIterator()
    -      ) { it =>
    -        var last = Double.NaN
    -        var currentIdx = -1L
    -        indices.map { idx =>
    -          if (idx == currentIdx) {
    -            last
    -          } else {
    -            val diff = idx - currentIdx
    -            currentIdx = idx
    -            if (it.skip(diff - 1)) {
    -              last = fn(it.next()).toDouble
    -              last
    -            } else {
    -              Double.NaN
    -            }
    -          }
    -        }.toIndexedSeq
    +      val quantileTasks = store.view(classOf[TaskDataWrapper])
    +        .parent(stageKey)
    +        .index(index)
    +        .first(0L)
    +        .asScala
    +        .filter(_.status == "SUCCESS") // Filter "SUCCESS" tasks
    +        .zipWithIndex
    +        .filter(x => indices.contains(x._2))
    --- End diff --
    
    It doesn't make a runtime different. That's why this is a style nit.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to