This is an automated email from the ASF dual-hosted git repository. meng pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new d98a5ce [SPARK-28115][CORE][TEST] Fix flaky test: SparkContextSuite.test resource scheduling under local-cluster mode d98a5ce is described below commit d98a5ce34d6b4b098d30c26c89a9d65d931f930d Author: Xingbo Jiang <xingbo.ji...@databricks.com> AuthorDate: Thu Jun 20 13:23:29 2019 -0700 [SPARK-28115][CORE][TEST] Fix flaky test: SparkContextSuite.test resource scheduling under local-cluster mode ## What changes were proposed in this pull request? The test `SparkContextSuite.test resource scheduling under local-cluster mode` has been flaky, because it expects the size of `sc.statusTracker.getExecutorInfos` be the same as the number of executors, while the returned list contains both the driver and executors. ## How was this patch tested? Updated existing tests. Closes #24917 from jiangxb1987/getExecutorInfos. Authored-by: Xingbo Jiang <xingbo.ji...@databricks.com> Signed-off-by: Xiangrui Meng <m...@databricks.com> --- .../src/main/scala/org/apache/spark/SparkStatusTracker.scala | 1 + core/src/test/scala/org/apache/spark/SparkContextSuite.scala | 12 +++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala b/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala index 815237e..555c085 100644 --- a/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala +++ b/core/src/main/scala/org/apache/spark/SparkStatusTracker.scala @@ -99,6 +99,7 @@ class SparkStatusTracker private[spark] (sc: SparkContext, store: AppStatusStore /** * Returns information of all known executors, including host, port, cacheSize, numRunningTasks * and memory metrics. + * Note this include information for both the driver and executors. */ def getExecutorInfos: Array[SparkExecutorInfo] = { store.executorList(true).map { exec => diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala index fa2c4bd..628ac60 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala @@ -750,9 +750,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu sc = new SparkContext(conf) // Ensure all executors has started - eventually(timeout(10.seconds)) { - assert(sc.statusTracker.getExecutorInfos.size == 1) - } + TestUtils.waitUntilExecutorsUp(sc, 1, 10000) assert(sc.resources.size === 1) assert(sc.resources.get(GPU).get.addresses === Array("5", "6")) assert(sc.resources.get(GPU).get.name === "gpu") @@ -780,9 +778,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu sc = new SparkContext(conf) // Ensure all executors has started - eventually(timeout(10.seconds)) { - assert(sc.statusTracker.getExecutorInfos.size == 1) - } + TestUtils.waitUntilExecutorsUp(sc, 1, 10000) // driver gpu resources file should take precedence over the script assert(sc.resources.size === 1) assert(sc.resources.get(GPU).get.addresses === Array("0", "1", "8")) @@ -855,9 +851,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu sc = new SparkContext(conf) // Ensure all executors has started - eventually(timeout(60.seconds)) { - assert(sc.statusTracker.getExecutorInfos.size == 3) - } + TestUtils.waitUntilExecutorsUp(sc, 3, 60000) val rdd = sc.makeRDD(1 to 10, 9).mapPartitions { it => val context = TaskContext.get() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org