This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new b76f262 [SPARK-27032][TEST] De-flake org.apache.spark.sql.execution.streaming.HDFSMetadataLogSuite.HDFSMetadataLog: metadata directory collision b76f262 is described below commit b76f262fc84a1fbe52bbc6bf74573dec4d1ae4df Author: Sean Owen <sean.o...@databricks.com> AuthorDate: Mon Mar 4 13:36:41 2019 +0900 [SPARK-27032][TEST] De-flake org.apache.spark.sql.execution.streaming.HDFSMetadataLogSuite.HDFSMetadataLog: metadata directory collision ## What changes were proposed in this pull request? Reduce work in HDFSMetadataLogSuite test to possibly de-flake it. ## How was this patch tested? Existing tests Closes #23937 from srowen/SPARK-27032. Authored-by: Sean Owen <sean.o...@databricks.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../spark/sql/execution/streaming/HDFSMetadataLogSuite.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala index 0e36e7f..3706835 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLogSuite.scala @@ -131,9 +131,10 @@ class HDFSMetadataLogSuite extends SparkFunSuite with SharedSQLContext { testQuietly("HDFSMetadataLog: metadata directory collision") { withTempDir { temp => - val waiter = new Waiter - val maxBatchId = 100 - for (id <- 0 until 10) { + val waiter = new Waiter() + val maxBatchId = 10 + val numThreads = 5 + for (id <- 0 until numThreads) { new UninterruptibleThread(s"HDFSMetadataLog: metadata directory collision - thread $id") { override def run(): Unit = waiter { val metadataLog = @@ -146,7 +147,7 @@ class HDFSMetadataLogSuite extends SparkFunSuite with SharedSQLContext { nextBatchId += 1 } } catch { - case e: ConcurrentModificationException => + case _: ConcurrentModificationException => // This is expected since there are multiple writers } finally { waiter.dismiss() @@ -155,7 +156,7 @@ class HDFSMetadataLogSuite extends SparkFunSuite with SharedSQLContext { }.start() } - waiter.await(timeout(10.seconds), dismissals(10)) + waiter.await(timeout(10.seconds), dismissals(numThreads)) val metadataLog = new HDFSMetadataLog[String](spark, temp.getAbsolutePath) assert(metadataLog.getLatest() === Some(maxBatchId -> maxBatchId.toString)) assert( --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org