This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 7373cd2 [SPARK-38619][TESTS] Clean up Junit api usage in scalatest 7373cd2 is described below commit 7373cd22210ab9ef865740059ba265f781978469 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Tue Mar 22 18:09:40 2022 -0500 [SPARK-38619][TESTS] Clean up Junit api usage in scalatest ### What changes were proposed in this pull request? This pr clean up Junit api usage in scalatest and replace them with native scalatest api. ### Why are the changes needed? Use scalatest api in scalatest and Junit api in Java test ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA Closes #35929 from LuciferYang/remove-junit-in-scalatest. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- .../scala/org/apache/spark/SparkContextSuite.scala | 11 ++-- .../spark/api/python/PythonHadoopUtilSuite.scala | 9 ++- .../spark/storage/BlockManagerMasterSuite.scala | 6 +- .../cluster/k8s/ExecutorRollPluginSuite.scala | 71 +++++++++++----------- .../scala/org/apache/spark/sql/QueryTest.scala | 3 +- 5 files changed, 46 insertions(+), 54 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala index 411a3b1..8671180 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala @@ -33,7 +33,6 @@ import org.apache.hadoop.mapred.TextInputFormat import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat} import org.apache.logging.log4j.{Level, LogManager} import org.json4s.{DefaultFormats, Extraction} -import org.junit.Assert.{assertEquals, assertFalse} import org.scalatest.concurrent.Eventually import org.scalatest.matchers.must.Matchers._ @@ -1257,12 +1256,12 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu test("SPARK-35383: Fill missing S3A magic committer configs if needed") { val c1 = new SparkConf().setAppName("s3a-test").setMaster("local") sc = new SparkContext(c1) - assertFalse(sc.getConf.contains("spark.hadoop.fs.s3a.committer.name")) + assert(!sc.getConf.contains("spark.hadoop.fs.s3a.committer.name")) resetSparkContext() val c2 = c1.clone.set("spark.hadoop.fs.s3a.bucket.mybucket.committer.magic.enabled", "false") sc = new SparkContext(c2) - assertFalse(sc.getConf.contains("spark.hadoop.fs.s3a.committer.name")) + assert(!sc.getConf.contains("spark.hadoop.fs.s3a.committer.name")) resetSparkContext() val c3 = c1.clone.set("spark.hadoop.fs.s3a.bucket.mybucket.committer.magic.enabled", "true") @@ -1277,7 +1276,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu "spark.sql.sources.commitProtocolClass" -> "org.apache.spark.internal.io.cloud.PathOutputCommitProtocol" ).foreach { case (k, v) => - assertEquals(v, sc.getConf.get(k)) + assert(v == sc.getConf.get(k)) } // Respect a user configuration @@ -1294,9 +1293,9 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu "spark.sql.sources.commitProtocolClass" -> null ).foreach { case (k, v) => if (v == null) { - assertFalse(sc.getConf.contains(k)) + assert(!sc.getConf.contains(k)) } else { - assertEquals(v, sc.getConf.get(k)) + assert(v == sc.getConf.get(k)) } } } diff --git a/core/src/test/scala/org/apache/spark/api/python/PythonHadoopUtilSuite.scala b/core/src/test/scala/org/apache/spark/api/python/PythonHadoopUtilSuite.scala index 039d49d..b4f7f1d 100644 --- a/core/src/test/scala/org/apache/spark/api/python/PythonHadoopUtilSuite.scala +++ b/core/src/test/scala/org/apache/spark/api/python/PythonHadoopUtilSuite.scala @@ -21,7 +21,6 @@ import java.util.HashMap import org.apache.hadoop.io.{BooleanWritable, BytesWritable, ByteWritable, DoubleWritable, FloatWritable, IntWritable, LongWritable, MapWritable, NullWritable, ShortWritable, Text, Writable} -import org.junit.Assert import org.mockito.Mockito.mock import org.apache.spark.SparkFunSuite @@ -34,13 +33,13 @@ class PythonHadoopUtilSuite extends SparkFunSuite { val writableToJavaConverter = new WritableToJavaConverter(broadcast) val result = writableToJavaConverter.convert(input) expected match { - case _: Array[Byte] => Assert.assertArrayEquals( - expected.asInstanceOf[Array[Byte]], result.asInstanceOf[Array[Byte]]) - case _ => Assert.assertEquals(expected, result) + case _: Array[Byte] => assert(expected.asInstanceOf[Array[Byte]] + sameElements result.asInstanceOf[Array[Byte]]) + case _ => assert(expected == result) } val javaToWritableConverter = new JavaToWritableConverter() val reConverted = javaToWritableConverter.convert(result) - Assert.assertEquals("Round trip conversion failed", input, reConverted) + assert(input == reConverted, "Round trip conversion failed") } test("Testing roundtrip conversion of various types") { diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala index 0d54726..2457aef 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala @@ -17,19 +17,17 @@ package org.apache.spark.storage -import org.junit.Assert.assertTrue - import org.apache.spark.{SparkConf, SparkFunSuite} class BlockManagerMasterSuite extends SparkFunSuite { test("SPARK-31422: getMemoryStatus should not fail after BlockManagerMaster stops") { val bmm = new BlockManagerMaster(null, null, new SparkConf, true) - assertTrue(bmm.getMemoryStatus.isEmpty) + assert(bmm.getMemoryStatus.isEmpty) } test("SPARK-31422: getStorageStatus should not fail after BlockManagerMaster stops") { val bmm = new BlockManagerMaster(null, null, new SparkConf, true) - assertTrue(bmm.getStorageStatus.isEmpty) + assert(bmm.getStorageStatus.isEmpty) } } diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorRollPluginSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorRollPluginSuite.scala index 886abc0..a9e9b66 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorRollPluginSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorRollPluginSuite.scala @@ -18,7 +18,6 @@ package org.apache.spark.scheduler.cluster.k8s import java.util.Date -import org.junit.Assert.assertEquals import org.scalatest.PrivateMethodTester import org.apache.spark.SparkFunSuite @@ -121,61 +120,59 @@ class ExecutorRollPluginSuite extends SparkFunSuite with PrivateMethodTester { test("Empty executor list") { ExecutorRollPolicy.values.foreach { value => - assertEquals(None, plugin.invokePrivate[Option[String]](_choose(Seq.empty, value))) + assert(plugin.invokePrivate[Option[String]](_choose(Seq.empty, value)).isEmpty) } } test("Driver summary should be ignored") { ExecutorRollPolicy.values.foreach { value => - assertEquals(plugin.invokePrivate(_choose(Seq(driverSummary), value)), None) + assert(plugin.invokePrivate(_choose(Seq(driverSummary), value)).isEmpty) } } test("A one-item executor list") { ExecutorRollPolicy.values.filter(_ != ExecutorRollPolicy.OUTLIER_NO_FALLBACK).foreach { value => - assertEquals( - Some(execWithSmallestID.id), - plugin.invokePrivate(_choose(Seq(execWithSmallestID), value))) + assert( + plugin.invokePrivate(_choose(Seq(execWithSmallestID), value)) + .contains(execWithSmallestID.id)) } } test("SPARK-37806: All policy should ignore executor if totalTasks < minTasks") { plugin.asInstanceOf[ExecutorRollDriverPlugin].minTasks = 1000 ExecutorRollPolicy.values.foreach { value => - assertEquals(None, plugin.invokePrivate(_choose(list, value))) + assert(plugin.invokePrivate(_choose(list, value)).isEmpty) } } test("Policy: ID") { - assertEquals(Some("1"), plugin.invokePrivate(_choose(list, ExecutorRollPolicy.ID))) + assert(plugin.invokePrivate(_choose(list, ExecutorRollPolicy.ID)).contains("1")) } test("Policy: ADD_TIME") { - assertEquals(Some("2"), plugin.invokePrivate(_choose(list, ExecutorRollPolicy.ADD_TIME))) + assert(plugin.invokePrivate(_choose(list, ExecutorRollPolicy.ADD_TIME)).contains("2")) } test("Policy: TOTAL_GC_TIME") { - assertEquals(Some("3"), plugin.invokePrivate(_choose(list, ExecutorRollPolicy.TOTAL_GC_TIME))) + assert(plugin.invokePrivate(_choose(list, ExecutorRollPolicy.TOTAL_GC_TIME)).contains("3")) } test("Policy: TOTAL_DURATION") { - assertEquals(Some("4"), plugin.invokePrivate(_choose(list, ExecutorRollPolicy.TOTAL_DURATION))) + assert(plugin.invokePrivate(_choose(list, ExecutorRollPolicy.TOTAL_DURATION)).contains("4")) } test("Policy: FAILED_TASKS") { - assertEquals(Some("5"), plugin.invokePrivate(_choose(list, ExecutorRollPolicy.FAILED_TASKS))) + assert(plugin.invokePrivate(_choose(list, ExecutorRollPolicy.FAILED_TASKS)).contains("5")) } test("Policy: AVERAGE_DURATION") { - assertEquals( - Some("6"), - plugin.invokePrivate(_choose(list, ExecutorRollPolicy.AVERAGE_DURATION))) + assert(plugin.invokePrivate(_choose(list, ExecutorRollPolicy.AVERAGE_DURATION)).contains("6")) } test("Policy: OUTLIER - Work like TOTAL_DURATION if there is no outlier") { - assertEquals( - plugin.invokePrivate(_choose(list, ExecutorRollPolicy.TOTAL_DURATION)), - plugin.invokePrivate(_choose(list, ExecutorRollPolicy.OUTLIER))) + assert( + plugin.invokePrivate(_choose(list, ExecutorRollPolicy.TOTAL_DURATION)) == + plugin.invokePrivate(_choose(list, ExecutorRollPolicy.OUTLIER))) } test("Policy: OUTLIER - Detect an average task duration outlier") { @@ -186,9 +183,9 @@ class ExecutorRollPluginSuite extends SparkFunSuite with PrivateMethodTester { 0, false, 0, new Date(1639300001000L), Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1, false, Set()) - assertEquals( - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.AVERAGE_DURATION)), - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER))) + assert( + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.AVERAGE_DURATION)) == + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER))) } test("Policy: OUTLIER - Detect a total task duration outlier") { @@ -199,9 +196,9 @@ class ExecutorRollPluginSuite extends SparkFunSuite with PrivateMethodTester { 0, false, 0, new Date(1639300001000L), Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1, false, Set()) - assertEquals( - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_DURATION)), - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER))) + assert( + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_DURATION)) == + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER))) } test("Policy: OUTLIER - Detect a total GC time outlier") { @@ -212,13 +209,13 @@ class ExecutorRollPluginSuite extends SparkFunSuite with PrivateMethodTester { 0, false, 0, new Date(1639300001000L), Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1, false, Set()) - assertEquals( - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_GC_TIME)), - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER))) + assert( + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_GC_TIME)) == + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER))) } test("Policy: OUTLIER_NO_FALLBACK - Return None if there are no outliers") { - assertEquals(None, plugin.invokePrivate(_choose(list, ExecutorRollPolicy.OUTLIER_NO_FALLBACK))) + assert(plugin.invokePrivate(_choose(list, ExecutorRollPolicy.OUTLIER_NO_FALLBACK)).isEmpty) } test("Policy: OUTLIER_NO_FALLBACK - Detect an average task duration outlier") { @@ -229,9 +226,9 @@ class ExecutorRollPluginSuite extends SparkFunSuite with PrivateMethodTester { 0, false, 0, new Date(1639300001000L), Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1, false, Set()) - assertEquals( - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.AVERAGE_DURATION)), - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER_NO_FALLBACK))) + assert( + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.AVERAGE_DURATION)) == + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER_NO_FALLBACK))) } test("Policy: OUTLIER_NO_FALLBACK - Detect a total task duration outlier") { @@ -242,9 +239,9 @@ class ExecutorRollPluginSuite extends SparkFunSuite with PrivateMethodTester { 0, false, 0, new Date(1639300001000L), Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1, false, Set()) - assertEquals( - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_DURATION)), - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER_NO_FALLBACK))) + assert( + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_DURATION)) == + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER_NO_FALLBACK))) } test("Policy: OUTLIER_NO_FALLBACK - Detect a total GC time outlier") { @@ -255,8 +252,8 @@ class ExecutorRollPluginSuite extends SparkFunSuite with PrivateMethodTester { 0, false, 0, new Date(1639300001000L), Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1, false, Set()) - assertEquals( - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_GC_TIME)), - plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER_NO_FALLBACK))) + assert( + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.TOTAL_GC_TIME)) == + plugin.invokePrivate(_choose(list :+ outlier, ExecutorRollPolicy.OUTLIER_NO_FALLBACK))) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala index 06f94c6..0bb5e52 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala @@ -21,7 +21,6 @@ import java.util.TimeZone import scala.collection.JavaConverters._ -import org.junit.Assert import org.scalatest.Assertions import org.apache.spark.sql.catalyst.plans._ @@ -420,7 +419,7 @@ object QueryTest extends Assertions { def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): Unit = { getErrorMessageInCheckAnswer(df, expectedAnswer.asScala.toSeq) match { - case Some(errorMessage) => Assert.fail(errorMessage) + case Some(errorMessage) => fail(errorMessage) case None => } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org