This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 7ec3a5730bd [SPARK-38678][TESTS][FOLLOWUP] Enable RocksDB tests in `AppStatusStoreSuite` and `StreamingQueryStatusListenerSuite` on Apple Silicon on MacOS 7ec3a5730bd is described below commit 7ec3a5730bd864089c22e19e823d87b107688378 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Wed Apr 13 10:45:22 2022 -0700 [SPARK-38678][TESTS][FOLLOWUP] Enable RocksDB tests in `AppStatusStoreSuite` and `StreamingQueryStatusListenerSuite` on Apple Silicon on MacOS ### What changes were proposed in this pull request? This pr aims to enable RocksDB tests in `AppStatusStoreSuite` and `StreamingQueryStatusListenerSuite` on Apple Silicon on MacOS, it's a followup of SPARK-38678. ### Why are the changes needed? Enable more RocksDB related test on Apple Silicon on MacOS ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - Pass GA - Manual test on Apple Silicon environment: ``` build/sbt "core/testOnly *AppStatusStoreSuite*" "sql/testOnly *StreamingQueryStatusListenerSuite*" ``` All tests passed. Closes #36139 from LuciferYang/SPARK-38678-FOLLOWUP. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../apache/spark/status/AppStatusStoreSuite.scala | 26 +++++++++++++--------- .../ui/StreamingQueryStatusListenerSuite.scala | 1 - 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala index 70852164b89..b05f2b799d2 100644 --- a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala +++ b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala @@ -90,10 +90,6 @@ class AppStatusStoreSuite extends SparkFunSuite { if (live) { return AppStatusStore.createLiveStore(conf) } - // LevelDB doesn't support Apple Silicon yet - if (Utils.isMacOnAppleSilicon && disk) { - return null - } val store: KVStore = if (disk) { conf.set(HYBRID_STORE_DISK_BACKEND, diskStoreType.toString) @@ -106,12 +102,22 @@ class AppStatusStoreSuite extends SparkFunSuite { new AppStatusStore(store) } - Seq( - "disk leveldb" -> createAppStore(disk = true, HybridStoreDiskBackend.LEVELDB, live = false), - "disk rocksdb" -> createAppStore(disk = true, HybridStoreDiskBackend.ROCKSDB, live = false), - "in memory" -> createAppStore(disk = false, live = false), - "in memory live" -> createAppStore(disk = false, live = true) - ).foreach { case (hint, appStore) => + private val cases = { + val baseCases = Seq( + "disk rocksdb" -> createAppStore(disk = true, HybridStoreDiskBackend.ROCKSDB, live = false), + "in memory" -> createAppStore(disk = false, live = false), + "in memory live" -> createAppStore(disk = false, live = true) + ) + if (Utils.isMacOnAppleSilicon) { + baseCases + } else { + Seq( + "disk leveldb" -> createAppStore(disk = true, HybridStoreDiskBackend.LEVELDB, live = false) + ) ++ baseCases + } + } + + cases.foreach { case (hint, appStore) => test(s"SPARK-26260: summary should contain only successful tasks' metrics (store = $hint)") { assume(appStore != null) val store = appStore.store diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala index 1d1b51354f8..1a51b58f4f6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala @@ -236,7 +236,6 @@ class StreamingQueryStatusListenerSuite extends StreamTest { } test("SPARK-38056: test writing StreamingQueryData to a RocksDB store") { - assume(!Utils.isMacOnAppleSilicon) val conf = new SparkConf() .set(HYBRID_STORE_DISK_BACKEND, HybridStoreDiskBackend.ROCKSDB.toString) val testDir = Utils.createTempDir() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org