This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 33f2cc8649 [GLUTEN-11911][VL][Spark4] Enable Structured Streaming
suites with GlutenPlugin fallback (#11957)
33f2cc8649 is described below
commit 33f2cc8649a368eadca237797af5a34ad9f7bbfb
Author: Mohammad Linjawi <[email protected]>
AuthorDate: Wed Apr 29 23:22:27 2026 +0300
[GLUTEN-11911][VL][Spark4] Enable Structured Streaming suites with
GlutenPlugin fallback (#11957)
* [VL][Spark4] enable structured streaming suites with fallback traits
* [VL][Spark4] make hash partition verify suite resolve valid
spark.test.home
* [VL][Spark4] fix spotless style in streaming hash partition suite
* [VL][Spark4] fix structured-streaming checkpoint resource loading
* [VL][Spark4] fix SPARK-49905 streaming shuffle assertion
---------
Co-authored-by: Mohammad Linjawi <[email protected]>
---
.../gluten/utils/velox/VeloxTestSettings.scala | 46 +++----
.../spark/sql/GlutenStreamingTestsTraits.scala | 134 +++++++++++++++++++++
.../GlutenAcceptsLatestSeenOffsetSuite.scala | 4 +-
.../streaming/GlutenEventTimeWatermarkSuite.scala | 6 +-
.../sql/streaming/GlutenFileStreamSinkSuite.scala | 6 +-
.../streaming/GlutenFileStreamSourceSuite.scala | 6 +-
.../streaming/GlutenFileStreamStressSuite.scala | 4 +-
...pGroupsInPandasWithStateDistributionSuite.scala | 4 +-
...GlutenFlatMapGroupsInPandasWithStateSuite.scala | 4 +-
...enFlatMapGroupsWithStateDistributionSuite.scala | 4 +-
.../GlutenFlatMapGroupsWithStateSuite.scala | 6 +-
...atMapGroupsWithStateWithInitialStateSuite.scala | 4 +-
.../streaming/GlutenMemorySourceStressSuite.scala | 6 +-
.../GlutenMultiStatefulOperatorsSuite.scala | 4 +-
.../streaming/GlutenReportSinkMetricsSuite.scala | 6 +-
.../spark/sql/streaming/GlutenStreamSuite.scala | 4 +-
...utenStreamingAggregationDistributionSuite.scala | 4 +-
.../GlutenStreamingAggregationSuite.scala | 8 +-
...enStreamingDeduplicationDistributionSuite.scala | 4 +-
.../GlutenStreamingDeduplicationSuite.scala | 6 +-
...treamingDeduplicationWithinWatermarkSuite.scala | 4 +-
.../sql/streaming/GlutenStreamingJoinSuite.scala | 14 ++-
...tenStreamingQueryHashPartitionVerifySuite.scala | 105 +++++++++++++++-
.../GlutenStreamingQueryListenerSuite.scala | 4 +-
.../GlutenStreamingQueryListenersConfSuite.scala | 4 +-
.../GlutenStreamingQueryManagerSuite.scala | 4 +-
...treamingQueryOptimizationCorrectnessSuite.scala | 4 +-
...lutenStreamingQueryStatusAndProgressSuite.scala | 4 +-
.../sql/streaming/GlutenStreamingQuerySuite.scala | 15 +--
.../streaming/GlutenStreamingSelfUnionSuite.scala | 6 +-
...enStreamingSessionWindowDistributionSuite.scala | 4 +-
.../GlutenStreamingSessionWindowSuite.scala | 4 +-
...reamingStateStoreFormatCompatibilitySuite.scala | 4 +-
...utenStreamingSymmetricHashJoinHelperSuite.scala | 4 +-
.../GlutenTransformWithListStateSuite.scala | 4 +-
.../GlutenTransformWithListStateTTLSuite.scala | 4 +-
.../GlutenTransformWithMapStateSuite.scala | 4 +-
.../GlutenTransformWithMapStateTTLSuite.scala | 4 +-
.../GlutenTransformWithStateAvroSuite.scala | 4 +-
.../GlutenTransformWithStateChainingSuite.scala | 4 +-
.../GlutenTransformWithStateClusterSuite.scala | 4 +-
...GlutenTransformWithStateInitialStateSuite.scala | 4 +-
.../streaming/GlutenTransformWithStateSuite.scala | 4 +-
.../GlutenTransformWithStateUnsafeRowSuite.scala | 4 +-
.../GlutenTransformWithValueStateTTLSuite.scala | 4 +-
.../streaming/GlutenTriggerAvailableNowSuite.scala | 6 +-
.../gluten/utils/velox/VeloxTestSettings.scala | 46 +++----
.../spark/sql/GlutenStreamingTestsTraits.scala | 134 +++++++++++++++++++++
.../GlutenAcceptsLatestSeenOffsetSuite.scala | 4 +-
.../streaming/GlutenEventTimeWatermarkSuite.scala | 6 +-
.../sql/streaming/GlutenFileStreamSinkSuite.scala | 6 +-
.../streaming/GlutenFileStreamSourceSuite.scala | 6 +-
.../streaming/GlutenFileStreamStressSuite.scala | 4 +-
...pGroupsInPandasWithStateDistributionSuite.scala | 4 +-
...GlutenFlatMapGroupsInPandasWithStateSuite.scala | 4 +-
...enFlatMapGroupsWithStateDistributionSuite.scala | 4 +-
.../GlutenFlatMapGroupsWithStateSuite.scala | 6 +-
...atMapGroupsWithStateWithInitialStateSuite.scala | 4 +-
.../streaming/GlutenMemorySourceStressSuite.scala | 6 +-
.../GlutenMultiStatefulOperatorsSuite.scala | 4 +-
.../streaming/GlutenReportSinkMetricsSuite.scala | 6 +-
.../GlutenStreamRealTimeModeAllowlistSuite.scala | 4 +-
.../GlutenStreamRealTimeModeE2ESuite.scala | 4 +-
.../streaming/GlutenStreamRealTimeModeSuite.scala | 6 +-
.../spark/sql/streaming/GlutenStreamSuite.scala | 4 +-
...utenStreamingAggregationDistributionSuite.scala | 4 +-
.../GlutenStreamingAggregationSuite.scala | 8 +-
...enStreamingDeduplicationDistributionSuite.scala | 4 +-
.../GlutenStreamingDeduplicationSuite.scala | 6 +-
...treamingDeduplicationWithinWatermarkSuite.scala | 4 +-
.../sql/streaming/GlutenStreamingJoinSuite.scala | 14 ++-
...tenStreamingQueryHashPartitionVerifySuite.scala | 105 +++++++++++++++-
.../GlutenStreamingQueryListenerSuite.scala | 4 +-
.../GlutenStreamingQueryListenersConfSuite.scala | 4 +-
.../GlutenStreamingQueryManagerSuite.scala | 4 +-
...treamingQueryOptimizationCorrectnessSuite.scala | 4 +-
...lutenStreamingQueryStatusAndProgressSuite.scala | 4 +-
.../sql/streaming/GlutenStreamingQuerySuite.scala | 15 +--
.../streaming/GlutenStreamingSelfUnionSuite.scala | 6 +-
...enStreamingSessionWindowDistributionSuite.scala | 4 +-
.../GlutenStreamingSessionWindowSuite.scala | 4 +-
...reamingStateStoreFormatCompatibilitySuite.scala | 4 +-
...utenStreamingSymmetricHashJoinHelperSuite.scala | 4 +-
.../GlutenTransformWithListStateSuite.scala | 4 +-
.../GlutenTransformWithListStateTTLSuite.scala | 4 +-
.../GlutenTransformWithMapStateSuite.scala | 4 +-
.../GlutenTransformWithMapStateTTLSuite.scala | 4 +-
.../GlutenTransformWithStateAvroSuite.scala | 4 +-
.../GlutenTransformWithStateChainingSuite.scala | 4 +-
.../GlutenTransformWithStateClusterSuite.scala | 4 +-
...GlutenTransformWithStateInitialStateSuite.scala | 4 +-
.../streaming/GlutenTransformWithStateSuite.scala | 4 +-
.../GlutenTransformWithStateUnsafeRowSuite.scala | 4 +-
.../GlutenTransformWithValueStateTTLSuite.scala | 4 +-
.../streaming/GlutenTriggerAvailableNowSuite.scala | 6 +-
95 files changed, 762 insertions(+), 252 deletions(-)
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index c6d77e28e2..ba8c773c97 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1167,8 +1167,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("detect escaped path and report the migration guide")
.exclude("ignore the escaped path check when the flag is off")
.excludeByPrefix("SPARK-51187")
- // Rewrite for the query plan check
- .excludeByPrefix("SPARK-49905")
enableSuite[GlutenQueryExecutionSuite]
// Rewritten to set root logger level to INFO so that logs can be parsed
.exclude("Logging plan changes for execution")
@@ -1181,45 +1179,49 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenSQLMetricsSuite]
enableSuite[GlutenAcceptsLatestSeenOffsetSuite]
enableSuite[GlutenCommitLogSuite]
- // TODO: 4.x enableSuite[GlutenEventTimeWatermarkSuite]
+ enableSuite[GlutenEventTimeWatermarkSuite]
enableSuite[GlutenFileStreamSinkV1Suite]
- // TODO: 4.x enableSuite[GlutenFileStreamSinkV2Suite] // 1 failure
+ enableSuite[GlutenFileStreamSinkV2Suite]
enableSuite[GlutenFileStreamSourceStressTestSuite]
- // TODO: 4.x enableSuite[GlutenFileStreamSourceSuite]
+ enableSuite[GlutenFileStreamSourceSuite]
enableSuite[GlutenFileStreamStressSuite]
- // TODO: 4.x
enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite] // failures
with GlutenPlugin
+ enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite]
enableSuite[GlutenFlatMapGroupsInPandasWithStateSuite]
- // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite]
- // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateSuite]
+ enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite]
+ enableSuite[GlutenFlatMapGroupsWithStateSuite]
enableSuite[GlutenFlatMapGroupsWithStateWithInitialStateSuite]
enableSuite[GlutenGroupStateSuite]
enableSuite[GlutenLongOffsetSuite]
enableSuite[GlutenMemorySourceStressSuite]
- // TODO: 4.x enableSuite[GlutenMultiStatefulOperatorsSuite] // 2 failures
+ enableSuite[GlutenMultiStatefulOperatorsSuite]
enableSuite[GlutenReportSinkMetricsSuite]
- // TODO: 4.x enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite]
- // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite]
- // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite]
- // TODO: 4.x enableSuite[GlutenStreamSuite]
- // TODO: 4.x enableSuite[GlutenStreamingAggregationDistributionSuite]
- // TODO: 4.x enableSuite[GlutenStreamingAggregationSuite]
- // TODO: 4.x enableSuite[GlutenStreamingDeduplicationDistributionSuite]
- // TODO: 4.x enableSuite[GlutenStreamingDeduplicationSuite]
+ enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite]
+ enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite]
+ // Spark 4.x: these cases can hang waiting for expected failure with
stateSchemaCheck off.
+ .excludeByPrefix("changing schema of state when restarting query - schema
check off")
+ enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite]
+ enableSuite[GlutenStreamSuite]
+ enableSuite[GlutenStreamingAggregationDistributionSuite]
+ enableSuite[GlutenStreamingAggregationSuite]
+ // Spark 4.x: these cases can hang waiting for expected failure with
stateSchemaCheck off.
+ .excludeByPrefix("changing schema of state when restarting query - schema
check off")
+ enableSuite[GlutenStreamingDeduplicationDistributionSuite]
+ enableSuite[GlutenStreamingDeduplicationSuite]
enableSuite[GlutenStreamingDeduplicationWithinWatermarkSuite]
enableSuite[GlutenStreamingFullOuterJoinSuite]
- // TODO: 4.x enableSuite[GlutenStreamingInnerJoinSuite]
+ enableSuite[GlutenStreamingInnerJoinSuite]
enableSuite[GlutenStreamingLeftSemiJoinSuite]
- // TODO: 4.x enableSuite[GlutenStreamingOuterJoinSuite]
- // TODO: 4.x enableSuite[GlutenStreamingQueryHashPartitionVerifySuite]
+ enableSuite[GlutenStreamingOuterJoinSuite]
+ enableSuite[GlutenStreamingQueryHashPartitionVerifySuite]
enableSuite[GlutenStreamingQueryListenerSuite]
enableSuite[GlutenStreamingQueryListenersConfSuite]
enableSuite[GlutenStreamingQueryManagerSuite]
enableSuite[GlutenStreamingQueryOptimizationCorrectnessSuite]
enableSuite[GlutenStreamingQueryStatusAndProgressSuite]
enableSuite[GlutenStreamingSelfUnionSuite]
- // TODO: 4.x enableSuite[GlutenStreamingSessionWindowDistributionSuite]
+ enableSuite[GlutenStreamingSessionWindowDistributionSuite]
enableSuite[GlutenStreamingSessionWindowSuite]
- // TODO: 4.x enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite]
+ enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite]
enableSuite[GlutenStreamingSymmetricHashJoinHelperSuite]
enableSuite[GlutenTransformWithListStateSuite]
enableSuite[GlutenTransformWithListStateTTLSuite]
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala
new file mode 100644
index 0000000000..210c124605
--- /dev/null
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+import org.apache.spark.SparkConf
+
+import java.net.JarURLConnection
+import java.nio.file.{Files, Paths, StandardCopyOption}
+import java.util.jar.JarFile
+
+private[sql] object GlutenStructuredStreamingResourceBootstrap {
+ private val StructuredStreamingRoot = "structured-streaming/"
+ private val ResourceProbe =
s"${StructuredStreamingRoot}partition-tests/randomSchemas"
+
+ @volatile private var initialized = false
+
+ def ensureResourcesOnFilesystem(): Unit = synchronized {
+ if (initialized) {
+ return
+ }
+
+ val maybeProbe = Option(getClass.getClassLoader.getResource(ResourceProbe))
+ maybeProbe.foreach {
+ probeUrl =>
+ if (probeUrl.getProtocol == "jar") {
+ copyStructuredStreamingResourcesFromJar(probeUrl)
+ }
+ }
+
+ initialized = true
+ }
+
+ private def copyStructuredStreamingResourcesFromJar(resourceUrl:
java.net.URL): Unit = {
+ val maybeTestClassesRoot = Option(getClass.getResource("/"))
+ .filter(_.getProtocol == "file")
+ .map(url => Paths.get(url.toURI))
+ if (maybeTestClassesRoot.isEmpty) {
+ return
+ }
+
+ val testClassesRoot = maybeTestClassesRoot.get
+ if (Files.exists(testClassesRoot.resolve(ResourceProbe))) {
+ return
+ }
+
+ val connection =
resourceUrl.openConnection().asInstanceOf[JarURLConnection]
+ val jarPath = Paths.get(connection.getJarFileURL.toURI)
+ val jarFile = new JarFile(jarPath.toFile)
+ try {
+ val entries = jarFile.entries()
+ while (entries.hasMoreElements) {
+ val entry = entries.nextElement()
+ val entryName = entry.getName
+ if (!entry.isDirectory &&
entryName.startsWith(StructuredStreamingRoot)) {
+ val targetPath = testClassesRoot.resolve(entryName)
+ Option(targetPath.getParent).foreach(Files.createDirectories(_))
+ val in = jarFile.getInputStream(entry)
+ try {
+ Files.copy(in, targetPath, StandardCopyOption.REPLACE_EXISTING)
+ } finally {
+ in.close()
+ }
+ }
+ }
+ } finally {
+ jarFile.close()
+ }
+ }
+}
+
+private[sql] object GlutenStreamingTestConf {
+ def withFallbackToVanilla(conf: SparkConf): SparkConf = {
+ GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem()
+ conf
+ .set("spark.driver.host", "127.0.0.1")
+ .set("spark.driver.bindAddress", "127.0.0.1")
+ .set("spark.sql.shuffle.partitions", "5")
+ .set("spark.shuffle.manager", "sort")
+ .set("spark.gluten.sql.columnar.batchscan", "false")
+ .set("spark.gluten.sql.columnar.filescan", "false")
+ .set("spark.gluten.sql.columnar.project", "false")
+ .set("spark.gluten.sql.columnar.filter", "false")
+ .set("spark.gluten.sql.columnar.sort", "false")
+ .set("spark.gluten.sql.columnar.window", "false")
+ .set("spark.gluten.sql.columnar.union", "false")
+ .set("spark.gluten.sql.columnar.expand", "false")
+ .set("spark.gluten.sql.columnar.generate", "false")
+ .set("spark.gluten.sql.columnar.coalesce", "false")
+ .set("spark.gluten.sql.columnar.range", "false")
+ .set("spark.gluten.sql.columnar.shuffle", "false")
+ .set("spark.gluten.sql.columnar.hashagg", "false")
+ .set("spark.gluten.sql.columnar.shuffledHashJoin", "false")
+ .set("spark.gluten.sql.columnar.sortMergeJoin", "false")
+ .set("spark.gluten.sql.columnar.broadcastExchange", "false")
+ .set("spark.gluten.sql.columnar.broadcastJoin", "false")
+ .set("spark.gluten.sql.columnar.appendData", "false")
+ .set("spark.gluten.sql.columnar.writeToDataSourceV2", "false")
+ .set("spark.gluten.sql.native.writer.enabled", "false")
+ .set("spark.gluten.sql.columnar.query.fallback.threshold", "0")
+ .set("spark.gluten.sql.columnar.wholeStage.fallback.threshold", "0")
+ .set("spark.gluten.sql.columnar.fallback.expressions.threshold", "0")
+ .set("spark.gluten.sql.columnar.fallback.preferColumnar", "false")
+ .set("spark.gluten.expression.blacklist", "collect_list,collect_set")
+ }
+}
+
+trait GlutenStreamingSQLTestsTrait extends GlutenSQLTestsTrait {
+ private val structuredStreamingResourcesInitialized: Unit =
+ GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem()
+
+ override def sparkConf: SparkConf = {
+ GlutenStreamingTestConf.withFallbackToVanilla(super.sparkConf)
+ }
+}
+
+trait GlutenStreamingVanillaFallbackTestsTrait extends
GlutenStreamingSQLTestsTrait {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ }
+}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
index 2bf23fbac1..f4f97cea5b 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenAcceptsLatestSeenOffsetSuite
extends AcceptsLatestSeenOffsetSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
index ae006ee38a..e2699e97ec 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenEventTimeWatermarkSuite extends EventTimeWatermarkSuite with
GlutenSQLTestsTrait {}
+class GlutenEventTimeWatermarkSuite
+ extends EventTimeWatermarkSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
index 74091348ea..9d4ae3f7ff 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with
GlutenStreamingSQLTestsTrait {}
-class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with
GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
index fd681ffd10..b953f5a4ff 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
@@ -16,10 +16,10 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with
GlutenStreamingSQLTestsTrait {}
class GlutenFileStreamSourceStressTestSuite
extends FileStreamSourceStressTestSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
index 566a5defe1..2df2b99101 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
@@ -16,6 +16,6 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenFileStreamStressSuite extends FileStreamStressSuite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamStressSuite extends FileStreamStressSuite with
GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
index 2199e32a40..5b2ec04590 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsInPandasWithStateDistributionSuite
extends FlatMapGroupsInPandasWithStateDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
index c653aa1d56..6d1ae5df23 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsInPandasWithStateSuite
extends FlatMapGroupsInPandasWithStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
index 343832f7ae..e9a0eee66c 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenFlatMapGroupsWithStateDistributionSuite
extends FlatMapGroupsWithStateDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
index 21d890a6cc..df7f30c14f 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
@@ -16,12 +16,12 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsWithStateSuite
extends FlatMapGroupsWithStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
class GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite
extends RocksDBStateStoreFlatMapGroupsWithStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
index 8ad6305992..f39b66480a 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsWithStateWithInitialStateSuite
extends FlatMapGroupsWithStateWithInitialStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
index ed28286fa6..468093cf03 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenMemorySourceStressSuite extends MemorySourceStressSuite with
GlutenSQLTestsTrait {}
+class GlutenMemorySourceStressSuite
+ extends MemorySourceStressSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
index 5860bf3b58..2177f48acb 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenMultiStatefulOperatorsSuite
extends MultiStatefulOperatorsSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
index 32a340d7f9..052a1f6aff 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenReportSinkMetricsSuite extends ReportSinkMetricsSuite with
GlutenSQLTestsTrait {}
+class GlutenReportSinkMetricsSuite
+ extends ReportSinkMetricsSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
index 7291881012..bd162bd6db 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
@@ -16,6 +16,6 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenStreamSuite extends StreamSuite with GlutenSQLTestsTrait {}
+class GlutenStreamSuite extends StreamSuite with GlutenStreamingSQLTestsTrait
{}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
index 37928f16ec..def343896f 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingAggregationDistributionSuite
extends StreamingAggregationDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
index d7d2af6e07..ab831aff95 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
@@ -16,10 +16,12 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
-class GlutenStreamingAggregationSuite extends StreamingAggregationSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingAggregationSuite
+ extends StreamingAggregationSuite
+ with GlutenStreamingVanillaFallbackTestsTrait {}
class GlutenRocksDBStateStoreStreamingAggregationSuite
extends RocksDBStateStoreStreamingAggregationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
index b8adfbc86a..8fcb8faeab 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingDeduplicationDistributionSuite
extends StreamingDeduplicationDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
index 67c2813141..bfc50fc5ea 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
@@ -16,12 +16,12 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingDeduplicationSuite
extends StreamingDeduplicationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
class GlutenRocksDBStateStoreStreamingDeduplicationSuite
extends RocksDBStateStoreStreamingDeduplicationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
index dd555fd72a..e6aec89246 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingDeduplicationWithinWatermarkSuite
extends StreamingDeduplicationWithinWatermarkSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
index 65489c981d..321c0fb559 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
@@ -16,16 +16,20 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
-class GlutenStreamingInnerJoinSuite extends StreamingInnerJoinSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingInnerJoinSuite
+ extends StreamingInnerJoinSuite
+ with GlutenStreamingVanillaFallbackTestsTrait {}
-class GlutenStreamingOuterJoinSuite extends StreamingOuterJoinSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingOuterJoinSuite
+ extends StreamingOuterJoinSuite
+ with GlutenStreamingVanillaFallbackTestsTrait {}
class GlutenStreamingFullOuterJoinSuite
extends StreamingFullOuterJoinSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
class GlutenStreamingLeftSemiJoinSuite
extends StreamingLeftSemiJoinSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
index 82dbf96c78..ec6b482533 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
@@ -16,8 +16,109 @@
*/
package org.apache.spark.sql.streaming
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.GlutenStreamingTestConf
import org.apache.spark.sql.GlutenTestsCommonTrait
+import java.io.{File, FileOutputStream}
+import java.nio.file.Files
+
class GlutenStreamingQueryHashPartitionVerifySuite
- extends StreamingQueryHashPartitionVerifySuite
- with GlutenTestsCommonTrait {}
+ extends {
+ private val sparkTestHomeInitialized: Unit =
+ GlutenStreamingQueryHashPartitionVerifySuite.initSparkTestHome()
+ }
+ with StreamingQueryHashPartitionVerifySuite
+ with GlutenTestsCommonTrait {
+
+ override def sparkConf: SparkConf = {
+ val conf = super.sparkConf
+ val warehousePath =
+ conf
+ .getOption("spark.sql.warehouse.dir")
+ .getOrElse(System.getProperty("java.io.tmpdir") + "/spark-warehouse")
+ GlutenStreamingTestConf.withFallbackToVanilla(
+ GlutenSQLTestsBaseTrait.nativeSparkConf(conf, warehousePath))
+ }
+}
+
+object GlutenStreamingQueryHashPartitionVerifySuite {
+ private val ResourcePath = "structured-streaming/partition-tests"
+ private val PartitionTestsRelativePath =
+ "sql/core/src/test/resources/structured-streaming/partition-tests"
+
+ private[streaming] def initSparkTestHome(): Unit = {
+ val existingValidSparkTestHome =
+ sys.props.get("spark.test.home").filter(isValidSparkTestHome)
+ if (existingValidSparkTestHome.isEmpty) {
+ val sparkTestHome = createSparkTestHomeFromResources()
+ .orElse(localSparkTestHome().filter(isValidSparkTestHome))
+ .orElse(sys.env.get("SPARK_HOME").filter(isValidSparkTestHome))
+
+ sparkTestHome.foreach(path => System.setProperty("spark.test.home",
path))
+ }
+ }
+
+ private def localSparkTestHome(): Option[String] = {
+ val userDir = sys.props.getOrElse("user.dir", ".")
+ val moduleRelativeSparkTestHome =
+ new File(userDir, "../common/src/test/resources/spark-home")
+ val localSparkTestHome =
+ new File(userDir, "src/test/resources/spark-home")
+
+ if (moduleRelativeSparkTestHome.exists()) {
+ Some(moduleRelativeSparkTestHome.getAbsolutePath)
+ } else if (localSparkTestHome.exists()) {
+ Some(localSparkTestHome.getAbsolutePath)
+ } else {
+ None
+ }
+ }
+
+ private def createSparkTestHomeFromResources(): Option[String] = {
+ val randomSchemasPath = s"$ResourcePath/randomSchemas"
+ val rowsAndPartIdsPath = s"$ResourcePath/rowsAndPartIds"
+
+ val rootDir = Files.createTempDirectory("gluten-spark-test-home").toFile
+ val partitionTestsDir = new File(rootDir, PartitionTestsRelativePath)
+
+ if (!partitionTestsDir.mkdirs() && !partitionTestsDir.exists()) {
+ None
+ } else {
+ val copiedRandomSchemas =
+ copyResource(randomSchemasPath, new File(partitionTestsDir,
"randomSchemas"))
+ val copiedRowsAndPartIds =
+ copyResource(rowsAndPartIdsPath, new File(partitionTestsDir,
"rowsAndPartIds"))
+
+ if (copiedRandomSchemas && copiedRowsAndPartIds) {
+ rootDir.deleteOnExit()
+ Some(rootDir.getAbsolutePath).filter(isValidSparkTestHome)
+ } else {
+ None
+ }
+ }
+ }
+
+ private def isValidSparkTestHome(path: String): Boolean = {
+ val partitionTestsDir = new File(path, PartitionTestsRelativePath)
+ val randomSchemas = new File(partitionTestsDir, "randomSchemas")
+ val rowsAndPartIds = new File(partitionTestsDir, "rowsAndPartIds")
+ randomSchemas.exists() && rowsAndPartIds.exists()
+ }
+
+ private def copyResource(resourcePath: String, targetFile: File): Boolean = {
+ val in = Option(getClass.getClassLoader.getResourceAsStream(resourcePath))
+ in.exists {
+ is =>
+ val out = new FileOutputStream(targetFile)
+ try {
+ is.transferTo(out)
+ true
+ } finally {
+ out.close()
+ is.close()
+ }
+ }
+ }
+}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
index 43793e599a..3f3122a648 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryListenerSuite
extends StreamingQueryListenerSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
index 785fa418ea..1db79b5424 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryListenersConfSuite
extends StreamingQueryListenersConfSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
index 5629f12bec..a93c888435 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryManagerSuite
extends StreamingQueryManagerSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
index 3503398148..a607af941f 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryOptimizationCorrectnessSuite
extends StreamingQueryOptimizationCorrectnessSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
index 921e20574f..bc3d6b20a2 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryStatusAndProgressSuite
extends StreamingQueryStatusAndProgressSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
index 7f7000c33c..0926fc00a6 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
@@ -16,12 +16,11 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
-import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec
-import org.apache.spark.sql.execution.exchange.REQUIRED_BY_STATEFUL_OPERATOR
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
+import org.apache.spark.sql.execution.exchange.{REQUIRED_BY_STATEFUL_OPERATOR,
ShuffleExchangeLike}
import org.apache.spark.sql.execution.streaming.MemoryStream
-class GlutenStreamingQuerySuite extends StreamingQuerySuite with
GlutenSQLTestsTrait {
+class GlutenStreamingQuerySuite extends StreamingQuerySuite with
GlutenStreamingSQLTestsTrait {
import testImplicits._
@@ -38,11 +37,13 @@ class GlutenStreamingQuerySuite extends StreamingQuerySuite
with GlutenSQLTestsT
Execute {
qe =>
val shuffleOpt = qe.lastExecution.executedPlan.collect {
- case s: ColumnarShuffleExchangeExec => s
+ case s: ShuffleExchangeLike if s.shuffleOrigin ==
REQUIRED_BY_STATEFUL_OPERATOR => s
}
- assert(shuffleOpt.nonEmpty, "No shuffle exchange found in the query
plan")
- assert(shuffleOpt.head.shuffleOrigin ===
REQUIRED_BY_STATEFUL_OPERATOR)
+ assert(
+ shuffleOpt.nonEmpty,
+ s"No stateful shuffle exchange found in the query plan: " +
+ qe.lastExecution.executedPlan)
}
)
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
index 4f5a84d642..fdab47efd7 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenStreamingSelfUnionSuite extends StreamingSelfUnionSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingSelfUnionSuite
+ extends StreamingSelfUnionSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
index de167d74f1..d21a95ff01 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingSessionWindowDistributionSuite
extends StreamingSessionWindowDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
index 00888b1a93..45e0c6eb2e 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingSessionWindowSuite
extends StreamingSessionWindowSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
index 2fa4b17fc9..acb7ebb3ce 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingStateStoreFormatCompatibilitySuite
extends StreamingStateStoreFormatCompatibilitySuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
index 5dd65c179f..9020a2702b 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingSymmetricHashJoinHelperSuite
extends StreamingSymmetricHashJoinHelperSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
index 907e048a33..8ad2aafaff 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithListStateSuite
extends TransformWithListStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
index b785c7db33..315021d65a 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithListStateTTLSuite
extends TransformWithListStateTTLSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
index b3742f6ee0..650a36271e 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithMapStateSuite
extends TransformWithMapStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
index 48b4d634bd..0c20e0ce62 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithMapStateTTLSuite
extends TransformWithMapStateTTLSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
index 61336607ed..95aa8433b0 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateAvroSuite
extends TransformWithStateAvroSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
index 852f4161d8..e2d475e997 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateChainingSuite
extends TransformWithStateChainingSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
index 3ba4436b0a..dbe1fefc5e 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateClusterSuite
extends TransformWithStateClusterSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
index 5727122b4a..1ecfb10525 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateInitialStateSuite
extends TransformWithStateInitialStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
index 6c02a69c2c..a0c6a2f990 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateValidationSuite
extends TransformWithStateValidationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
index 40802ce46c..fde363a580 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateUnsafeRowSuite
extends TransformWithStateUnsafeRowSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
index 6aeb84f9f1..1c5ec31909 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithValueStateTTLSuite
extends TransformWithValueStateTTLSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
index 89fb3cf5f4..efab0a270b 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenTriggerAvailableNowSuite extends TriggerAvailableNowSuite with
GlutenSQLTestsTrait {}
+class GlutenTriggerAvailableNowSuite
+ extends TriggerAvailableNowSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 5baf6e186c..616cb9ac7d 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1171,8 +1171,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("detect escaped path and report the migration guide")
.exclude("ignore the escaped path check when the flag is off")
.excludeByPrefix("SPARK-51187")
- // Rewrite for the query plan check
- .excludeByPrefix("SPARK-49905")
// TODO: fix on Spark-4.1 introduced by
https://github.com/apache/spark/pull/52645
.exclude("SPARK-53942: changing the number of stateless shuffle partitions
via config")
.exclude("SPARK-53942: stateful shuffle partitions are retained from old
checkpoint")
@@ -1205,45 +1203,49 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenSQLMetricsSuite]
enableSuite[GlutenAcceptsLatestSeenOffsetSuite]
enableSuite[GlutenCommitLogSuite]
- // TODO: 4.x enableSuite[GlutenEventTimeWatermarkSuite]
+ enableSuite[GlutenEventTimeWatermarkSuite]
enableSuite[GlutenFileStreamSinkV1Suite]
- // TODO: 4.x enableSuite[GlutenFileStreamSinkV2Suite] // 1 failure
+ enableSuite[GlutenFileStreamSinkV2Suite]
enableSuite[GlutenFileStreamSourceStressTestSuite]
- // TODO: 4.x enableSuite[GlutenFileStreamSourceSuite]
+ enableSuite[GlutenFileStreamSourceSuite]
enableSuite[GlutenFileStreamStressSuite]
- // TODO: 4.x
enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite] // failures
with GlutenPlugin
+ enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite]
enableSuite[GlutenFlatMapGroupsInPandasWithStateSuite]
- // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite]
- // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateSuite]
+ enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite]
+ enableSuite[GlutenFlatMapGroupsWithStateSuite]
enableSuite[GlutenFlatMapGroupsWithStateWithInitialStateSuite]
enableSuite[GlutenGroupStateSuite]
enableSuite[GlutenLongOffsetSuite]
enableSuite[GlutenMemorySourceStressSuite]
- // TODO: 4.x enableSuite[GlutenMultiStatefulOperatorsSuite] // 2 failures
+ enableSuite[GlutenMultiStatefulOperatorsSuite]
enableSuite[GlutenReportSinkMetricsSuite]
- // TODO: 4.x enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite]
- // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite]
- // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite]
- // TODO: 4.x enableSuite[GlutenStreamSuite]
- // TODO: 4.x enableSuite[GlutenStreamingAggregationDistributionSuite]
- // TODO: 4.x enableSuite[GlutenStreamingAggregationSuite]
- // TODO: 4.x enableSuite[GlutenStreamingDeduplicationDistributionSuite]
- // TODO: 4.x enableSuite[GlutenStreamingDeduplicationSuite]
+ enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite]
+ enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite]
+ // Spark 4.x: these cases can hang waiting for expected failure with
stateSchemaCheck off.
+ .excludeByPrefix("changing schema of state when restarting query - schema
check off")
+ enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite]
+ enableSuite[GlutenStreamSuite]
+ enableSuite[GlutenStreamingAggregationDistributionSuite]
+ enableSuite[GlutenStreamingAggregationSuite]
+ // Spark 4.x: these cases can hang waiting for expected failure with
stateSchemaCheck off.
+ .excludeByPrefix("changing schema of state when restarting query - schema
check off")
+ enableSuite[GlutenStreamingDeduplicationDistributionSuite]
+ enableSuite[GlutenStreamingDeduplicationSuite]
enableSuite[GlutenStreamingDeduplicationWithinWatermarkSuite]
enableSuite[GlutenStreamingFullOuterJoinSuite]
- // TODO: 4.x enableSuite[GlutenStreamingInnerJoinSuite]
+ enableSuite[GlutenStreamingInnerJoinSuite]
enableSuite[GlutenStreamingLeftSemiJoinSuite]
- // TODO: 4.x enableSuite[GlutenStreamingOuterJoinSuite]
- // TODO: 4.x enableSuite[GlutenStreamingQueryHashPartitionVerifySuite]
+ enableSuite[GlutenStreamingOuterJoinSuite]
+ enableSuite[GlutenStreamingQueryHashPartitionVerifySuite]
enableSuite[GlutenStreamingQueryListenerSuite]
enableSuite[GlutenStreamingQueryListenersConfSuite]
enableSuite[GlutenStreamingQueryManagerSuite]
enableSuite[GlutenStreamingQueryOptimizationCorrectnessSuite]
enableSuite[GlutenStreamingQueryStatusAndProgressSuite]
enableSuite[GlutenStreamingSelfUnionSuite]
- // TODO: 4.x enableSuite[GlutenStreamingSessionWindowDistributionSuite]
+ enableSuite[GlutenStreamingSessionWindowDistributionSuite]
enableSuite[GlutenStreamingSessionWindowSuite]
- // TODO: 4.x enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite]
+ enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite]
enableSuite[GlutenStreamingSymmetricHashJoinHelperSuite]
enableSuite[GlutenTransformWithListStateSuite]
enableSuite[GlutenTransformWithListStateTTLSuite]
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala
new file mode 100644
index 0000000000..210c124605
--- /dev/null
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql
+
+import org.apache.spark.SparkConf
+
+import java.net.JarURLConnection
+import java.nio.file.{Files, Paths, StandardCopyOption}
+import java.util.jar.JarFile
+
+private[sql] object GlutenStructuredStreamingResourceBootstrap {
+ private val StructuredStreamingRoot = "structured-streaming/"
+ private val ResourceProbe =
s"${StructuredStreamingRoot}partition-tests/randomSchemas"
+
+ @volatile private var initialized = false
+
+ def ensureResourcesOnFilesystem(): Unit = synchronized {
+ if (initialized) {
+ return
+ }
+
+ val maybeProbe = Option(getClass.getClassLoader.getResource(ResourceProbe))
+ maybeProbe.foreach {
+ probeUrl =>
+ if (probeUrl.getProtocol == "jar") {
+ copyStructuredStreamingResourcesFromJar(probeUrl)
+ }
+ }
+
+ initialized = true
+ }
+
+ private def copyStructuredStreamingResourcesFromJar(resourceUrl:
java.net.URL): Unit = {
+ val maybeTestClassesRoot = Option(getClass.getResource("/"))
+ .filter(_.getProtocol == "file")
+ .map(url => Paths.get(url.toURI))
+ if (maybeTestClassesRoot.isEmpty) {
+ return
+ }
+
+ val testClassesRoot = maybeTestClassesRoot.get
+ if (Files.exists(testClassesRoot.resolve(ResourceProbe))) {
+ return
+ }
+
+ val connection =
resourceUrl.openConnection().asInstanceOf[JarURLConnection]
+ val jarPath = Paths.get(connection.getJarFileURL.toURI)
+ val jarFile = new JarFile(jarPath.toFile)
+ try {
+ val entries = jarFile.entries()
+ while (entries.hasMoreElements) {
+ val entry = entries.nextElement()
+ val entryName = entry.getName
+ if (!entry.isDirectory &&
entryName.startsWith(StructuredStreamingRoot)) {
+ val targetPath = testClassesRoot.resolve(entryName)
+ Option(targetPath.getParent).foreach(Files.createDirectories(_))
+ val in = jarFile.getInputStream(entry)
+ try {
+ Files.copy(in, targetPath, StandardCopyOption.REPLACE_EXISTING)
+ } finally {
+ in.close()
+ }
+ }
+ }
+ } finally {
+ jarFile.close()
+ }
+ }
+}
+
+private[sql] object GlutenStreamingTestConf {
+ def withFallbackToVanilla(conf: SparkConf): SparkConf = {
+ GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem()
+ conf
+ .set("spark.driver.host", "127.0.0.1")
+ .set("spark.driver.bindAddress", "127.0.0.1")
+ .set("spark.sql.shuffle.partitions", "5")
+ .set("spark.shuffle.manager", "sort")
+ .set("spark.gluten.sql.columnar.batchscan", "false")
+ .set("spark.gluten.sql.columnar.filescan", "false")
+ .set("spark.gluten.sql.columnar.project", "false")
+ .set("spark.gluten.sql.columnar.filter", "false")
+ .set("spark.gluten.sql.columnar.sort", "false")
+ .set("spark.gluten.sql.columnar.window", "false")
+ .set("spark.gluten.sql.columnar.union", "false")
+ .set("spark.gluten.sql.columnar.expand", "false")
+ .set("spark.gluten.sql.columnar.generate", "false")
+ .set("spark.gluten.sql.columnar.coalesce", "false")
+ .set("spark.gluten.sql.columnar.range", "false")
+ .set("spark.gluten.sql.columnar.shuffle", "false")
+ .set("spark.gluten.sql.columnar.hashagg", "false")
+ .set("spark.gluten.sql.columnar.shuffledHashJoin", "false")
+ .set("spark.gluten.sql.columnar.sortMergeJoin", "false")
+ .set("spark.gluten.sql.columnar.broadcastExchange", "false")
+ .set("spark.gluten.sql.columnar.broadcastJoin", "false")
+ .set("spark.gluten.sql.columnar.appendData", "false")
+ .set("spark.gluten.sql.columnar.writeToDataSourceV2", "false")
+ .set("spark.gluten.sql.native.writer.enabled", "false")
+ .set("spark.gluten.sql.columnar.query.fallback.threshold", "0")
+ .set("spark.gluten.sql.columnar.wholeStage.fallback.threshold", "0")
+ .set("spark.gluten.sql.columnar.fallback.expressions.threshold", "0")
+ .set("spark.gluten.sql.columnar.fallback.preferColumnar", "false")
+ .set("spark.gluten.expression.blacklist", "collect_list,collect_set")
+ }
+}
+
+trait GlutenStreamingSQLTestsTrait extends GlutenSQLTestsTrait {
+ private val structuredStreamingResourcesInitialized: Unit =
+ GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem()
+
+ override def sparkConf: SparkConf = {
+ GlutenStreamingTestConf.withFallbackToVanilla(super.sparkConf)
+ }
+}
+
+trait GlutenStreamingVanillaFallbackTestsTrait extends
GlutenStreamingSQLTestsTrait {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ }
+}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
index 2bf23fbac1..f4f97cea5b 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenAcceptsLatestSeenOffsetSuite
extends AcceptsLatestSeenOffsetSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
index ae006ee38a..e2699e97ec 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenEventTimeWatermarkSuite extends EventTimeWatermarkSuite with
GlutenSQLTestsTrait {}
+class GlutenEventTimeWatermarkSuite
+ extends EventTimeWatermarkSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
index 74091348ea..9d4ae3f7ff 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with
GlutenStreamingSQLTestsTrait {}
-class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with
GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
index fd681ffd10..b953f5a4ff 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala
@@ -16,10 +16,10 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with
GlutenStreamingSQLTestsTrait {}
class GlutenFileStreamSourceStressTestSuite
extends FileStreamSourceStressTestSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
index 566a5defe1..2df2b99101 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala
@@ -16,6 +16,6 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenFileStreamStressSuite extends FileStreamStressSuite with
GlutenSQLTestsTrait {}
+class GlutenFileStreamStressSuite extends FileStreamStressSuite with
GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
index 2199e32a40..5b2ec04590 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsInPandasWithStateDistributionSuite
extends FlatMapGroupsInPandasWithStateDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
index c653aa1d56..6d1ae5df23 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsInPandasWithStateSuite
extends FlatMapGroupsInPandasWithStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
index 343832f7ae..e9a0eee66c 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenFlatMapGroupsWithStateDistributionSuite
extends FlatMapGroupsWithStateDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
index 21d890a6cc..df7f30c14f 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala
@@ -16,12 +16,12 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsWithStateSuite
extends FlatMapGroupsWithStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
class GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite
extends RocksDBStateStoreFlatMapGroupsWithStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
index 8ad6305992..f39b66480a 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenFlatMapGroupsWithStateWithInitialStateSuite
extends FlatMapGroupsWithStateWithInitialStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
index ed28286fa6..468093cf03 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenMemorySourceStressSuite extends MemorySourceStressSuite with
GlutenSQLTestsTrait {}
+class GlutenMemorySourceStressSuite
+ extends MemorySourceStressSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
index 5860bf3b58..2177f48acb 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenMultiStatefulOperatorsSuite
extends MultiStatefulOperatorsSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
index 32a340d7f9..052a1f6aff 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenReportSinkMetricsSuite extends ReportSinkMetricsSuite with
GlutenSQLTestsTrait {}
+class GlutenReportSinkMetricsSuite
+ extends ReportSinkMetricsSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala
index 2aaf0d8eea..75268ae860 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamRealTimeModeAllowlistSuite
extends StreamRealTimeModeAllowlistSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala
index c9effaf52a..92919883e3 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamRealTimeModeE2ESuite
extends StreamRealTimeModeE2ESuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala
index de80b65b64..687d0e819a 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenStreamRealTimeModeSuite extends StreamRealTimeModeSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamRealTimeModeSuite
+ extends StreamRealTimeModeSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
index 7291881012..bd162bd6db 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala
@@ -16,6 +16,6 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenStreamSuite extends StreamSuite with GlutenSQLTestsTrait {}
+class GlutenStreamSuite extends StreamSuite with GlutenStreamingSQLTestsTrait
{}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
index 37928f16ec..def343896f 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingAggregationDistributionSuite
extends StreamingAggregationDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
index d7d2af6e07..ab831aff95 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala
@@ -16,10 +16,12 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
-class GlutenStreamingAggregationSuite extends StreamingAggregationSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingAggregationSuite
+ extends StreamingAggregationSuite
+ with GlutenStreamingVanillaFallbackTestsTrait {}
class GlutenRocksDBStateStoreStreamingAggregationSuite
extends RocksDBStateStoreStreamingAggregationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
index b8adfbc86a..8fcb8faeab 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingDeduplicationDistributionSuite
extends StreamingDeduplicationDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
index 67c2813141..bfc50fc5ea 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala
@@ -16,12 +16,12 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingDeduplicationSuite
extends StreamingDeduplicationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
class GlutenRocksDBStateStoreStreamingDeduplicationSuite
extends RocksDBStateStoreStreamingDeduplicationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
index dd555fd72a..e6aec89246 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingDeduplicationWithinWatermarkSuite
extends StreamingDeduplicationWithinWatermarkSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
index 65489c981d..321c0fb559 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala
@@ -16,16 +16,20 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
-class GlutenStreamingInnerJoinSuite extends StreamingInnerJoinSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingInnerJoinSuite
+ extends StreamingInnerJoinSuite
+ with GlutenStreamingVanillaFallbackTestsTrait {}
-class GlutenStreamingOuterJoinSuite extends StreamingOuterJoinSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingOuterJoinSuite
+ extends StreamingOuterJoinSuite
+ with GlutenStreamingVanillaFallbackTestsTrait {}
class GlutenStreamingFullOuterJoinSuite
extends StreamingFullOuterJoinSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
class GlutenStreamingLeftSemiJoinSuite
extends StreamingLeftSemiJoinSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
index 82dbf96c78..ec6b482533 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala
@@ -16,8 +16,109 @@
*/
package org.apache.spark.sql.streaming
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.GlutenStreamingTestConf
import org.apache.spark.sql.GlutenTestsCommonTrait
+import java.io.{File, FileOutputStream}
+import java.nio.file.Files
+
class GlutenStreamingQueryHashPartitionVerifySuite
- extends StreamingQueryHashPartitionVerifySuite
- with GlutenTestsCommonTrait {}
+ extends {
+ private val sparkTestHomeInitialized: Unit =
+ GlutenStreamingQueryHashPartitionVerifySuite.initSparkTestHome()
+ }
+ with StreamingQueryHashPartitionVerifySuite
+ with GlutenTestsCommonTrait {
+
+ override def sparkConf: SparkConf = {
+ val conf = super.sparkConf
+ val warehousePath =
+ conf
+ .getOption("spark.sql.warehouse.dir")
+ .getOrElse(System.getProperty("java.io.tmpdir") + "/spark-warehouse")
+ GlutenStreamingTestConf.withFallbackToVanilla(
+ GlutenSQLTestsBaseTrait.nativeSparkConf(conf, warehousePath))
+ }
+}
+
+object GlutenStreamingQueryHashPartitionVerifySuite {
+ private val ResourcePath = "structured-streaming/partition-tests"
+ private val PartitionTestsRelativePath =
+ "sql/core/src/test/resources/structured-streaming/partition-tests"
+
+ private[streaming] def initSparkTestHome(): Unit = {
+ val existingValidSparkTestHome =
+ sys.props.get("spark.test.home").filter(isValidSparkTestHome)
+ if (existingValidSparkTestHome.isEmpty) {
+ val sparkTestHome = createSparkTestHomeFromResources()
+ .orElse(localSparkTestHome().filter(isValidSparkTestHome))
+ .orElse(sys.env.get("SPARK_HOME").filter(isValidSparkTestHome))
+
+ sparkTestHome.foreach(path => System.setProperty("spark.test.home",
path))
+ }
+ }
+
+ private def localSparkTestHome(): Option[String] = {
+ val userDir = sys.props.getOrElse("user.dir", ".")
+ val moduleRelativeSparkTestHome =
+ new File(userDir, "../common/src/test/resources/spark-home")
+ val localSparkTestHome =
+ new File(userDir, "src/test/resources/spark-home")
+
+ if (moduleRelativeSparkTestHome.exists()) {
+ Some(moduleRelativeSparkTestHome.getAbsolutePath)
+ } else if (localSparkTestHome.exists()) {
+ Some(localSparkTestHome.getAbsolutePath)
+ } else {
+ None
+ }
+ }
+
+ private def createSparkTestHomeFromResources(): Option[String] = {
+ val randomSchemasPath = s"$ResourcePath/randomSchemas"
+ val rowsAndPartIdsPath = s"$ResourcePath/rowsAndPartIds"
+
+ val rootDir = Files.createTempDirectory("gluten-spark-test-home").toFile
+ val partitionTestsDir = new File(rootDir, PartitionTestsRelativePath)
+
+ if (!partitionTestsDir.mkdirs() && !partitionTestsDir.exists()) {
+ None
+ } else {
+ val copiedRandomSchemas =
+ copyResource(randomSchemasPath, new File(partitionTestsDir,
"randomSchemas"))
+ val copiedRowsAndPartIds =
+ copyResource(rowsAndPartIdsPath, new File(partitionTestsDir,
"rowsAndPartIds"))
+
+ if (copiedRandomSchemas && copiedRowsAndPartIds) {
+ rootDir.deleteOnExit()
+ Some(rootDir.getAbsolutePath).filter(isValidSparkTestHome)
+ } else {
+ None
+ }
+ }
+ }
+
+ private def isValidSparkTestHome(path: String): Boolean = {
+ val partitionTestsDir = new File(path, PartitionTestsRelativePath)
+ val randomSchemas = new File(partitionTestsDir, "randomSchemas")
+ val rowsAndPartIds = new File(partitionTestsDir, "rowsAndPartIds")
+ randomSchemas.exists() && rowsAndPartIds.exists()
+ }
+
+ private def copyResource(resourcePath: String, targetFile: File): Boolean = {
+ val in = Option(getClass.getClassLoader.getResourceAsStream(resourcePath))
+ in.exists {
+ is =>
+ val out = new FileOutputStream(targetFile)
+ try {
+ is.transferTo(out)
+ true
+ } finally {
+ out.close()
+ is.close()
+ }
+ }
+ }
+}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
index 43793e599a..3f3122a648 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryListenerSuite
extends StreamingQueryListenerSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
index 785fa418ea..1db79b5424 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryListenersConfSuite
extends StreamingQueryListenersConfSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
index 5629f12bec..a93c888435 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryManagerSuite
extends StreamingQueryManagerSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
index 3503398148..a607af941f 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryOptimizationCorrectnessSuite
extends StreamingQueryOptimizationCorrectnessSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
index 921e20574f..bc3d6b20a2 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingQueryStatusAndProgressSuite
extends StreamingQueryStatusAndProgressSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
index 87e7924ce7..bf9c890c78 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala
@@ -16,12 +16,11 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
-import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec
-import org.apache.spark.sql.execution.exchange.REQUIRED_BY_STATEFUL_OPERATOR
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
+import org.apache.spark.sql.execution.exchange.{REQUIRED_BY_STATEFUL_OPERATOR,
ShuffleExchangeLike}
import org.apache.spark.sql.execution.streaming.runtime.MemoryStream
-class GlutenStreamingQuerySuite extends StreamingQuerySuite with
GlutenSQLTestsTrait {
+class GlutenStreamingQuerySuite extends StreamingQuerySuite with
GlutenStreamingSQLTestsTrait {
import testImplicits._
@@ -38,11 +37,13 @@ class GlutenStreamingQuerySuite extends StreamingQuerySuite
with GlutenSQLTestsT
Execute {
qe =>
val shuffleOpt = qe.lastExecution.executedPlan.collect {
- case s: ColumnarShuffleExchangeExec => s
+ case s: ShuffleExchangeLike if s.shuffleOrigin ==
REQUIRED_BY_STATEFUL_OPERATOR => s
}
- assert(shuffleOpt.nonEmpty, "No shuffle exchange found in the query
plan")
- assert(shuffleOpt.head.shuffleOrigin ===
REQUIRED_BY_STATEFUL_OPERATOR)
+ assert(
+ shuffleOpt.nonEmpty,
+ s"No stateful shuffle exchange found in the query plan: " +
+ qe.lastExecution.executedPlan)
}
)
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
index 4f5a84d642..fdab47efd7 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenStreamingSelfUnionSuite extends StreamingSelfUnionSuite with
GlutenSQLTestsTrait {}
+class GlutenStreamingSelfUnionSuite
+ extends StreamingSelfUnionSuite
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
index de167d74f1..d21a95ff01 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingSessionWindowDistributionSuite
extends StreamingSessionWindowDistributionSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
index 00888b1a93..45e0c6eb2e 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingSessionWindowSuite
extends StreamingSessionWindowSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
index 2fa4b17fc9..acb7ebb3ce 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait
class GlutenStreamingStateStoreFormatCompatibilitySuite
extends StreamingStateStoreFormatCompatibilitySuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingVanillaFallbackTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
index 5dd65c179f..9020a2702b 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenStreamingSymmetricHashJoinHelperSuite
extends StreamingSymmetricHashJoinHelperSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
index 907e048a33..8ad2aafaff 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithListStateSuite
extends TransformWithListStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
index b785c7db33..315021d65a 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithListStateTTLSuite
extends TransformWithListStateTTLSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
index b3742f6ee0..650a36271e 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithMapStateSuite
extends TransformWithMapStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
index 48b4d634bd..0c20e0ce62 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithMapStateTTLSuite
extends TransformWithMapStateTTLSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
index 61336607ed..95aa8433b0 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateAvroSuite
extends TransformWithStateAvroSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
index 852f4161d8..e2d475e997 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateChainingSuite
extends TransformWithStateChainingSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
index 3ba4436b0a..dbe1fefc5e 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateClusterSuite
extends TransformWithStateClusterSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
index 5727122b4a..1ecfb10525 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateInitialStateSuite
extends TransformWithStateInitialStateSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
index 6c02a69c2c..a0c6a2f990 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateValidationSuite
extends TransformWithStateValidationSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
index 40802ce46c..fde363a580 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithStateUnsafeRowSuite
extends TransformWithStateUnsafeRowSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
index 6aeb84f9f1..1c5ec31909 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala
@@ -16,8 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
class GlutenTransformWithValueStateTTLSuite
extends TransformWithValueStateTTLSuite
- with GlutenSQLTestsTrait {}
+ with GlutenStreamingSQLTestsTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
index 89fb3cf5f4..efab0a270b 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.streaming
-import org.apache.spark.sql.GlutenSQLTestsTrait
+import org.apache.spark.sql.GlutenStreamingSQLTestsTrait
-class GlutenTriggerAvailableNowSuite extends TriggerAvailableNowSuite with
GlutenSQLTestsTrait {}
+class GlutenTriggerAvailableNowSuite
+ extends TriggerAvailableNowSuite
+ with GlutenStreamingSQLTestsTrait {}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
