This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6c49c4128b [GLUTEN-11088][VL] fix CoalesceShufflePartitionsSuite
(#11201)
6c49c4128b is described below
commit 6c49c4128b03ac34fa7c5ca3b9e2c95cc1d27611
Author: Rong Ma <[email protected]>
AuthorDate: Thu Nov 27 17:58:16 2025 +0000
[GLUTEN-11088][VL] fix CoalesceShufflePartitionsSuite (#11201)
---
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 1 +
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 1 +
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 1 +
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 1 +
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 5 +++--
.../execution/GlutenCoalesceShufflePartitionsSuite.scala | 16 ++++------------
6 files changed, 11 insertions(+), 14 deletions(-)
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 584905e493..48315d2697 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -422,6 +422,7 @@ class VeloxTestSettings extends BackendTestSettings {
.excludeByPrefix("SPARK-35675")
enableSuite[GlutenCoalesceShufflePartitionsSuite]
.excludeByPrefix("SPARK-24705")
+ // Rewrite for Gluten. Change details are in the inline comments in
individual tests.
.excludeByPrefix("determining the number of reducers")
enableSuite[GlutenFileSourceCharVarcharTestSuite]
// Following tests are excluded as these are overridden in Gluten test
suite..
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index e036f539e8..4a850690d3 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -544,6 +544,7 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[FallbackStrategiesSuite]
enableSuite[GlutenBroadcastExchangeSuite]
enableSuite[GlutenCoalesceShufflePartitionsSuite]
+ // Rewrite for Gluten. Change details are in the inline comments in
individual tests.
.excludeByPrefix("determining the number of reducers")
enableSuite[GlutenExchangeSuite]
// ColumnarShuffleExchangeExec does not support doExecute() method
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index d351e707ca..87d085ec51 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -564,6 +564,7 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenBroadcastExchangeSuite]
enableSuite[GlutenLocalBroadcastExchangeSuite]
enableSuite[GlutenCoalesceShufflePartitionsSuite]
+ // Rewrite for Gluten. Change details are in the inline comments in
individual tests.
.excludeByPrefix("determining the number of reducers")
enableSuite[GlutenExchangeSuite]
// ColumnarShuffleExchangeExec does not support doExecute() method
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index d21e39fafd..f4427d7d43 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -580,6 +580,7 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenBroadcastExchangeSuite]
enableSuite[GlutenLocalBroadcastExchangeSuite]
enableSuite[GlutenCoalesceShufflePartitionsSuite]
+ // Rewrite for Gluten. Change details are in the inline comments in
individual tests.
.excludeByPrefix("determining the number of reducers")
enableSuite[GlutenExchangeSuite]
// ColumnarShuffleExchangeExec does not support doExecute() method
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index b889422fc4..2e1984dc58 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -620,8 +620,9 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[FallbackStrategiesSuite]
enableSuite[GlutenBroadcastExchangeSuite]
enableSuite[GlutenLocalBroadcastExchangeSuite]
- // TODO: fix in Spark-4.0
- // enableSuite[GlutenCoalesceShufflePartitionsSuite]
+ enableSuite[GlutenCoalesceShufflePartitionsSuite]
+ // Rewrite for Gluten. Change details are in the inline comments in
individual tests.
+ .excludeByPrefix("determining the number of reducers")
enableSuite[GlutenExchangeSuite]
// ColumnarShuffleExchangeExec does not support doExecute() method
.exclude("shuffling UnsafeRows in exchange")
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenCoalesceShufflePartitionsSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenCoalesceShufflePartitionsSuite.scala
index 8e5c71735a..fec67b9a27 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenCoalesceShufflePartitionsSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenCoalesceShufflePartitionsSuite.scala
@@ -92,9 +92,7 @@ class GlutenCoalesceShufflePartitionsSuite
// Then, let's look at the number of post-shuffle partitions
estimated
// by the ExchangeCoordinator.
- val finalPlan = agg.queryExecution.executedPlan
- .asInstanceOf[AdaptiveSparkPlanExec]
- .executedPlan
+ val finalPlan = stripAQEPlan(agg.queryExecution.executedPlan)
val shuffleReads = finalPlan.collect { case r @
CoalescedShuffleRead() => r }
minNumPostShufflePartitions match {
@@ -135,9 +133,7 @@ class GlutenCoalesceShufflePartitionsSuite
// Then, let's look at the number of post-shuffle partitions
estimated
// by the ExchangeCoordinator.
- val finalPlan = join.queryExecution.executedPlan
- .asInstanceOf[AdaptiveSparkPlanExec]
- .executedPlan
+ val finalPlan = stripAQEPlan(join.queryExecution.executedPlan)
val shuffleReads = finalPlan.collect { case r @
CoalescedShuffleRead() => r }
minNumPostShufflePartitions match {
@@ -184,9 +180,7 @@ class GlutenCoalesceShufflePartitionsSuite
// Then, let's look at the number of post-shuffle partitions
estimated
// by the ExchangeCoordinator.
- val finalPlan = join.queryExecution.executedPlan
- .asInstanceOf[AdaptiveSparkPlanExec]
- .executedPlan
+ val finalPlan = stripAQEPlan(join.queryExecution.executedPlan)
val shuffleReads = finalPlan.collect { case r @
CoalescedShuffleRead() => r }
minNumPostShufflePartitions match {
@@ -234,9 +228,7 @@ class GlutenCoalesceShufflePartitionsSuite
// Then, let's look at the number of post-shuffle partitions
estimated
// by the ExchangeCoordinator.
- val finalPlan = join.queryExecution.executedPlan
- .asInstanceOf[AdaptiveSparkPlanExec]
- .executedPlan
+ val finalPlan = stripAQEPlan(join.queryExecution.executedPlan)
val shuffleReads = finalPlan.collect { case r @
CoalescedShuffleRead() => r }
minNumPostShufflePartitions match {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
