This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 90a2d4e364b4236166ad2abec8f7f3b158ae9e69 Author: Chang chen <[email protected]> AuthorDate: Fri Apr 10 06:41:33 2026 +0000 [GLUTEN-10134][VL] Handle ANSI mode exceptions in runQueryAndCompare and add testWithAnsiMode - GlutenQueryComparisonTest: wrap vanilla Spark execution in Try; when ANSI mode is on and vanilla Spark throws, verify Gluten also throws. - GlutenQueryTest: add testWithAnsiMode(name, ansiEnabled) to skip tests that are not designed for a specific ANSI mode. - FallbackSuite: use testWithAnsiMode(_, false) for nested loop join test whose data triggers ANSI cast error after fallback. Co-Authored-By: Claude Opus 4 <[email protected]> --- .../apache/gluten/execution/FallbackSuite.scala | 2 +- .../execution/GlutenQueryComparisonTest.scala | 33 ++++++++++++++++------ .../org/apache/spark/sql/GlutenQueryTest.scala | 10 +++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala index b77c8c2f08..500085c9e8 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala @@ -353,7 +353,7 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl } } - test("fallback when nested loop join has unsupported expression") { + testWithAnsiMode("fallback when nested loop join has unsupported expression", false) { val events = new ArrayBuffer[GlutenPlanFallbackEvent] val listener = new SparkListener { override def onOtherEvent(event: SparkListenerEvent): Unit = { diff --git a/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala b/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala index 8a22b706db..d8641db985 100644 --- a/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala +++ b/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala @@ -25,6 +25,7 @@ import org.apache.gluten.test.FallbackUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.{DataFrame, GlutenQueryTest, Row} +import org.apache.spark.sql.internal.SQLConf import java.util.concurrent.atomic.AtomicBoolean @@ -96,10 +97,8 @@ abstract class GlutenQueryComparisonTest extends GlutenQueryTest { customCheck: DataFrame => Unit, noFallBack: Boolean = true, cache: Boolean = false): DataFrame = { - var expected: Seq[Row] = null - withSQLConf(vanillaSparkConfs(): _*) { - val df = dataframe() - expected = df.collect() + val expected = withSQLConf(vanillaSparkConfs(): _*) { + scala.util.Try(dataframe().collect()) } // By default, we will fallback complex type scan but here we should allow // to test support of complex type @@ -109,17 +108,33 @@ abstract class GlutenQueryComparisonTest extends GlutenQueryTest { df.cache() } try { - if (compareResult) { - checkAnswer(df, expected) - } else { - df.collect() + expected match { + case scala.util.Failure(_) if SQLConf.get.ansiEnabled => + // In ANSI mode, if vanilla Spark throws an exception on the test data, + // verify that Gluten also throws an exception. + intercept[Exception](df.collect()) + // TODO: once exception type mapping is implemented, enable this assertion + // to verify that the exception type matches vanilla Spark's. + // assert( + // glutenException.getClass == expected.failed.get.getClass, + // s"Expected ${expected.failed.get.getClass.getName} " + + // s"but got ${glutenException.getClass.getName}" + // ) + case scala.util.Failure(e) => + throw e + case scala.util.Success(rows) => + if (compareResult) { + checkAnswer(df, rows) + } else { + df.collect() + } + checkDataFrame(noFallBack, customCheck, df) } } finally { if (cache) { df.unpersist() } } - checkDataFrame(noFallBack, customCheck, df) df } diff --git a/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala b/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala index 93c3fe631f..16a02d37fb 100644 --- a/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala +++ b/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.execution.{CommandResultExec, SparkPlan, SQLExecutio import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.columnar.InMemoryRelation +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.storage.StorageLevel import org.junit.Assert @@ -137,6 +138,15 @@ abstract class GlutenQueryTest extends PlanTest with AdaptiveSparkPlanHelper { } } + /** Run the test only when ANSI mode matches the expected value. */ + def testWithAnsiMode(testName: String, ansiEnabled: Boolean)(testFun: => Any): Unit = { + if (SQLConf.get.ansiEnabled == ansiEnabled) { + test(testName) { + testFun + } + } + } + /** Run the test on the specified spark version */ def testWithSpecifiedSparkVersion(testName: String, versions: String*)(testFun: => Any): Unit = { if (versions.exists(v => matchSparkVersion(Some(v), Some(v)))) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
