This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit df01c5038badd198901e2f362f3218627811c5fb Author: Chang chen <[email protected]> AuthorDate: Tue Apr 14 04:05:13 2026 +0000 [GLUTEN-10134][VL] Open ANSI excludes and add verify-ansi-expressions.sh - Remove 6 ANSI-related .exclude() entries in spark41 VeloxTestSettings (TryCastSuite, QueryExecutionAnsiErrorsSuite, DataFrameAggregateSuite) - Remove 5 ANSI-related .exclude() entries in spark40 VeloxTestSettings - Add dev/verify-ansi-expressions.sh for ANSI expression matrix verification with --jvm-arg ansiFallback.enabled=false and --clean support Note: local testing with ansiFallback=false shows these excludes still fail on Velox path. This commit is to verify CI results match local behavior. Co-Authored-By: Claude Opus 4 <[email protected]> --- dev/verify-ansi-expressions.sh | 122 +++++++++++++++++++++ .../gluten/utils/velox/VeloxTestSettings.scala | 10 -- .../gluten/utils/velox/VeloxTestSettings.scala | 11 -- 3 files changed, 122 insertions(+), 21 deletions(-) diff --git a/dev/verify-ansi-expressions.sh b/dev/verify-ansi-expressions.sh new file mode 100755 index 0000000000..00dfe62ec1 --- /dev/null +++ b/dev/verify-ansi-expressions.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# verify-ansi-expressions.sh — 验证 73 个 ANSI 表达式 +# +# 基于 expression-matrix.md v3，运行所有覆盖 ANSI 影响表达式的 Gluten UT +# 分 4 段运行：spark41 gluten-ut + backends-velox，spark40 gluten-ut + backends-velox +# +# 用法： +# cd /root/SourceCode/gluten +# bash dev/verify-ansi-expressions.sh [spark41|spark40|all] +# +# 输出： +# /tmp/ansi-verify-spark41-ut.log +# /tmp/ansi-verify-spark41-backends.log +# /tmp/ansi-verify-spark40-ut.log +# /tmp/ansi-verify-spark40-backends.log + +set -uo pipefail + +export SPARK_ANSI_SQL_MODE=true +export SPARK_TESTING=true + +TARGET="${1:-all}" + +SUITES_UT=( + -s org.apache.spark.sql.catalyst.expressions.GlutenArithmeticExpressionSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOnSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOffSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenTryCastSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenMathExpressionsSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenCollectionExpressionsSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenStringExpressionsSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenDateExpressionsSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenDecimalExpressionSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenDecimalPrecisionSuite + -s org.apache.spark.sql.GlutenMathFunctionsSuite + -s org.apache.spark.sql.GlutenDataFrameAggregateSuite + -s org.apache.spark.sql.GlutenStringFunctionsSuite + -s org.apache.spark.sql.errors.GlutenQueryExecutionAnsiErrorsSuite +) + +SUITES_BACKENDS=( + -s org.apache.gluten.functions.ArithmeticAnsiValidateSuite + -s org.apache.gluten.functions.MathFunctionsValidateSuiteAnsiOn + -s org.apache.gluten.functions.FunctionsValidateSuite + -s org.apache.spark.sql.catalyst.expressions.VeloxCastSuite +) + +echo "========================================" +echo "ANSI Expression Matrix Verification" +echo "Date: $(date '+%Y-%m-%d %H:%M:%S')" +echo "Target: ${TARGET}" +echo "SPARK_ANSI_SQL_MODE=${SPARK_ANSI_SQL_MODE}" +echo "SPARK_TESTING=${SPARK_TESTING}" +echo "spark.gluten.sql.ansiFallback.enabled=false" +echo "========================================" + +run_spark41() { + local PROFILES="-Pjava-17,spark-4.1,scala-2.13,backends-velox,hadoop-3.3" + local ANSI_ARG='--jvm-arg -Dspark.gluten.sql.ansiFallback.enabled=false' + + echo "" + echo "=== Part 1/4: spark41 gluten-ut (14 suites) ===" + ./dev/run-scala-test.sh \ + --clean \ + ${ANSI_ARG} \ + ${PROFILES},spark-ut \ + -pl gluten-ut/spark41 \ + "${SUITES_UT[@]}" \ + 2>&1 | tee /tmp/ansi-verify-spark41-ut.log + + echo "" + echo "=== Part 2/4: spark41 backends-velox (4 suites) ===" + ./dev/run-scala-test.sh \ + ${ANSI_ARG} \ + ${PROFILES} \ + -pl backends-velox \ + "${SUITES_BACKENDS[@]}" \ + 2>&1 | tee /tmp/ansi-verify-spark41-backends.log +} + +run_spark40() { + local PROFILES="-Pjava-17,spark-4.0,scala-2.13,backends-velox,hadoop-3.3" + local ANSI_ARG='--jvm-arg -Dspark.gluten.sql.ansiFallback.enabled=false' + + echo "" + echo "=== Part 3/4: spark40 gluten-ut (14 suites) ===" + ./dev/run-scala-test.sh \ + --clean \ + ${ANSI_ARG} \ + ${PROFILES},spark-ut \ + -pl gluten-ut/spark40 \ + "${SUITES_UT[@]}" \ + 2>&1 | tee /tmp/ansi-verify-spark40-ut.log + + echo "" + echo "=== Part 4/4: spark40 backends-velox (4 suites) ===" + ./dev/run-scala-test.sh \ + ${ANSI_ARG} \ + ${PROFILES} \ + -pl backends-velox \ + "${SUITES_BACKENDS[@]}" \ + 2>&1 | tee /tmp/ansi-verify-spark40-backends.log +} + +case "${TARGET}" in + spark41) run_spark41 ;; + spark40) run_spark40 ;; + all) run_spark41; run_spark40 ;; + *) echo "Usage: $0 [spark41|spark40|all]"; exit 1 ;; +esac + +echo "" +echo "========================================" +echo "Verification Complete" +echo "Results:" +for f in /tmp/ansi-verify-spark4{1,0}-{ut,backends}.log; do + if [ -f "$f" ]; then + echo " $(basename $f):" + grep -E "Tests:.*succeeded|All tests passed|FAILED" "$f" | tail -2 | sed 's/^/ /' + fi +done +echo "========================================" diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 7f734b5820..b87160d132 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -109,8 +109,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-39749: cast Decimal to string") enableSuite[GlutenTryCastSuite] .exclude("cast from timestamp II") // Rewrite test for Gluten not supported with ANSI mode - // Velox try_cast(float→long) returns null for boundary values near Long.MaxValue/MinValue - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") // Set timezone through config. .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. @@ -265,9 +263,6 @@ class VeloxTestSettings extends BackendTestSettings { // SparkArithmeticException). Tracked for future fix via exception type mapping. .exclude("SPARK-50290: Add a flag to disable DataFrame context") enableSuite[GlutenQueryExecutionAnsiErrorsSuite] - // ANSI mode: exception type mismatch -- Velox throws GlutenException, test expects specific - // Spark exception types (SparkArithmeticException, SparkRuntimeException). - .exclude("SPARK-46922: user-facing runtime errors") enableSuite[VeloxAdaptiveQueryExecSuite] .includeAllGlutenTests() .includeByPrefix( @@ -887,11 +882,6 @@ class VeloxTestSettings extends BackendTestSettings { "SPARK-31620: agg with subquery (whole-stage-codegen = true)", "SPARK-31620: agg with subquery (whole-stage-codegen = false)" ) - // ANSI mode: exception type mismatch -- Velox throws GlutenException, test expects - // SparkArithmeticException. Tracked for future fix via exception type mapping. - .exclude("SPARK-28067: Aggregate sum should not return wrong results for decimal overflow") - .exclude("SPARK-35955: Aggregate avg should not return wrong results for decimal overflow") - .exclude("SPARK-28224: Aggregate sum big decimal overflow") enableSuite[GlutenDataFrameAsOfJoinSuite] enableSuite[GlutenDataFrameComplexTypeSuite] enableSuite[GlutenDataFrameFunctionsSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 312263f9f1..5788527de0 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -114,8 +114,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-39749: cast Decimal to string") enableSuite[GlutenTryCastSuite] .exclude("cast from timestamp II") // Rewrite test for Gluten not supported with ANSI mode - // Velox try_cast(float→long) returns null for boundary values near Long.MaxValue/MinValue - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") // Set timezone through config. .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. @@ -276,10 +274,6 @@ class VeloxTestSettings extends BackendTestSettings { // SparkArithmeticException). Tracked for future fix via exception type mapping. .exclude("SPARK-50290: Add a flag to disable DataFrame context") enableSuite[GlutenQueryExecutionAnsiErrorsSuite] - // ANSI mode: exception type mismatch -- Velox throws GlutenException, test expects specific - // Spark exception types (SparkArithmeticException, SparkRuntimeException). - .exclude("SPARK-46922: user-facing runtime errors") - .exclude("INVALID_DATETIME_PATTERN with non-constant pattern") enableSuite[VeloxAdaptiveQueryExecSuite] .includeAllGlutenTests() .includeByPrefix( @@ -876,11 +870,6 @@ class VeloxTestSettings extends BackendTestSettings { "SPARK-31620: agg with subquery (whole-stage-codegen = true)", "SPARK-31620: agg with subquery (whole-stage-codegen = false)" ) - // ANSI mode: exception type mismatch -- Velox throws GlutenException, test expects - // SparkArithmeticException. Tracked for future fix via exception type mapping. - .exclude("SPARK-28067: Aggregate sum should not return wrong results for decimal overflow") - .exclude("SPARK-35955: Aggregate avg should not return wrong results for decimal overflow") - .exclude("SPARK-28224: Aggregate sum big decimal overflow") enableSuite[GlutenDataFrameAsOfJoinSuite] enableSuite[GlutenDataFrameComplexTypeSuite] enableSuite[GlutenDataFrameFunctionsSuite] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
