This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 32d3112210ff860e1c2c0c1d5067cbc7f85079ed Author: Chang chen <[email protected]> AuthorDate: Tue Mar 31 13:36:00 2026 +0000 Fix GlutenCastWithAnsiOnSuite failures #4-#5 for spark40/spark41 Enable ANSI mode session config and disable ANSI fallback in GlutenCastWithAnsiOnSuite so Velox receives the correct ANSI flag. This fixes: - #3 Casting to char/varchar (now ANSI config propagates correctly) - #5 SPARK-39749: cast Decimal to string (Velox isScientific() reads sparkAnsiEnabled from session config) - #4 out-of-range to long (Velox Conversions.h float→int64 boundary fix, committed separately in Velox upstream) Velox C++ changes (not in this commit, require Velox upstream PR): - SparkCastHooks::isScientific() returns !config_.sparkAnsiEnabled() - Conversions.h: fix float/double→int64 boundary detection UB by comparing in double precision against exact 2^63 boundary Co-Authored-By: Claude Opus 4.6 <[email protected]> --- .../expression/UnaryExpressionTransformer.scala | 38 ++++++++++++++++++++-- .../gluten/utils/velox/VeloxTestSettings.scala | 3 -- .../expressions/GlutenCastWithAnsiOnSuite.scala | 8 +++++ .../gluten/utils/velox/VeloxTestSettings.scala | 3 -- .../expressions/GlutenCastWithAnsiOnSuite.scala | 8 +++++ 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala index b762ec95b6..a076c46b40 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala @@ -44,11 +44,43 @@ case class ChildTransformer( case class CastTransformer(substraitExprName: String, child: ExpressionTransformer, original: Cast) extends UnaryExpressionTransformer { override def doTransform(context: SubstraitContext): ExpressionNode = { - val typeNode = ConverterUtils.getTypeNode(dataType, original.nullable) - ExpressionBuilder.makeCast( - typeNode, + original.dataType match { + case CharType(length) => + buildCastWithWriteSideCheck( + context, + ExpressionNames.CHAR_TYPE_WRITE_SIDE_CHECK, + length) + case VarcharType(length) => + buildCastWithWriteSideCheck( + context, + ExpressionNames.VARCHAR_TYPE_WRITE_SIDE_CHECK, + length) + case _ => + val typeNode = ConverterUtils.getTypeNode(dataType, original.nullable) + ExpressionBuilder.makeCast( + typeNode, + child.doTransform(context), + SparkShimLoader.getSparkShims.withTryEvalMode(original)) + } + } + + /** Builds Cast(child, StringType) + write-side check for CharType/VarcharType. */ + private def buildCastWithWriteSideCheck( + context: SubstraitContext, + checkFuncName: String, + length: Int): ExpressionNode = { + val stringTypeNode = ConverterUtils.getTypeNode(StringType, original.nullable) + val castNode = ExpressionBuilder.makeCast( + stringTypeNode, child.doTransform(context), SparkShimLoader.getSparkShims.withTryEvalMode(original)) + val funcName = + ConverterUtils.makeFuncName(checkFuncName, Seq(StringType, IntegerType)) + val functionId = context.registerFunction(funcName) + val lengthNode = + ExpressionBuilder.makeLiteral(length.asInstanceOf[java.lang.Integer], IntegerType, false) + val childNodes: java.util.List[ExpressionNode] = Lists.newArrayList(castNode, lengthNode) + ExpressionBuilder.makeScalarFunction(functionId, childNodes, stringTypeNode) } } diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 5728f851bc..bcc78a2481 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -213,9 +213,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenCastWithAnsiOnSuite] .exclude("data type casting") .exclude("cast string to timestamp") - .exclude("Casting to char/varchar") - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") - .exclude("SPARK-39749: cast Decimal to string") enableSuite[GlutenCodeGenerationSuite] enableSuite[GlutenCodeGeneratorWithInterpretedFallbackSuite] enableSuite[GlutenCollationExpressionSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala index 0c236a803f..93162ce8fa 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.catalyst.expressions +import org.apache.gluten.config.GlutenConfig + import org.apache.spark.sql.GlutenTestsTrait import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT} import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC} @@ -30,6 +32,12 @@ class GlutenCastWithAnsiOnSuite extends CastWithAnsiOnSuite with GlutenTestsTrai override def beforeAll(): Unit = { super.beforeAll() conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true) + // CastWithAnsiOnSuite creates Cast expressions with EvalMode.ANSI but does not set the + // session-level ANSI config. Velox reads ANSI mode from session config to decide cast + // behavior (e.g., scientific notation for Decimal→String). We must sync session config + // with the expression-level evalMode and disable ANSI fallback so Velox actually executes. + conf.setConf(SQLConf.ANSI_ENABLED, true) + conf.setConfString(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key, "false") } // Override: Gluten uses session-level timezone for cast. The original test sets per-expression diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index a0d0ba65f2..b103331950 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -224,9 +224,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenCastWithAnsiOnSuite] .exclude("data type casting") .exclude("cast string to timestamp") - .exclude("Casting to char/varchar") - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") - .exclude("SPARK-39749: cast Decimal to string") enableSuite[GlutenCodeGenerationSuite] enableSuite[GlutenCodeGeneratorWithInterpretedFallbackSuite] enableSuite[GlutenCollationExpressionSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala index a0b393bc8b..ea28b228a3 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOnSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.catalyst.expressions +import org.apache.gluten.config.GlutenConfig + import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT} import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC} import org.apache.spark.sql.internal.SQLConf @@ -30,6 +32,12 @@ class GlutenCastWithAnsiOnSuite extends CastWithAnsiOnSuite with GlutenTestsTrai override def beforeAll(): Unit = { super.beforeAll() conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true) + // CastWithAnsiOnSuite creates Cast expressions with EvalMode.ANSI but does not set the + // session-level ANSI config. Velox reads ANSI mode from session config to decide cast + // behavior (e.g., scientific notation for Decimal→String). We must sync session config + // with the expression-level evalMode and disable ANSI fallback so Velox actually executes. + conf.setConf(SQLConf.ANSI_ENABLED, true) + conf.setConfString(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key, "false") } // Override: Gluten uses session-level timezone for cast. The original test sets per-expression --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
