This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 1314fbf4a3499b41e3775cb0bb74d5544c12068b Author: Chang chen <[email protected]> AuthorDate: Mon Apr 13 15:20:38 2026 +0000 [GLUTEN-10134][VL] Implement Cast three-state EvalMode and remove isAnsiSupported whitelist Replace boolean isTryCast with int evalMode (0=LEGACY, 1=TRY, 2=ANSI) in CastNode/ExpressionBuilder, and add Cast to withAnsiEvalMode() in Spark34/35 shims. This enables the full three-state Cast semantics end-to-end: - LEGACY (evalMode=0): kCast path, isTryCast=true, allowOverflow=true (truncate) - ANSI (evalMode=2): kCast path, isTryCast=false, allowOverflow=false (throw) - TRY (evalMode=1): kTryCast path, isTryCast=true, allowOverflow=false (null) The Velox C++ side (SparkCastExpr.cpp) removes the isAnsiSupported() whitelist so all cast types now respect ANSI mode uniformly. C++ changes are in the Velox subproject and managed separately. Co-Authored-By: Claude Opus 4 <[email protected]> --- .../backendsapi/velox/VeloxTransformerApi.scala | 2 +- .../execution/HashAggregateExecTransformer.scala | 4 ++-- .../apache/gluten/substrait/expression/CastNode.java | 18 ++++++++---------- .../substrait/expression/ExpressionBuilder.java | 5 ++--- .../expression/UnaryExpressionTransformer.scala | 20 ++++++++++++-------- .../gluten/sql/shims/spark34/Spark34Shims.scala | 1 + .../gluten/sql/shims/spark35/Spark35Shims.scala | 1 + 7 files changed, 27 insertions(+), 24 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index 3a1d53154f..42f50b3064 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -105,7 +105,7 @@ class VeloxTransformerApi extends TransformerApi with Logging { childNode } else { val typeNode = ConverterUtils.getTypeNode(dataType, nullable) - ExpressionBuilder.makeCast(typeNode, childNode, nullOnOverflow) + ExpressionBuilder.makeCast(typeNode, childNode, if (nullOnOverflow) 1 else 0) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala index a0c52c7909..d252ab8bb8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala @@ -136,7 +136,7 @@ abstract class HashAggregateExecTransformer( .makeCast( ConverterUtils.getTypeNode(sparkType, nullable = false), ExpressionBuilder.makeSelection(colIdx, adjustedOrders(idx)), - SQLConf.get.ansiEnabled)) + if (SQLConf.get.ansiEnabled) 2 else 0)) } else { // Velox and Spark have the same type expressionNodes.add(ExpressionBuilder.makeSelection(colIdx, adjustedOrders(idx))) @@ -337,7 +337,7 @@ abstract class HashAggregateExecTransformer( ExpressionBuilder.makeCast( ConverterUtils.getTypeNode(veloxType, attr.nullable), aggFuncInputAttrNode, - SQLConf.get.ansiEnabled) + if (SQLConf.get.ansiEnabled) 2 else 0) } else { newInputAttributes += attr aggFuncInputAttrNode diff --git a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java index 1984c44d74..037777ce9c 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java +++ b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java @@ -26,12 +26,16 @@ public class CastNode implements ExpressionNode, Serializable { private final TypeNode typeNode; private final ExpressionNode expressionNode; - public final boolean isTryCast; + // Maps directly to Substrait FailureBehavior: + // 0 = UNSPECIFIED (LEGACY: truncate on overflow) + // 1 = RETURN_NULL (TRY: return null on failure) + // 2 = THROW_EXCEPTION (ANSI: throw on failure) + public final int evalMode; - CastNode(TypeNode typeNode, ExpressionNode expressionNode, boolean isTryCast) { + CastNode(TypeNode typeNode, ExpressionNode expressionNode, int evalMode) { this.typeNode = typeNode; this.expressionNode = expressionNode; - this.isTryCast = isTryCast; + this.evalMode = evalMode; } @Override @@ -39,13 +43,7 @@ public class CastNode implements ExpressionNode, Serializable { Expression.Cast.Builder castBuilder = Expression.Cast.newBuilder(); castBuilder.setType(typeNode.toProtobuf()); castBuilder.setInput(expressionNode.toProtobuf()); - if (!isTryCast) { - // Throw exception on failure. - castBuilder.setFailureBehaviorValue(2); - } else { - // Return null on failure. - castBuilder.setFailureBehaviorValue(1); - } + castBuilder.setFailureBehaviorValue(evalMode); Expression.Builder builder = Expression.newBuilder(); builder.setCast(castBuilder.build()); return builder.build(); diff --git a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java index 4bdef37878..2ee83ac53f 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java +++ b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java @@ -237,9 +237,8 @@ public class ExpressionBuilder { return new AggregateFunctionNode(functionId, expressionNodes, phase, outputTypeNode); } - public static CastNode makeCast( - TypeNode typeNode, ExpressionNode expressionNode, boolean isTryCast) { - return new CastNode(typeNode, expressionNode, isTryCast); + public static CastNode makeCast(TypeNode typeNode, ExpressionNode expressionNode, int evalMode) { + return new CastNode(typeNode, expressionNode, evalMode); } public static StringMapNode makeStringMap(Map<String, String> values) { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala index d8404cd84d..ecb96afc53 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala @@ -51,23 +51,27 @@ case class CastTransformer(substraitExprName: String, child: ExpressionTransform buildCastWithWriteSideCheck(context, ExpressionNames.VARCHAR_TYPE_WRITE_SIDE_CHECK, length) case _ => val typeNode = ConverterUtils.getTypeNode(dataType, original.nullable) - ExpressionBuilder.makeCast( - typeNode, - child.doTransform(context), - !SparkShimLoader.getSparkShims.withAnsiEvalMode(original)) + ExpressionBuilder.makeCast(typeNode, child.doTransform(context), castEvalMode) } } + // Maps Spark Cast EvalMode to Substrait FailureBehavior: + // LEGACY=0(UNSPECIFIED), TRY=1(RETURN_NULL), ANSI=2(THROW_EXCEPTION) + private val castEvalMode: Int = { + val shims = SparkShimLoader.getSparkShims + if (shims.withAnsiEvalMode(original)) 2 + else if (shims.withTryEvalMode(original)) 1 + else 0 + } + /** Builds Cast(child, StringType) + write-side check for CharType/VarcharType. */ private def buildCastWithWriteSideCheck( context: SubstraitContext, checkFuncName: String, length: Int): ExpressionNode = { val stringTypeNode = ConverterUtils.getTypeNode(StringType, original.nullable) - val castNode = ExpressionBuilder.makeCast( - stringTypeNode, - child.doTransform(context), - !SparkShimLoader.getSparkShims.withAnsiEvalMode(original)) + val castNode = + ExpressionBuilder.makeCast(stringTypeNode, child.doTransform(context), castEvalMode) val funcName = ConverterUtils.makeFuncName(checkFuncName, Seq(StringType, IntegerType)) val functionId = context.registerFunction(funcName) diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 61bc3bc945..9033738764 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -445,6 +445,7 @@ class Spark34Shims extends SparkShims { case s: Subtract => s.evalMode == EvalMode.ANSI case d: Divide => d.evalMode == EvalMode.ANSI case m: Multiply => m.evalMode == EvalMode.ANSI + case c: Cast => c.evalMode == EvalMode.ANSI case i: IntegralDivide => i.evalMode == EvalMode.ANSI case _ => false } diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 7e31af9b67..181aad9919 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -482,6 +482,7 @@ class Spark35Shims extends SparkShims { case s: Subtract => s.evalMode == EvalMode.ANSI case d: Divide => d.evalMode == EvalMode.ANSI case m: Multiply => m.evalMode == EvalMode.ANSI + case c: Cast => c.evalMode == EvalMode.ANSI case i: IntegralDivide => i.evalMode == EvalMode.ANSI case _ => false } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
