This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 12de508ecc180660937601ea6560428e6c292b9e Author: Chang chen <[email protected]> AuthorDate: Wed Apr 15 04:37:34 2026 +0000 [GLUTEN-10134][VL] Translate Velox ANSI exceptions to Spark exception types in ColumnarBatchOutIterator Add VeloxCastExceptionTranslator to parse Velox error messages (from the JNI VeloxUserError "Reason:" field) and construct the correct Spark exception types: - Cast VARCHAR→BOOLEAN: SparkRuntimeException (CAST_INVALID_INPUT) - Cast VARCHAR→numeric: SparkNumberFormatException (CAST_INVALID_INPUT) - Cast VARCHAR→date/timestamp: SparkDateTimeException (CAST_INVALID_INPUT) - Cast DECIMAL→DECIMAL: SparkArithmeticException (NUMERIC_VALUE_OUT_OF_RANGE) - Cast numeric overflow: SparkArithmeticException (CAST_OVERFLOW) - Division by zero: SparkArithmeticException (DIVIDE_BY_ZERO) - Remainder by zero: SparkArithmeticException (REMAINDER_BY_ZERO) - Arithmetic overflow: SparkArithmeticException (ARITHMETIC_OVERFLOW) Update ArithmeticAnsiValidateSuite to expect ArithmeticException instead of SparkException for division/remainder by zero tests, matching the new translated exception types. Co-Authored-By: Claude Opus 4 <[email protected]> --- .../functions/ArithmeticAnsiValidateSuite.scala | 30 +--- .../vectorized/ColumnarBatchOutIterator.java | 6 + .../spark/sql/VeloxCastExceptionTranslator.scala | 172 +++++++++++++++++++++ 3 files changed, 185 insertions(+), 23 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala index 0fb59fe9c0..06f95e5225 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala @@ -20,7 +20,6 @@ import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.spark.SparkConf -import org.apache.spark.SparkException import org.apache.spark.sql.internal.SQLConf class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { @@ -39,15 +38,8 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { } val df = sql("SELECT 2147483647 + 1") - - if (isSparkVersionGE("4.0")) { - intercept[SparkException] { - df.collect() - } - } else { - intercept[ArithmeticException] { - df.collect() - } + intercept[ArithmeticException] { + df.collect() } } @@ -63,14 +55,8 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { } val df = sql("SELECT 2147483647 + 1") - if (isSparkVersionGE("4.0")) { - intercept[SparkException] { - df.collect() - } - } else { - intercept[ArithmeticException] { - df.collect() - } + intercept[ArithmeticException] { + df.collect() } } @@ -79,12 +65,10 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { checkGlutenPlan[ProjectExecTransformer] } if (isSparkVersionGE("3.4")) { - // Spark 3.4+ throws exception for division by zero in ANSI mode - intercept[SparkException] { + intercept[ArithmeticException] { sql("SELECT 1 / 0").collect() } } else { - // Spark 3.2 and 3.3 don't throw exception for division by zero in ANSI mode sql("SELECT 1 / 0").collect() } } @@ -94,7 +78,7 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { checkGlutenPlan[ProjectExecTransformer] } if (isSparkVersionGE("3.4")) { - intercept[SparkException] { + intercept[ArithmeticException] { sql("SELECT 1 div 0 ").collect() } } @@ -105,7 +89,7 @@ class ArithmeticAnsiValidateSuite extends FunctionsValidateSuite { checkGlutenPlan[ProjectExecTransformer] } if (isSparkVersionGE("3.4")) { - intercept[SparkException] { + intercept[ArithmeticException] { sql("SELECT 1 % 0").collect() } } diff --git a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index d251371841..db7b8e1842 100644 --- a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -22,6 +22,7 @@ import org.apache.gluten.iterator.ClosableIterator; import org.apache.gluten.runtime.Runtime; import org.apache.gluten.runtime.RuntimeAware; +import org.apache.spark.sql.VeloxCastExceptionTranslator; import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException; import org.apache.spark.sql.vectorized.ColumnarBatch; @@ -152,6 +153,11 @@ public class ColumnarBatchOutIterator extends ClosableIterator<ColumnarBatch> schemaEx.initCause(e); return schemaEx; } + RuntimeException castEx = VeloxCastExceptionTranslator.translate(msg); + if (castEx != null) { + castEx.initCause(e); + return castEx; + } } return new GlutenException(e); } diff --git a/gluten-arrow/src/main/scala/org/apache/spark/sql/VeloxCastExceptionTranslator.scala b/gluten-arrow/src/main/scala/org/apache/spark/sql/VeloxCastExceptionTranslator.scala new file mode 100644 index 0000000000..27da0d80b7 --- /dev/null +++ b/gluten-arrow/src/main/scala/org/apache/spark/sql/VeloxCastExceptionTranslator.scala @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.sql.errors.QueryExecutionErrors +import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.UTF8String + +import scala.util.matching.Regex + +/** + * Translates Velox cast error messages into the corresponding Spark exception types. + * + * Velox cast errors follow the format: "Cannot cast FROMTYPE 'VALUE' to TOTYPE. DETAILS" + * (produced by makeErrorMessage in CastExpr-inl.h). This translator parses that format and + * constructs the correct Spark exception using QueryExecutionErrors factory methods, so that + * the exception type and message match what Spark's native Cast expression would produce. + * + * Velox errors arrive via JNI as GlutenException with a structured message: + * {{{ + * Exception: VeloxUserError + * Error Source: USER + * Error Code: INVALID_ARGUMENT + * Reason: Cannot cast VARCHAR 'abc' to BOOLEAN. ... + * Retriable: False + * Context: ... + * }}} + * The translator extracts the "Reason:" line to find the actual error message. + * + * Also translates Velox arithmetic errors (division by zero, overflow) into + * SparkArithmeticException. + * + * Must be in org.apache.spark.sql package to access private[sql] QueryExecutionErrors. + */ +object VeloxCastExceptionTranslator { + + private val CastErrorRegex: Regex = + """(?s)Cannot cast (\S+) '(.*?)' to (\S+(?:\(\d+,\s*\d+\))?)\.?\s*(.*)""".r + + private val DecimalTypeRegex: Regex = """DECIMAL\((\d+),\s*(\d+)\)""".r + + private val ReasonLineRegex: Regex = """(?m)^Reason:\s*(.*)""".r + + private val DivisionByZeroRegex: Regex = + """(?i)(?:\([^)]*\)\s*)?[Dd]ivision by zero""".r + + private val ArithmeticOverflowRegex: Regex = + """(?i)(?:Arithmetic overflow|Overflow in .+?)(?::?\s*(.+))?""".r + + private val VeloxTypeMap: Map[String, DataType] = Map( + "BOOLEAN" -> BooleanType, + "TINYINT" -> ByteType, + "SMALLINT" -> ShortType, + "INTEGER" -> IntegerType, + "BIGINT" -> LongType, + "REAL" -> FloatType, + "DOUBLE" -> DoubleType, + "TIMESTAMP" -> TimestampType, + "DATE" -> DateType, + "VARCHAR" -> StringType + ) + + private val NumericTypes: Set[DataType] = + Set(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType) + + /** + * Translates a Velox cast error message into the corresponding Spark exception. + * Returns null if the message is not a recognized Velox cast error. + * + * Called from ColumnarBatchOutIterator.translateException() (Java). + */ + def translate(msg: String): RuntimeException = { + if (msg == null) { + return null + } + + val reason = extractReason(msg) + + translateCast(reason) + .orElse(translateArithmetic(reason, msg)) + .orNull + } + + private def extractReason(msg: String): String = { + ReasonLineRegex.findFirstMatchIn(msg) match { + case Some(m) => + val reason = m.group(1).trim + // Velox sometimes produces "Reason: Reason: ..." — strip the inner prefix + if (reason.startsWith("Reason:")) reason.substring("Reason:".length).trim + else reason + case None => msg + } + } + + private def translateCast(reason: String): Option[RuntimeException] = { + if (!reason.startsWith("Cannot cast ")) { + return None + } + + reason match { + case CastErrorRegex(fromTypeStr, value, toTypeStr, _) => + val fromType = resolveType(fromTypeStr) + val toType = resolveType(toTypeStr) + if (toType == null) return None + + (fromType, toType) match { + case (StringType, BooleanType) => + Some(QueryExecutionErrors.invalidInputSyntaxForBooleanError( + UTF8String.fromString(value), null)) + + case (StringType, _: IntegralType | FloatType | DoubleType | _: DecimalType) => + Some(QueryExecutionErrors.invalidInputInCastToNumberError( + toType, UTF8String.fromString(value), null)) + + case (StringType, TimestampType | DateType) => + Some(QueryExecutionErrors.invalidInputInCastToDatetimeError( + UTF8String.fromString(value), toType, null)) + + case (_: DecimalType, dt: DecimalType) => + Some(QueryExecutionErrors.cannotChangeDecimalPrecisionError( + Decimal(value), dt.precision, dt.scale, null)) + + case (ft, tt) if ft != null && ft != StringType && NumericTypes.contains(tt) => + Some(QueryExecutionErrors.castingCauseOverflowError(value, ft, tt)) + + case _ => None + } + + case _ => None + } + } + + private val ContextLineRegex: Regex = + """(?m)^Context:.*?(checked_remainder|checked_modulus)""".r + + private def translateArithmetic(reason: String, fullMsg: String): Option[RuntimeException] = { + if (DivisionByZeroRegex.findFirstIn(reason).isDefined) { + if (ContextLineRegex.findFirstIn(fullMsg).isDefined) { + Some(QueryExecutionErrors.remainderByZeroError(null)) + } else { + Some(QueryExecutionErrors.divideByZeroError(null)) + } + } else { + reason match { + case ArithmeticOverflowRegex(_) => + Some(QueryExecutionErrors.overflowInIntegralDivideError(null)) + case _ => None + } + } + } + + private def resolveType(veloxTypeName: String): DataType = { + veloxTypeName match { + case DecimalTypeRegex(p, s) => DecimalType(p.toInt, s.toInt) + case other => VeloxTypeMap.getOrElse(other, null) + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
