This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit f4e6232d4e0caa8d6752bb72887e0492220fb980 Author: Chang chen <[email protected]> AuthorDate: Wed Apr 15 10:27:07 2026 +0000 [GLUTEN-10134][VL] Fix TYPE_MISMATCH exceptions, add C2R translation path, and improve test framework - Add translateOther() to VeloxCastExceptionTranslator for MapFromEntries (NULL_MAP_KEY), make_date (DateTimeException), timestamp_seconds/millis (ArithmeticException), and make_ym_interval (INTERVAL_ARITHMETIC_OVERFLOW) - Add exception translation in VeloxColumnarToRowExec for C2R path (timestamp_millis) - Use forceCause() reflection to preserve native exception stack when Spark exception constructors set cause=null making initCause() unusable - Add SparkThrowable/GlutenException pass-through guards in ColumnarBatchOutIterator - Rewrite glutenCheckExceptionInExpression to use intercept[Exception] + findCause traversing cause chain, fixing fail()-inside-try and SparkArithmeticException not extending SparkException issues - Restructure verify-ansi-expressions.sh to support per-category testing Co-Authored-By: Claude Opus 4 <[email protected]> --- .../gluten/execution/VeloxColumnarToRowExec.scala | 12 +- dev/verify-ansi-expressions.sh | 294 +++++++++++++++------ .../vectorized/ColumnarBatchOutIterator.java | 31 ++- .../spark/sql/VeloxCastExceptionTranslator.scala | 82 ++++-- .../org/apache/spark/sql/GlutenTestsTrait.scala | 36 +-- 5 files changed, 346 insertions(+), 109 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala index 2354ebf39f..e55e015186 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala @@ -25,6 +25,7 @@ import org.apache.gluten.vectorized.{NativeColumnarToRowInfo, NativeColumnarToRo import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD +import org.apache.spark.sql.VeloxCastExceptionTranslator import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.UnsafeRow import org.apache.spark.sql.execution.{BroadcastUtils, SparkPlan} @@ -199,7 +200,16 @@ object VeloxColumnarToRowExec { baseLength + info.lengths.length } val before = System.currentTimeMillis() - info = jniWrapper.nativeColumnarToRowConvert(c2rId, batchHandle, rowId) + try { + info = jniWrapper.nativeColumnarToRowConvert(c2rId, batchHandle, rowId) + } catch { + case e: Exception => + val translated = VeloxCastExceptionTranslator.translate(e.getMessage) + if (translated != null) { + throw translated + } + throw e + } convertTime += (System.currentTimeMillis() - before) } val (offset, length) = diff --git a/dev/verify-ansi-expressions.sh b/dev/verify-ansi-expressions.sh index 00dfe62ec1..c2fd9ed8b8 100755 --- a/dev/verify-ansi-expressions.sh +++ b/dev/verify-ansi-expressions.sh @@ -1,122 +1,268 @@ #!/usr/bin/env bash -# verify-ansi-expressions.sh — 验证 73 个 ANSI 表达式 -# -# 基于 expression-matrix.md v3，运行所有覆盖 ANSI 影响表达式的 Gluten UT -# 分 4 段运行：spark41 gluten-ut + backends-velox，spark40 gluten-ut + backends-velox +# verify-ansi-expressions.sh — 按 expression-matrix v7 分类验证 ANSI 表达式 # # 用法： # cd /root/SourceCode/gluten -# bash dev/verify-ansi-expressions.sh [spark41|spark40|all] +# bash dev/verify-ansi-expressions.sh <category> [spark41|spark40|all] [--clean] +# +# category（对应矩阵第三节）： +# cast — §3.1.1 Cast + §3.3.1 try_cast +# arithmetic — §3.1.2 算术 + §3.2.6 Abs/UnaryMinus + §3.3.1 try 算术 +# collection — §3.2.1 集合 + §3.3.2 try_element_at +# datetime — §3.2.2 日期时间/Interval + §3.3.2 try_to_timestamp 等 +# math — §3.2.3 数学（Round/BRound/conv） +# decimal — §3.2.4 Decimal（CheckOverflow） +# string — §3.2.5 字符串 + §3.3.2 try_parse_url +# aggregate — §3.1.3 聚合 + §3.4 间接（Sum/Avg/VAR/STDDEV，需人工校验） +# errors — QueryExecutionAnsiErrorsSuite +# all — 以上全部 +# +# spark version（默认 spark41）： +# spark41 — Spark 4.1 +# spark40 — Spark 4.0 +# all — 先 spark41 再 spark40 # -# 输出： -# /tmp/ansi-verify-spark41-ut.log -# /tmp/ansi-verify-spark41-backends.log -# /tmp/ansi-verify-spark40-ut.log -# /tmp/ansi-verify-spark40-backends.log set -uo pipefail export SPARK_ANSI_SQL_MODE=true export SPARK_TESTING=true -TARGET="${1:-all}" +CATEGORY="${1:?Usage: $0 <category> [spark41|spark40|all] [--clean]}" +SPARK_VER="${2:-spark41}" +CLEAN_FLAG="" +if [[ "${3:-}" == "--clean" ]] || [[ "${2:-}" == "--clean" ]]; then + CLEAN_FLAG="--clean" + # if --clean was $2, default spark version + if [[ "${2:-}" == "--clean" ]]; then + SPARK_VER="spark41" + fi +fi -SUITES_UT=( - -s org.apache.spark.sql.catalyst.expressions.GlutenArithmeticExpressionSuite +case "${SPARK_VER}" in + spark41) PROFILES="-Pjava-17,spark-4.1,scala-2.13,backends-velox,hadoop-3.3"; UT_MODULE="gluten-ut/spark41" ;; + spark40) PROFILES="-Pjava-17,spark-4.0,scala-2.13,backends-velox,hadoop-3.3"; UT_MODULE="gluten-ut/spark40" ;; + all) ;; # handled in main entry + *) echo "Unknown spark version: ${SPARK_VER}"; echo "Usage: $0 <category> [spark41|spark40|all] [--clean]"; exit 1 ;; +esac + +ANSI_ARG="--jvm-arg -Dspark.gluten.sql.ansiFallback.enabled=false" +LOG_DIR="/tmp/ansi-matrix" +mkdir -p "${LOG_DIR}" + +# ── Suite 定义 ────────────────────────────────────────────── +# 按矩阵 v7 第三节，强相关 Suite 映射 + +# §3.1.1 Cast + §3.3.1 try_cast +# 强相关 Suite: CastWithAnsiOnSuite(42), CastWithAnsiOffSuite(30), TryCastSuite(3), ComplexTypeSuite(2) +CAST_UT=( -s org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOnSuite -s org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOffSuite -s org.apache.spark.sql.catalyst.expressions.GlutenTryCastSuite - -s org.apache.spark.sql.catalyst.expressions.GlutenMathExpressionsSuite +) +CAST_BACKENDS=( + -s org.apache.spark.sql.catalyst.expressions.VeloxCastSuite +) + +# §3.1.2 算术 + §3.2.6 Abs/UnaryMinus + §3.3.1 try 算术 +# 强相关 Suite: ArithmeticExpressionSuite(37), TryEvalSuite(1) +ARITHMETIC_UT=( + -s org.apache.spark.sql.catalyst.expressions.GlutenArithmeticExpressionSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenTryEvalSuite +) +ARITHMETIC_BACKENDS=( + -s org.apache.gluten.functions.ArithmeticAnsiValidateSuite + -s org.apache.gluten.functions.MathFunctionsValidateSuiteAnsiOn +) + +# §3.2.1 集合 + §3.3.2 try_element_at +# 强相关 Suite: CollectionExpressionsSuite(21) +COLLECTION_UT=( -s org.apache.spark.sql.catalyst.expressions.GlutenCollectionExpressionsSuite - -s org.apache.spark.sql.catalyst.expressions.GlutenStringExpressionsSuite +) + +# §3.2.2 日期时间/Interval + §3.3.2 try_to_timestamp/try_to_date/try_make_timestamp/try_make_interval +# 强相关 Suite: DateExpressionsSuite(22), IntervalExpressionsSuite(9), DateFunctionsSuite(6*) +DATETIME_UT=( -s org.apache.spark.sql.catalyst.expressions.GlutenDateExpressionsSuite + -s org.apache.spark.sql.catalyst.expressions.GlutenIntervalExpressionsSuite + -s org.apache.spark.sql.GlutenDateFunctionsSuite +) + +# §3.2.3 数学 +# 强相关 Suite: MathExpressionsSuite(2) +MATH_UT=( + -s org.apache.spark.sql.catalyst.expressions.GlutenMathExpressionsSuite +) + +# §3.2.4 Decimal +# 强相关 Suite: DecimalExpressionSuite(2) +DECIMAL_UT=( -s org.apache.spark.sql.catalyst.expressions.GlutenDecimalExpressionSuite - -s org.apache.spark.sql.catalyst.expressions.GlutenDecimalPrecisionSuite - -s org.apache.spark.sql.GlutenMathFunctionsSuite - -s org.apache.spark.sql.GlutenDataFrameAggregateSuite - -s org.apache.spark.sql.GlutenStringFunctionsSuite - -s org.apache.spark.sql.errors.GlutenQueryExecutionAnsiErrorsSuite ) -SUITES_BACKENDS=( - -s org.apache.gluten.functions.ArithmeticAnsiValidateSuite - -s org.apache.gluten.functions.MathFunctionsValidateSuiteAnsiOn - -s org.apache.gluten.functions.FunctionsValidateSuite - -s org.apache.spark.sql.catalyst.expressions.VeloxCastSuite +# §3.2.5 字符串 + §3.3.2 try_parse_url +# 强相关 Suite: StringExpressionsSuite(3), UrlFunctionsSuite(2*) +STRING_UT=( + -s org.apache.spark.sql.catalyst.expressions.GlutenStringExpressionsSuite + -s org.apache.spark.sql.GlutenUrlFunctionsSuite ) -echo "========================================" -echo "ANSI Expression Matrix Verification" -echo "Date: $(date '+%Y-%m-%d %H:%M:%S')" -echo "Target: ${TARGET}" -echo "SPARK_ANSI_SQL_MODE=${SPARK_ANSI_SQL_MODE}" -echo "SPARK_TESTING=${SPARK_TESTING}" -echo "spark.gluten.sql.ansiFallback.enabled=false" -echo "========================================" +# §3.1.3 聚合 + §3.4 间接（VAR/STDDEV）— 需人工校验 +# 强相关 Suite: DataFrameAggregateSuite(2* intercept) +AGGREGATE_UT=( + -s org.apache.spark.sql.GlutenDataFrameAggregateSuite +) -run_spark41() { - local PROFILES="-Pjava-17,spark-4.1,scala-2.13,backends-velox,hadoop-3.3" - local ANSI_ARG='--jvm-arg -Dspark.gluten.sql.ansiFallback.enabled=false' +# ANSI 错误语义 +# 强相关 Suite: QueryExecutionAnsiErrorsSuite(3* intercept) +ERRORS_UT=( + -s org.apache.spark.sql.errors.GlutenQueryExecutionAnsiErrorsSuite +) + +# ── 运行函数 ────────────────────────────────────────────── +run_ut() { + local name="$1" + shift + local log="${LOG_DIR}/${name}-${SPARK_VER}-ut.log" echo "" - echo "=== Part 1/4: spark41 gluten-ut (14 suites) ===" + echo "=== ${name}: ${UT_MODULE} ===" ./dev/run-scala-test.sh \ - --clean \ + ${CLEAN_FLAG} \ ${ANSI_ARG} \ ${PROFILES},spark-ut \ - -pl gluten-ut/spark41 \ - "${SUITES_UT[@]}" \ - 2>&1 | tee /tmp/ansi-verify-spark41-ut.log + -pl "${UT_MODULE}" \ + "$@" \ + 2>&1 | tee "${log}" + # 只第一次 clean + CLEAN_FLAG="" +} +run_backends() { + local name="$1" + shift + local log="${LOG_DIR}/${name}-${SPARK_VER}-backends.log" echo "" - echo "=== Part 2/4: spark41 backends-velox (4 suites) ===" + echo "=== ${name}: backends-velox (${SPARK_VER}) ===" ./dev/run-scala-test.sh \ ${ANSI_ARG} \ ${PROFILES} \ -pl backends-velox \ - "${SUITES_BACKENDS[@]}" \ - 2>&1 | tee /tmp/ansi-verify-spark41-backends.log + "$@" \ + 2>&1 | tee "${log}" } -run_spark40() { - local PROFILES="-Pjava-17,spark-4.0,scala-2.13,backends-velox,hadoop-3.3" - local ANSI_ARG='--jvm-arg -Dspark.gluten.sql.ansiFallback.enabled=false' +# ── 分类执行 ────────────────────────────────────────────── - echo "" - echo "=== Part 3/4: spark40 gluten-ut (14 suites) ===" - ./dev/run-scala-test.sh \ - --clean \ - ${ANSI_ARG} \ - ${PROFILES},spark-ut \ - -pl gluten-ut/spark40 \ - "${SUITES_UT[@]}" \ - 2>&1 | tee /tmp/ansi-verify-spark40-ut.log +run_cast() { + run_ut cast "${CAST_UT[@]}" + run_backends cast "${CAST_BACKENDS[@]}" +} - echo "" - echo "=== Part 4/4: spark40 backends-velox (4 suites) ===" - ./dev/run-scala-test.sh \ - ${ANSI_ARG} \ - ${PROFILES} \ - -pl backends-velox \ - "${SUITES_BACKENDS[@]}" \ - 2>&1 | tee /tmp/ansi-verify-spark40-backends.log +run_arithmetic() { + run_ut arithmetic "${ARITHMETIC_UT[@]}" + run_backends arithmetic "${ARITHMETIC_BACKENDS[@]}" } -case "${TARGET}" in - spark41) run_spark41 ;; - spark40) run_spark40 ;; - all) run_spark41; run_spark40 ;; - *) echo "Usage: $0 [spark41|spark40|all]"; exit 1 ;; -esac +run_collection() { + run_ut collection "${COLLECTION_UT[@]}" +} + +run_datetime() { + run_ut datetime "${DATETIME_UT[@]}" +} + +run_math() { + run_ut math "${MATH_UT[@]}" +} + +run_decimal() { + run_ut decimal "${DECIMAL_UT[@]}" +} + +run_string() { + run_ut string "${STRING_UT[@]}" +} + +run_aggregate() { + echo "⚠️ aggregate: §3.1.3 + §3.4 — DataFrame 级 intercept，需人工校验结果" + run_ut aggregate "${AGGREGATE_UT[@]}" +} + +run_errors() { + run_ut errors "${ERRORS_UT[@]}" +} + +run_all() { + run_cast + run_arithmetic + run_collection + run_datetime + run_math + run_decimal + run_string + run_aggregate + run_errors +} + +# ── 主入口 ────────────────────────────────────────────── + +run_category() { + case "${CATEGORY}" in + cast) run_cast ;; + arithmetic) run_arithmetic ;; + collection) run_collection ;; + datetime) run_datetime ;; + math) run_math ;; + decimal) run_decimal ;; + string) run_string ;; + aggregate) run_aggregate ;; + errors) run_errors ;; + all) run_all ;; + *) + echo "Unknown category: ${CATEGORY}" + echo "Usage: $0 <category> [spark41|spark40|all] [--clean]" + exit 1 + ;; + esac +} + +echo "========================================" +echo "ANSI Expression Matrix Verification (v7)" +echo "Date: $(date '+%Y-%m-%d %H:%M:%S')" +echo "Category: ${CATEGORY}" +echo "Spark: ${SPARK_VER}" +echo "SPARK_ANSI_SQL_MODE=${SPARK_ANSI_SQL_MODE}" +echo "SPARK_TESTING=${SPARK_TESTING}" +echo "ansiFallback=false" +echo "Logs: ${LOG_DIR}/" +echo "========================================" + +if [[ "${SPARK_VER}" == "all" ]]; then + # Run spark41 first, then spark40 + SPARK_VER="spark41" + PROFILES="-Pjava-17,spark-4.1,scala-2.13,backends-velox,hadoop-3.3" + UT_MODULE="gluten-ut/spark41" + run_category + + SPARK_VER="spark40" + PROFILES="-Pjava-17,spark-4.0,scala-2.13,backends-velox,hadoop-3.3" + UT_MODULE="gluten-ut/spark40" + CLEAN_FLAG="--clean" + run_category +else + run_category +fi echo "" echo "========================================" -echo "Verification Complete" +echo "Verification Complete — ${CATEGORY}" echo "Results:" -for f in /tmp/ansi-verify-spark4{1,0}-{ut,backends}.log; do +for f in "${LOG_DIR}"/*.log; do if [ -f "$f" ]; then - echo " $(basename $f):" - grep -E "Tests:.*succeeded|All tests passed|FAILED" "$f" | tail -2 | sed 's/^/ /' + echo " $(basename "$f"):" + grep -E "Tests:.*succeeded|Tests run|BUILD SUCCESS|BUILD FAILURE|FAILED" "$f" | tail -3 | sed 's/^/ /' fi done echo "========================================" diff --git a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index db7b8e1842..96bfa14fe1 100644 --- a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -22,11 +22,13 @@ import org.apache.gluten.iterator.ClosableIterator; import org.apache.gluten.runtime.Runtime; import org.apache.gluten.runtime.RuntimeAware; +import org.apache.spark.SparkThrowable; import org.apache.spark.sql.VeloxCastExceptionTranslator; import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException; import org.apache.spark.sql.vectorized.ColumnarBatch; import java.io.IOException; +import java.lang.reflect.Field; public class ColumnarBatchOutIterator extends ClosableIterator<ColumnarBatch> implements RuntimeAware { @@ -155,13 +157,40 @@ public class ColumnarBatchOutIterator extends ClosableIterator<ColumnarBatch> } RuntimeException castEx = VeloxCastExceptionTranslator.translate(msg); if (castEx != null) { - castEx.initCause(e); + forceCause(castEx, e); return castEx; } } + + if (e instanceof GlutenException) { + return (GlutenException) e; + } + + if (e instanceof RuntimeException && e instanceof SparkThrowable) { + return (RuntimeException) e; + } return new GlutenException(e); } + /** + * Forces the cause of a Throwable via reflection. Spark exception constructors pass cause=null to + * super(message, null), which sets the internal cause field to null rather than the sentinel + * value "this", making initCause() unusable. Reflection bypasses this limitation so the original + * native exception is preserved in the chain for debugging. + */ + private static void forceCause(Throwable target, Throwable cause) { + if (target.getCause() != null) { + return; + } + try { + Field f = Throwable.class.getDeclaredField("cause"); + f.setAccessible(true); + f.set(target, cause); + } catch (NoSuchFieldException | IllegalAccessException ignored) { + // Should not happen with standard JDK Throwable; fall through without cause. + } + } + private static String findFirstNonNullMessage(Throwable t) { while (t != null) { String msg = t.getMessage(); diff --git a/gluten-arrow/src/main/scala/org/apache/spark/sql/VeloxCastExceptionTranslator.scala b/gluten-arrow/src/main/scala/org/apache/spark/sql/VeloxCastExceptionTranslator.scala index 27da0d80b7..de44402cc5 100644 --- a/gluten-arrow/src/main/scala/org/apache/spark/sql/VeloxCastExceptionTranslator.scala +++ b/gluten-arrow/src/main/scala/org/apache/spark/sql/VeloxCastExceptionTranslator.scala @@ -25,10 +25,10 @@ import scala.util.matching.Regex /** * Translates Velox cast error messages into the corresponding Spark exception types. * - * Velox cast errors follow the format: "Cannot cast FROMTYPE 'VALUE' to TOTYPE. DETAILS" - * (produced by makeErrorMessage in CastExpr-inl.h). This translator parses that format and - * constructs the correct Spark exception using QueryExecutionErrors factory methods, so that - * the exception type and message match what Spark's native Cast expression would produce. + * Velox cast errors follow the format: "Cannot cast FROMTYPE 'VALUE' to TOTYPE. DETAILS" (produced + * by makeErrorMessage in CastExpr-inl.h). This translator parses that format and constructs the + * correct Spark exception using QueryExecutionErrors factory methods, so that the exception type + * and message match what Spark's native Cast expression would produce. * * Velox errors arrive via JNI as GlutenException with a structured message: * {{{ @@ -42,7 +42,8 @@ import scala.util.matching.Regex * The translator extracts the "Reason:" line to find the actual error message. * * Also translates Velox arithmetic errors (division by zero, overflow) into - * SparkArithmeticException. + * SparkArithmeticException, and Velox datetime/collection errors into the corresponding Spark + * exception types. * * Must be in org.apache.spark.sql package to access private[sql] QueryExecutionErrors. */ @@ -61,6 +62,22 @@ object VeloxCastExceptionTranslator { private val ArithmeticOverflowRegex: Regex = """(?i)(?:Arithmetic overflow|Overflow in .+?)(?::?\s*(.+))?""".r + // Velox: "Invalid value for MAKE_DATE: year=X, month=Y, day=Z" + private val MakeDateRegex: Regex = + """Invalid value for MAKE_DATE: year=(-?\d+), month=(-?\d+), day=(-?\d+)""".r + + // Velox: "map key cannot be null" + private val NullMapKeyRegex: Regex = + """(?i)map key cannot be null""".r + + // Velox: "Timestamp seconds out of range" or "Could not convert Timestamp... to microseconds" + private val TimestampOverflowRegex: Regex = + """(?i)(?:Timestamp seconds out of range|Could not convert Timestamp.+to microseconds)""".r + + // Velox: "Integer overflow in make_ym_interval(X, Y)" + private val MakeYMIntervalOverflowRegex: Regex = + """Integer overflow in make_ym_interval""".r + private val VeloxTypeMap: Map[String, DataType] = Map( "BOOLEAN" -> BooleanType, "TINYINT" -> ByteType, @@ -78,8 +95,8 @@ object VeloxCastExceptionTranslator { Set(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType) /** - * Translates a Velox cast error message into the corresponding Spark exception. - * Returns null if the message is not a recognized Velox cast error. + * Translates a Velox cast error message into the corresponding Spark exception. Returns null if + * the message is not a recognized Velox cast error. * * Called from ColumnarBatchOutIterator.translateException() (Java). */ @@ -92,6 +109,7 @@ object VeloxCastExceptionTranslator { translateCast(reason) .orElse(translateArithmetic(reason, msg)) + .orElse(translateOther(reason)) .orNull } @@ -99,7 +117,7 @@ object VeloxCastExceptionTranslator { ReasonLineRegex.findFirstMatchIn(msg) match { case Some(m) => val reason = m.group(1).trim - // Velox sometimes produces "Reason: Reason: ..." — strip the inner prefix + // Velox sometimes produces "Reason: Reason: ..." -- strip the inner prefix if (reason.startsWith("Reason:")) reason.substring("Reason:".length).trim else reason case None => msg @@ -119,20 +137,24 @@ object VeloxCastExceptionTranslator { (fromType, toType) match { case (StringType, BooleanType) => - Some(QueryExecutionErrors.invalidInputSyntaxForBooleanError( - UTF8String.fromString(value), null)) + Some( + QueryExecutionErrors + .invalidInputSyntaxForBooleanError(UTF8String.fromString(value), null)) case (StringType, _: IntegralType | FloatType | DoubleType | _: DecimalType) => - Some(QueryExecutionErrors.invalidInputInCastToNumberError( - toType, UTF8String.fromString(value), null)) + Some( + QueryExecutionErrors + .invalidInputInCastToNumberError(toType, UTF8String.fromString(value), null)) case (StringType, TimestampType | DateType) => - Some(QueryExecutionErrors.invalidInputInCastToDatetimeError( - UTF8String.fromString(value), toType, null)) + Some( + QueryExecutionErrors + .invalidInputInCastToDatetimeError(UTF8String.fromString(value), toType, null)) case (_: DecimalType, dt: DecimalType) => - Some(QueryExecutionErrors.cannotChangeDecimalPrecisionError( - Decimal(value), dt.precision, dt.scale, null)) + Some( + QueryExecutionErrors + .cannotChangeDecimalPrecisionError(Decimal(value), dt.precision, dt.scale, null)) case (ft, tt) if ft != null && ft != StringType && NumericTypes.contains(tt) => Some(QueryExecutionErrors.castingCauseOverflowError(value, ft, tt)) @@ -163,6 +185,34 @@ object VeloxCastExceptionTranslator { } } + private def translateOther(reason: String): Option[RuntimeException] = { + // MapFromEntries: "map key cannot be null" -> SparkRuntimeException(NULL_MAP_KEY) + if (NullMapKeyRegex.findFirstIn(reason).isDefined) { + return Some(QueryExecutionErrors.nullAsMapKeyNotAllowedError()) + } + + // make_date: "Invalid value for MAKE_DATE: year=X, month=Y, day=Z" -> SparkDateTimeException + reason match { + case MakeDateRegex(year, month, day) => + val dtException = new java.time.DateTimeException( + s"Invalid value for MAKE_DATE: year=$year, month=$month, day=$day") + return Some(QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(dtException)) + case _ => + } + + // SecondsToTimestamp/MillisToTimestamp overflow -> ArithmeticException("long overflow") + if (TimestampOverflowRegex.findFirstIn(reason).isDefined) { + return Some(new ArithmeticException("long overflow")) + } + + // make_ym_interval overflow -> SparkArithmeticException(INTERVAL_ARITHMETIC_OVERFLOW) + if (MakeYMIntervalOverflowRegex.findFirstIn(reason).isDefined) { + return Some(QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(null)) + } + + None + } + private def resolveType(veloxTypeName: String): DataType = { veloxTypeName match { case DecimalTypeRegex(p, s) => DecimalType(p.toInt, s.toInt) diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala index 58bb96c511..925624f159 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala @@ -332,23 +332,14 @@ trait GlutenTestsTrait extends GlutenTestsCommonTrait { _spark.createDataFrame(_spark.sparkContext.parallelize(empData), schema) } val resultDF = df.select(ClassicColumn(expression)) - val exception = - try { - resultDF.collect() - fail(s"Expected ${clazz.getSimpleName} but no exception was thrown for: $expression") - } catch { - case e: SparkException if e.getCause != null && clazz.isInstance(e.getCause) => - e.getCause - case e: Throwable if clazz.isInstance(e) => - e - case e: SparkException if e.getCause != null => - e.getCause - case e: Throwable => - fail( - s"Expected ${clazz.getSimpleName} but got ${e.getClass.getSimpleName}: " + - s"${e.getMessage}", - e) - } + val thrown = intercept[Exception](resultDF.collect()) + // Walk the cause chain to find the target exception type + val exception = findCause(thrown, clazz).getOrElse { + fail( + s"Expected ${clazz.getSimpleName} but got ${thrown.getClass.getSimpleName}: " + + s"${thrown.getMessage}", + thrown) + } if (expectedErrMsg != null && exception.getMessage != null) { assert( exception.getMessage.contains(expectedErrMsg), @@ -357,6 +348,17 @@ trait GlutenTestsTrait extends GlutenTestsCommonTrait { } } + private def findCause(e: Throwable, clazz: Class[_]): Option[Throwable] = { + var current: Throwable = e + while (current != null) { + if (clazz.isAssignableFrom(current.getClass)) { + return Some(current) + } + current = current.getCause + } + None + } + def shouldNotFallback(): Unit = { TestStats.offloadGluten = false if (!BackendTestUtils.isCHBackendLoaded()) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
