This is an automated email from the ASF dual-hosted git repository.
jackylee-ch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a5c46feb22 [GLUTEN-12863][TEST] Document test exclusions and restore
cast-from-timestamp coverage for Spark 3.4/3.5 (#12898)
a5c46feb22 is described below
commit a5c46feb22810c9b56c125ed6644532b465a1938
Author: YangJie <[email protected]>
AuthorDate: Wed Aug 26 14:21:55 2026 +0800
[GLUTEN-12863][TEST] Document test exclusions and restore
cast-from-timestamp coverage for Spark 3.4/3.5 (#12898)
---
.../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 3 +++
.../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ++++
.../spark/sql/catalyst/expressions/GlutenCastSuite.scala | 12 ++++++++++++
.../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 11 +++++++++++
.../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ++++
.../spark/sql/catalyst/expressions/GlutenCastSuite.scala | 12 ++++++++++++
.../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ++++
.../catalyst/expressions/GlutenCastWithAnsiOffSuite.scala | 5 ++---
.../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ++++
.../catalyst/expressions/GlutenCastWithAnsiOffSuite.scala | 5 ++---
10 files changed, 58 insertions(+), 6 deletions(-)
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 214d677cdd..3493681aad 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -551,6 +551,9 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("cast from boolean")
.exclude("data type casting")
.excludeGlutenTest("data type casting")
+ // The Gluten rewrite of "cast from timestamp II" is not vetted on
ClickHouse;
+ // the vanilla case is excluded separately in this block.
+ .excludeGlutenTest("cast from timestamp II")
.exclude("cast between string and interval")
.exclude("SPARK-27671: cast from nested null type in struct")
.exclude("Process Infinity, -Infinity, NaN in case insensitive manner")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 635ff6cbc5..c4bc847af5 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -100,6 +100,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("data type casting")
// Revised by setting timezone through config and commented unsupported
cases.
.exclude("cast string to timestamp")
+ // Excluded in favour of the GlutenCastSuite rewrite, which drops the
Long.MinValue
+ // assertion: collect() -> toJavaTimestamp ->
rebaseGregorianToJulianMicros overflows.
.exclude("cast from timestamp II")
.exclude("SPARK-36286: invalid string cast to timestamp")
.exclude("SPARK-39749: cast Decimal to string")
@@ -629,6 +631,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenInsertSuite]
// the native write staing dir is differnt with vanilla Spark for coustom
partition paths
.exclude("SPARK-35106: Throw exception when rename custom partition paths
returns false")
+ // The case expects a SparkException; Gluten surfaces the raw
+ // FileAlreadyExistsException instead.
.exclude("Stop task set if FileAlreadyExistsException was thrown")
// Rewrite: Additional support for file scan with default values has been
added in Spark-3.4.
// It appends the default value in record if it is not present while
scanning.
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index 0e1af4a4e2..0be74fbda7 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -165,6 +165,18 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with
GlutenTestsTrait {
checkEvaluation(cast(false, TimestampType), tsFalse)
}
+ // Gluten's glutenCheckExpression uses collect(), which triggers
+ // toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros
(~292000 BC) overflows
+ // during rebase, so the vanilla case's Long.MinValue assertion is dropped
here.
+ testGluten("cast from timestamp II") {
+ checkEvaluation(cast(Double.NaN, TimestampType), null)
+ checkEvaluation(cast(1.0 / 0.0, TimestampType), null)
+ checkEvaluation(cast(Float.NaN, TimestampType), null)
+ checkEvaluation(cast(1.0f / 0.0f, TimestampType), null)
+ checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue)
+ // Long.MinValue is not asserted; see the comment above the test.
+ }
+
testGluten("cast string to timestamp") {
DebuggableThreadUtils.parmap(
ALL_TIMEZONES
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 920554f91e..49d637681b 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -374,6 +374,9 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("SPARK-33291: Cast struct with null elements to string")
.excludeCH("SPARK-35111: Cast string to year-month interval")
.excludeCH("Gluten - data type casting")
+ // The Gluten rewrite of "cast from timestamp II" is not vetted on
ClickHouse;
+ // the vanilla case is excluded separately in this block.
+ .excludeGlutenTest("cast from timestamp II")
.exclude("cast string to date #2")
.exclude("casting to fixed-precision decimals")
.exclude("SPARK-28470: Cast should honor nullOnOverflow property")
@@ -1180,6 +1183,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenMathExpressionsSuite]
// Spark round UT for round(3.1415,3) is not correct.
.exclude("round/bround/floor/ceil")
+ // TANH(-0.1) returns -0.0996695958408681 on ClickHouse; the case expects
+ // -0.09966799462495582.
.excludeCH("tanh")
.excludeCH("unhex")
.excludeCH("atan2")
@@ -2195,8 +2200,14 @@ class ClickHouseTestSettings extends BackendTestSettings
{
.excludeCH("cast from timestamp II")
.excludeCH("cast a timestamp before the epoch 1970-01-01 00:00:00Z II")
.excludeCH("cast a timestamp before the epoch 1970-01-01 00:00:00Z")
+ // Casting the string array ("123", "true", "f") to array<boolean> should
yield
+ // [null, true, false] under try_cast; ClickHouse throws instead.
.excludeCH("cast from array II")
+ // TRY-mode overflow inside a complex type wraps instead of yielding null
on
+ // ClickHouse: try_cast([2.147483648E9] as array<int>) returns
[-2147483648].
.excludeCH("cast from array III")
+ // Same as "cast from array III": try_cast([2.147483648E9] as
struct<a:int>)
+ // returns [-2147483648] on ClickHouse.
.excludeCH("cast from struct III")
.excludeCH("ANSI mode: cast string to timestamp with parse error")
.excludeCH("ANSI mode: cast string to date with parse error")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 69ab3bc0d9..538ca28417 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -104,6 +104,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("data type casting")
// Revised by setting timezone through config and commented unsupported
cases.
.exclude("cast string to timestamp")
+ // Excluded in favour of the GlutenCastSuite rewrite, which drops the
Long.MinValue
+ // assertion: collect() -> toJavaTimestamp ->
rebaseGregorianToJulianMicros overflows.
.exclude("cast from timestamp II")
.exclude("SPARK-36286: invalid string cast to timestamp")
.exclude("SPARK-39749: cast Decimal to string")
@@ -589,6 +591,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenInsertSuite]
// the native write staing dir is differnt with vanilla Spark for coustom
partition paths
.exclude("SPARK-35106: Throw exception when rename custom partition paths
returns false")
+ // The case expects a SparkException; Gluten surfaces the raw
+ // FileAlreadyExistsException instead.
.exclude("Stop task set if FileAlreadyExistsException was thrown")
// Rewrite: Additional support for file scan with default values has been
added in Spark-3.4.
// It appends the default value in record if it is not present while
scanning.
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index ef06a98539..2e9990fe33 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -168,6 +168,18 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with
GlutenTestsTrait {
checkEvaluation(cast(false, TimestampType), tsFalse)
}
+ // Gluten's glutenCheckExpression uses collect(), which triggers
+ // toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros
(~292000 BC) overflows
+ // during rebase, so the vanilla case's Long.MinValue assertion is dropped
here.
+ testGluten("cast from timestamp II") {
+ checkEvaluation(cast(Double.NaN, TimestampType), null)
+ checkEvaluation(cast(1.0 / 0.0, TimestampType), null)
+ checkEvaluation(cast(Float.NaN, TimestampType), null)
+ checkEvaluation(cast(1.0f / 0.0f, TimestampType), null)
+ checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue)
+ // Long.MinValue is not asserted; see the comment above the test.
+ }
+
testGluten("cast string to timestamp") {
DebuggableThreadUtils.parmap(
ALL_TIMEZONES
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index bbfcca85c6..053369d2be 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -114,6 +114,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("data type casting")
// Revised by setting timezone through config and commented unsupported
cases.
.exclude("cast string to timestamp")
+ // Excluded in favour of the GlutenCastWithAnsiOffSuite rewrite, which
drops the Long.MinValue
+ // assertion: collect() -> toJavaTimestamp ->
rebaseGregorianToJulianMicros overflows.
.exclude("cast from timestamp II")
.exclude("SPARK-36286: invalid string cast to timestamp")
.exclude("SPARK-39749: cast Decimal to string")
@@ -859,6 +861,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-24583 Wrong schema type in InsertIntoDataSourceCommand")
// the native write staing dir is differnt with vanilla Spark for coustom
partition paths
.exclude("SPARK-35106: Throw exception when rename custom partition paths
returns false")
+ // The case expects a SparkException; Gluten surfaces the raw
+ // FileAlreadyExistsException instead.
.exclude("Stop task set if FileAlreadyExistsException was thrown")
// Rewrite: Additional support for file scan with default values has been
added in Spark-3.4.
// It appends the default value in record if it is not present while
scanning.
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
index 3e19554265..9a8f5567d0 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
@@ -134,15 +134,14 @@ class GlutenCastWithAnsiOffSuite extends
CastWithAnsiOffSuite with GlutenExpress
// Gluten's glutenCheckExpression uses collect(), which triggers
// toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros
(~292000 BC) overflows
- // during rebase. Velox computes correctly; only the collect path fails.
Skip Long.MinValue.
+ // during rebase, so the vanilla case's Long.MinValue assertion is dropped
here.
testGluten("cast from timestamp II") {
checkEvaluation(cast(Double.NaN, TimestampType), null)
checkEvaluation(cast(1.0 / 0.0, TimestampType), null)
checkEvaluation(cast(Float.NaN, TimestampType), null)
checkEvaluation(cast(1.0f / 0.0f, TimestampType), null)
checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue)
- // Skip Long.MinValue: Velox result is correct but collect() path
overflows in
- // rebaseGregorianToJulianMicros when converting extreme timestamp to
java.sql.Timestamp.
+ // Long.MinValue is not asserted; see the comment above the test.
}
// Sync session timezone with per-expression timezone and run
single-threaded.
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index c1e7c89804..199068737c 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -117,6 +117,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("data type casting")
// Revised by setting timezone through config and commented unsupported
cases.
.exclude("cast string to timestamp")
+ // Excluded in favour of the GlutenCastWithAnsiOffSuite rewrite, which
drops the Long.MinValue
+ // assertion: collect() -> toJavaTimestamp ->
rebaseGregorianToJulianMicros overflows.
.exclude("cast from timestamp II")
.exclude("SPARK-36286: invalid string cast to timestamp")
.exclude("SPARK-39749: cast Decimal to string")
@@ -840,6 +842,8 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-24583 Wrong schema type in InsertIntoDataSourceCommand")
// the native write staing dir is differnt with vanilla Spark for coustom
partition paths
.exclude("SPARK-35106: Throw exception when rename custom partition paths
returns false")
+ // The case expects a SparkException; Gluten surfaces the raw
+ // FileAlreadyExistsException instead.
.exclude("Stop task set if FileAlreadyExistsException was thrown")
// Rewrite: Additional support for file scan with default values has been
added in Spark-3.4.
// It appends the default value in record if it is not present while
scanning.
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
index a0d758aaea..446fa995aa 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala
@@ -132,15 +132,14 @@ class GlutenCastWithAnsiOffSuite
// Gluten's glutenCheckExpression uses collect(), which triggers
// toJavaTimestamp -> rebaseGregorianToJulianMicros. Long.MinValue micros
(~292000 BC) overflows
- // during rebase. Velox computes correctly; only the collect path fails.
Skip Long.MinValue.
+ // during rebase, so the vanilla case's Long.MinValue assertion is dropped
here.
testGluten("cast from timestamp II") {
checkEvaluation(cast(Double.NaN, TimestampType), null)
checkEvaluation(cast(1.0 / 0.0, TimestampType), null)
checkEvaluation(cast(Float.NaN, TimestampType), null)
checkEvaluation(cast(1.0f / 0.0f, TimestampType), null)
checkEvaluation(cast(Literal(Long.MaxValue), TimestampType), Long.MaxValue)
- // Skip Long.MinValue: Velox result is correct but collect() path
overflows in
- // rebaseGregorianToJulianMicros when converting extreme timestamp to
java.sql.Timestamp.
+ // Long.MinValue is not asserted; see the comment above the test.
}
// Sync session timezone with per-expression timezone and run
single-threaded.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
