This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c4f0c4c2a80d fix(spark): make the data skipping cast allowlist
deny-by-default (#20513)
c4f0c4c2a80d is described below
commit c4f0c4c2a80d8ac2ef11939a3301ef38b7415567
Author: voonhous <[email protected]>
AuthorDate: Thu Nov 5 17:09:33 2026 +0800
fix(spark): make the data skipping cast allowlist deny-by-default (#20513)
* fix(spark): make the data skipping cast allowlist deny-by-default
CastRule.preservesOrder rejected a list of known-bad casts and
returned true for everything else, so every unlisted pair was
re-applied to min/max stats. Each soundness bug found so far came
through that fallthrough, for ex timestamp to timestamp_ntz, which
takes the local wall-clock time and re-orders across a DST fall-back.
It now accepts only an allowlist: identity, numeric up-casts, decimal
to float/double, boolean to integral/float/double, timestamp to
bigint/float/double, date <-> timestamp(_ntz) truncation, and default
string <-> binary. Every other cast falls back to no pruning.
String to date/timestamp stays allowed, though unpadded strings parse
out of order: the optimizer rewrites to_date/to_timestamp without a
format into that cast, and expression indexes on them would stop
pruning. Tracked in #19481.
Refs #19445
* fix(spark): drop string to date/timestamp from the cast allowlist
The arm was kept so that expression indexes on to_date/to_timestamp
keep pruning, but they never reach it: ExpressionIndexSupport strips
the indexed function and rebuilds the query over the bare column, so
the stored f(col) bounds are compared as is. Only plain column stats
on a string column used it, where unpadded strings sort differently
from what they parse to ("2022-1-5" > "2022-01-10" as strings, but
parses to an earlier date).
* test(spark): update golden filters for the cast allowlist
Dropping string to date/timestamp from the allowlist changes 9
golden lines, all cast(s as date), which now compile to TRUE.
---
.../sql/hudi/dataskipping/rules/CastRule.scala | 76 ++++++++++++----------
.../data-skipping/golden-index-filters.txt | 18 ++---
.../org/apache/hudi/TestDataSkippingUtils.scala | 10 ++-
.../hudi/TestHoodieCatalystExpressionUtils.scala | 31 +++++++++
4 files changed, 91 insertions(+), 44 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
index cde92d492f77..7f6f86d5411b 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
@@ -19,52 +19,45 @@ package org.apache.spark.sql.hudi.dataskipping.rules
import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
-import org.apache.spark.sql.types.{BooleanType, DataType, DecimalType,
DoubleType, FloatType, LongType, NumericType, StringType, TimestampType}
+import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType,
DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType,
NumericType, ShortType, StringType, TimestampNTZType, TimestampType}
import scala.util.Try
/**
* `Cast`.
*
- * Shared guard: [[preservesOrder]] of the child type to the target type
(rejects, for ex, string to
- * numeric, timestamp to string, narrowing numeric casts, and collation
changes). The (from, to)
- * allowlist lives here, at the top of this object.
- *
- * NOTE: The trailing arm still accepts every pair not listed; making the
allowlist deny-by-default
- * (trailing `case _ => false`) is pending in #20513
+ * Shared guard: [[preservesOrder]] of the child type to the target type. The
(from, to) allowlist
+ * lives here, at the top of this object, and is deny-by-default: a cast pair
not listed has no range.
*/
object CastRule extends RangeRule {
/**
* Checks whether casting expression of [[from]] [[DataType]] to [[to]]
[[DataType]] will
- * preserve ordering of the elements
+ * preserve ordering of the elements.
+ *
+ * Only casts known to preserve ordering are accepted; anything else is
rejected. A rejection only
+ * costs pruning opportunity, never correctness, and makes new types (for
ex, Spark 4 collations or
+ * TimeType) safe by construction. Notable rejected classes:
+ * - string to/from numeric re-orders values (for ex, "10" < "9"
lexicographically)
+ * - timestamp to string formats in the session time zone, so local time
repeats across a DST
+ * fall-back (for ex, 01:30 EDT is followed by 01:10 EST)
+ * - collation-changing string casts (for ex, "B" < "a" in UTF8_BINARY but
"a" < "B" in UTF8_LCASE)
+ * - casts to boolean (for ex, min -5 and max 5 both cast to true while 0
casts to false)
+ * - narrowing numeric casts, which overflow and wrap around in non-ANSI
mode (for ex, bigint to int)
+ * - timestamp to/from timestamp_ntz, which shifts across DST (for ex, a
local time in the
+ * spring-forward gap shifts forward, so 02:30 maps after 03:00)
+ * - string to date/timestamp, since unpadded strings sort differently from
what they parse to (for
+ * ex, "2022-1-5" > "2022-01-10" as strings, but parses to an earlier
date). Expression indexes on
+ * to_date/to_timestamp do not need it: `ExpressionIndexSupport` strips
the function first
*/
private[dataskipping] def preservesOrder(from: DataType, to: DataType):
Boolean =
(from, to) match {
- // NOTE: Casting between String and Numeric types re-orders elements
(for ex, "10" < "9"
- // lexicographically). These match by type, so that collated
strings on Spark 4 are
- // rejected as well
- case (_: StringType, _: NumericType) => false
- case (_: NumericType, _: StringType) => false
- // NOTE: Timestamps format in the session time zone, so local time
repeats across a DST
- // fall-back (for ex, 01:30 EDT is followed by 01:10 EST) and the
strings re-order
- case (TimestampType, _: StringType) => false
- // NOTE: On Spark 4 StringType carries a collation, and casting to a
different collation changes
- // the sort order (for ex, "B" < "a" in UTF8_BINARY but "a" < "B"
in UTF8_LCASE). Equality
- // also compares the char/varchar length constraint, so those
casts are conservatively
- // rejected too. On Spark 3 StringType is a singleton, making this
arm trivially true
- case (fromStr: StringType, toStr: StringType) => fromStr == toStr
- // NOTE: Every type casts to a string, so collated targets are rejected
whatever the source
- // (for ex, binary bounds are in byte order, which UTF8_LCASE does
not follow)
- case (_, toStr: StringType) => toStr == StringType
- // NOTE: Casting to boolean maps every non-zero (or "true"-like) value
to true, so it is not
- // monotonic (for ex, min -5 and max 5 both cast to true while 0
casts to false)
- case (BooleanType, BooleanType) => true
- case (_, BooleanType) => false
- // NOTE: A collated string does not sort by its bytes, so casting it to
any non-string type
- // (for ex, binary, which compares bytes) can re-order values.
String targets are
- // handled above, so this only rejects collated sources
- case (fromStr: StringType, _) => fromStr == StringType
+ // NOTE: On Spark 4 StringType equality includes the collation and the
char/varchar length
+ // constraint, so a collation-changing cast is not an identity
(for ex, "B" < "a" in
+ // UTF8_BINARY but "a" < "B" in UTF8_LCASE). On Spark 3 StringType
is a singleton
+ case _ if from == to => true
+
+ // Numeric
// NOTE: Cast.canUpCast rejects decimal to float/double as lossy, but
rounding is monotonic
// and every decimal fits within float range. Spark's type
coercion inserts this cast
// when a decimal column is compared with a float/double (for ex,
`price > 99.5D`)
@@ -74,13 +67,28 @@ object CastRule extends RangeRule {
// This is conservative (for ex, double to float rounds
monotonically but is
// rejected), which only costs pruning opportunity, never
correctness
case (fromNum: NumericType, toNum: NumericType) =>
Cast.canUpCast(fromNum, toNum)
+ // NOTE: false < true maps to 0 < 1. Decimal targets are excluded since,
for ex,
+ // DECIMAL(1, 1) cannot hold 1 and overflows into NULL
+ case (BooleanType, ByteType | ShortType | IntegerType | LongType |
FloatType | DoubleType) => true
+
+ // Date and time
// NOTE: Timestamps cast to numbers through their epoch seconds, which
only fit bigint and
// floating-point targets; narrower integral targets wrap around
and decimals overflow
// into NULL in non-ANSI mode
case (TimestampType, LongType | FloatType | DoubleType) => true
- case (TimestampType, _: NumericType) => false
+ // NOTE: A date maps to the start of that day, which increases with the
date
+ case (DateType, TimestampType | TimestampNTZType) => true
+ // NOTE: Truncating to the (local) date is non-decreasing, also across a
DST fall-back since
+ // the date does not repeat
+ case (TimestampType | TimestampNTZType, DateType) => true
+
+ // Binary and strings
+ // NOTE: The default (UTF8_BINARY) string compares by its bytes, same as
binary. Matching the
+ // StringType companion excludes collated strings on Spark 4
+ case (StringType, BinaryType) => true
+ case (BinaryType, StringType) => true
- case _ => true
+ case _ => false
}
override def steps(term: Expression): Seq[Step] = term match {
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
index b51e4170e117..0c4745b09939 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
+++
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
@@ -626,17 +626,17 @@ cs startswith(cast(dec as double), 'ab')
true:boolean
ei startswith(cast(dec as double), 'ab') ((cast(dec_minValue as double)
<= 'ab') AND (cast(dec_maxValue as double) >= 'ab'))
cs NOT startswith(cast(dec as double), 'ab') true:boolean
ei NOT startswith(cast(dec as double), 'ab') NOT
(startswith(cast(dec_minValue as double), 'ab') AND
startswith(cast(dec_maxValue as double), 'ab'))
-cs (cast(s as date) = 0:date) ((cast(s_minValue as date) <= 0:date)
AND (cast(s_maxValue as date) >= 0:date))
-cs (0:date < cast(s as date)) (cast(s_maxValue as date) > 0:date)
-cs (cast(s as date) >= 0:date) (cast(s_maxValue as date) >= 0:date)
-cs NOT (cast(s as date) = 0:date) NOT ((cast(s_minValue as date) =
0:date) AND (cast(s_maxValue as date) = 0:date))
-cs (cast(s as date) = cast(0:date as date)) ((cast(s_minValue as
date) <= cast(0:date as date)) AND (cast(s_maxValue as date) >= cast(0:date as
date)))
-cs cast(s as date) IN (0:date, null:date) (((cast(s_minValue as date) <=
0:date) AND (cast(s_maxValue as date) >= 0:date)) OR ((cast(s_minValue as date)
<= null:date) AND (cast(s_maxValue as date) >= null:date)))
-cs NOT cast(s as date) IN (0:date) NOT ((cast(s_minValue as date) =
0:date) AND (cast(s_maxValue as date) = 0:date))
+cs (cast(s as date) = 0:date) true:boolean
+cs (0:date < cast(s as date)) true:boolean
+cs (cast(s as date) >= 0:date) true:boolean
+cs NOT (cast(s as date) = 0:date) true:boolean
+cs (cast(s as date) = cast(0:date as date)) true:boolean
+cs cast(s as date) IN (0:date, null:date) true:boolean
+cs NOT cast(s as date) IN (0:date) true:boolean
cs startswith(cast(s as date), 'ab') true:boolean
-ei startswith(cast(s as date), 'ab') ((cast(s_minValue as date) <=
'ab') AND (cast(s_maxValue as date) >= 'ab'))
+ei startswith(cast(s as date), 'ab') true:boolean
cs NOT startswith(cast(s as date), 'ab') true:boolean
-ei NOT startswith(cast(s as date), 'ab') NOT (startswith(cast(s_minValue
as date), 'ab') AND startswith(cast(s_maxValue as date), 'ab'))
+ei NOT startswith(cast(s as date), 'ab') true:boolean
cs (cast(d as timestamp) = 0:timestamp) ((cast(d_minValue as timestamp)
<= 0:timestamp) AND (cast(d_maxValue as timestamp) >= 0:timestamp))
cs (0:timestamp < cast(d as timestamp)) (cast(d_maxValue as timestamp)
> 0:timestamp)
cs (cast(d as timestamp) >= 0:timestamp) (cast(d_maxValue as timestamp)
>= 0:timestamp)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index 4ac23be2174b..eb275eb1e539 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -138,7 +138,7 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
}
@Test
- def testTimestampToStringCastAcrossDstFallBack(): Unit = {
+ def testTimestampCastsAcrossDstFallBack(): Unit = {
spark.sqlContext.setConf(SESSION_LOCAL_TIMEZONE.key, "America/New_York")
try {
// 05:30Z formats as 01:30 EDT but the later 06:10Z formats as 01:10
EST, so the string max
@@ -152,6 +152,14 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
val resolvedFilterExpr = resolveExpr(spark, "CAST(C AS STRING) >
'2026-11-01 01:20:00'", sourceTableSchema)
assertEquals(Seq("file_1"), applyFilterExpr(resolvedFilterExpr, input))
+
+ // Converting to timestamp_ntz takes the same local wall-clock time, so
it re-orders the same way
+ if (HoodieSparkUtils.gteqSpark3_4) {
+ val ntzFilterExpr = resolveExpr(spark,
+ "CAST(C AS TIMESTAMP_NTZ) > TIMESTAMP_NTZ '2026-11-01 01:20:00'",
sourceTableSchema)
+
+ assertEquals(Seq("file_1"), applyFilterExpr(ntzFilterExpr, input))
+ }
} finally {
spark.sqlContext.setConf(SESSION_LOCAL_TIMEZONE.key, "UTC")
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 324d01e8c78b..33d0eb7bb040 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -41,6 +41,9 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
private val tsAttr = AttributeReference("ts", TimestampType)()
private val boolAttr = AttributeReference("b", BooleanType)()
private val binAttr = AttributeReference("bin", BinaryType)()
+ // TimestampNTZType is private[spark] on Spark 3.3, where TIMESTAMP_NTZ is
unsupported, so it is
+ // parsed from DDL and only built on Spark 3.4+.
+ private lazy val ntzAttr = AttributeReference("ntz",
DataType.fromDDL("timestamp_ntz"))()
/**
* The column `expr` is ranged through, found by the production range walk
([[TermRange.derive]])
@@ -204,6 +207,34 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(Some(collatedAttr), matched(Cast(collatedAttr, lcase)))
}
+ @Test
+ def testCastsOutsideTheAllowlistDoNotPreserveOrdering(): Unit = {
+ // Dates and booleans only cast to the explicitly allowed targets;
DECIMAL(1, 1) cannot hold true.
+ assertEquals(None, matched(Cast(dateAttr, StringType)))
+ assertEquals(None, matched(Cast(dateAttr, IntegerType)))
+ assertEquals(None, matched(Cast(boolAttr, DecimalType(1, 1))))
+ // A date maps to the start of its day, and truncating a timestamp to its
date is non-decreasing.
+ assertEquals(Some(dateAttr), matched(Cast(dateAttr, TimestampType)))
+ assertEquals(Some(tsAttr), matched(Cast(tsAttr, DateType)))
+ // The default string and binary both compare by bytes.
+ assertEquals(Some(binAttr), matched(Cast(binAttr, StringType)))
+ // Boolean to bigint keeps false < true as 0 < 1, and an identity cast is
always allowed.
+ assertEquals(Some(boolAttr), matched(Cast(boolAttr, LongType)))
+ assertEquals(Some(intAttr), matched(Cast(intAttr, IntegerType)))
+ // Unpadded strings sort differently from what they parse to ("2022-1-5" >
"2022-01-10").
+ assertEquals(None, matched(Cast(strAttr, DateType)))
+ assertEquals(None, matched(Cast(strAttr, TimestampType)))
+ if (HoodieSparkUtils.gteqSpark3_4) {
+ // Converting between timestamp and timestamp_ntz shifts local times in
the DST spring-forward gap.
+ assertEquals(None, matched(Cast(tsAttr, ntzAttr.dataType)))
+ assertEquals(None, matched(Cast(ntzAttr, TimestampType)))
+ assertEquals(None, matched(Cast(strAttr, ntzAttr.dataType)))
+ // Dates map to the start of the day and back without a time zone shift.
+ assertEquals(Some(dateAttr), matched(Cast(dateAttr, ntzAttr.dataType)))
+ assertEquals(Some(ntzAttr), matched(Cast(ntzAttr, DateType)))
+ }
+ }
+
@Test
def testNestedComposition(): Unit = {
assertEquals(Some(intAttr), matched(Add(Multiply(intAttr, Literal(2)),
Literal(3))))