This is an automated email from the ASF dual-hosted git repository.

voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new a9dcd6807105 fix(spark): reject collation-changing casts in data 
skipping (#20458)
a9dcd6807105 is described below

commit a9dcd6807105cbcc739f993b51c507d58334c4ed
Author: voonhous <[email protected]>
AuthorDate: Wed Nov 4 10:01:23 2026 +0800

    fix(spark): reject collation-changing casts in data skipping (#20458)
    
    * fix(spark): reject collation-changing casts in data skipping
    
    On Spark 4 StringType carries a collation. isCastPreservingOrdering
    matched String <-> Numeric with the StringType companion, which only
    equals the default collation, so every cast involving a collated
    string fell through to `case _ => true`. A cast to another collation
    changes the sort order ("B" < "a" in UTF8_BINARY, "a" < "B" in
    UTF8_LCASE), so re-applying it to min/max stats could prune files
    holding matching rows.
    
    - String <-> Numeric arms now match by type, covering collated strings
    - string -> string casts require an identical StringType; on Spark 3
      StringType is a singleton, so nothing changes there
    
    Refs #19445
    
    * fix(spark): reject casts from any type to a collated string
    
    Every type casts to a string, so a cast into a collated target is
    rejected whatever its source, not only from another string. A binary
    column's byte-ordered bounds would otherwise be compared in
    UTF8_LCASE order.
    
    * fix(spark): reject casts from a collated string to non-string types
    
    A collated string does not sort by its bytes, so a cast from it to a
    type that does, such as binary, can re-order values. The earlier arms
    only covered string targets and numeric targets; a collated source cast
    to binary or date still fell through to the order-preserving default.
    
    Reject every cast from a collated string to a non-string type. Default
    strings are unchanged, so Spark 3 behaves as before.
---
 .../apache/spark/sql/HoodieSparkTypeUtils.scala    | 21 ++++++++++++---
 .../org/apache/hudi/TestDataSkippingUtils.scala    | 13 ++++++++++
 .../hudi/TestHoodieCatalystExpressionUtils.scala   | 30 +++++++++++++++++++++-
 3 files changed, 59 insertions(+), 5 deletions(-)

diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
index 355bbcd1881e..ca4c73330454 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/HoodieSparkTypeUtils.scala
@@ -37,16 +37,29 @@ object HoodieSparkTypeUtils {
   def isCastPreservingOrdering(from: DataType, to: DataType): Boolean =
     (from, to) match {
       // NOTE: Casting between String and Numeric types re-orders elements 
(for ex, "10" < "9"
-      //       lexicographically)
-      case (StringType, _: NumericType) => false
-      case (_: NumericType, StringType) => false
+      //       lexicographically). These match by type, so that collated 
strings on Spark 4 are
+      //       rejected as well
+      case (_: StringType, _: NumericType) => false
+      case (_: NumericType, _: StringType) => false
       // NOTE: Timestamps format in the session time zone, so local time 
repeats across a DST
       //       fall-back (for ex, 01:30 EDT is followed by 01:10 EST) and the 
strings re-order
-      case (TimestampType, StringType) => false
+      case (TimestampType, _: StringType) => false
+      // NOTE: On Spark 4 StringType carries a collation, and casting to a 
different collation changes
+      //       the sort order (for ex, "B" < "a" in UTF8_BINARY but "a" < "B" 
in UTF8_LCASE). Equality
+      //       also compares the char/varchar length constraint, so those 
casts are conservatively
+      //       rejected too. On Spark 3 StringType is a singleton, making this 
arm trivially true
+      case (fromStr: StringType, toStr: StringType) => fromStr == toStr
+      // NOTE: Every type casts to a string, so collated targets are rejected 
whatever the source
+      //       (for ex, binary bounds are in byte order, which UTF8_LCASE does 
not follow)
+      case (_, toStr: StringType) => toStr == StringType
       // NOTE: Casting to boolean maps every non-zero (or "true"-like) value 
to true, so it is not
       //       monotonic (for ex, min -5 and max 5 both cast to true while 0 
casts to false)
       case (BooleanType, BooleanType) => true
       case (_, BooleanType) => false
+      // NOTE: A collated string does not sort by its bytes, so casting it to 
any non-string type
+      //       (for ex, binary, which compares bytes) can re-order values. 
String targets are
+      //       handled above, so this only rejects collated sources
+      case (fromStr: StringType, _) => fromStr == StringType
       // NOTE: Cast.canUpCast rejects decimal to float/double as lossy, but 
rounding is monotonic
       //       and every decimal fits within float range. Spark's type 
coercion inserts this cast
       //       when a decimal column is compared with a float/double (for ex, 
`price > 99.5D`)
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index a09b91a54ed7..41c04681fd54 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -35,6 +35,7 @@ import 
org.apache.spark.sql.internal.SQLConf.SESSION_LOCAL_TIMEZONE
 import org.apache.spark.sql.types._
 import org.junit.jupiter.api.{BeforeEach, Test}
 import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assumptions.assumeTrue
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.{Arguments, MethodSource}
 import org.junit.jupiter.params.provider.Arguments.arguments
@@ -108,6 +109,18 @@ class TestDataSkippingUtils extends 
HoodieSparkClientTestBase with SparkAdapterS
     assertEquals(expectedOutput, rows)
   }
 
+  @Test
+  def testCollationChangingCastDoesNotPrune(): Unit = {
+    assumeTrue(HoodieSparkUtils.gteqSpark4_0, "String collations only exist on 
Spark 4.x")
+    // Stats are in UTF8_BINARY order ("B" < "a"), but the filter compares in 
UTF8_LCASE order
+    // ("a" < "B"): re-applying the cast to the bounds gives "B" <= "a" = 
false and would wrongly
+    // prune file_1, which holds "a". Must fall back to always-true, keeping 
the file
+    testLookupFilterExpressions(
+      "CAST(B AS STRING COLLATE UTF8_LCASE) = 'a'",
+      Seq(IndexRow("file_1", valueCount = 2, B_minValue = "B", B_maxValue = 
"a", B_nullCount = 0)),
+      Seq("file_1"))
+  }
+
   @ParameterizedTest
   @MethodSource(Array(
     "testMiscLookupFilterExpressionsSource"
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 9b79c24a6340..d937ed51f7ca 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -18,8 +18,9 @@
 package org.apache.hudi
 
 import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference, 
BitwiseOr, Cast, DateAdd, DateDiff, DateSub, Divide, Exp, Expression, Literal, 
Log, Lower, Multiply, ParseToDate, ShiftLeft, Sqrt, Upper}
-import org.apache.spark.sql.types.{BooleanType, ByteType, DateType, 
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, 
StringType, TimestampType}
+import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, 
DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, 
ShortType, StringType, TimestampType}
 import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assumptions.assumeTrue
 import org.junit.jupiter.api.Test
 
 /**
@@ -38,6 +39,7 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
   private val decAttr = AttributeReference("dec", DecimalType(10, 2))()
   private val tsAttr = AttributeReference("ts", TimestampType)()
   private val boolAttr = AttributeReference("b", BooleanType)()
+  private val binAttr = AttributeReference("bin", BinaryType)()
 
   private def matched(expr: Expression): Option[AttributeReference] =
     
sparkAdapter.getCatalystExpressionUtils.tryMatchAttributeOrderingPreservingTransformation(expr)
@@ -134,6 +136,32 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
     assertEquals(Some(boolAttr), matched(Cast(boolAttr, IntegerType)))
   }
 
+  @Test
+  def testCollationChangingStringCastsDoNotPreserveOrdering(): Unit = {
+    assumeTrue(HoodieSparkUtils.gteqSpark4_0, "String collations only exist on 
Spark 4.x")
+    // StringType("UTF8_LCASE") is a Spark 4 API, so it is obtained 
reflectively to keep this
+    // file compiling against the Spark 3 profiles.
+    val lcase = StringType.getClass.getMethod("apply", classOf[String])
+      .invoke(StringType, "UTF8_LCASE").asInstanceOf[DataType]
+    val collatedAttr = AttributeReference("cs", lcase)()
+    // Changing collation changes the sort order, in either direction.
+    assertEquals(None, matched(Cast(collatedAttr, StringType)))
+    assertEquals(None, matched(Cast(strAttr, lcase)))
+    // Collated strings and numbers re-order each other like default strings 
do.
+    assertEquals(None, matched(Cast(collatedAttr, IntegerType)))
+    assertEquals(None, matched(Cast(intAttr, lcase)))
+    // Any other source cast to a collated string is rejected too, binary 
bounds being in byte order.
+    assertEquals(None, matched(Cast(binAttr, lcase)))
+    assertEquals(None, matched(Cast(dateAttr, lcase)))
+    // A collated string cast to any non-string type is rejected, for ex 
binary compares bytes.
+    assertEquals(None, matched(Cast(collatedAttr, BinaryType)))
+    assertEquals(None, matched(Cast(collatedAttr, DateType)))
+    // The default string still casts to binary, which keeps its byte order.
+    assertEquals(Some(strAttr), matched(Cast(strAttr, BinaryType)))
+    // A collation-preserving cast keeps the ordering.
+    assertEquals(Some(collatedAttr), matched(Cast(collatedAttr, lcase)))
+  }
+
   @Test
   def testNestedComposition(): Unit = {
     assertEquals(Some(intAttr), matched(Add(Multiply(intAttr, Literal(2)), 
Literal(3))))

Reply via email to