wombatu-kun commented on code in PR #20759:
URL: https://github.com/apache/hudi/pull/20759#discussion_r4652221437


##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -409,23 +410,41 @@ class ExpressionIndexSupport(spark: SparkSession,
   /**
    * Bloom filter keys: the values of `f(col) = v` and `f(col) IN (...)`, as 
strings like the writer added them. A file
    * passes if it may hold any key, which is conservative across conjuncts. A 
conjunct with a value that is not a
-   * non-null literal gives no keys, and is skipped. So does a term whose 
values the writer and the literal print
-   * differently, for ex a DATE is `2023-01-02` to the writer and `19359` here 
(#20567).
+   * non-null literal gives no keys, and is skipped. So does a term whose keys 
cannot be rebuilt here, see
+   * [[hasWriterKeys]].
    */
   private def bloomFilterKeys(use: IndexUse): List[String] =
     use.conjuncts.flatMap(Leaf.parse).filter(leaf => 
use.indexed.matches(leaf.term) && hasWriterKeys(leaf.term.dataType)).flatMap { 
leaf =>
       val values: Seq[Any] = leaf.values.map {
         case Literal(value, _) => value
         case _ => null
       }
-      if (values.isEmpty || values.contains(null)) Seq.empty else 
values.map(_.toString)
+      if (values.isEmpty || values.contains(null)) Seq.empty else 
values.flatMap(writerKeys(_, leaf.term.dataType))
     }.toList
 
+  /**
+   * Whether the writer's keys, `toString` of the external value, can be 
rebuilt from a literal of this type.
+   *
+   * NOTE: Not TIMESTAMP: the writer prints a `java.sql.Timestamp` in the 
executor's default time zone, or an `Instant`
+   * with `spark.sql.datetime.java8API.enabled`, which cannot be rebuilt here.
+   * NOTE: Not FLOAT/DOUBLE: `-0.0` and `0.0` print apart though they compare 
equal. Not DECIMAL: it prints its scale.
+   */
   private def hasWriterKeys(dataType: DataType): Boolean = dataType match {
-    case _: StringType | ByteType | ShortType | IntegerType | LongType => true
+    case _: StringType | ByteType | ShortType | IntegerType | LongType | 
DateType => true
     case _ => false
   }
 
+  /**
+   * The writer's keys for an internal literal value. A DATE is epoch days 
here, and the writer prints a `java.sql.Date`,
+   * or a `LocalDate` with `spark.sql.datetime.java8API.enabled`; both are 
probed, as they differ for edge years.
+   */
+  private def writerKeys(value: Any, dataType: DataType): Seq[String] = 
dataType match {
+    case DateType =>
+      val days = value.asInstanceOf[Int]
+      Seq(DateTimeUtils.toJavaDate(days).toString, 
DateTimeUtils.daysToLocalDate(days).toString).distinct

Review Comment:
   `toJavaDate(days)` uses the reader's default zone, while the legacy writer 
key uses the executor's zone; for `2011-12-30`, a Pacific/Apia writer stores 
`2011-12-31` but a UTC reader probes only `2011-12-30`. Please guard legacy 
DATE pruning on a known writer zone, with a conservative fallback for existing 
indexes, and cover this cross-zone case.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala:
##########
@@ -1934,14 +1934,25 @@ class TestExpressionIndex extends 
HoodieSparkSqlTestBase with SparkAdapterSuppor
         spark.sql(s"INSERT INTO $tableName VALUES ('2023-09-23 
09:42:17','u1','rider-F',34.15,'sao_paulo')")
         spark.sql(s"INSERT INTO $tableName VALUES ('2023-09-24 
09:00:00','u2','rider-Z',12.50,'sao_paulo')")
         spark.sql(s"CREATE INDEX ts_year ON $tableName USING column_stats(ts) 
options(expr='date_format', format='yyyy')")
-        spark.sql(s"CREATE INDEX ts_two_days ON $tableName USING 
bloom_filters(ts) options(expr='date_add', days='2')")
 
         withSQLConf(DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true", 
HoodieMetadataConfig.ENABLE.key -> "true",
           "spark.sql.adaptive.enabled" -> "false") {
           // Spark casts the indexed string to INT here, which no rule ranges: 
the index cannot prune this filter
           assertScan(tableName, "date_format(ts, 'yyyy') = 2023 AND fare = 
12.5", Seq("rider-Z"), expectedFiles = 1)
-          // A bloom filter holds a DATE as the writer's string, not as the 
epoch days of the literal: it cannot answer this
-          assertScan(tableName, "date_add(ts, 2) IN (DATE'2023-09-25', 
DATE'2023-09-27')", Seq("rider-F"), expectedFiles = 2)
+        }
+
+        // A bloom filter holds a DATE as the writer's string, a java.sql.Date 
or, with the java8 API, a LocalDate,
+        // not as the epoch days of the literal (#20567)
+        Seq("false", "true").foreach { java8Api =>
+          withSQLConf("spark.sql.datetime.java8API.enabled" -> java8Api) {
+            spark.sql(s"CREATE INDEX ts_two_days ON $tableName USING 
bloom_filters(ts) options(expr='date_add', days='2')")
+          }
+          withSQLConf(DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> 
"true", HoodieMetadataConfig.ENABLE.key -> "true",
+            "spark.sql.adaptive.enabled" -> "false") {
+            assertScan(tableName, "date_add(ts, 2) = DATE'2023-09-25'", 
Seq("rider-F"), expectedFiles = 1)

Review Comment:
   These 2023 dates produce the same `java.sql.Date` and `LocalDate` key, so 
both writer-setting iterations would still pass if either conversion were 
removed. Add a year-zero case under both settings to exercise the distinct keys.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to