This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20526-843a0ab6510aca890e192a6cd3018dd722d683e5
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 9a7728c5862a8939f58f47c35abb1d838fc2dcb3
Author: Vova Kolmakov <[email protected]>
AuthorDate: Wed Nov 11 03:21:20 2026 +0000

    fix(spark): guard data skipping on a timestamp's local date against 
fall-backs across midnight (#20526)
    
    * fix(spark): guard data skipping on a timestamp's local date against 
fall-backs across midnight
    
    * addressed review comments: move the local date reversal guard into 
fileGuards on the rendering arms
    
    * addressed review comments: move localDateReversals to TimeZones, read 
derived guard bounds through TryEval in Step, lead the reversal guard with a 
range check, and sharpen the guard tests
---
 .../spark/sql/hudi/dataskipping/RangeRule.scala    | 31 +++++++-
 .../spark/sql/hudi/dataskipping/TimeZones.scala    | 44 ++++++++++-
 .../sql/hudi/dataskipping/rules/CastRule.scala     | 19 ++++-
 .../dataskipping/rules/DateTimeFormatRule.scala    | 19 +++--
 .../hudi/dataskipping/rules/DateTimeOrdering.scala | 47 ++++++++++--
 .../data-skipping/golden-index-filters.txt         | 28 +++----
 .../org/apache/hudi/TestDataSkippingStages.scala   | 70 +++++++++++++++++-
 .../org/apache/hudi/TestDataSkippingUtils.scala    | 86 ++++++++++++++++++++++
 .../hudi/TestHoodieCatalystExpressionUtils.scala   | 11 ++-
 9 files changed, 312 insertions(+), 43 deletions(-)

diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
index 3d2a71a89a79..57a2798099f1 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.hudi.dataskipping
 
-import org.apache.spark.sql.catalyst.expressions.{Expression, If, Literal, 
TryEval}
+import org.apache.spark.sql.catalyst.expressions.{And, Expression, If, 
LeafExpression, Literal, TryEval}
 import org.apache.spark.sql.hudi.dataskipping.rules.{CastRule, 
ConstantOperandRule, DateTimeFormatRule, MonotonicFunctionRule, PrefixRule, 
TimeZoneShiftRule}
 
 /**
@@ -61,7 +61,8 @@ case object Decreasing extends Direction
  * @param child     the operand whose range is pushed through, for ex `x`
  * @param direction whether `term` grows or shrinks with `child`
  * @param fileGuard per-file condition under which the mapping holds (for ex, 
no overflow); outside it
- *                  the bounds become NULL, i.e. unknown, and 
[[StatsFilterCompiler]] keeps the file
+ *                  the bounds become NULL, i.e. unknown, and 
[[StatsFilterCompiler]] keeps the file. It
+ *                  reads a bound of `child` that fails to evaluate as NULL
  * @param partial   whether `term` can be undefined for a non-NULL `child` 
(for ex, `log` of a negative,
  *                  or a parse of an unparseable string). Spark reports that 
as NULL or, depending on
  *                  the session (for ex, ANSI), as an exception; the rebuilt 
bounds are wrapped in
@@ -71,7 +72,7 @@ case object Decreasing extends Direction
 case class Step(term: Expression,
                 child: Expression,
                 direction: Direction,
-                fileGuard: Option[TermRange => Expression],
+                fileGuard: Option[Step.Guard],
                 partial: Boolean) {
 
   def apply(r: TermRange): TermRange = {
@@ -89,10 +90,20 @@ case class Step(term: Expression,
 
   // NOTE: Uses `term.dataType` (same as the rebuilt bound's) since 
stats-table bounds are unresolved here
   private def guarded(bound: Expression, r: TermRange): Expression = fileGuard 
match {
-    case Some(guard) => If(guard(r), bound, Literal(null, term.dataType))
+    case Some(guard) =>
+      val input = r.copy(lo = readInGuard(r.lo), hi = readInGuard(r.hi))
+      If(guard(input), bound, Literal(null, term.dataType))
     case None => bound
   }
 
+  // NOTE: The guard reads both bounds, while a one-sided verdict (for ex, 
`<`) reads one, so a derived bound
+  //       that fails to evaluate (for ex, from_utc_timestamp of the largest 
timestamp) is NULL there, which
+  //       keeps the file. A leaf (a stats column or a literal) cannot fail, 
so it is read as is
+  private def readInGuard(bound: Expression): Expression = bound match {
+    case leaf: LeafExpression => leaf
+    case _ => TryEval(bound)
+  }
+
   // NOTE: TryEval is in every supported Spark version, so no version-specific 
shape (for ex, a
   //       `failOnError = false` copy of each parse expression) is needed
   private def nullOnFailure(bound: Expression): Expression = if (partial) 
TryEval(bound) else bound
@@ -109,6 +120,18 @@ case class Step(term: Expression,
     }
 }
 
+object Step {
+
+  /** A per-file condition over the range of the step's child, see `fileGuard` 
of [[Step]]. */
+  type Guard = TermRange => Expression
+
+  /** Both guards: the mapping holds where each one does. A missing guard 
leaves the other as is. */
+  def andGuards(a: Option[Guard], b: Option[Guard]): Option[Guard] = (a, b) 
match {
+    case (Some(x), Some(y)) => Some(r => And(x(r), y(r)))
+    case _ => a.orElse(b)
+  }
+}
+
 /**
  * A family of functions sharing one guard. Not sealed: families live one per 
file under `rules`.
  *
diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
index 7b84d0d70105..d6e771f7558d 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
@@ -19,11 +19,16 @@ package org.apache.spark.sql.hudi.dataskipping
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 
+import java.time.Instant
+import java.time.zone.{ZoneOffsetTransition, ZoneRules}
+import java.util.concurrent.ConcurrentHashMap
+
+import scala.collection.JavaConverters._
 import scala.util.Try
 
 /**
- * Time zone checks shared by the rules whose ordering depends on a zone's 
offset: `TimeZoneShiftRule`, and
- * `DateTimeFormatRule` through `DateTimeOrdering`.
+ * Time zone checks shared by the rules whose ordering depends on a zone's 
offset: `TimeZoneShiftRule`,
+ * `DateTimeFormatRule` through `DateTimeOrdering`, and `CastRule` for a 
timestamp's local date.
  */
 object TimeZones {
 
@@ -32,6 +37,37 @@ object TimeZones {
    * evaluates it, so for ex +8:00 is +08:00 and the short ID PST is 
America/Los_Angeles; an unknown zone is not
    * fixed.
    */
-  def isFixedOffsetZone(timeZoneId: String): Boolean =
-    
Try(DateTimeUtils.getZoneId(timeZoneId).getRules.isFixedOffset).getOrElse(false)
+  def isFixedOffsetZone(timeZoneId: String): Boolean = 
rulesOf(timeZoneId).exists(_.isFixedOffset)
+
+  private val localDateReversalsByZoneId = new ConcurrentHashMap[String, 
Option[Seq[Instant]]]()
+
+  /**
+   * Instants at which the local date runs backwards in the zone, resolved the 
way Spark evaluates it and
+   * cached per zone id (see the overload over `ZoneRules`). None, so that the 
rendering takes no step, if
+   * there is no zone (not analyzed), the zone is unknown, or its current 
rules keep doing it.
+   */
+  def localDateReversals(timeZoneId: Option[String]): Option[Seq[Instant]] =
+    timeZoneId.flatMap { id =>
+      localDateReversalsByZoneId.computeIfAbsent(id, _ => 
rulesOf(id).flatMap(rules => localDateReversals(rules)))
+    }
+
+  /**
+   * Instants at which the local date runs backwards in a zone with the given 
rules, that is where its clocks fell back
+   * across midnight, or None if its current rules keep doing it. Falling back 
at midnight itself (for ex, Asia/Beirut)
+   * repeats the previous date without running backwards, since the local date 
before the transition is that of the
+   * instant just before it. The instants are in chronological order, as 
`ZoneRules.getTransitions` lists them
+   */
+  def localDateReversals(rules: ZoneRules): Option[Seq[Instant]] = {
+    def isReversal(transition: ZoneOffsetTransition): Boolean = 
transition.isOverlap &&
+      
transition.getDateTimeAfter.toLocalDate.isBefore(transition.getDateTimeBefore.minusNanos(1).toLocalDate)
+    // NOTE: A rule falls back at the same time of day every year, so a single 
year shows whether it reverses the date
+    if (rules.getTransitionRules.asScala.exists(rule => 
isReversal(rule.createTransition(2000)))) {
+      None
+    } else {
+      
Some(rules.getTransitions.asScala.filter(isReversal).map(_.getInstant).toList)
+    }
+  }
+
+  // The rules of the zone, resolved the way Spark evaluates it; None for an 
unknown zone
+  private def rulesOf(timeZoneId: String): Option[ZoneRules] = 
Try(DateTimeUtils.getZoneId(timeZoneId).getRules).toOption
 }
diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
index 144737c5ac9c..ae388e7894ca 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.hudi.dataskipping.rules
 
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
-import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
+import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step, 
TimeZones}
 import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, 
DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, 
NumericType, ShortType, StringType, TimestampNTZType, TimestampType}
 
 import scala.util.Try
@@ -28,6 +28,7 @@ import scala.util.Try
  *
  * Shared guard: [[preservesOrder]] of the child type to the target type. The 
(from, to) allowlist
  * lives here, at the top of this object, and is deny-by-default: a cast pair 
not listed has no range.
+ * A timestamp to date cast is ranged apart from it, with a per-file guard 
(see [[steps]]).
  */
 object CastRule extends RangeRule {
 
@@ -49,6 +50,9 @@ object CastRule extends RangeRule {
    *  - string to date/timestamp, since unpadded strings sort differently from 
what they parse to (for
    *    ex, "2022-1-5" > "2022-01-10" as strings, but parses to an earlier 
date). Expression indexes on
    *    to_date/to_timestamp do not need it: [[TermRange]] asks the source for 
the cast itself first
+   *  - timestamp to date, since the local date runs backwards where the 
clocks fell back across
+   *    midnight (for ex, America/St_Johns went from 00:01 on 2009-11-01 back 
to 23:01 on 2009-10-31).
+   *    [[steps]] ranges it with a per-file guard instead
    */
   private[dataskipping] def preservesOrder(from: DataType, to: DataType): 
Boolean =
     (from, to) match {
@@ -78,9 +82,8 @@ object CastRule extends RangeRule {
       case (TimestampType, LongType | FloatType | DoubleType) => true
       // NOTE: A date maps to the start of that day, which increases with the 
date
       case (DateType, TimestampType | TimestampNTZType) => true
-      // NOTE: Truncating to the (local) date is non-decreasing, also across a 
DST fall-back since
-      //       the date does not repeat
-      case (TimestampType | TimestampNTZType, DateType) => true
+      // NOTE: A timestamp_ntz truncates to its wall-clock date, which never 
runs backwards
+      case (TimestampNTZType, DateType) => true
 
       // Binary and strings
       // NOTE: The default (UTF8_BINARY) string compares by its bytes, same as 
binary. Matching the
@@ -92,6 +95,14 @@ object CastRule extends RangeRule {
     }
 
   override def steps(term: Expression): Seq[Step] = term match {
+    // NOTE: The allowlist rejects this cast: a file spanning an instant at 
which the local date runs
+    //       backwards takes no bounds, and a zone whose current rules keep 
doing it takes no step
+    case Cast(child, DateType, timeZoneId, _) if hasDataType(child) && 
child.dataType == TimestampType =>
+      TimeZones.localDateReversals(timeZoneId).toSeq.map { instants =>
+        Step(term, child, Increasing,
+          fileGuard = DateTimeOrdering.noLocalDateReversal(instants, 
DateTimeOrdering.asTimestamp), partial = false)
+      }
+
     // NOTE: Some casts (for ex, string to date in non-ANSI mode) return NULL 
on malformed input;
     //       `partial` is left false for now, as it has no behavioral effect 
yet
     case cast @ Cast(child, _, _, _) if hasDataType(child) && 
preservesOrder(child.dataType, cast.dataType) =>
diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
index 340e19994389..91c0b92096a5 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.hudi.dataskipping.rules
 
 import org.apache.spark.sql.catalyst.expressions.{And, DateFormatClass, 
Expression, FromUnixTime, GetTimestamp, GreaterThanOrEqual, LessThan, Literal, 
ToUnixTimestamp, UnixTimestamp}
-import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step, 
TermRange}
+import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step, 
TermRange, TimeZones}
 import org.apache.spark.sql.types.{DataType, DateType, LongType, StringType, 
TimestampNTZType, TimestampType}
 
 import java.time.{LocalDate, ZoneOffset}
@@ -32,11 +32,12 @@ import scala.util.Try
  *
  * Shared guard: [[DateTimeOrdering]] (allowlisted literal format, non-LEGACY 
parser policy, and for a
  * time-of-day pattern a fixed-offset time zone), plus a per-file guard on the 
values the format or
- * parse is sound for. No parsing step without a time zone: the analyzer sets 
one on every resolved
- * expression.
+ * parse is sound for. No parsing or formatting step without a time zone: the 
analyzer sets one on every
+ * resolved expression.
  *
  *  - Formatting (`DateFormatClass`, `FromUnixTime`): the instant lies in 
years 0001 to 9999 (see
- *    [[formattableInstants]]).
+ *    [[formattableInstants]]), and the file spans no instant at which the 
zone's local date runs
+ *    backwards (see [[DateTimeOrdering.noLocalDateReversal]]).
  *  - Parsing a string (`ToUnixTimestamp`, `UnixTimestamp`, `GetTimestamp`): 
every string of the file
  *    starts with a digit (see [[digitFirstStrings]]). Parsing is `partial`: 
an unparseable string is
  *    NULL, or throws under ANSI or the EXCEPTION parser policy; `Step` turns 
a throwing bound into NULL,
@@ -53,10 +54,16 @@ object DateTimeFormatRule extends RangeRule {
   override def steps(term: Expression): Seq[Step] = term match {
     case DateFormatClass(timestamp, format, timeZoneId)
       if dataTypeOf(timestamp).contains(TimestampType) && 
DateTimeOrdering.formatPreservesOrder(format, timeZoneId) =>
-      Seq(Step(term, timestamp, Increasing, fileGuard = 
Some(formattableInstants), partial = false))
+      TimeZones.localDateReversals(timeZoneId).toSeq.map { instants =>
+        Step(term, timestamp, Increasing, fileGuard = 
Step.andGuards(Some(formattableInstants),
+          DateTimeOrdering.noLocalDateReversal(instants, 
DateTimeOrdering.asTimestamp)), partial = false)
+      }
     case FromUnixTime(seconds, format, timeZoneId)
       if dataTypeOf(seconds).contains(LongType) && 
DateTimeOrdering.formatPreservesOrder(format, timeZoneId) =>
-      Seq(Step(term, seconds, Increasing, fileGuard = 
Some(formattableSeconds), partial = false))
+      TimeZones.localDateReversals(timeZoneId).toSeq.map { instants =>
+        Step(term, seconds, Increasing, fileGuard = 
Step.andGuards(Some(formattableSeconds),
+          DateTimeOrdering.noLocalDateReversal(instants, 
DateTimeOrdering.asEpochSeconds)), partial = false)
+      }
 
     // NOTE: `to_unix_timestamp` and `unix_timestamp` evaluate the same way, 
and take the same guards
     case ToUnixTimestamp(time, format, timeZoneId, _) =>
diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
index 66afeeddf661..884c901e149d 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
@@ -17,15 +17,19 @@
 
 package org.apache.spark.sql.hudi.dataskipping.rules
 
-import org.apache.spark.sql.catalyst.expressions.{Expression, Literal}
+import org.apache.spark.sql.catalyst.expressions.{And, Expression, 
GreaterThanOrEqual, LessThan, Literal, Not, Or}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.hudi.dataskipping.Step.Guard
 import org.apache.spark.sql.hudi.dataskipping.TimeZones
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.StringType
+import org.apache.spark.sql.types.{LongType, StringType, TimestampType}
 import org.apache.spark.unsafe.types.UTF8String
 
+import java.time.Instant
+
 /**
- * Shared guards of [[DateTimeFormatRule]]: when does turning a timestamp into 
text (formatting) or
- * text into a timestamp (parsing) preserve ordering?
+ * Shared guards of [[DateTimeFormatRule]], and of [[CastRule]] for a 
timestamp's local date: when does
+ * turning a timestamp into text (formatting) or text into a timestamp 
(parsing) preserve ordering?
  *
  *  - Pattern: the format is a literal on the allowlist below. Every 
allowlisted pattern is year-first
  *    and fixed-width, so the lexicographic order of the strings is the 
chronological order of the
@@ -42,10 +46,11 @@ import org.apache.spark.unsafe.types.UTF8String
  *    EST). Across a spring-forward gap, a wall clock inside the gap is 
shifted forward past later ones
  *    (for ex, in America/Los_Angeles '2022-03-13 02:30:00' parses after 
'2022-03-13 03:05:00'). A
  *    date-only parse is monotone in every zone (a gap shifts a start of day 
by less than a day).
- *
- * NOTE: A date-only format is assumed monotone in every zone: the local date 
only runs backwards where
- *       the clocks fall back across midnight, which tzdb last records in 2010 
(America/St_Johns at
- *       00:01; see #20525)
+ *  - Zone, a timestamp's local date (formatting, and `CAST(ts AS DATE)`): the 
local date runs
+ *    backwards where the clocks fell back across midnight (for ex, 
America/St_Johns went from 00:01
+ *    on 2009-11-01 back to 23:01 on 2009-10-31). A file spanning such an 
instant takes no bounds (see
+ *    [[noLocalDateReversal]]), and a zone whose current rules keep doing it 
takes no step (see
+ *    `TimeZones.localDateReversals`).
  */
 object DateTimeOrdering {
 
@@ -125,4 +130,30 @@ object DateTimeOrdering {
   // the zone never changes its offset (an unknown zone is not fixed)
   private def isZoneSafe(p: Pattern, timeZoneId: Option[String]): Boolean =
     !p.hasTimeOfDay || timeZoneId.exists(TimeZones.isFixedOffsetZone)
+
+  /**
+   * File guard of rendering a timestamp's local date: the file's range spans 
none of `instants`, since only
+   * a file spanning one can hold rows that render outside of [f(lo), f(hi)]. 
None, i.e. unguarded, without
+   * instants.
+   *
+   * @param instants  in chronological order, as 
`TimeZones.localDateReversals` lists them
+   * @param toLiteral an instant as a value of the rendered input: 
[[asTimestamp]] or [[asEpochSeconds]]
+   */
+  def noLocalDateReversal(instants: Seq[Instant], toLiteral: Instant => 
Literal): Option[Guard] =
+    if (instants.isEmpty) {
+      None
+    } else {
+      val literals = instants.map(toLiteral)
+      // NOTE: A file wholly before the first instant or from the last one on 
spans none, which settles most files
+      //       (for ex, any after 2010) in two comparisons
+      Some(r => Or(
+        Or(LessThan(r.hi, literals.head), GreaterThanOrEqual(r.lo, 
literals.last)),
+        literals.map(t => Not(And(LessThan(r.lo, t), GreaterThanOrEqual(r.hi, 
t)))).reduce[Expression](And(_, _))))
+    }
+
+  /** An instant as a timestamp, the input of `CAST(ts AS DATE)` and 
`date_format`. */
+  val asTimestamp: Instant => Literal = instant => 
Literal(DateTimeUtils.instantToMicros(instant), TimestampType)
+
+  /** An instant as seconds since the epoch, the input of `from_unixtime`. */
+  val asEpochSeconds: Instant => Literal = instant => 
Literal(instant.getEpochSecond, LongType)
 }
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
 
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
index ac02db7cb444..3f3ff8950463 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
@@ -812,22 +812,22 @@ cs        cast(to_date(s) as timestamp) IN (0:timestamp, 
null:timestamp)  true:boolean
 cs     NOT cast(to_date(s) as timestamp) IN (0:timestamp)      true:boolean
 cs     startswith(cast(to_date(s) as timestamp), 'ab') true:boolean
 cs     NOT startswith(cast(to_date(s) as timestamp), 'ab')     true:boolean
-cs     ((cast(a as bigint) + 1:bigint) = 0:bigint)     
coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND 
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)), 
true:boolean)
-cs     (0:bigint < (cast(a as bigint) + 1:bigint))     
coalesce((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) > 0:bigint), true:boolean)
-cs     ((cast(a as bigint) + 1:bigint) >= 0:bigint)    
coalesce((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint), 
true:boolean)
-cs     NOT ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(NOT 
((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND 
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), 
true:boolean)
-cs     ((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint))     
coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= cast(0:bigint as 
bigint)) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= cast(0:bigint as 
bigint))), true:boolean)
-cs     (cast(a as bigint) + 1:bigint) IN (0:bigint, null:bigint)       
coalesce((((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND 
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)) OR 
((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= null:bigint) AND 
(if((cast(a [...]
-cs     NOT (cast(a as bigint) + 1:bigint) IN (0:bigint)        coalesce(NOT 
((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND 
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), 
true:boolean)
+cs     ((cast(a as bigint) + 1:bigint) = 0:bigint)     
coalesce(((if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), 
null:bigint) <= 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), 
null:bigint) >= 0:bigint)), true:boolean)
+cs     (0:bigint < (cast(a as bigint) + 1:bigint))     
coalesce((if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), 
null:bigint) > 0:bigint), true:boolean)
+cs     ((cast(a as bigint) + 1:bigint) >= 0:bigint)    
coalesce((if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), 
null:bigint) >= 0:bigint), true:boolean)
+cs     NOT ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(NOT 
((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND 
(if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), 
true:boolean)
+cs     ((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint))     
coalesce(((if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), 
null:bigint) <= cast(0:bigint as bigint)) AND (if((tryeval(cast(a_maxValue as 
bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 
1:bigint), null:bigint) >= cast(0:bigint as bigint))), true:boolean)
+cs     (cast(a as bigint) + 1:bigint) IN (0:bigint, null:bigint)       
coalesce((((if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), 
null:bigint) <= 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), 
null:bigint) >= 0:bigint)) OR ((if((tryeval(cast(a_maxValue as bigint)) <= 
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), 
null:bigint) <= n [...]
+cs     NOT (cast(a as bigint) + 1:bigint) IN (0:bigint)        coalesce(NOT 
((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), 
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND 
(if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), 
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), 
true:boolean)
 cs     startswith((cast(a as bigint) + 1:bigint), 'ab')        true:boolean
 cs     NOT startswith((cast(a as bigint) + 1:bigint), 'ab')    true:boolean
-cs     ((datediff(d, 19723:date) + 1:int) = 0:int)     
coalesce(((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND 
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 
19723:date) + 1:int), null:int) >= 0:int)), true:boolean)
-cs     (0:int < (datediff(d, 19723:date) + 1:int))     
coalesce((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) > 0:int), true:boolean)
-cs     ((datediff(d, 19723:date) + 1:int) >= 0:int)    
coalesce((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int), true:boolean)
-cs     NOT ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(NOT 
((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND 
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 
19723:date) + 1:int), null:int) = 0:int)), true:boolean)
-cs     ((datediff(d, 19723:date) + 1:int) = cast(0:int as int))        
coalesce(((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= cast(0:int as int)) 
AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= cast(0:int as int))), 
true:boolean)
-cs     (datediff(d, 19723:date) + 1:int) IN (0:int, null:int)  
coalesce((((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND 
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 
19723:date) + 1:int), null:int) >= 0:int)) OR ((if((datediff(d_maxValue, 
19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), 
null:int) <= null:int) AND (if((datediff(d_maxValue, 19723:date) <=  [...]
-cs     NOT (datediff(d, 19723:date) + 1:int) IN (0:int)        coalesce(NOT 
((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND 
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 
19723:date) + 1:int), null:int) = 0:int)), true:boolean)
+cs     ((datediff(d, 19723:date) + 1:int) = 0:int)     
coalesce(((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND 
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)), true:boolean)
+cs     (0:int < (datediff(d, 19723:date) + 1:int))     
coalesce((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) > 0:int), true:boolean)
+cs     ((datediff(d, 19723:date) + 1:int) >= 0:int)    
coalesce((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int), true:boolean)
+cs     NOT ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(NOT 
((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND 
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean)
+cs     ((datediff(d, 19723:date) + 1:int) = cast(0:int as int))        
coalesce(((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= cast(0:int as int)) 
AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= cast(0:int as int))), 
true:boolean)
+cs     (datediff(d, 19723:date) + 1:int) IN (0:int, null:int)  
coalesce((((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND 
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)) OR 
((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= null:int) AND 
(if((tryeval(d [...]
+cs     NOT (datediff(d, 19723:date) + 1:int) IN (0:int)        coalesce(NOT 
((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND 
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), 
(datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean)
 cs     startswith((datediff(d, 19723:date) + 1:int), 'ab')     true:boolean
 cs     NOT startswith((datediff(d, 19723:date) + 1:int), 'ab') true:boolean
 cs     (concat(s, s) = '')     true:boolean
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
index 7fde323f01e5..cd5d9b1c4935 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.expressions.{Abs, Add, And, 
AttributeReference, Cast, Coalesce, DateFormatClass, EqualNullSafe, EqualTo, 
Expression, FromUnixTime, GreaterThan, GreaterThanOrEqual, Hour, If, In, InSet, 
IsNull, LeafExpression, Length, LessThan, LessThanOrEqual, Literal, Log, 
Minute, Not, Or, StartsWith, TryEval, UnaryMinus, UnixTimestamp, Upper}
 import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
-import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, Decreasing, 
ExpressionIndexSource, Increasing, IndexedExpression, Leaf, Op, RangeRule, 
StatsFilterCompiler, Step, TermRange, Verdict}
+import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, Decreasing, 
ExpressionIndexSource, Increasing, IndexedExpression, Leaf, Op, RangeRule, 
StatsFilterCompiler, Step, TermRange, TimeZones, Verdict}
 import org.apache.spark.sql.hudi.dataskipping.rules.DateTimeOrdering
 import org.apache.spark.sql.hudi.dataskipping.rules.DateTimeOrdering.Pattern
 import org.apache.spark.sql.internal.SQLConf
@@ -30,6 +30,13 @@ import org.apache.spark.unsafe.types.UTF8String
 import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
 import org.junit.jupiter.api.Test
 
+import java.time.{DayOfWeek, Duration, Instant, LocalDate, LocalTime, Month, 
ZoneId, ZoneOffset}
+import java.time.zone.{ZoneOffsetTransition, ZoneOffsetTransitionRule, 
ZoneRules}
+import java.time.zone.ZoneOffsetTransitionRule.TimeDefinition
+import java.util.Collections
+
+import scala.collection.JavaConverters._
+
 /**
  * Unit tests of the stages on their own: [[Leaf.parse]], [[Verdict]], 
[[Step.apply]] and
  * [[TermRange.derive]] over a stub rule table. End-to-end outputs are pinned 
by
@@ -394,6 +401,12 @@ class TestDataSkippingStages {
     val ts = AttributeReference("ts", TimestampType)()
     val formatted = EqualTo(DateFormatClass(ts, Literal("yyyy-MM-dd"), 
Some("UTC")), Literal("2024-01-01"))
     assertEquals(true, overNullStats(formatted, ColumnStatsSource(Seq("ts")), 
TimestampType))
+    // A cast to the local date: unguarded in UTC, guarded where the local 
date has run backwards
+    // (America/St_Johns)
+    def castToDateIn(zone: String): Expression =
+      In(Cast(ts, DateType, Some(zone)), Seq(Literal(LocalDate.of(2009, 10, 
31)), Literal(LocalDate.of(2009, 11, 1))))
+    assertEquals(null, overNullStats(castToDateIn("UTC"), 
ColumnStatsSource(Seq("ts")), TimestampType))
+    assertEquals(true, overNullStats(castToDateIn("America/St_Johns"), 
ColumnStatsSource(Seq("ts")), TimestampType))
   }
 
   @Test
@@ -460,6 +473,61 @@ class TestDataSkippingStages {
     assertFalse(DateTimeOrdering.parsePreservesOrder(Literal("yyyy-M-d"), 
Some("UTC"), zoneDependent = true))
   }
 
+  @Test
+  def testLocalDateReversals(): Unit = {
+    def reversals(zoneId: String): Option[Seq[Instant]] = 
TimeZones.localDateReversals(ZoneId.of(zoneId).getRules)
+
+    // America/St_Johns fell back across midnight every year from 1987 to 2010
+    val stJohns = reversals("America/St_Johns").get
+    assertTrue(stJohns.contains(Instant.parse("2009-11-01T02:31:00Z")))
+    assertEquals(Instant.parse("2010-11-07T02:31:00Z"), stJohns.last)
+    // Falling back at midnight itself (Asia/Beirut, or Pacific/Apia repeating 
a whole day in 1892) repeats the previous
+    // date without running backwards, and a fall-back within the day 
(America/New_York) does not change the date
+    Seq("Asia/Beirut", "Pacific/Apia", "America/New_York", "UTC").foreach { 
zoneId =>
+      assertEquals(Some(Seq.empty), reversals(zoneId), zoneId)
+    }
+
+    // A current rule falling back from 00:30 to 23:30 runs the date backwards 
every year, without bound
+    val (standard, summer) = (ZoneOffset.ofHours(-4), ZoneOffset.ofHours(-3))
+    val rules = ZoneRules.of(standard, standard,
+      Collections.emptyList[ZoneOffsetTransition](), 
Collections.emptyList[ZoneOffsetTransition](), Seq(
+        ZoneOffsetTransitionRule.of(Month.MARCH, 8, DayOfWeek.SUNDAY, 
LocalTime.of(2, 0), false, TimeDefinition.WALL,
+          standard, standard, summer),
+        ZoneOffsetTransitionRule.of(Month.NOVEMBER, 1, DayOfWeek.SUNDAY, 
LocalTime.of(0, 30), false, TimeDefinition.WALL,
+          standard, summer, standard)).asJava)
+    assertEquals(None, TimeZones.localDateReversals(rules))
+
+    // By zone id, resolved the way Spark evaluates it: the short id CNT is 
America/St_Johns
+    assertEquals(Some(stJohns), TimeZones.localDateReversals(Some("CNT")))
+    assertEquals(Some(Seq.empty), TimeZones.localDateReversals(Some("UTC")))
+    // No time zone (not analyzed) or an unknown one: no step
+    assertEquals(None, TimeZones.localDateReversals(None))
+    assertEquals(None, TimeZones.localDateReversals(Some("Not/A_Zone")))
+  }
+
+  @Test
+  def testNoLocalDateReversalGuard(): Unit = {
+    // America/Phoenix ran its date backwards at 06:01Z on 1944-01-01 and on 
1944-10-01
+    val (first, last) = (Instant.parse("1944-01-01T06:01:00Z"), 
Instant.parse("1944-10-01T06:01:00Z"))
+    val guard = DateTimeOrdering.noLocalDateReversal(Seq(first, last), 
DateTimeOrdering.asTimestamp).get
+    // A bound of the file, None standing for one that failed to evaluate
+    def bound(instant: Option[Instant]): Expression =
+      instant.map(DateTimeOrdering.asTimestamp).getOrElse(Literal(null, 
TimestampType))
+    def overFile(lo: Option[Instant], hi: Option[Instant]): Any =
+      guard(TermRange(bound(lo), bound(hi), None, None, nullBoundIsUnknown = 
false)).eval()
+    val minute = Duration.ofMinutes(1)
+
+    // A file ending at an instant spans it, and one starting at it does not
+    assertEquals(false, overFile(Some(first.minus(minute)), Some(first)))
+    assertEquals(false, overFile(Some(last.minus(minute)), Some(last)))
+    assertEquals(true, overFile(Some(last), Some(last.plus(minute))))
+    assertEquals(true, overFile(Some(first), Some(last.minus(minute))))
+    // Without one bound, the file is known to span none only if the other 
bound alone rules them all out
+    assertEquals(true, overFile(None, Some(first.minus(minute))))
+    assertEquals(true, overFile(Some(last), None))
+    assertEquals(null, overFile(Some(first.minus(minute)), None))
+  }
+
   @Test
   def testDateTimeLetterLiteralIsFormatOnly(): Unit = {
     // The parser matches 'T' in any case: '2022-03-05t09:00:00' parses, and 
sorts after
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index c03d099802fe..7d2926afbde7 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -346,6 +346,92 @@ class TestDataSkippingUtils extends 
HoodieSparkClientTestBase with SparkAdapterS
     }
   }
 
+  @Test
+  def testLocalDateAcrossMidnightFallBack(): Unit = {
+    withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "America/St_Johns") {
+      // At 02:31Z on 2009-11-01 the clocks went from 00:01 NDT back to 23:01 
NST of the previous day, so 02:30:30Z
+      // renders as 2009-11-01 but 02:31Z as 2009-10-31. Re-applying a date 
rendering over min/max stats puts the max
+      // below the min and would wrongly prune file_1 for either date; must 
keep the file
+      val acrossFallBack = IndexRow("file_1", valueCount = 2,
+        A_minValue = 1257042630L, A_maxValue = 1257042660L, A_nullCount = 0, 
// epoch seconds of C's bounds
+        C_minValue = new Timestamp(1257042630000L), // 2009-11-01 02:30:30Z
+        C_maxValue = new Timestamp(1257042660000L), // 2009-11-01 02:31:00Z, 
the first instant after the fall-back
+        C_nullCount = 0)
+      // Files that do not span the fall-back are still pruned
+      val afterFallBack = IndexRow("file_2", valueCount = 2,
+        A_minValue = 1257042660L, A_maxValue = 1257043200L, A_nullCount = 0,
+        C_minValue = new Timestamp(1257042660000L), // 2009-11-01 02:31:00Z, 
renders as 2009-10-31
+        C_maxValue = new Timestamp(1257043200000L), // 2009-11-01 02:40:00Z, 
renders as 2009-10-31
+        C_nullCount = 0)
+      val earlierDay = IndexRow("file_3", valueCount = 2,
+        A_minValue = 1256904000L, A_maxValue = 1256907600L, A_nullCount = 0,
+        C_minValue = new Timestamp(1256904000000L), // 2009-10-30 12:00Z
+        C_maxValue = new Timestamp(1256907600000L), // 2009-10-30 13:00Z
+        C_nullCount = 0)
+      val files = Seq(acrossFallBack, afterFallBack, earlierDay)
+      Seq(
+        "date_format(C, 'yyyy-MM-dd') = '2009-10-31'" -> Seq("file_1", 
"file_2"),
+        "date_format(C, 'yyyy-MM-dd') = '2009-11-01'" -> Seq("file_1"),
+        "from_unixtime(A, 'yyyy-MM-dd') = '2009-10-31'" -> Seq("file_1", 
"file_2"),
+        "CAST(C AS DATE) IN (DATE'2009-10-29', DATE'2009-10-31')" -> 
Seq("file_1", "file_2"),
+        "date_add(CAST(C AS DATE), 1) = DATE'2009-11-01'" -> Seq("file_1", 
"file_2")
+      ).foreach { case (filter, expected) =>
+        assertEquals(expected, applyFilterExpr(resolveExpr(spark, filter, 
sourceTableSchema), files), filter)
+      }
+      // to_date reaches data skipping as the cast it stands for, once the 
optimizer replaced it. Two dates: with one,
+      // the optimizer turns IN into "=", which Spark 3.5+ rewrites into a 
range on C itself
+      assertEquals(Seq("file_1", "file_2"),
+        lookupThroughFullOptimizer("to_date(C) IN (DATE'2009-10-29', 
DATE'2009-10-31')", files))
+
+      // The same date at both ends does not rule out a row of the previous 
date in between
+      val aroundFallBack = IndexRow("file_4", valueCount = 3,
+        C_minValue = new Timestamp(1257042630000L), // 2009-11-01 02:30:30Z, 
renders as 2009-11-01
+        C_maxValue = new Timestamp(1257046800000L), // 2009-11-01 03:40:00Z, 
renders as 2009-11-01
+        C_nullCount = 0)
+      assertEquals(Seq("file_4"), applyFilterExpr(
+        resolveExpr(spark, "NOT (date_format(C, 'yyyy-MM-dd') = 
'2009-11-01')", sourceTableSchema), Seq(aroundFallBack)))
+    }
+  }
+
+  @Test
+  def testLocalDateGuardToleratesFailingBounds(): Unit = {
+    // America/Phoenix ran its date backwards twice in 1944, so the cast to 
the local date is guarded there, and the
+    // guard reads both bounds of the cast's input, while "<" only needs the 
min. The max of each file overflows,
+    // which must not fail the lookup. Without its max, file_1 may span the 
first instant, so it is kept: its min
+    // renders as 1944-01-01, but a row 30 seconds later renders as 
1943-12-31. The min of file_2 is past both
+    // instants, which settles the guard without the max, so file_2 is still 
pruned
+    withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "America/Phoenix") {
+      assertEquals(Seq("file_1"), applyFilterExpr(
+        resolveExpr(spark, "CAST(from_utc_timestamp(C, '+08:00') AS DATE) < 
DATE'1944-01-01'", sourceTableSchema),
+        filesWithFailingMax))
+    }
+  }
+
+  @Test
+  def testFormatGuardToleratesFailingBounds(): Unit = {
+    // Formatting is guarded in every zone, to years 0001 to 9999, and that 
guard reads both bounds too. The lookup
+    // must not fail; as the guard needs the max, both files are kept
+    withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+      assertEquals(Seq("file_1", "file_2"), applyFilterExpr(
+        resolveExpr(spark, "date_format(from_utc_timestamp(C, '+08:00'), 
'yyyy-MM-dd') < '2000-01-01'", sourceTableSchema),
+        filesWithFailingMax))
+    }
+  }
+
+  // Shifting the max of these files, the largest timestamp, by 
from_utc_timestamp overflows
+  private def filesWithFailingMax: Seq[IndexRow] = {
+    val failingMax = new Timestamp(Long.MaxValue / 1000) // in year 294247
+    Seq(
+      IndexRow("file_1", valueCount = 2,
+        C_minValue = new Timestamp(-820547970000L), // 1943-12-31 22:00:30Z, 
shifted to 06:00:30Z on 1944-01-01
+        C_maxValue = failingMax,
+        C_nullCount = 0),
+      IndexRow("file_2", valueCount = 2,
+        C_minValue = new Timestamp(978307200000L), // 2001-01-01 00:00Z
+        C_maxValue = failingMax,
+        C_nullCount = 0))
+  }
+
   @ParameterizedTest
   @MethodSource(Array("testStringsLookupFilterExpressionsSource"))
   def testStringsLookupFilterExpressions(sourceExpr: Expression, input: 
Seq[IndexRow], output: Seq[String]): Unit = {
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 68e597b6516c..93569202e0ce 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -247,6 +247,8 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
       assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr, 
Literal(pattern), newYork)), pattern)
       assertEquals(Some(longAttr), matched(FromUnixTime(longAttr, 
Literal(pattern), newYork)), pattern)
     }
+    // In America/St_Johns the local date has run backwards, so a date-only 
pattern is guarded per file there.
+    assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr, 
Literal("yyyy-MM-dd"), Some("America/St_Johns"))))
     // PST resolves to America/Los_Angeles; Kolkata has no DST but historical 
offset changes.
     assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd 
HH:mm"), Some("PST"))))
     assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd 
HH:mm"), Some("Asia/Kolkata"))))
@@ -255,6 +257,7 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
     assertEquals(None, matched(DateFormatClass(tsAttr, Literal(null, 
StringType), newYork)))
     assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd 
HH:mm"), Some("Not/AZone"))))
     assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd 
HH:mm"), None)))
+    assertEquals(None, matched(FromUnixTime(longAttr, Literal("yyyy-MM-dd"), 
None)))
   }
 
   @Test
@@ -342,9 +345,13 @@ class TestHoodieCatalystExpressionUtils extends 
SparkAdapterSupport {
     assertEquals(None, matched(Cast(dateAttr, StringType)))
     assertEquals(None, matched(Cast(dateAttr, IntegerType)))
     assertEquals(None, matched(Cast(boolAttr, DecimalType(1, 1))))
-    // A date maps to the start of its day, and truncating a timestamp to its 
date is non-decreasing.
+    // A date maps to the start of its day.
     assertEquals(Some(dateAttr), matched(Cast(dateAttr, TimestampType)))
-    assertEquals(Some(tsAttr), matched(Cast(tsAttr, DateType)))
+    // A timestamp's local date runs backwards where the clocks fell back 
across midnight, so it is
+    // ranged with a per-file guard there (America/St_Johns), and not at all 
without a time zone.
+    assertEquals(Some(false), guarded(Cast(tsAttr, DateType, Some("UTC")), 
tsAttr))
+    assertEquals(Some(true), guarded(Cast(tsAttr, DateType, 
Some("America/St_Johns")), tsAttr))
+    assertEquals(None, matched(Cast(tsAttr, DateType)))
     // The default string and binary both compare by bytes.
     assertEquals(Some(binAttr), matched(Cast(binAttr, StringType)))
     // Boolean to bigint keeps false < true as 0 < 1, and an identity cast is 
always allowed.

Reply via email to