This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8e5608ecaf6e fix(spark): skip partition stats pruning for time travel
queries (#20451)
8e5608ecaf6e is described below
commit 8e5608ecaf6e3f78b33d83ac0bd4e4712cb81bd6
Author: Shuo Cheng <[email protected]>
AuthorDate: Wed Nov 4 23:59:13 2026 +0800
fix(spark): skip partition stats pruning for time travel queries (#20451)
---
.../scala/org/apache/hudi/HoodieFileIndex.scala | 9 ++--
.../apache/hudi/PartitionStatsIndexSupport.scala | 13 +++++
.../hudi/functional/TestTimeTravelQuery.scala | 62 +++++++++++++++++++++-
3 files changed, 77 insertions(+), 7 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
index eafff4e8ddb8..c0be0c49b33b 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
@@ -341,6 +341,7 @@ case class HoodieFileIndex(spark: SparkSession,
partitionFilters: Seq[Expression]):
(Boolean, Seq[(Option[BaseHoodieTableFileIndex.PartitionPath],
Seq[FileSlice])]) = {
val isPartitionedTable = getPartitionColumns.length > 0
+ lazy val partitionStatsIndexSupport = new
PartitionStatsIndexSupport(spark, schema, rawHoodieSchema, metadataConfig,
metaClient)
val prunedPartitionsTuple: (Boolean, Seq[PartitionPath]) =
if (isPartitionedTable && partitionFilters.nonEmpty) {
// For partitioned table and partition filters, prune the partitions
by the partition filters
@@ -349,11 +350,9 @@ case class HoodieFileIndex(spark: SparkSession,
} else {
(true, listMatchingPartitionPaths(partitionFilters))
}
- } else if (isPartitionedTable && isDataSkippingEnabled) {
- // For partitioned table and no partition filters, if data skipping is
enabled,
- // try using the PARTITION_STATS index to prune the partitions
- val prunedPartitionPaths = new PartitionStatsIndexSupport(spark,
schema, rawHoodieSchema, metadataConfig, metaClient)
- .prunePartitions(this, dataFilters)
+ } else if (isPartitionedTable && isDataSkippingEnabled &&
partitionStatsIndexSupport.supportsQueryType(options)) {
+ // Both PARTITION_STATS and the expression index fallback use
latest-snapshot partition statistics.
+ val prunedPartitionPaths =
partitionStatsIndexSupport.prunePartitions(this, dataFilters)
if (prunedPartitionPaths.nonEmpty) {
try {
(true, prunedPartitionPaths.get.map(e =>
convertToPartitionPath(e)).toSeq)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
index 82a8f15f3663..d738fde19a2e 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
@@ -19,6 +19,7 @@
package org.apache.hudi
+import org.apache.hudi.DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT
import org.apache.hudi.HoodieConversionUtils.toScalaOption
import org.apache.hudi.HoodieFileIndex.collectReferencedColumns
import org.apache.hudi.avro.model.{HoodieMetadataColumnStats,
HoodieMetadataRecord}
@@ -27,6 +28,7 @@ import org.apache.hudi.common.data.{HoodieData,
HoodieListData}
import org.apache.hudi.common.model.{FileSlice, HoodieRecord}
import org.apache.hudi.common.schema.HoodieSchema
import org.apache.hudi.common.table.HoodieTableMetaClient
+import
org.apache.hudi.common.table.timeline.InstantComparison.{compareTimestamps,
GREATER_THAN_OR_EQUALS}
import org.apache.hudi.common.util.ValidationUtils.checkState
import org.apache.hudi.core.read.BaseHoodieTableFileIndex
import org.apache.hudi.metadata.{ColumnStatsIndexPrefixRawKey,
HoodieMetadataPayload, HoodieTableMetadataUtil}
@@ -38,6 +40,7 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.expressions.{And, DateAdd,
DateFormatClass, DateSub, Expression, FromUnixTime, ParseToDate,
ParseToTimestamp, RegExpExtract, RegExpReplace, StringSplit, StringTrim,
StringTrimLeft, StringTrimRight, Substring, UnaryExpression, UnixTimestamp}
import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
import
org.apache.spark.sql.hudi.DataSkippingUtils.{containsNullOrValueCountBasedFilters,
translateIntoColumnStatsIndexFilterExpr}
+import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
import org.apache.spark.sql.types.StructType
import org.apache.spark.storage.StorageLevel
@@ -59,6 +62,16 @@ class PartitionStatsIndexSupport(spark: SparkSession,
metaClient.getIndexMetadata.isPresent
}
+ override def supportsQueryType(options: Map[String, String]): Boolean = {
+ // Partition statistics describe the latest snapshot. Their bounds can
shrink after updates
+ // or partition replacement and exclude historical records.
+ options.get(TIME_TRAVEL_AS_OF_INSTANT.key).forall { instant =>
+ val latestCommit =
metaClient.getCommitsTimeline.filterCompletedInstants.lastInstant
+ latestCommit.isPresent &&
compareTimestamps(HoodieSqlCommonUtils.formatQueryInstant(instant),
+ GREATER_THAN_OR_EQUALS, latestCommit.get.requestedTime)
+ }
+ }
+
override def computeCandidateFileNames(fileIndex: HoodieFileIndex,
queryFilters: Seq[Expression],
queryReferencedColumns: Seq[String],
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
index 4593e21839a7..aefc02f072be 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
@@ -17,11 +17,11 @@
package org.apache.hudi.functional
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions,
ScalaAssertionSupport}
+import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions,
HoodieFileIndex, ScalaAssertionSupport}
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.{HoodieCleaningPolicy, HoodieTableType}
import org.apache.hudi.common.model.HoodieTableType.{COPY_ON_WRITE,
MERGE_ON_READ}
-import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient,
TableSchemaResolver}
import org.apache.hudi.common.table.timeline.TimelineUtils
import org.apache.hudi.common.testutils.{HoodieTestTable, HoodieTestUtils}
import org.apache.hudi.config.{HoodieArchivalConfig, HoodieCleanConfig,
HoodieCompactionConfig, HoodieWriteConfig}
@@ -30,6 +30,8 @@ import org.apache.hudi.testutils.HoodieSparkClientTestBase
import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
import org.apache.spark.sql.SaveMode.{Append, Overwrite}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo,
Literal}
+import org.apache.spark.sql.types.IntegerType
import org.junit.jupiter.api.{AfterEach, BeforeEach}
import org.junit.jupiter.api.Assertions.{assertEquals, assertNotNull,
assertNull, assertTrue}
import org.junit.jupiter.params.ParameterizedTest
@@ -226,6 +228,62 @@ class TestTimeTravelQuery extends
HoodieSparkClientTestBase with ScalaAssertionS
assertTrue(result4.isEmpty)
}
+ @ParameterizedTest
+ @EnumSource(value = classOf[HoodieTableType])
+ def testTimeTravelWithPartitionStats(tableType: HoodieTableType): Unit = {
+ initMetaClient(tableType)
+ val _spark = spark
+ import _spark.implicits._
+
+ val opts = commonOpts ++ Map(
+ DataSourceWriteOptions.TABLE_TYPE.key -> tableType.name,
+ DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "dt",
+ HoodieMetadataConfig.ENABLE.key -> "true",
+ HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
+ HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
+ HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> "value",
+ HoodieCompactionConfig.INLINE_COMPACT.key -> (tableType ==
MERGE_ON_READ).toString,
+ HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key -> "1"
+ )
+ val firstCommit = writeBatch(
+ Seq((1, 10, 1000, "p1"), (2, 100, 1000, "p2")).toDF("id", "value",
"version", "dt"), opts, Overwrite)
+ // Update record id=1 from 10 to 50. The two snapshots contain:
+ // partition dt='p1' partition dt='p2'
+ // firstCommit: (id=1, value=10) (id=2, value=100)
+ // latestCommit: (id=1, value=50) (id=2, value=100)
+ // COW rewrites the base file; MOR compacts inline. The latest min/max for
column 'value'
+ // are therefore [50, 50] in partition dt='p1' and [100, 100] in partition
dt='p2'.
+ val latestCommit = writeBatch(
+ Seq((1, 50, 1001, "p1")).toDF("id", "value", "version", "dt"), opts)
+
+ val readOpts = Map(
+ "path" -> basePath,
+ HoodieMetadataConfig.ENABLE.key -> "true",
+ DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true")
+ val valueFilter = EqualTo(AttributeReference("value", IntegerType)(),
Literal(10))
+
+ metaClient = HoodieTableMetaClient.reload(metaClient)
+ // Query the latest snapshot WHERE value = 10: neither record matches, so
statistics should
+ // prune both partitions, with as.of.instant either omitted or set to
latestCommit.
+ Seq(None, Some(latestCommit)).foreach { instant =>
+ val options = readOpts ++
instant.map(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT.key -> _)
+ val fileIndex = HoodieFileIndex(spark, metaClient, None, options)
+ val (pruned, partitions) =
fileIndex.prunePartitionsAndGetFileSlices(Seq(valueFilter), Seq.empty)
+ assertTrue(pruned)
+ assertTrue(partitions.isEmpty, s"Unexpected partitions: $partitions")
+ }
+
+ // Query AS OF firstCommit WHERE value = 10: expect (id=1, value=10,
version=1000, dt='p1').
+ // Applying the latest [50, 50] bounds would skip partition dt='p1' and
incorrectly return no rows.
+ // Check both the commit instant string and its formatted date-time
representation.
+ Seq(firstCommit, defaultDateTimeFormat(firstCommit)).foreach { instant =>
+ val options = readOpts +
(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT.key -> instant)
+ val result = spark.read.format("hudi").options(options).load()
+ .filter("value = 10").select("id", "value", "version",
"dt").collect().toSeq
+ assertEquals(Seq(Row(1, 10, 1000, "p1")), result)
+ }
+ }
+
private def writeBatch(df: DataFrame, options: Map[String, String], mode:
SaveMode = Append): String = {
df.write.format("hudi").options(options).mode(mode).save(basePath)
metaClient.reloadActiveTimeline()