wombatu-kun commented on code in PR #20672:
URL: https://github.com/apache/hudi/pull/20672#discussion_r4611870151


##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BloomFiltersIndexSupport.scala:
##########
@@ -41,41 +57,29 @@ class BloomFiltersIndexSupport(spark: SparkSession,
                                          prunedPartitionsAndFileSlices: 
Seq[(Option[BaseHoodieTableFileIndex.PartitionPath], Seq[FileSlice])],
                                          shouldPushDownFilesFilter: Boolean
                                         ): Option[Set[String]] = {
-    lazy val (_, recordKeys) = filterQueriesWithRecordKey(queryFilters)
-    if (recordKeys.nonEmpty) {
-      val prunedPartitionAndFileNames = 
getPrunedPartitionAndFileNames(prunedPartitionsAndFileSlices)
-      
Option.apply(getCandidateFilesForSecondaryKeys(prunedPartitionAndFileNames, 
recordKeys))
+    val (_, recordKeys) = filterQueriesWithRecordKey(queryFilters)
+    if (recordKeys.isEmpty) {
+      None
     } else {
-      Option.empty
+      val candidateFileNames = prunedPartitionsAndFileSlices.flatMap { case 
(partition, fileSlices) =>
+        val partitionPath = partition.map(_.getPath).getOrElse("")
+        fileSlices.flatMap { slice =>
+          if (mustRetainSlice(slice)) {
+            // A log filename retains the entire slice without querying its 
base bloom.
+            slice.getLogFiles.iterator().asScala.map(_.getFileName).toSeq
+          } else {
+            toScalaOption(slice.getBaseFile).filter { baseFile =>
+              // A missing bloom cannot rule out a match.
+              toScalaOption(metadataTable.getBloomFilter(partitionPath, 
baseFile.getFileName))

Review Comment:
   The rewritten lookup has no regression for a missing bloom, so the 
conservative Option.forall fallback remains unverified. Please add a base-file 
case with absent bloom metadata and assert that its slice is retained.



##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BloomFiltersIndexSupport.scala:
##########
@@ -23,16 +23,32 @@ import org.apache.hudi.HoodieConversionUtils.toScalaOption
 import org.apache.hudi.common.config.HoodieMetadataConfig
 import org.apache.hudi.common.model.FileSlice
 import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.config.HoodieIndexConfig
 import org.apache.hudi.core.read.BaseHoodieTableFileIndex
+import org.apache.hudi.index.HoodieIndex.IndexType
 import org.apache.hudi.metadata.HoodieTableMetadataUtil
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.expressions.Expression
 
+import scala.collection.JavaConverters._
+
 class BloomFiltersIndexSupport(spark: SparkSession,
                                metadataConfig: HoodieMetadataConfig,
                                metaClient: HoodieTableMetaClient) extends 
SparkBaseIndexSupport(spark, metadataConfig, metaClient) {
 
+  // Like bucket pruning, trust the declared reader index type to match the 
writer. These Spark
+  // indexes write new keys to base files. A custom class overrides the type 
and has no such guarantee.
+  // Do not infer the writer's default SIMPLE index when the reader has not 
declared a type.
+  private val canPruneLogSlices = {
+    val indexType = 
metadataConfig.getStringOrDefault(HoodieIndexConfig.INDEX_TYPE, "")
+    metadataConfig.getStringOrDefault(HoodieIndexConfig.INDEX_CLASS_NAME, 
"").isEmpty &&
+      
BloomFiltersIndexSupport.BASE_BLOOM_COVERED_INDEX_TYPES.exists(_.name().equalsIgnoreCase(indexType))
+  }
+
+  private def mustRetainSlice(fileSlice: FileSlice): Boolean =
+    fileSlice.hasLogFiles && (fileSlice.getBaseFile.isEmpty || 
!canPruneLogSlices)

Review Comment:
   A reader declaring SIMPLE can still prune base+log slices written earlier 
with BUCKET/INMEMORY or by Flink, none of which guarantees that new log keys 
appear in the base bloom. Please retain log-bearing slices unless base-bloom 
coverage is established for the writes that produced them.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala:
##########
@@ -124,6 +124,125 @@ class TestBloomFiltersIndexSupport extends 
HoodieSparkClientTestBase {
     verifyQueryPredicate(hudiOpts, "_row_key")
   }
 
+  /**
+   * Regression for #20668: base-file blooms do not cover new keys in logs. A 
negative base
+   * bloom must not prune a whole base+log slice, and log-only slices must 
also survive.
+   * Layout: base-and-log = base(a) + log(b), base-only = base(c), log-only = 
log(d).
+   */
+  @Test
+  def testBloomFiltersIndexWithLogOnlyKeys(): Unit = {
+    val session = spark
+    import session.implicits._
+
+    // One bucket sends b to a's file group. Disable other query indexes and 
compaction
+    // so that metadata bloom pruning is tested against records that remain in 
logs.
+    val opts = commonOpts ++ Map(
+      TABLE_TYPE.key -> MOR_TABLE_TYPE_OPT_VAL,
+      HoodieIndexConfig.INDEX_TYPE.key -> "BUCKET",
+      HoodieIndexConfig.BUCKET_INDEX_NUM_BUCKETS.key -> "1",
+      HoodieIndexConfig.BUCKET_QUERY_INDEX.key -> "false",
+      HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
"false",
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> "false",
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false",
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> 
"false",
+      "hoodie.compact.inline" -> "false",
+      "hoodie.compact.schedule.inline" -> "false",
+      DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true")
+
+    Seq(("a", "base-and-log", 1L), ("c", "base-only", 1L)).toDF("_row_key", 
"partition", "timestamp")
+      .write.format("hudi").options(opts)
+      .option(OPERATION.key, 
INSERT_OVERWRITE_OPERATION_OPT_VAL).mode(SaveMode.Overwrite).save(basePath)
+    Seq(("b", "base-and-log", 2L)).toDF("_row_key", "partition", "timestamp")
+      .write.format("hudi").options(opts)
+      .option(OPERATION.key, 
UPSERT_OPERATION_OPT_VAL).mode(SaveMode.Append).save(basePath)
+    // A new bucket would create a base file; INMEMORY allows a genuine 
log-only insert.
+    Seq(("d", "log-only", 2L)).toDF("_row_key", "partition", "timestamp")
+      .write.format("hudi").options(opts)
+      .option(HoodieIndexConfig.INDEX_TYPE.key, "INMEMORY")
+      .option(OPERATION.key, 
INSERT_OPERATION_OPT_VAL).mode(SaveMode.Append).save(basePath)
+
+    def query(keys: Seq[String], skipping: Boolean = true, readOptimized: 
Boolean = false): Set[String] = {
+      spark.read.format("hudi").options(opts)
+        .option(DataSourceReadOptions.ENABLE_DATA_SKIPPING.key, 
skipping.toString)
+        .option(DataSourceReadOptions.QUERY_TYPE.key,
+          if (readOptimized) 
DataSourceReadOptions.QUERY_TYPE_READ_OPTIMIZED_OPT_VAL else 
DataSourceReadOptions.QUERY_TYPE_SNAPSHOT_OPT_VAL)
+        .load(basePath).filter(col("_row_key").isin(keys: 
_*)).select("_row_key").as[String].collect().toSet
+    }
+
+    // Without skipping all four keys are visible. Bloom pruning must preserve 
both log keys
+    // and the base record a in the retained base+log slice (not just its log 
files).
+    val allKeys = Seq("a", "b", "c", "d")
+    assertEquals(allKeys.toSet, query(allKeys, skipping = false))
+    assertEquals(allKeys.toSet, query(allKeys))
+    assertEquals(Set("b"), query(Seq("b")))

Review Comment:
   This regression never verifies that key b is absent from the base bloom, so 
a false positive can let it pass without exercising the pruning bug. Please 
assert the negative bloom result before running the query, or use a controlled 
bloom fixture.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to