wombatu-kun commented on code in PR #20672:
URL: https://github.com/apache/hudi/pull/20672#discussion_r4611870151
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BloomFiltersIndexSupport.scala:
##########
@@ -41,41 +57,29 @@ class BloomFiltersIndexSupport(spark: SparkSession,
prunedPartitionsAndFileSlices:
Seq[(Option[BaseHoodieTableFileIndex.PartitionPath], Seq[FileSlice])],
shouldPushDownFilesFilter: Boolean
): Option[Set[String]] = {
- lazy val (_, recordKeys) = filterQueriesWithRecordKey(queryFilters)
- if (recordKeys.nonEmpty) {
- val prunedPartitionAndFileNames =
getPrunedPartitionAndFileNames(prunedPartitionsAndFileSlices)
-
Option.apply(getCandidateFilesForSecondaryKeys(prunedPartitionAndFileNames,
recordKeys))
+ val (_, recordKeys) = filterQueriesWithRecordKey(queryFilters)
+ if (recordKeys.isEmpty) {
+ None
} else {
- Option.empty
+ val candidateFileNames = prunedPartitionsAndFileSlices.flatMap { case
(partition, fileSlices) =>
+ val partitionPath = partition.map(_.getPath).getOrElse("")
+ fileSlices.flatMap { slice =>
+ if (mustRetainSlice(slice)) {
+ // A log filename retains the entire slice without querying its
base bloom.
+ slice.getLogFiles.iterator().asScala.map(_.getFileName).toSeq
+ } else {
+ toScalaOption(slice.getBaseFile).filter { baseFile =>
+ // A missing bloom cannot rule out a match.
+ toScalaOption(metadataTable.getBloomFilter(partitionPath,
baseFile.getFileName))
Review Comment:
The rewritten lookup has no regression for a missing bloom, so the
conservative Option.forall fallback remains unverified. Please add a base-file
case with absent bloom metadata and assert that its slice is retained.
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BloomFiltersIndexSupport.scala:
##########
@@ -23,16 +23,32 @@ import org.apache.hudi.HoodieConversionUtils.toScalaOption
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.FileSlice
import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.config.HoodieIndexConfig
import org.apache.hudi.core.read.BaseHoodieTableFileIndex
+import org.apache.hudi.index.HoodieIndex.IndexType
import org.apache.hudi.metadata.HoodieTableMetadataUtil
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.expressions.Expression
+import scala.collection.JavaConverters._
+
class BloomFiltersIndexSupport(spark: SparkSession,
metadataConfig: HoodieMetadataConfig,
metaClient: HoodieTableMetaClient) extends
SparkBaseIndexSupport(spark, metadataConfig, metaClient) {
+ // Like bucket pruning, trust the declared reader index type to match the
writer. These Spark
+ // indexes write new keys to base files. A custom class overrides the type
and has no such guarantee.
+ // Do not infer the writer's default SIMPLE index when the reader has not
declared a type.
+ private val canPruneLogSlices = {
+ val indexType =
metadataConfig.getStringOrDefault(HoodieIndexConfig.INDEX_TYPE, "")
+ metadataConfig.getStringOrDefault(HoodieIndexConfig.INDEX_CLASS_NAME,
"").isEmpty &&
+
BloomFiltersIndexSupport.BASE_BLOOM_COVERED_INDEX_TYPES.exists(_.name().equalsIgnoreCase(indexType))
+ }
+
+ private def mustRetainSlice(fileSlice: FileSlice): Boolean =
+ fileSlice.hasLogFiles && (fileSlice.getBaseFile.isEmpty ||
!canPruneLogSlices)
Review Comment:
A reader declaring SIMPLE can still prune base+log slices written earlier
with BUCKET/INMEMORY or by Flink, none of which guarantees that new log keys
appear in the base bloom. Please retain log-bearing slices unless base-bloom
coverage is established for the writes that produced them.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala:
##########
@@ -124,6 +124,125 @@ class TestBloomFiltersIndexSupport extends
HoodieSparkClientTestBase {
verifyQueryPredicate(hudiOpts, "_row_key")
}
+ /**
+ * Regression for #20668: base-file blooms do not cover new keys in logs. A
negative base
+ * bloom must not prune a whole base+log slice, and log-only slices must
also survive.
+ * Layout: base-and-log = base(a) + log(b), base-only = base(c), log-only =
log(d).
+ */
+ @Test
+ def testBloomFiltersIndexWithLogOnlyKeys(): Unit = {
+ val session = spark
+ import session.implicits._
+
+ // One bucket sends b to a's file group. Disable other query indexes and
compaction
+ // so that metadata bloom pruning is tested against records that remain in
logs.
+ val opts = commonOpts ++ Map(
+ TABLE_TYPE.key -> MOR_TABLE_TYPE_OPT_VAL,
+ HoodieIndexConfig.INDEX_TYPE.key -> "BUCKET",
+ HoodieIndexConfig.BUCKET_INDEX_NUM_BUCKETS.key -> "1",
+ HoodieIndexConfig.BUCKET_QUERY_INDEX.key -> "false",
+ HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key ->
"false",
+ HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> "false",
+ HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false",
+ HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key ->
"false",
+ "hoodie.compact.inline" -> "false",
+ "hoodie.compact.schedule.inline" -> "false",
+ DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true")
+
+ Seq(("a", "base-and-log", 1L), ("c", "base-only", 1L)).toDF("_row_key",
"partition", "timestamp")
+ .write.format("hudi").options(opts)
+ .option(OPERATION.key,
INSERT_OVERWRITE_OPERATION_OPT_VAL).mode(SaveMode.Overwrite).save(basePath)
+ Seq(("b", "base-and-log", 2L)).toDF("_row_key", "partition", "timestamp")
+ .write.format("hudi").options(opts)
+ .option(OPERATION.key,
UPSERT_OPERATION_OPT_VAL).mode(SaveMode.Append).save(basePath)
+ // A new bucket would create a base file; INMEMORY allows a genuine
log-only insert.
+ Seq(("d", "log-only", 2L)).toDF("_row_key", "partition", "timestamp")
+ .write.format("hudi").options(opts)
+ .option(HoodieIndexConfig.INDEX_TYPE.key, "INMEMORY")
+ .option(OPERATION.key,
INSERT_OPERATION_OPT_VAL).mode(SaveMode.Append).save(basePath)
+
+ def query(keys: Seq[String], skipping: Boolean = true, readOptimized:
Boolean = false): Set[String] = {
+ spark.read.format("hudi").options(opts)
+ .option(DataSourceReadOptions.ENABLE_DATA_SKIPPING.key,
skipping.toString)
+ .option(DataSourceReadOptions.QUERY_TYPE.key,
+ if (readOptimized)
DataSourceReadOptions.QUERY_TYPE_READ_OPTIMIZED_OPT_VAL else
DataSourceReadOptions.QUERY_TYPE_SNAPSHOT_OPT_VAL)
+ .load(basePath).filter(col("_row_key").isin(keys:
_*)).select("_row_key").as[String].collect().toSet
+ }
+
+ // Without skipping all four keys are visible. Bloom pruning must preserve
both log keys
+ // and the base record a in the retained base+log slice (not just its log
files).
+ val allKeys = Seq("a", "b", "c", "d")
+ assertEquals(allKeys.toSet, query(allKeys, skipping = false))
+ assertEquals(allKeys.toSet, query(allKeys))
+ assertEquals(Set("b"), query(Seq("b")))
Review Comment:
This regression never verifies that key b is absent from the base bloom, so
a false positive can let it pass without exercising the pruning bug. Please
assert the negative bloom result before running the query, or use a controlled
bloom fixture.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]