codope commented on code in PR #784:
URL: https://github.com/apache/hudi-rs/pull/784#discussion_r4022722089
##########
crates/core/src/table/partition.rs:
##########
@@ -174,6 +174,71 @@ impl PartitionPruner {
})
}
+ /// Returns `true` if a partition directory prefix should be descended
into.
+ ///
+ /// Unlike `should_include`, `partial_path` may resolve fewer segments than
+ /// the schema has fields (we're mid-descent, not at a leaf yet). Any
segment
+ /// this can't safely reason about — not enough schema fields to compare
+ /// against, a single opaque path field (e.g. under a timestamp-based key
+ /// generator, where a segment carries no independently meaningful value),
+ /// a hive-style segment that doesn't parse as `key=value`, or a
cast/compare
+ /// error — is treated as unconstrained, never as a reason to reject. Only
a
+ /// segment that positively fails an already-resolved filter returns
`false`.
+ pub fn should_include_prefix(&self, partial_path: &str) -> bool {
Review Comment:
I think this can drop a partition that actually matches. Take
`region=us%2Feast/dt=2024-01-01` with a filter `region = 'us/east'`: the prefix
check comes back false, but `should_include` on the full path comes back true.
Values with a `/` in them are not that rare. Am I reading it right?
I suspect the main reason is that `parse_segments` decodes the whole path
and then splits on `/`. Java goes the other way, splitting first and unescaping
each fragment (`AbstractHoodieTableMetadata.extractPartitionValues`). Would it
make sense to fix that ordering first? Looks like it would help
`should_include` too.
Once segments decode independently, i would suggest gating this on
`is_hive_style && is_url_encoded`? That is what Java requires
(`FileSystemBackedTableMetadata.java:157`), and it seems to be the thing that
makes the positional binding safe.
##########
crates/core/src/table/partition.rs:
##########
@@ -174,6 +174,71 @@ impl PartitionPruner {
})
}
+ /// Returns `true` if a partition directory prefix should be descended
into.
+ ///
+ /// Unlike `should_include`, `partial_path` may resolve fewer segments than
+ /// the schema has fields (we're mid-descent, not at a leaf yet). Any
segment
+ /// this can't safely reason about — not enough schema fields to compare
+ /// against, a single opaque path field (e.g. under a timestamp-based key
+ /// generator, where a segment carries no independently meaningful value),
+ /// a hive-style segment that doesn't parse as `key=value`, or a
cast/compare
+ /// error — is treated as unconstrained, never as a reason to reject. Only
a
+ /// segment that positively fails an already-resolved filter returns
`false`.
+ pub fn should_include_prefix(&self, partial_path: &str) -> bool {
+ if self.and_filters.is_empty() || self.schema.fields().len() <= 1 {
+ return true;
+ }
+
+ let decoded;
Review Comment:
minor: `let decoded; ... &decoded` reads a bit awkwardly. `Cow<str>` would
do this directly.
##########
crates/core/src/table/partition.rs:
##########
@@ -401,6 +466,113 @@ mod tests {
assert!(!pruner.should_include("date=2023-02-01/category=B/count=10"));
}
+ #[test]
+ fn test_partition_pruner_should_include_prefix() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(true, false);
+
+ let filter_gt_date = Filter::try_from(("date", ">",
"2023-01-01")).unwrap();
+ let filter_eq_a = Filter::try_from(("category", "=", "A")).unwrap();
+ let filter_lte_100 = Filter::try_from(("count", "<=", "100")).unwrap();
+
+ let pruner = PartitionPruner::new(
+ &[filter_gt_date, filter_eq_a, filter_lte_100],
+ &schema,
+ &configs,
+ )
+ .unwrap();
+
+ // A resolved segment that already violates its filter is rejected
+ // without needing to resolve the rest of the path.
+ assert!(!pruner.should_include_prefix("date=2022-12-31"));
+ assert!(!pruner.should_include_prefix("date=2023-02-01/category=B"));
+
+ // A resolved segment that satisfies its filter, with later fields not
+ // yet resolved, is not rejected.
+ assert!(pruner.should_include_prefix("date=2023-02-01"));
+ assert!(pruner.should_include_prefix("date=2023-02-01/category=A"));
+
assert!(pruner.should_include_prefix("date=2023-02-01/category=A/count=10"));
+ }
+
+ #[test]
+ fn test_partition_pruner_should_include_prefix_no_filters() {
+ let pruner = PartitionPruner::empty();
+ assert!(pruner.should_include_prefix("date=2022-12-31"));
+ assert!(pruner.should_include_prefix(""));
+ }
+
+ #[test]
+ fn
test_partition_pruner_should_include_prefix_malformed_segment_fails_open() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(true, false);
+ let filter_eq_a = Filter::try_from(("category", "=", "A")).unwrap();
+ let pruner = PartitionPruner::new(&[filter_eq_a], &schema,
&configs).unwrap();
+
+ // A segment that doesn't parse as `key=value` under hive-style
+ // partitioning can't be safely compared, so it must not be rejected.
+ assert!(pruner.should_include_prefix("not-a-kv-pair"));
+ }
+
+ #[test]
+ fn test_partition_pruner_should_include_prefix_non_hive_style() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(false, false);
+ let filter_gt_date = Filter::try_from(("date", ">",
"2023-01-01")).unwrap();
+ let pruner = PartitionPruner::new(&[filter_gt_date], &schema,
&configs).unwrap();
+
+ assert!(!pruner.should_include_prefix("2022-12-31"));
+ assert!(pruner.should_include_prefix("2023-02-01"));
+ }
+
+ #[test]
+ fn test_partition_pruner_should_include_prefix_url_encoded() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(true, true);
+ let filter_eq_a = Filter::try_from(("category", "=", "A")).unwrap();
+ let pruner = PartitionPruner::new(&[filter_eq_a], &schema,
&configs).unwrap();
+
+
assert!(!pruner.should_include_prefix("date%3D2023-02-01%2Fcategory%3DB"));
Review Comment:
I don't think Hudi writes paths like this. It escapes values but leaves `=`
and `/` alone. Something like `category=A%2FB` would be closer, and it would
cover the slash case too.
##########
crates/core/src/table/listing.rs:
##########
@@ -185,9 +185,14 @@ impl FileLister {
.filter(|dir| !LAKE_FORMAT_METADATA_DIRS.contains(&dir.as_str()))
.collect();
+ let should_descend = |prefix: &str|
self.partition_pruner.should_include_prefix(prefix);
+
let mut partition_paths = Vec::new();
for dir in top_level_dirs {
- partition_paths.extend(get_leaf_dirs(&self.storage,
Some(&dir)).await?);
+ if !should_descend(&dir) {
Review Comment:
Could this fold into the `top_level_dirs` filter chain next to the
`LAKE_FORMAT_METADATA_DIRS` one, rather than a separate `continue`?
##########
crates/core/src/table/partition.rs:
##########
@@ -174,6 +174,71 @@ impl PartitionPruner {
})
}
+ /// Returns `true` if a partition directory prefix should be descended
into.
+ ///
+ /// Unlike `should_include`, `partial_path` may resolve fewer segments than
+ /// the schema has fields (we're mid-descent, not at a leaf yet). Any
segment
+ /// this can't safely reason about — not enough schema fields to compare
+ /// against, a single opaque path field (e.g. under a timestamp-based key
+ /// generator, where a segment carries no independently meaningful value),
+ /// a hive-style segment that doesn't parse as `key=value`, or a
cast/compare
+ /// error — is treated as unconstrained, never as a reason to reject. Only
a
+ /// segment that positively fails an already-resolved filter returns
`false`.
+ pub fn should_include_prefix(&self, partial_path: &str) -> bool {
+ if self.and_filters.is_empty() || self.schema.fields().len() <= 1 {
Review Comment:
This also switches the optimization off for single-column `dt=...` tables,
which is probably the most common layout out there. I tried it on
`V6SimplekeygenHivestyleNoMetafields` and got 4 list calls either way. Keying
on the `_hoodie_partition_path` field instead of the field count got it down to
2, with the suite still green. Worth a try?
##########
crates/core/src/table/listing.rs:
##########
@@ -311,6 +316,61 @@ mod test {
)
}
+ /// Regression test for descent-time partition pruning: a selective filter
+ /// must skip the storage `list` calls under directories it already rules
+ /// out, not merely filter the leaf paths after every directory is listed.
+ ///
+ /// For `V6ComplexkeygenHivestyle` (3 leaf partitions under two hive-style
+ /// levels, `byteField`/`shortField`), an unfiltered run issues 7 `list`
+ /// calls: 1 for the top-level `byteField=*` dirs, then for each of the 3,
+ /// 1 to find its `shortField=*` child and 1 more on that leaf to confirm
+ /// it has no children. A filter matching only `byteField=10` issues 3:
+ /// the same unavoidable top-level call, then only `byteField=10`'s two
+ /// levels are ever listed — `byteField=20`/`byteField=30` are pruned
+ /// before incurring any `list` call under them.
+ #[tokio::test]
+ async fn partition_filter_pruning_reduces_storage_list_calls() {
+ use crate::storage::counting::CountingObjectStore;
+ use object_store::local::LocalFileSystem;
+
+ let base_url = SampleTable::V6ComplexkeygenHivestyle.url_to_cow();
+ let hudi_table = Table::new(base_url.path()).await.unwrap();
+ let hudi_configs = hudi_table.hudi_configs.clone();
+ let partition_schema =
hudi_table.get_partition_schema().await.unwrap();
+
+ let list_calls_for = |pruner: PartitionPruner| {
+ let base_url = base_url.clone();
+ let hudi_configs = hudi_configs.clone();
+ async move {
+ let (object_store, counts) =
+ CountingObjectStore::new(Arc::new(LocalFileSystem::new()));
+ let storage =
+ Storage::new_with_object_store(base_url, object_store,
hudi_configs);
+ let lister = FileLister::new(storage.hudi_configs.clone(),
storage, pruner);
+ let partition_paths =
lister.list_relevant_partition_paths().await.unwrap();
+ (partition_paths.len(), counts.lists())
+ }
+ };
+
+ let (unfiltered_partitions, unfiltered_lists) =
+ list_calls_for(PartitionPruner::empty()).await;
+ assert_eq!(unfiltered_partitions, 3);
+ assert_eq!(unfiltered_lists, 7);
+
+ let filter_eq_10 = crate::expr::filter::Filter::try_from(("byteField",
"=", "10")).unwrap();
+ let selective_pruner =
+ PartitionPruner::new(&[filter_eq_10], &partition_schema,
&hudi_configs).unwrap();
+ let (filtered_partitions, filtered_lists) =
list_calls_for(selective_pruner).await;
+ assert_eq!(filtered_partitions, 1);
+ assert_eq!(filtered_lists, 3);
+
+ assert!(
Review Comment:
minor: this cannot fail once the two exact counts above are asserted.
##########
crates/core/src/storage/mod.rs:
##########
@@ -409,8 +409,16 @@ impl Storage {
/// - /usr/hudi/table_name/dt=2025/month=02
///
/// the result is \[".hoodie", "dt=2024/mont=01/day=01", "dt=2025/month=02"\]
+///
+/// `should_descend` is checked before recursing into each child directory, so
a
+/// directory a filter already rules out never incurs the storage `list` call
+/// its own children would otherwise need.
#[async_recursion]
-pub async fn get_leaf_dirs(storage: &Storage, subdir: Option<&str>) ->
Result<Vec<String>> {
+pub async fn get_leaf_dirs(
Review Comment:
Just noting this is `pub`, so the added parameter is a signature break.
Nothing outside the crate calls it, so probably fine. Maybe worth a line in the
description?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
