This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 9740674c fix: preserve Java empty string predicates for search
refinement (#1096)
9740674c is described below
commit 9740674cfff33a8908f1cebc7997bc48b4f76b87
Author: Jingsong Lee <[email protected]>
AuthorDate: Sat Oct 10 22:32:05 2026 +0800
fix: preserve Java empty string predicates for search refinement (#1096)
---
bindings/python/src/predicate.rs | 22 +++-
crates/paimon/src/spec/predicate.rs | 115 ++++++++++++---------
.../table/full_text_search_builder/parity_tests.rs | 115 +++++++++++++++++++++
3 files changed, 198 insertions(+), 54 deletions(-)
diff --git a/bindings/python/src/predicate.rs b/bindings/python/src/predicate.rs
index 1986bad5..df15470b 100644
--- a/bindings/python/src/predicate.rs
+++ b/bindings/python/src/predicate.rs
@@ -906,13 +906,29 @@ mod tests {
}
#[test]
- fn string_op_empty_pattern_folds_to_is_not_null() {
+ fn string_op_empty_pattern_preserves_java_operator() {
Python::attach(|py| {
let fields = test_fields();
- for method in ["startsWith", "endsWith", "contains"] {
+ for (method, op) in [
+ ("startsWith", PredicateOperator::StartsWith),
+ ("endsWith", PredicateOperator::EndsWith),
+ ("contains", PredicateOperator::Contains),
+ ] {
let dict = str_leaf_dict(py, method, "name", &[""]);
let pred = dict_to_predicate(&dict, &fields, true).unwrap();
- expect_leaf_op(&pred, PredicateOperator::IsNotNull);
+ expect_leaf_op(&pred, op);
+ }
+ });
+ }
+
+ #[test]
+ fn like_empty_and_all_wildcard_patterns_stay_like() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ for pattern in ["", "%", "%%", "%%%"] {
+ let dict = str_leaf_dict(py, "like", "name", &[pattern]);
+ let pred = dict_to_predicate(&dict, &fields, true).unwrap();
+ expect_leaf_op(&pred, PredicateOperator::Like);
}
});
}
diff --git a/crates/paimon/src/spec/predicate.rs
b/crates/paimon/src/spec/predicate.rs
index e3107e8e..21c5ec81 100644
--- a/crates/paimon/src/spec/predicate.rs
+++ b/crates/paimon/src/spec/predicate.rs
@@ -1436,19 +1436,18 @@ impl PredicateBuilder {
// -- string operators --
- /// `field LIKE 'pat%'` shape. Empty pattern → `IsNotNull(field)` (every
- /// non-null string starts with the empty string). Non-string `pattern`
- /// → [`Error::ConfigInvalid`].
+ /// `field LIKE 'pat%'` shape. Retains an empty pattern as a string leaf,
+ /// matching Java. Non-string `pattern` → [`Error::ConfigInvalid`].
pub fn starts_with(&self, field: &str, pattern: Datum) ->
Result<Predicate> {
self.string_leaf(field, PredicateOperator::StartsWith, pattern)
}
- /// `field LIKE '%pat'` shape. Empty pattern → `IsNotNull(field)`.
+ /// `field LIKE '%pat'` shape, including an empty pattern.
pub fn ends_with(&self, field: &str, pattern: Datum) -> Result<Predicate> {
self.string_leaf(field, PredicateOperator::EndsWith, pattern)
}
- /// `field LIKE '%pat%'` shape. Empty pattern → `IsNotNull(field)`.
+ /// `field LIKE '%pat%'` shape, including an empty pattern.
pub fn contains(&self, field: &str, pattern: Datum) -> Result<Predicate> {
self.string_leaf(field, PredicateOperator::Contains, pattern)
}
@@ -1477,14 +1476,13 @@ impl PredicateBuilder {
/// `%mid%` / no-wildcard patterns into [`PredicateOperator::StartsWith`] /
/// [`PredicateOperator::EndsWith`] / [`PredicateOperator::Contains`] /
/// [`PredicateOperator::Eq`]; falls back to a [`PredicateOperator::Like`]
- /// leaf for anything more complex (`_`, multi-segment `%`, escaped
- /// wildcards). The `Like` evaluator follows arrow_string `like` kernel
- /// semantics for the residual cases.
+ /// leaf for empty or all-wildcard patterns and anything more complex
+ /// (`_`, multi-segment `%`, escaped wildcards). The `Like` evaluator
follows
+ /// arrow_string `like` kernel semantics for the residual cases.
///
/// `escape == None` defaults to `\`. Any other ESCAPE character is
/// rejected with [`Error::ConfigInvalid`] (the DataFusion translator turns
- /// that into a fall-open). Empty pattern → [`PredicateOperator::Eq`] of
the
- /// empty string (SQL semantics: only the empty string matches).
+ /// that into a fall-open).
pub fn like(&self, field: &str, pattern: Datum, escape: Option<char>) ->
Result<Predicate> {
let pattern_str = match &pattern {
Datum::String(s) => s.clone(),
@@ -1504,7 +1502,7 @@ impl PredicateBuilder {
}
match optimize_like_pattern(&pattern_str) {
- LikeShape::EmptyOrLiteral(s) => self.equal(field,
Datum::String(s)),
+ LikeShape::Literal(s) => self.equal(field, Datum::String(s)),
LikeShape::StartsWith(prefix) => self.starts_with(field,
Datum::String(prefix)),
LikeShape::EndsWith(suffix) => self.ends_with(field,
Datum::String(suffix)),
LikeShape::Contains(mid) => self.contains(field,
Datum::String(mid)),
@@ -1539,17 +1537,11 @@ impl PredicateBuilder {
matches!(datum_cmp(low, high), Some(Ordering::Greater))
}
- /// Shared body for the three string operators: empty-string short-circuit
- /// and literal-type guard ([`leaf`] still cross-checks against the column
- /// type, so non-string columns are rejected there).
+ /// Shared literal-type guard for the three string operators. Preserve
+ /// empty string leaves: search refinement depends on the original
operator,
+ /// even when row evaluation is equivalent to `IsNotNull`.
fn string_leaf(&self, field: &str, op: PredicateOperator, pattern: Datum)
-> Result<Predicate> {
match &pattern {
- // Every non-null string starts with / ends with / contains the
- // empty string, and a NULL value matches none of them — i.e. the
- // empty pattern is exactly `IsNotNull`. Folding to `AlwaysTrue`
- // would wrongly retain NULL rows (and drop the field reference,
- // keeping the predicate out of the data-pruning path).
- Datum::String(s) if s.is_empty() => return self.is_not_null(field),
Datum::String(_) => {}
other => {
return Err(Error::ConfigInvalid {
@@ -1944,10 +1936,8 @@ fn eval_leaf(op: PredicateOperator, datum:
Option<&Datum>, literals: &[Datum]) -
/// literal substrings extracted from the pattern (with escape sequences
/// already decoded).
enum LikeShape {
- /// Empty pattern, or a pattern that contains no wildcards / escapes —
- /// equivalent to `Eq <literal>` (where the literal is the unescaped
- /// pattern, possibly empty).
- EmptyOrLiteral(String),
+ /// Nonempty pattern with no wildcards or escapes, equivalent to `Eq`.
+ Literal(String),
/// `prefix%` (exactly one trailing `%`, no other wildcards or escapes).
StartsWith(String),
/// `%suffix`.
@@ -1965,14 +1955,14 @@ enum LikeShape {
/// [`LikeShape::Residual`] (the simple shape rules don't account for escaped
/// wildcards).
fn optimize_like_pattern(pattern: &str) -> LikeShape {
- if pattern.contains('\\') || pattern.contains('_') {
+ if pattern.is_empty() || pattern.contains('\\') || pattern.contains('_') {
return LikeShape::Residual;
}
let bytes = pattern.as_bytes();
let percent_count = bytes.iter().filter(|b| **b == b'%').count();
match percent_count {
- 0 => LikeShape::EmptyOrLiteral(pattern.to_string()),
- 1 => {
+ 0 => LikeShape::Literal(pattern.to_string()),
+ 1 if pattern.len() > 1 => {
if let Some(prefix) = pattern.strip_suffix('%') {
LikeShape::StartsWith(prefix.to_string())
} else if let Some(suffix) = pattern.strip_prefix('%') {
@@ -1981,10 +1971,7 @@ fn optimize_like_pattern(pattern: &str) -> LikeShape {
LikeShape::Residual
}
}
- 2 if pattern.starts_with('%') && pattern.ends_with('%') => {
- // `%%` reduces to `Contains('')`, which itself short-circuits to
- // `IsNotNull` at the StartsWith/EndsWith/Contains builder boundary
- // — so no special-casing here.
+ 2 if pattern.len() > 2 && pattern.starts_with('%') &&
pattern.ends_with('%') => {
LikeShape::Contains(pattern[1..pattern.len() - 1].to_string())
}
_ => LikeShape::Residual,
@@ -3273,22 +3260,28 @@ mod tests {
}
#[test]
- fn test_builder_string_ops_empty_pattern_is_not_null() {
+ fn test_builder_string_ops_preserve_empty_pattern() {
let pb = PredicateBuilder::new(&test_fields());
- // An empty pattern matches every non-null string and no NULL, so it is
- // exactly `IsNotNull` (not `AlwaysTrue`, which would retain NULL
rows).
- for build in [
- pb.starts_with("name", Datum::String(String::new())),
- pb.ends_with("name", Datum::String(String::new())),
- pb.contains("name", Datum::String(String::new())),
+ for (build, op) in [
+ (
+ pb.starts_with("name", Datum::String(String::new())),
+ PredicateOperator::StartsWith,
+ ),
+ (
+ pb.ends_with("name", Datum::String(String::new())),
+ PredicateOperator::EndsWith,
+ ),
+ (
+ pb.contains("name", Datum::String(String::new())),
+ PredicateOperator::Contains,
+ ),
] {
- assert!(matches!(
- build.unwrap(),
- Predicate::Leaf {
- op: PredicateOperator::IsNotNull,
- ..
- }
- ));
+ assert_leaf(&build.unwrap(), op, "");
+ let literals = [Datum::String(String::new())];
+ for value in ["", "paimon"] {
+ assert!(eval_leaf(op, Some(&Datum::String(value.into())),
&literals));
+ }
+ assert!(!eval_leaf(op, None, &literals));
}
}
@@ -3369,12 +3362,6 @@ mod tests {
PredicateOperator::Eq,
"foo",
);
- // Empty pattern → Eq("") (only empty string matches).
- assert_leaf(
- &pb.like("name", Datum::String(String::new()), None).unwrap(),
- PredicateOperator::Eq,
- "",
- );
// prefix% → StartsWith.
assert_leaf(
&pb.like("name", Datum::String("foo%".to_string()), None)
@@ -3424,6 +3411,32 @@ mod tests {
);
}
+ #[test]
+ fn test_like_empty_and_all_wildcard_patterns_preserve_java_leaves() {
+ let pb = PredicateBuilder::new(&test_fields());
+ for pattern in ["", "%", "%%", "%%%"] {
+ let predicate = pb
+ .like("name", Datum::String(pattern.into()), None)
+ .unwrap();
+ assert_leaf(&predicate, PredicateOperator::Like, pattern);
+ let literals = [Datum::String(pattern.into())];
+ assert!(eval_leaf(
+ PredicateOperator::Like,
+ Some(&Datum::String(String::new())),
+ &literals
+ ));
+ assert_eq!(
+ eval_leaf(
+ PredicateOperator::Like,
+ Some(&Datum::String("paimon".into())),
+ &literals
+ ),
+ !pattern.is_empty()
+ );
+ assert!(!eval_leaf(PredicateOperator::Like, None, &literals));
+ }
+ }
+
#[test]
fn test_like_rejects_custom_escape_char() {
let pb = PredicateBuilder::new(&test_fields());
diff --git a/crates/paimon/src/table/full_text_search_builder/parity_tests.rs
b/crates/paimon/src/table/full_text_search_builder/parity_tests.rs
index fde501e5..b8948262 100644
--- a/crates/paimon/src/table/full_text_search_builder/parity_tests.rs
+++ b/crates/paimon/src/table/full_text_search_builder/parity_tests.rs
@@ -101,6 +101,32 @@ async fn index(table: &Table, kind: &str, column: &str) {
.unwrap();
}
+async fn append_labels(table: &Table, start: i32, labels: &[Option<&str>]) {
+ let schema = Arc::new(ArrowSchema::new(vec![
+ Field::new("id", ArrowType::Int32, true),
+ Field::new("text", ArrowType::Utf8, true),
+ Field::new("label", ArrowType::Utf8, true),
+ Field::new("pt", ArrowType::Int32, true),
+ ]));
+ let columns: Vec<ArrayRef> = vec![
+ Arc::new(Int32Array::from_iter_values(
+ start..start + labels.len() as i32,
+ )),
+ Arc::new(StringArray::from(vec!["paimon"; labels.len()])),
+ Arc::new(StringArray::from(labels.to_vec())),
+ Arc::new(Int32Array::from(vec![0; labels.len()])),
+ ];
+ let mut write = TableWrite::new(table, "data".into()).unwrap();
+ write
+ .write_arrow_batch(&RecordBatch::try_new(schema, columns).unwrap())
+ .await
+ .unwrap();
+ TableCommit::new(table.clone(), "data".into())
+ .commit(write.prepare_commit().await.unwrap())
+ .await
+ .unwrap();
+}
+
fn query(table: &Table) -> FullTextSearchBuilder<'_> {
let mut builder = table.new_full_text_search_builder();
builder
@@ -178,6 +204,95 @@ async fn
scalar_modes_and_refinement_follow_java_before_top_k() {
assert_eq!(ids(search.execute_scored().await.unwrap()), vec![5]);
}
+#[tokio::test]
+async fn empty_string_filters_preserve_java_search_refinement() {
+ for kind in ["btree", "bitmap"] {
+ let table = new_table(false);
+ append_labels(
+ &table,
+ 0,
+ &[
+ Some(""),
+ None,
+ Some("keep"),
+ Some("drop"),
+ Some(""),
+ Some("keep"),
+ Some("drop"),
+ None,
+ ],
+ )
+ .await;
+ index(&table, "full-text", "text").await;
+ index(&table, kind, "label").await;
+ let pb = PredicateBuilder::new(table.schema().fields());
+ let filters = [
+ (
+ "like-empty",
+ pb.like("label", Datum::String("".into()), None).unwrap(),
+ ),
+ (
+ "like-percent",
+ pb.like("label", Datum::String("%".into()), None).unwrap(),
+ ),
+ (
+ "like-double-percent",
+ pb.like("label", Datum::String("%%".into()), None).unwrap(),
+ ),
+ (
+ "contains",
+ pb.contains("label", Datum::String("".into())).unwrap(),
+ ),
+ (
+ "ends-with",
+ pb.ends_with("label", Datum::String("".into())).unwrap(),
+ ),
+ (
+ "starts-with",
+ pb.starts_with("label", Datum::String("".into())).unwrap(),
+ ),
+ ];
+ for partial in [false, true] {
+ if partial {
+ append_labels(&table, 8, &[Some(""), None, Some("keep"),
Some("drop")]).await;
+ }
+ for refine in [false, true] {
+ let table = table.copy_with_options(HashMap::from([(
+ "global-index.filter.refine-from-data".into(),
+ refine.to_string(),
+ )]));
+ for (name, filter) in &filters {
+ let mut expected = if refine || *name == "starts-with" {
+ if *name == "like-empty" {
+ vec![0, 4]
+ } else {
+ vec![0, 2, 3, 4, 5, 6]
+ }
+ } else {
+ vec![]
+ };
+ if partial {
+ expected.extend(if *name == "like-empty" {
+ vec![8]
+ } else {
+ vec![8, 10, 11]
+ });
+ }
+ assert_eq!(
+ ids(query(&table)
+ .with_filter(filter.clone())
+ .execute_scored()
+ .await
+ .unwrap()),
+ expected,
+ "index={kind}, filter={name}, refine={refine},
partial={partial}",
+ );
+ }
+ }
+ }
+ }
+}
+
async fn user_ids(table: &Table, result: SearchResult) -> Vec<i32> {
let mut builder = table.new_read_builder();
builder.with_projection(&["id", ROW_ID_FIELD_NAME]).unwrap();
