This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 38598f6041 [core] Decide full-text row filters from data when no
scalar index can evaluate them (#10381)
38598f6041 is described below
commit 38598f6041efb4f0daca3ee1cdd28980693cb952
Author: Eunbin Son <[email protected]>
AuthorDate: Mon Oct 5 21:41:07 2026 +0900
[core] Decide full-text row filters from data when no scalar index can
evaluate them (#10381)
---
.../table/source/DataEvolutionFullTextRead.java | 9 +++++-
.../table/source/FullTextSearchBuilderTest.java | 35 ++++++++++++++++++----
2 files changed, 38 insertions(+), 6 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
index a3bbc6d1a6..1ed32cb51f 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java
@@ -178,7 +178,9 @@ public class DataEvolutionFullTextRead implements
FullTextRead {
* dropped, or a {@code contains} / {@code endsWith} / {@code like}
leaf, which BTree
* answers with every non-null row), the candidates are refined by
reading their filter
* columns if {@code global-index.filter.refine-from-data} allows
it, and excluded
- * otherwise.
+ * otherwise. When every index covering the filter columns declines
the predicate, the
+ * rows are excluded in {@code fast} and otherwise decided by
reading their filter
+ * columns.
* <li>Rows whose filter columns are not covered follow {@code
scalar-index.search-mode}:
* excluded in {@code fast}, otherwise decided by reading their
filter columns.
* </ul>
@@ -247,6 +249,11 @@ public class DataEvolutionFullTextRead implements
FullTextRead {
matched.or(fromIndex);
} else {
warnUnindexedFilter();
+ if (table.coreOptions().scalarIndexSearchMode() !=
GlobalIndexSearchMode.FAST) {
+ // No scalar index could evaluate the filter: decide these
rows from the data,
+ // like rows whose filter columns have no index.
+ unindexed.or(decidedByIndex);
+ }
}
}
if (!unindexed.isEmpty()) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
index 630c82b15d..7baeed2e10 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
@@ -31,6 +31,7 @@ import
org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory;
+import org.apache.paimon.globalindex.fmindex.FMGlobalIndexerFactory;
import
org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexReader;
import
org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory;
import org.apache.paimon.index.GlobalIndexMeta;
@@ -914,6 +915,28 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
.containsExactly(5L);
}
+ @ParameterizedTest(name = "scalar-index.search-mode={0}")
+ @CsvSource({"fast", "full", "detail"})
+ public void
testFullTextSearchReadsFilterColumnsWhenScalarIndexDeclinesFilter(String
scalarMode)
+ throws Exception {
+ // FM covers the filter column but cannot evaluate equality, so no
index decides the
+ // rows. Like rows without an index, full and detail decide them from
the data.
+ FileStoreTable table = createTable("full_text_declined_" + scalarMode,
scalarMode, false);
+ writeDocuments(table, RANKED_DOCUMENTS);
+ buildAndCommitIndex(table, RANKED_DOCUMENTS);
+ buildAndCommitBTreeIndex(table, RANKED_DOCUMENTS,
FMGlobalIndexerFactory.IDENTIFIER);
+
+ Predicate filter =
+ new PredicateBuilder(table.rowType())
+ .equal(1, BinaryString.fromString("paimon zeta"));
+ RoaringNavigableMap64 rows = searchWithFilter(table, filter,
1).results();
+ if (scalarMode.equals("fast")) {
+ assertThat(rows).isEmpty();
+ } else {
+ assertThat(rows).containsExactly(5L);
+ }
+ }
+
@ParameterizedTest(name = "scalar-index.search-mode={0},
refine-from-data={1}")
@CsvSource({"fast, true", "full, true", "fast, false", "full, false"})
public void
testFullTextSearchPartiallyIndexedConjunctionIsNeverRankedAsSuperset(
@@ -1947,16 +1970,18 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
private void buildAndCommitBTreeIndex(FileStoreTable table, String[]
documents)
throws Exception {
+ buildAndCommitBTreeIndex(table, documents,
BTreeGlobalIndexerFactory.IDENTIFIER);
+ }
+
+ private void buildAndCommitBTreeIndex(
+ FileStoreTable table, String[] documents, String indexType) throws
Exception {
Options options = table.coreOptions().toConfiguration();
DataField textField = table.rowType().getField(TEXT_FIELD_NAME);
GlobalIndexSingleColumnWriter writer =
(GlobalIndexSingleColumnWriter)
GlobalIndexBuilderUtils.createIndexWriter(
- table,
- BTreeGlobalIndexerFactory.IDENTIFIER,
- Collections.singletonList(textField),
- options);
+ table, indexType,
Collections.singletonList(textField), options);
for (int i = 0; i < documents.length; i++) {
writer.write(BinaryString.fromString(documents[i]), i);
}
@@ -1970,7 +1995,7 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
table.coreOptions(),
rowRange,
textField.id(),
- BTreeGlobalIndexerFactory.IDENTIFIER,
+ indexType,
entries);
DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);
