leaves12138 commented on code in PR #10494:
URL: https://github.com/apache/paimon/pull/10494#discussion_r4237931664
##########
paimon-python/pypaimon/table/source/full_text_read.py:
##########
@@ -91,27 +92,72 @@ def read(self, splits: List[FullTextSearchSplit]) ->
GlobalIndexResult:
splits_by_column.setdefault(split.column_name, []).append(split)
live_rows = global_index_live_row_filter.live_rows(
self._table, self._partition_filter)
- if self._filter is not None:
- from pypaimon.table.source.global_index_row_filter import
matching_rows
-
- candidates = RoaringBitmap64()
- for split in index_splits:
- candidates.add_range(split.row_range_start,
split.row_range_end)
- for row_range in _raw_row_ranges(raw_splits):
- candidates.add_range(row_range.from_, row_range.to)
- # Scalar indexes may prune this read, but incomplete scalar
coverage
- # must not exclude rows covered by the full-text search plan.
- table = self._table.copy({"scalar-index.search-mode": "full"})
- matched = matching_rows(table, self._filter, candidates,
self._partition_filter)
- live_rows = matched if live_rows is None else
RoaringBitmap64.and_(live_rows, matched)
- indexed_result = self._eval_column_query(splits_by_column, live_rows)
+ matched = self._matched_indexed_rows(index_splits, live_rows)
+ indexed_result = self._eval_column_query(splits_by_column, matched if
self._filter is not None else live_rows)
raw_ranges = _raw_row_ranges(raw_splits)
if self._filter is None:
raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits))
else:
- raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits), live_rows)
+ candidates = GlobalIndexResult.from_ranges(raw_ranges).results()
+ if live_rows is not None:
+ candidates = RoaringBitmap64.and_(candidates, live_rows)
+ matched = self._matching_rows(candidates)
+ raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits), matched)
return indexed_result.or_(raw_result).top_k(self._limit)
+ def _matching_rows(self, candidates):
+ from pypaimon.table.source.global_index_row_filter import matching_rows
+
+ table = self._table.copy({'scalar-index.search-mode': 'full'})
+ return matching_rows(table, self._filter, candidates,
self._partition_filter)
+
+ def _matched_indexed_rows(self, splits, live_rows):
+ if self._filter is None:
+ return live_rows
+ from pypaimon.common.options.core_options import GlobalIndexSearchMode
+ from pypaimon.globalindex.data_evolution_global_index_coverage import
DataEvolutionGlobalIndexCoverage
+ from pypaimon.globalindex.data_evolution_global_index_scanner import
DataEvolutionGlobalIndexScanner
+ from pypaimon.snapshot.time_travel_util import TimeTravelUtil
+ from pypaimon.table.source.global_index_row_filter import is_exact
+
+ covered = RoaringBitmap64()
+ files = {}
+ for split in splits:
+ covered.add_range(split.row_range_start, split.row_range_end)
+ files.update((file.file_name, file) for file in
split.scalar_index_files)
+ if live_rows is not None:
+ covered = RoaringBitmap64.and_(covered, live_rows)
+ if covered.is_empty():
+ return covered
+ snapshot = getattr(self, '_snapshot', None)
+ if snapshot is None:
+ snapshot = TimeTravelUtil.resolve_snapshot(self._table)
+ mode = self._table.options.scalar_index_search_mode()
+ scalar_files = list(files.values())
+ coverage = DataEvolutionGlobalIndexCoverage(self._table, snapshot,
self._partition_filter, scalar_files)
+ raw_ranges = coverage.unindexed_ranges(self._table.fields,
self._filter, mode)
+ unindexed =
RoaringBitmap64.and_(GlobalIndexResult.from_ranges(raw_ranges).results(),
covered)
+ decided = RoaringBitmap64.remove_all(covered, unindexed)
+ matched = RoaringBitmap64()
+ if not decided.is_empty():
+ scanner = DataEvolutionGlobalIndexScanner.create(self._table,
index_files=scalar_files,
+
snapshot=snapshot, partition_filter=self._partition_filter)
+ evaluation = None
+ if scanner is not None:
+ try:
+ evaluation = scanner.scan_with_coverage(self._filter)
+ finally:
+ scanner.close()
+ if evaluation is not None:
+ candidates = RoaringBitmap64.and_(evaluation.result.results(),
decided)
+ if is_exact(self._filter, evaluation.result):
Review Comment:
[P2] Provide exact BTree prefix results before dropping indexed candidates
This refinement gate now drops BTree `startsWith` candidates, including
optimized `LIKE 'ke%'`, when `global-index.filter.refine-from-data=false`. The
classic `BTreeIndexReader.visit_starts_with` still returns every non-null row
with `is_exact=False` (btree_index_reader.py:250-251), so the new shared helper
rejects it. In contrast, Java `BTreeIndexReader.visitStartsWith` performs an
exact prefix interval query (lines 280-297), and Java
`FilteredRowIdReader.isExact` does not classify StartsWith as candidate-only.
Rust also performs the exact prefix query and is correct to retain those
indexed matches. Consequently, ordinary prefix filters still change their
result set when `read.native.enabled` is toggled. The option defaults to false.
This is also a classic-route regression: before this PR the classic full-text
reader refined these candidates instead of excluding them.
Reproduced on Python head `5d2c640095b7d88f8b6641cb424007aa784f2076` using a
freshly rebuilt Rust wheel from `1ccf5baa1ad29e35f6220c3595cc24fdf4488b02`,
with the existing REST/full-text fixture, FULL text/scalar modes and a label
BTree index:
- With complete coverage, `label startsWith 'ke'` and `label LIKE 'ke%'`
both return empty on classic but `{1,5,7}` on native.
- After appending the unindexed tail, classic returns `{9,11}` but native
returns `{1,5,7,9,11}`.
- The independent 32-case prefix matrix (BTree/bitmap x StartsWith/optimized
LIKE x matching/missing prefix x complete/partial coverage x refinement off/on)
has four failures, all for matching BTree prefixes with refinement disabled.
The bitmap, missing-prefix and refinement-enabled controls pass. The native
comparison forbids a Python full-text scan retry.
- The four corresponding classic prefix cases pass on the PR base
`1b49a70cfaf5d4430298c825b5ba9d499bddad01` with refinement disabled. Empty
StartsWith adds two failures with the same BTree cause.
Please implement an exact prefix range query in the Python BTree reader,
matching Java/Rust, and only then return an exact result. Simply marking the
existing all-non-null bitmap exact would introduce false positives. Extend
`test_optimized_bitmap_like_filters_stay_exact_without_refinement` to BTree
too, covering both StartsWith and optimized prefix LIKE and an unindexed tail;
keep Rust's Java-aligned behavior unchanged.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
