leaves12138 commented on code in PR #10494:
URL: https://github.com/apache/paimon/pull/10494#discussion_r4237671641
##########
paimon-python/pypaimon/table/source/full_text_read.py:
##########
@@ -91,27 +92,71 @@ def read(self, splits: List[FullTextSearchSplit]) ->
GlobalIndexResult:
splits_by_column.setdefault(split.column_name, []).append(split)
live_rows = global_index_live_row_filter.live_rows(
self._table, self._partition_filter)
- if self._filter is not None:
- from pypaimon.table.source.global_index_row_filter import
matching_rows
-
- candidates = RoaringBitmap64()
- for split in index_splits:
- candidates.add_range(split.row_range_start,
split.row_range_end)
- for row_range in _raw_row_ranges(raw_splits):
- candidates.add_range(row_range.from_, row_range.to)
- # Scalar indexes may prune this read, but incomplete scalar
coverage
- # must not exclude rows covered by the full-text search plan.
- table = self._table.copy({"scalar-index.search-mode": "full"})
- matched = matching_rows(table, self._filter, candidates,
self._partition_filter)
- live_rows = matched if live_rows is None else
RoaringBitmap64.and_(live_rows, matched)
- indexed_result = self._eval_column_query(splits_by_column, live_rows)
+ matched = self._matched_indexed_rows(index_splits, live_rows)
+ indexed_result = self._eval_column_query(splits_by_column, matched if
self._filter is not None else live_rows)
raw_ranges = _raw_row_ranges(raw_splits)
if self._filter is None:
raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits))
else:
- raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits), live_rows)
+ candidates = GlobalIndexResult.from_ranges(raw_ranges).results()
+ if live_rows is not None:
+ candidates = RoaringBitmap64.and_(candidates, live_rows)
+ matched = self._matching_rows(candidates)
+ raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits), matched)
return indexed_result.or_(raw_result).top_k(self._limit)
+ def _matching_rows(self, candidates):
+ from pypaimon.table.source.global_index_row_filter import matching_rows
+
+ table = self._table.copy({'scalar-index.search-mode': 'full'})
+ return matching_rows(table, self._filter, candidates,
self._partition_filter)
+
+ def _matched_indexed_rows(self, splits, live_rows):
+ if self._filter is None:
+ return live_rows
+ from pypaimon.common.options.core_options import GlobalIndexSearchMode
+ from pypaimon.globalindex.data_evolution_global_index_coverage import
DataEvolutionGlobalIndexCoverage
+ from pypaimon.globalindex.data_evolution_global_index_scanner import
DataEvolutionGlobalIndexScanner
+ from pypaimon.snapshot.time_travel_util import TimeTravelUtil
+
+ covered = RoaringBitmap64()
+ files = {}
+ for split in splits:
+ covered.add_range(split.row_range_start, split.row_range_end)
+ files.update((file.file_name, file) for file in
split.scalar_index_files)
+ if live_rows is not None:
+ covered = RoaringBitmap64.and_(covered, live_rows)
+ if covered.is_empty():
+ return covered
+ snapshot = getattr(self, '_snapshot', None)
+ if snapshot is None:
+ snapshot = TimeTravelUtil.resolve_snapshot(self._table)
+ mode = self._table.options.scalar_index_search_mode()
+ scalar_files = list(files.values())
+ coverage = DataEvolutionGlobalIndexCoverage(self._table, snapshot,
self._partition_filter, scalar_files)
+ raw_ranges = coverage.unindexed_ranges(self._table.fields,
self._filter, mode)
+ unindexed =
RoaringBitmap64.and_(GlobalIndexResult.from_ranges(raw_ranges).results(),
covered)
+ decided = RoaringBitmap64.remove_all(covered, unindexed)
+ matched = RoaringBitmap64()
+ if not decided.is_empty():
+ scanner = DataEvolutionGlobalIndexScanner.create(self._table,
index_files=scalar_files,
+
snapshot=snapshot, partition_filter=self._partition_filter)
+ evaluation = None
+ if scanner is not None:
+ try:
+ evaluation = scanner.scan_with_coverage(self._filter)
+ finally:
+ scanner.close()
+ if evaluation is not None:
+ candidates = RoaringBitmap64.and_(evaluation.result.results(),
decided)
+ if evaluation.result.is_exact():
Review Comment:
[P2] Match Java/Rust candidate policy for bitmap text predicates
Using evaluation.result.is_exact() alone admits indexed bitmap matches when
global-index.filter.refine-from-data=false. The bitmap reader marks contains,
endsWith and LIKE results exact, but Java FilteredRowIdReader.isExact
(paimon-core, lines 85-108) and Rust GlobalIndexScanner::is_exact_predicate
deliberately classify these predicate leaves as candidate-only regardless of
the selected index reader. Consequently the new native route excludes those
indexed rows while the classic Python route retains them. This option defaults
to false, so toggling read.native.enabled changes the result set for an
otherwise supported REST table.
Reproduced at these fixed heads using the REST fixture from
native_full_text_search_test.py: build the full-text content index and a bitmap
index on label, use label contains 'eep' / endsWith 'eep' / LIKE '%eep', set
refinement=false, and search 'paimon' with limit 64. With complete index
coverage, classic returns row IDs {1,5,7}, native returns empty. After
appending the unindexed tail, classic returns {1,5,7,9,11}, native returns
{9,11}. I used the existing _compare helper, which also forbids a Python scan
retry during native execution. All six bitmap/refinement-disabled comparisons
fail; the 18 BTree/refinement-enabled controls pass.
Please apply the same recursive predicate-level exactness policy as current
Java before accepting the scalar result here, rather than exempting bitmap
predicates solely because the reader returns an exact bitmap. Also change the
new full_text_scalar_filter_test.py expectation at lines 76-78, which currently
encodes the bitmap exemption, and extend the REST/native parity tests to bitmap
contains/endsWith/LIKE with refinement both disabled and enabled, including an
unindexed tail. Rust already follows the Java policy and should not be relaxed
just to match the classic path.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
