leaves12138 commented on code in PR #10494:
URL: https://github.com/apache/paimon/pull/10494#discussion_r4237619465
##########
paimon-python/pypaimon/table/source/full_text_read.py:
##########
@@ -91,27 +92,71 @@ def read(self, splits: List[FullTextSearchSplit]) ->
GlobalIndexResult:
splits_by_column.setdefault(split.column_name, []).append(split)
live_rows = global_index_live_row_filter.live_rows(
self._table, self._partition_filter)
- if self._filter is not None:
- from pypaimon.table.source.global_index_row_filter import
matching_rows
-
- candidates = RoaringBitmap64()
- for split in index_splits:
- candidates.add_range(split.row_range_start,
split.row_range_end)
- for row_range in _raw_row_ranges(raw_splits):
- candidates.add_range(row_range.from_, row_range.to)
- # Scalar indexes may prune this read, but incomplete scalar
coverage
- # must not exclude rows covered by the full-text search plan.
- table = self._table.copy({"scalar-index.search-mode": "full"})
- matched = matching_rows(table, self._filter, candidates,
self._partition_filter)
- live_rows = matched if live_rows is None else
RoaringBitmap64.and_(live_rows, matched)
- indexed_result = self._eval_column_query(splits_by_column, live_rows)
+ matched = self._matched_indexed_rows(index_splits, live_rows)
+ indexed_result = self._eval_column_query(splits_by_column, matched if
self._filter is not None else live_rows)
raw_ranges = _raw_row_ranges(raw_splits)
if self._filter is None:
raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits))
else:
- raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits), live_rows)
+ candidates = GlobalIndexResult.from_ranges(raw_ranges).results()
+ if live_rows is not None:
+ candidates = RoaringBitmap64.and_(candidates, live_rows)
+ matched = self._matching_rows(candidates)
+ raw_result = self._read_raw_search(raw_ranges,
_index_type(index_splits), matched)
return indexed_result.or_(raw_result).top_k(self._limit)
+ def _matching_rows(self, candidates):
+ from pypaimon.table.source.global_index_row_filter import matching_rows
+
+ table = self._table.copy({'scalar-index.search-mode': 'full'})
+ return matching_rows(table, self._filter, candidates,
self._partition_filter)
+
+ def _matched_indexed_rows(self, splits, live_rows):
+ if self._filter is None:
+ return live_rows
+ from pypaimon.common.options.core_options import GlobalIndexSearchMode
+ from pypaimon.globalindex.data_evolution_global_index_coverage import
DataEvolutionGlobalIndexCoverage
+ from pypaimon.globalindex.data_evolution_global_index_scanner import
DataEvolutionGlobalIndexScanner
+ from pypaimon.snapshot.time_travel_util import TimeTravelUtil
+
+ covered = RoaringBitmap64()
+ files = {}
+ for split in splits:
+ covered.add_range(split.row_range_start, split.row_range_end)
+ files.update((file.file_name, file) for file in
split.scalar_index_files)
+ if live_rows is not None:
+ covered = RoaringBitmap64.and_(covered, live_rows)
+ if covered.is_empty():
+ return covered
+ snapshot = getattr(self, '_snapshot', None)
+ if snapshot is None:
+ snapshot = TimeTravelUtil.resolve_snapshot(self._table)
+ mode = self._table.options.scalar_index_search_mode()
+ scalar_files = list(files.values())
+ coverage = DataEvolutionGlobalIndexCoverage(self._table, snapshot,
self._partition_filter, scalar_files)
+ raw_ranges = coverage.unindexed_ranges(self._table.fields,
self._filter, mode)
+ unindexed =
RoaringBitmap64.and_(GlobalIndexResult.from_ranges(raw_ranges).results(),
covered)
+ decided = RoaringBitmap64.remove_all(covered, unindexed)
+ matched = RoaringBitmap64()
+ if not decided.is_empty():
+ scanner = DataEvolutionGlobalIndexScanner.create(self._table,
index_files=scalar_files,
+
snapshot=snapshot, partition_filter=self._partition_filter)
+ evaluation = None
+ if scanner is not None:
+ try:
+ evaluation = scanner.scan_with_coverage(self._filter)
+ finally:
+ scanner.close()
+ if evaluation is not None:
+ candidates = RoaringBitmap64.and_(evaluation.result.results(),
decided)
+ if evaluation.result.is_exact():
+ matched = candidates
+ elif
self._table.options.global_index_filter_refine_from_data():
Review Comment:
[P2] Migrate the existing full-text tests to the new Java semantics
Honoring candidate refinement and scalar coverage here, together with the
no-definition early return in full_text_scan.py, changes the classic/Python
route too. The existing full_text_scalar_filter_test.py still asserts the
previous behavior, so this PR leaves the normal Python suite red even though
the new native tests pass.
With Python 3.10, PyArrow 19, and all native test gates disabled, running
full_text_scalar_filter_test.py, full_text_reader_cleanup_test.py, and
vector_filter_exactness_test.py gives 71 passed on base
1b49a70cfaf5d4430298c825b5ba9d499bddad01, but 67 passed / 4 failed on this
head. The same four failures are confirmed in the PR's Python 3.11 CI job
114208339144 (7,021 passed / 4 failed):
-
test_full_text_data_filters_precede_top_k_and_preserve_scores[False-btree]:
candidate-only LIKE now returns empty with refinement disabled instead of row 2.
- test_partial_scalar_coverage_does_not_drop_full_text_matches: scalar FAST
excludes the uncovered row 4; the test still expects rows 2 and 4.
- test_partition_and_data_predicates_keep_boolean_semantics[pt = 'b' AND
label = 'target'-expected0]: the partition has no selected full-text
definition, so the result is empty instead of row 4.
-
test_partition_without_full_text_index_respects_search_mode[full-expected1]:
the same definition requirement conflicts with its expected raw-row result.
Please update these existing fixtures/configurations and assertions to the
intended Java contract in this PR: explicitly enable refinement when testing
refined LIKE matches, distinguish FAST from FULL scalar coverage, and seed a
full-text definition in the selected partition when testing raw fallback. Keep
the implementation Java-aligned rather than reverting the new semantics just to
satisfy the old assertions, and rerun the existing suite as well as the new
REST/native cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
