This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a4ed3a037c0 [Vector Upsert 4/5] Apply the query's visible-document set
before vector candidate generation (#19300)
a4ed3a037c0 is described below
commit a4ed3a037c012b07c2255a3f83240ea3ea07e10b
Author: Xiang Fu <[email protected]>
AuthorDate: Wed Aug 26 17:28:58 2026 -0700
[Vector Upsert 4/5] Apply the query's visible-document set before vector
candidate generation (#19300)
FilterPlanNode constructed and executed VECTOR_SIMILARITY before adding
SegmentContext.getDocIdsSnapshot() as an outer AND. Obsolete physical
versions
of upserted rows could therefore occupy per-segment ANN top-K slots and be
removed only afterward, producing fewer than K rows or omitting nearer
current
rows.
Pass the snapshot to vector predicates as a required VectorCandidateScope
so it
constrains candidate generation, while retaining the outer bitmap AND as
defense
in depth. FilterPlanNode remains the only place that knows the scope comes
from
the segment's queryable-document snapshot; the operators only see 'documents
this predicate may consider'.
Choose the execution path at plan time, where both the reader capability and
forward index availability are known:
- a filter-aware reader receives the scope and does filtered ANN;
- a reader that cannot restrict its search is bypassed for
ExactVectorScanFilterOperator over the allowed documents, which reports it
through fallbackReason;
- an empty scope needs no candidate generation and no reader capability at
all,
so it short-circuits to an empty operator;
- when neither path is available the query fails clearly.
Because the choice is made once, VectorSimilarityFilterOperator no longer
needs
a runtime exact-scan branch, and it captures the reader capability once
instead
of re-reading it per execution, so a plan built on one answer can never
execute
against another. Required scopes stay separate from optimizer-selected
metadata
filters and are intersected with them before candidate generation.
Non-upsert
queries keep the adaptive metadata behavior, and non-vector plans do not
copy
the snapshot.
---
.../filter/VectorSimilarityFilterOperator.java | 170 ++++++++++---
.../org/apache/pinot/core/plan/FilterPlanNode.java | 65 ++++-
.../filter/FilterAwareVectorSearchTest.java | 122 ++++++++++
.../apache/pinot/core/plan/FilterPlanNodeTest.java | 271 +++++++++++++++++++++
4 files changed, 581 insertions(+), 47 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/VectorSimilarityFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/VectorSimilarityFilterOperator.java
index 62e814d243c..1b850c80641 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/VectorSimilarityFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/VectorSimilarityFilterOperator.java
@@ -19,6 +19,7 @@
package org.apache.pinot.core.operator.filter;
import com.google.common.base.CaseFormat;
+import com.google.common.base.Preconditions;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -58,9 +59,12 @@ import org.slf4j.LoggerFactory;
/// using exact distance from the forward index and re-sorted before
final top-K selection.
/// - **maxCandidates:** Controls how many ANN candidates are retrieved before
rerank. Only
/// meaningful when rerank is enabled.
-/// - **Pre-filter:** For backends that implement
FilterAwareVectorIndexReader, a pre-filter
-/// bitmap from sibling filter operators can be passed in to improve
search quality under
-/// highly selective filters.
+/// - **Pre-filter:** For backends that implement
FilterAwareVectorIndexReader, an optimizer-selected metadata bitmap
+/// can improve search quality under highly selective filters.
+/// - **Required candidate documents:** When the query restricts which
documents are visible in a segment, that set
+/// is applied before candidate selection rather than intersected
afterwards, because vector top-K is not
+/// monotonic. This operator requires a filter-aware reader in that case
-- the planner picks
+/// [ExactVectorScanFilterOperator] when the reader cannot do filtered
search.
///
/// When no query options are specified, behavior is identical to the previous
HNSW-only path
/// (full backward compatibility).
@@ -84,13 +88,24 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
private final boolean _hasThresholdPredicate;
private final float _distanceThreshold;
private final int _effectiveSearchCount;
+ /// Documents this predicate is allowed to consider as candidates, or null
when the query places no such
+ /// restriction. Applied before top-K selection rather than intersected with
the result, because vector top-K is
+ /// not monotonic: restricting the corpus changes which documents win.
+ @Nullable
+ private final ImmutableRoaringBitmap _requiredDocIds;
+ /// Captured once at construction: re-reading the reader capability per
execution would let a plan built on one
+ /// answer execute against another.
+ private final boolean _readerSupportsPreFilter;
private volatile VectorExplainContext _vectorExplainContext;
private volatile int _annCandidateCount;
private volatile int _rerankedCandidateCount;
private ImmutableRoaringBitmap _matches;
@Nullable
- private volatile ImmutableRoaringBitmap _preFilterBitmap;
+ private volatile ImmutableRoaringBitmap _optionalPreFilterBitmap;
+ @Nullable
+ private volatile ImmutableRoaringBitmap _effectiveAllowedDocIds;
private volatile VectorSearchMode _vectorSearchMode;
+ private volatile boolean _candidateGenerationSkipped;
/// Backward-compatible constructor that uses default search params and no
forward index.
/// Existing callers that do not pass query options continue to work
unchanged.
@@ -129,6 +144,20 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
public VectorSimilarityFilterOperator(VectorIndexReader vectorIndexReader,
VectorSimilarityPredicate predicate,
int numDocs, VectorSearchParams searchParams, @Nullable
ForwardIndexReader<?> forwardIndexReader,
@Nullable VectorIndexConfig vectorIndexConfig, boolean
hasMetadataFilter) {
+ this(vectorIndexReader, predicate, numDocs, searchParams,
forwardIndexReader, vectorIndexConfig,
+ hasMetadataFilter, null);
+ }
+
+ /// Full constructor with a required candidate scope and optional
optimizer-selected metadata.
+ ///
+ /// @param requiredDocIds documents this predicate may consider, applied
before top-K selection, or null when the
+ /// query places no such restriction. The reader must support
filtered search when these are supplied;
+ /// [org.apache.pinot.core.plan.FilterPlanNode] selects
[ExactVectorScanFilterOperator] when it does not.
+ /// Held by reference: the caller must not modify the bitmap
afterwards.
+ public VectorSimilarityFilterOperator(VectorIndexReader vectorIndexReader,
VectorSimilarityPredicate predicate,
+ int numDocs, VectorSearchParams searchParams, @Nullable
ForwardIndexReader<?> forwardIndexReader,
+ @Nullable VectorIndexConfig vectorIndexConfig, boolean hasMetadataFilter,
+ @Nullable ImmutableRoaringBitmap requiredDocIds) {
super(numDocs, false);
_vectorIndexReader = vectorIndexReader;
_predicate = predicate;
@@ -142,9 +171,14 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
_hasMetadataFilter = hasMetadataFilter;
_hasThresholdPredicate = searchParams.hasDistanceThreshold();
_distanceThreshold = searchParams.getDistanceThreshold();
- // When metadata filter is present, over-fetch ANN candidates to
compensate for filter loss.
- // Default over-fetch factor is 2x topK for filtered queries without
explicit maxCandidates.
- // For threshold queries, use a larger candidate pool since we need
distance refinement.
+ _requiredDocIds = requiredDocIds;
+ _readerSupportsPreFilter = vectorIndexReader instanceof
FilterAwareVectorIndexReader
+ && ((FilterAwareVectorIndexReader)
vectorIndexReader).supportsPreFilter();
+ // An empty scope makes candidate generation unnecessary, so only a
non-empty one demands filtered search.
+ Preconditions.checkState(_requiredDocIds == null ||
_requiredDocIds.isEmpty() || _readerSupportsPreFilter,
+ "Cannot honor required candidate doc IDs on vector column: %s --
vector index reader does not support "
+ + "filtered search", predicate.getLhs().getIdentifier());
+ // Threshold and exact-rerank queries use a larger candidate pool for
refinement.
int baseSearchCount;
if (_hasThresholdPredicate) {
// Threshold queries need a larger candidate pool for exact distance
refinement.
@@ -156,27 +190,34 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
// For plain top-K and filtered queries (no rerank, no threshold),
always ask
// the ANN index for exactly topK candidates. Over-fetching would change
the
// predicate semantics: vectorSimilarity(col, q, 10) must return at most
10 docs.
- // The metadata filter (bitmap AND) reduces this set further, which is
correct.
+ // Adaptive metadata that remains post-filtered may reduce this set
further. A required candidate scope is
+ // instead applied before this top-K selection.
baseSearchCount = predicate.getTopK();
}
_effectiveSearchCount = baseSearchCount;
- refreshExplainContext(null);
_annCandidateCount = -1;
_rerankedCandidateCount = -1;
_matches = null;
- _preFilterBitmap = null;
- _vectorSearchMode = VectorSearchMode.POST_FILTER_ANN;
+ _optionalPreFilterBitmap = null;
+ recomputeEffectiveAllowedDocIds();
+ _vectorSearchMode = hasRequiredCandidateScope() ?
VectorSearchMode.FILTER_THEN_ANN
+ : VectorSearchMode.POST_FILTER_ANN;
+ _candidateGenerationSkipped = hasRequiredCandidateScope() &&
_effectiveAllowedDocIds.isEmpty();
+ refreshExplainContext();
}
- /// Sets a pre-filter bitmap to restrict the ANN search to a subset of
documents.
- /// When set, the operator will use FILTER_THEN_ANN mode if the underlying
reader
- /// supports [FilterAwareVectorIndexReader].
+ /// Sets an optimizer-selected optional metadata bitmap. When a required
candidate scope is also present, the
+ /// operator intersects private copies of the two bitmaps before candidate
generation. When no scope is present,
+ /// readers that cannot pre-filter retain the existing unfiltered ANN
behavior.
///
/// This method must be called before any search execution (getTrues,
getBitmaps, etc.).
///
/// @param preFilterBitmap the bitmap of document IDs to restrict the search
to
public void setPreFilterBitmap(@Nullable ImmutableRoaringBitmap
preFilterBitmap) {
- _preFilterBitmap = preFilterBitmap;
+ _optionalPreFilterBitmap = copyBitmap(preFilterBitmap);
+ recomputeEffectiveAllowedDocIds();
+ _candidateGenerationSkipped = hasRequiredCandidateScope() &&
_effectiveAllowedDocIds.isEmpty();
+ refreshExplainContext();
}
@Override
@@ -244,6 +285,12 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
if (explainContext.getFilterSelectivity() >= 0) {
sb.append(", filterSelectivity:").append(String.format("%.4f",
explainContext.getFilterSelectivity()));
}
+ if (hasRequiredCandidateScope()) {
+ sb.append(", requiredDocIdFilterApplied:true")
+ .append(",
requiredDocIdFilterCardinality:").append(_requiredDocIds.getCardinality())
+ .append(",
effectiveAllowedDocIdsCardinality:").append(getEffectiveAllowedDocIdsCardinality())
+ .append(",
candidateGenerationSkipped:").append(_candidateGenerationSkipped);
+ }
sb.append(')');
return sb.toString();
}
@@ -292,12 +339,19 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
if (explainContext.getFilterSelectivity() >= 0) {
attributeBuilder.putString("filterSelectivity", String.format("%.4f",
explainContext.getFilterSelectivity()));
}
+ if (hasRequiredCandidateScope()) {
+ attributeBuilder.putBool("requiredDocIdFilterApplied", true);
+ // Additive, not idempotent: these counts differ per segment, and
IDEMPOTENT attributes stop PlanNodeMerger
+ // from merging nodes whose values differ, which would expand a large
explain to one node per segment.
+ attributeBuilder.putLong("requiredDocIdFilterCardinality",
_requiredDocIds.getCardinality());
+ attributeBuilder.putLong("effectiveAllowedDocIdsCardinality",
getEffectiveAllowedDocIdsCardinality());
+ attributeBuilder.putBool("candidateGenerationSkipped",
_candidateGenerationSkipped);
+ }
}
/// Returns true if the underlying vector index reader supports pre-filter
ANN search.
public boolean supportsPreFilter() {
- return _vectorIndexReader instanceof FilterAwareVectorIndexReader
- && ((FilterAwareVectorIndexReader)
_vectorIndexReader).supportsPreFilter();
+ return _readerSupportsPreFilter;
}
/// Executes the vector search with backend-specific parameter dispatch and
optional rerank.
@@ -308,32 +362,37 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
boolean backendParamsNeedCleanup = false;
boolean searchExecuted = false;
try {
+ ImmutableRoaringBitmap effectiveAllowedDocIds = _effectiveAllowedDocIds;
+ if (hasRequiredCandidateScope() && effectiveAllowedDocIds.isEmpty()) {
+ // Nothing in this segment is visible to the query, so candidate
generation would be pure waste.
+ _candidateGenerationSkipped = true;
+ _annCandidateCount = 0;
+ _rerankedCandidateCount = 0;
+ return new MutableRoaringBitmap();
+ }
+ _candidateGenerationSkipped = false;
+
// 1. Configure backend-specific parameters via interfaces
// Claim cleanup before the first setter because configuration can fail
after partially updating reader state.
backendParamsNeedCleanup = true;
configureBackendParams(column);
- refreshExplainContext(null);
+ refreshExplainContext();
explainContext = _vectorExplainContext;
// 2. Determine effective search count (higher if rerank is enabled)
int searchCount = explainContext.getEffectiveSearchCount();
// 3. Execute ANN search (with pre-filter if available)
- ImmutableRoaringBitmap preFilter = _preFilterBitmap;
+ ImmutableRoaringBitmap preFilter = effectiveAllowedDocIds;
ImmutableRoaringBitmap annResults;
searchExecuted = true;
- if (preFilter != null && _vectorIndexReader instanceof
FilterAwareVectorIndexReader) {
+ if (preFilter != null && _readerSupportsPreFilter) {
FilterAwareVectorIndexReader filterAwareReader =
(FilterAwareVectorIndexReader) _vectorIndexReader;
- if (filterAwareReader.supportsPreFilter()) {
- _vectorSearchMode = VectorSearchMode.FILTER_THEN_ANN;
- annResults = filterAwareReader.getDocIds(queryVector, searchCount,
preFilter);
- LOGGER.debug("Pre-filter ANN search on column: {},
filterCardinality: {}, filterSelectivity: {}",
- column, preFilter.getCardinality(),
- _numDocs > 0 ? (double) preFilter.getCardinality() / _numDocs :
0.0);
- } else {
- _vectorSearchMode = VectorSearchMode.POST_FILTER_ANN;
- annResults = _vectorIndexReader.getDocIds(queryVector, searchCount);
- }
+ _vectorSearchMode = VectorSearchMode.FILTER_THEN_ANN;
+ annResults = filterAwareReader.getDocIds(queryVector, searchCount,
preFilter);
+ LOGGER.debug("Pre-filter ANN search on column: {}, filterCardinality:
{}, filterSelectivity: {}",
+ column, preFilter.getCardinality(),
+ _numDocs > 0 ? (double) preFilter.getCardinality() / _numDocs :
0.0);
} else {
_vectorSearchMode = VectorSearchMode.POST_FILTER_ANN;
annResults = _vectorIndexReader.getDocIds(queryVector, searchCount);
@@ -391,7 +450,7 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
VectorSearchMetrics.getInstance().recordSearch(_vectorSearchMode,
_backendType);
}
// Refresh explain context with the final search mode decided during
execution
- refreshExplainContext(null);
+ refreshExplainContext();
} finally {
if (backendParamsNeedCleanup) {
clearBackendParams(column);
@@ -521,14 +580,19 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
return _backendType.name();
}
- private void refreshExplainContext(@Nullable String fallbackReason) {
- VectorExecutionMode executionMode =
VectorQueryExecutionContext.selectExecutionMode(
- true, _hasMetadataFilter, _hasThresholdPredicate,
_effectiveExactRerank);
- ImmutableRoaringBitmap preFilter = _preFilterBitmap;
+ private void refreshExplainContext() {
+ VectorExecutionMode executionMode;
+ if (_vectorSearchMode == VectorSearchMode.FILTER_THEN_ANN) {
+ executionMode = VectorExecutionMode.FILTER_THEN_ANN;
+ } else {
+ executionMode = VectorQueryExecutionContext.selectExecutionMode(
+ true, _hasMetadataFilter, _hasThresholdPredicate,
_effectiveExactRerank);
+ }
+ ImmutableRoaringBitmap preFilter = _effectiveAllowedDocIds;
double filterSelectivity = (preFilter != null && _numDocs > 0)
? (double) preFilter.getCardinality() / _numDocs : -1.0;
- Map<String, Object> indexDebugInfo =
- _backendType.supportsNprobe() ? _vectorIndexReader.getIndexDebugInfo()
: Map.of();
+ Map<String, Object> indexDebugInfo = !_candidateGenerationSkipped &&
_backendType.supportsNprobe()
+ ? _vectorIndexReader.getIndexDebugInfo() : Map.of();
int effectiveEfSearch =
resolveEffectiveEfSearch(_backendType, _searchParams);
Boolean effectiveHnswUseRelativeDistance =
@@ -539,11 +603,41 @@ public class VectorSimilarityFilterOperator extends
BaseFilterOperator {
float effectiveThreshold = threshold != null ? threshold : -1f;
_vectorExplainContext = new VectorExplainContext(_backendType,
_distanceFunction, executionMode,
resolveEffectiveNprobe(_backendType, _searchParams, indexDebugInfo),
- _effectiveExactRerank, _effectiveSearchCount, fallbackReason, null,
+ _effectiveExactRerank, _effectiveSearchCount, null, null,
effectiveEfSearch, effectiveThreshold, _vectorSearchMode,
filterSelectivity,
effectiveHnswUseRelativeDistance, effectiveHnswUseBoundedQueue);
}
+ private boolean hasRequiredCandidateScope() {
+ return _requiredDocIds != null;
+ }
+
+ private void recomputeEffectiveAllowedDocIds() {
+ if (_requiredDocIds == null) {
+ _effectiveAllowedDocIds = _optionalPreFilterBitmap;
+ return;
+ }
+ if (_optionalPreFilterBitmap == null) {
+ _effectiveAllowedDocIds = _requiredDocIds;
+ return;
+ }
+ MutableRoaringBitmap effective = _requiredDocIds.toMutableRoaringBitmap();
+ effective.and(_optionalPreFilterBitmap);
+ _effectiveAllowedDocIds = effective.toImmutableRoaringBitmap();
+ }
+
+ private int getEffectiveAllowedDocIdsCardinality() {
+ return _effectiveAllowedDocIds != null ?
_effectiveAllowedDocIds.getCardinality() : -1;
+ }
+
+ @Nullable
+ private static ImmutableRoaringBitmap copyBitmap(@Nullable
ImmutableRoaringBitmap bitmap) {
+ if (bitmap == null) {
+ return null;
+ }
+ return bitmap.toMutableRoaringBitmap().toImmutableRoaringBitmap();
+ }
+
private static int resolveEffectiveNprobe(VectorBackendType backendType,
VectorSearchParams searchParams,
Map<String, Object> indexDebugInfo) {
if (!backendType.supportsNprobe()) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
index 34b77301a25..013774f3c03 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
@@ -65,6 +65,7 @@ import org.apache.pinot.segment.spi.index.IndexType;
import org.apache.pinot.segment.spi.index.creator.VectorBackendType;
import org.apache.pinot.segment.spi.index.creator.VectorIndexConfig;
import
org.apache.pinot.segment.spi.index.multicolumntext.MultiColumnTextMetadata;
+import org.apache.pinot.segment.spi.index.reader.FilterAwareVectorIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
@@ -72,6 +73,7 @@ import
org.apache.pinot.segment.spi.index.reader.TextIndexReader;
import org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.exception.BadQueryRequestException;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
@@ -80,6 +82,10 @@ public class FilterPlanNode implements PlanNode {
private final SegmentContext _segmentContext;
private final QueryContext _queryContext;
private final FilterContext _filter;
+ /// Documents a vector predicate may consider as candidates for this query,
or null when the query places no such
+ /// restriction. Held by reference; the snapshot is not modified while the
plan executes.
+ @Nullable
+ private ImmutableRoaringBitmap _requiredVectorDocIds;
// Cache the predicate evaluators
private final List<Pair<Predicate, PredicateEvaluator>> _predicateEvaluators
= new ArrayList<>(4);
@@ -97,20 +103,31 @@ public class FilterPlanNode implements PlanNode {
@Override
public BaseFilterOperator run() {
+ // Read the snapshot before the document count, never the other way round.
A consuming segment publishes a row by
+ // adding it, then raising the count, then marking it valid, so reading
the count last guarantees every document in
+ // the snapshot is below it. See FilterPlanNodeTest#testConsistentSnapshot.
MutableRoaringBitmap docIdsSnapshot = _segmentContext.getDocIdsSnapshot();
int numDocs = _indexSegment.getSegmentMetadata().getTotalDocs();
+ // Vector top-K is not monotonic, so a restricted visible-document set
must reach candidate generation instead of
+ // only being intersected with the result. Test the snapshot first so
ordinary queries skip the filter tree walk.
+ _requiredVectorDocIds = docIdsSnapshot != null && _filter != null &&
containsVectorPredicate(_filter)
+ ? docIdsSnapshot : null;
+
+ // Candidate generation and the outer valid-document AND must observe the
same document set for this query.
+ ImmutableRoaringBitmap outerDocIdsSnapshot = docIdsSnapshot;
if (_filter != null) {
BaseFilterOperator filterOperator = constructPhysicalOperator(_filter,
numDocs);
- if (docIdsSnapshot != null) {
- BaseFilterOperator validDocFilter = new
BitmapBasedFilterOperator(docIdsSnapshot, false, numDocs);
+ if (outerDocIdsSnapshot != null) {
+ BaseFilterOperator validDocFilter =
+ new BitmapBasedFilterOperator(outerDocIdsSnapshot, false, numDocs);
return FilterOperatorUtils.getAndFilterOperator(_queryContext,
Arrays.asList(filterOperator, validDocFilter),
numDocs);
} else {
return filterOperator;
}
- } else if (docIdsSnapshot != null) {
- return new BitmapBasedFilterOperator(docIdsSnapshot, false, numDocs);
+ } else if (outerDocIdsSnapshot != null) {
+ return new BitmapBasedFilterOperator(outerDocIdsSnapshot, false,
numDocs);
} else {
return new MatchAllFilterOperator(numDocs);
}
@@ -367,8 +384,26 @@ public class FilterPlanNode implements PlanNode {
boolean isMutableSegment =
_indexSegment.getSegmentMetadata().isMutableSegment();
VectorSearchParams searchParams =
VectorSearchParams.fromQueryOptions(_queryContext.getQueryOptions());
+ // Nothing in this segment is visible to the query, so no candidate
generation is needed and no reader
+ // capability is required. The outer valid-document AND would discard any
result anyway.
+ if (_requiredVectorDocIds != null && _requiredVectorDocIds.isEmpty()) {
+ return EmptyFilterOperator.getInstance();
+ }
+
if (vectorIndex != null) {
- // ANN index path: pass forward index reader if rerank or threshold
search requires exact distances
+ // A required candidate scope can only be honored by a reader that does
filtered search. When it cannot, the
+ // ANN index is unusable for this query and an exact scan over the
allowed documents is the correct plan --
+ // decided here, where both the reader capability and the forward index
availability are known.
+ if (_requiredVectorDocIds != null && !supportsPreFilter(vectorIndex)) {
+ ForwardIndexReader<?> exactScanReader = dataSource.getForwardIndex();
+ Preconditions.checkState(exactScanReader != null,
+ "Cannot honor required candidate doc IDs on vector column: %s --
vector index reader does not support "
+ + "filtered search and no forward index is available", column);
+ return new ExactVectorScanFilterOperator(exactScanReader, predicate,
column, numDocs, vectorIndexConfig,
+ getFilteredSearchUnsupportedReason(isMutableSegment),
searchParams, _requiredVectorDocIds);
+ }
+
+ // ANN index path: pass the forward index when rerank or threshold needs
exact distances.
ForwardIndexReader<?> forwardIndexReader = null;
VectorBackendType backendType =
VectorDistanceUtils.resolveBackendType(vectorIndexConfig);
if (searchParams.isExactRerank(backendType) ||
searchParams.hasDistanceThreshold()) {
@@ -378,7 +413,7 @@ public class FilterPlanNode implements PlanNode {
column);
}
return new VectorSimilarityFilterOperator(vectorIndex, predicate,
numDocs, searchParams, forwardIndexReader,
- vectorIndexConfig, hasMetadataFilter);
+ vectorIndexConfig, hasMetadataFilter, _requiredVectorDocIds);
}
// Exact scan fallback: no vector index on this segment
@@ -386,7 +421,7 @@ public class FilterPlanNode implements PlanNode {
Preconditions.checkState(forwardIndexReader != null,
"Cannot apply VECTOR_SIMILARITY on column: %s -- no vector index and
no forward index available", column);
return new ExactVectorScanFilterOperator(forwardIndexReader, predicate,
column, numDocs, vectorIndexConfig,
- getVectorFallbackReason(vectorIndexConfig, isMutableSegment),
searchParams, null);
+ getVectorFallbackReason(vectorIndexConfig, isMutableSegment),
searchParams, _requiredVectorDocIds);
}
/// Constructs a vector operator for a VECTOR_SIMILARITY predicate that is
part of an AND
@@ -460,8 +495,8 @@ public class FilterPlanNode implements PlanNode {
/// The [VectorSearchStrategy] selectivity check below ensures queries only
pay this cost when the estimated
/// cardinality suggests pre-filtering is worthwhile.
///
- /// If no vector operators are found, this method is a no-op. A query whose
reader does not support
- /// pre-filtering retains the default post-filter path.
+ /// If no vector operators are found, this method is a no-op. A query whose
reader does not support pre-filtering
+ /// retains the default post-filter path.
///
/// @param childOperators the list of child filter operators under an AND
node
/// @param numDocs total documents in the segment
@@ -560,6 +595,18 @@ public class FilterPlanNode implements PlanNode {
}
}
+ private static boolean supportsPreFilter(VectorIndexReader vectorIndex) {
+ return vectorIndex instanceof FilterAwareVectorIndexReader
+ && ((FilterAwareVectorIndexReader) vectorIndex).supportsPreFilter();
+ }
+
+ /// Reason reported by [ExactVectorScanFilterOperator] when a vector index
exists but cannot restrict its search to
+ /// the documents the query is allowed to see.
+ private static String getFilteredSearchUnsupportedReason(boolean
isMutableSegment) {
+ return isMutableSegment ? "mutable_vector_index_not_filter_aware"
+ : "vector_index_not_filter_aware";
+ }
+
private static String getVectorFallbackReason(@Nullable VectorIndexConfig
vectorIndexConfig,
boolean isMutableSegment) {
if (vectorIndexConfig == null || vectorIndexConfig.isDisabled()) {
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/FilterAwareVectorSearchTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/FilterAwareVectorSearchTest.java
index bef51f7e303..fb0e4a5a148 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/FilterAwareVectorSearchTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/filter/FilterAwareVectorSearchTest.java
@@ -22,7 +22,12 @@ import
org.apache.pinot.common.request.context.ExpressionContext;
import
org.apache.pinot.common.request.context.predicate.VectorSimilarityPredicate;
import org.apache.pinot.segment.spi.index.creator.VectorIndexConfig;
import org.apache.pinot.segment.spi.index.reader.FilterAwareVectorIndexReader;
+import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
+import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
import org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.testng.Assert;
@@ -248,6 +253,98 @@ public class FilterAwareVectorSearchTest {
verify(mockReader, never()).getDocIds(eq(queryVector), anyInt(),
any(ImmutableRoaringBitmap.class));
}
+ @Test
+ public void testRequiredCandidateScopeAlwaysUsesFilteredReader() {
+ FilterAwareVectorIndexReader mockReader =
mock(FilterAwareVectorIndexReader.class);
+ float[] queryVector = {1.0f, 0.0f};
+ MutableRoaringBitmap requiredSnapshot = bitmapOf(2, 3);
+ MutableRoaringBitmap expectedResult = bitmapOf(2, 3);
+ when(mockReader.supportsPreFilter()).thenReturn(true);
+ when(mockReader.getDocIds(eq(queryVector), eq(2),
any(ImmutableRoaringBitmap.class)))
+ .thenReturn(expectedResult);
+
+ VectorSimilarityPredicate predicate = new VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), queryVector, 2);
+
+ VectorSimilarityFilterOperator operator = new
VectorSimilarityFilterOperator(mockReader, predicate, 4,
+ VectorSearchParams.DEFAULT, null, null, false, requiredSnapshot);
+
+ ImmutableRoaringBitmap result = operator.getBitmaps().reduce();
+ Assert.assertEquals(result, expectedResult);
+
+ ArgumentCaptor<ImmutableRoaringBitmap> filterCaptor =
ArgumentCaptor.forClass(ImmutableRoaringBitmap.class);
+ verify(mockReader).getDocIds(eq(queryVector), eq(2),
filterCaptor.capture());
+ Assert.assertEquals(filterCaptor.getValue(), bitmapOf(2, 3));
+ verify(mockReader, never()).getDocIds(queryVector, 2);
+
Assert.assertTrue(operator.toExplainString().contains("requiredDocIdFilterApplied:true"));
+
Assert.assertTrue(operator.toExplainString().contains("requiredDocIdFilterCardinality:2"));
+
Assert.assertTrue(operator.toExplainString().contains("searchMode:FILTER_THEN_ANN"));
+
Assert.assertTrue(operator.getExplainInfo().getAttributes().get("requiredDocIdFilterApplied").getBool());
+ Assert.assertEquals(
+
operator.getExplainInfo().getAttributes().get("requiredDocIdFilterCardinality").getLong(),
2L);
+
Assert.assertEquals(operator.getExplainInfo().getAttributes().get("searchMode").getString(),
+ "FILTER_THEN_ANN");
+ }
+
+ @Test
+ public void testEmptyRequiredCandidateScopeSkipsReader() {
+ FilterAwareVectorIndexReader mockReader =
mock(FilterAwareVectorIndexReader.class);
+ float[] queryVector = {1.0f, 0.0f};
+ VectorSimilarityPredicate predicate = new VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), queryVector, 2);
+ VectorSimilarityFilterOperator operator = new
VectorSimilarityFilterOperator(mockReader, predicate, 4,
+ VectorSearchParams.DEFAULT, null, null, false, new
MutableRoaringBitmap());
+
+ Assert.assertTrue(operator.getBitmaps().reduce().isEmpty());
+ // The reader is only asked about its capabilities, never to search.
+ verify(mockReader, never()).getDocIds(any(float[].class), anyInt());
+ verify(mockReader, never()).getDocIds(any(float[].class), anyInt(),
any(ImmutableRoaringBitmap.class));
+
Assert.assertTrue(operator.toExplainString().contains("candidateGenerationSkipped:true"));
+
Assert.assertTrue(operator.getExplainInfo().getAttributes().get("candidateGenerationSkipped").getBool());
+ }
+
+ @Test
+ public void testRequiredAndOptionalFiltersAreIntersected() {
+ FilterAwareVectorIndexReader mockReader =
mock(FilterAwareVectorIndexReader.class);
+ float[] queryVector = {1.0f, 0.0f};
+ MutableRoaringBitmap requiredSnapshot = bitmapOf(2, 3);
+ MutableRoaringBitmap optionalMetadata = bitmapOf(1, 3);
+ when(mockReader.supportsPreFilter()).thenReturn(true);
+ when(mockReader.getDocIds(eq(queryVector), eq(2),
any(ImmutableRoaringBitmap.class)))
+ .thenReturn(bitmapOf(3));
+
+ VectorSimilarityPredicate predicate = new VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), queryVector, 2);
+ VectorSimilarityFilterOperator operator = new
VectorSimilarityFilterOperator(mockReader, predicate, 4,
+ VectorSearchParams.DEFAULT, null, null, true,
+ requiredSnapshot);
+ operator.setPreFilterBitmap(optionalMetadata);
+
+ optionalMetadata.add(2);
+ ImmutableRoaringBitmap result = operator.getBitmaps().reduce();
+ Assert.assertEquals(result, bitmapOf(3));
+ Assert.assertEquals(requiredSnapshot, bitmapOf(2, 3), "Required snapshot
must not be mutated");
+
+ ArgumentCaptor<ImmutableRoaringBitmap> filterCaptor =
ArgumentCaptor.forClass(ImmutableRoaringBitmap.class);
+ verify(mockReader).getDocIds(eq(queryVector), eq(2),
filterCaptor.capture());
+ Assert.assertEquals(filterCaptor.getValue(), bitmapOf(3));
+ verify(mockReader, never()).getDocIds(queryVector, 2);
+ }
+
+ /// A reader that cannot do filtered search can never honor a required
candidate scope. FilterPlanNode selects
+ /// ExactVectorScanFilterOperator in that case, so reaching this operator at
all is a planning error.
+ @Test(expectedExceptions = IllegalStateException.class,
+ expectedExceptionsMessageRegExp = ".*required candidate doc IDs.*does
not support filtered search.*")
+ public void testRequiredCandidateScopeRejectsReaderWithoutFilteredSearch() {
+ VectorIndexReader mockReader = mock(VectorIndexReader.class);
+ float[] queryVector = {1.0f, 0.0f};
+ VectorSimilarityPredicate predicate = new VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), queryVector, 2);
+
+ new VectorSimilarityFilterOperator(mockReader, predicate, 4,
VectorSearchParams.DEFAULT, null, null, false,
+ bitmapOf(2, 3));
+ }
+
@Test
public void testExplainStringIncludesSearchMode() {
FilterAwareVectorIndexReader mockReader =
mock(FilterAwareVectorIndexReader.class);
@@ -289,4 +386,29 @@ public class FilterAwareVectorSearchTest {
Assert.assertEquals(context.getVectorSearchMode(),
VectorSearchMode.POST_FILTER_ANN);
Assert.assertEquals(context.getFilterSelectivity(), -1.0, 0.001);
}
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static ForwardIndexReader<?> createMockForwardIndexReader(float[][]
vectors) {
+ ForwardIndexReader mockReader = mock(ForwardIndexReader.class);
+ ForwardIndexReaderContext mockContext =
mock(ForwardIndexReaderContext.class);
+ when(mockReader.createContext()).thenReturn(mockContext);
+ when(mockReader.isSingleValue()).thenReturn(false);
+ when(mockReader.isDictionaryEncoded()).thenReturn(false);
+ when(mockReader.getStoredType()).thenReturn(DataType.FLOAT);
+ for (int i = 0; i < vectors.length; i++) {
+ when(mockReader.getFloatMV(Mockito.eq(i),
Mockito.any())).thenReturn(vectors[i]);
+ }
+ return mockReader;
+ }
+
+ private static VectorIndexConfig createVectorIndexConfig(
+ VectorIndexConfig.VectorDistanceFunction distanceFunction) {
+ return new VectorIndexConfig(false, "HNSW", 2, 1, distanceFunction,
java.util.Map.of());
+ }
+
+ private static MutableRoaringBitmap bitmapOf(int... docIds) {
+ MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+ bitmap.add(docIds);
+ return bitmap;
+ }
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/plan/FilterPlanNodeTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/plan/FilterPlanNodeTest.java
index ec094f4ff53..a74e72d28b5 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/plan/FilterPlanNodeTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/plan/FilterPlanNodeTest.java
@@ -21,6 +21,7 @@ package org.apache.pinot.core.plan;
import java.lang.reflect.Method;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.FilterContext;
@@ -30,8 +31,10 @@ import
org.apache.pinot.common.request.context.predicate.RegexpLikePredicate;
import
org.apache.pinot.common.request.context.predicate.VectorSimilarityPredicate;
import org.apache.pinot.core.common.BlockDocIdIterator;
import org.apache.pinot.core.common.BlockDocIdSet;
+import org.apache.pinot.core.common.Operator;
import org.apache.pinot.core.operator.blocks.FilterBlock;
import org.apache.pinot.core.operator.filter.BaseFilterOperator;
+import org.apache.pinot.core.operator.filter.ExactVectorScanFilterOperator;
import
org.apache.pinot.core.operator.filter.predicate.BaseDictIdBasedRegexpLikePredicateEvaluator;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import org.apache.pinot.core.query.request.context.QueryContext;
@@ -47,12 +50,15 @@ import
org.apache.pinot.segment.spi.index.mutable.ThreadSafeMutableRoaringBitmap
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.FilterAwareVectorIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
+import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
+import org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
@@ -66,6 +72,7 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
@@ -74,11 +81,45 @@ import static org.testng.Assert.assertTrue;
public class FilterPlanNodeTest {
+ @DataProvider(name = "vectorFilterShapes")
+ public Object[][] vectorFilterShapes() {
+ FilterContext vectorFilter = FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), new float[]{1.0f, 0.0f},
2));
+ return new Object[][]{
+ {vectorFilter},
+ {FilterContext.forAnd(List.of(vectorFilter,
FilterContext.CONSTANT_TRUE))},
+ {FilterContext.forOr(List.of(vectorFilter,
FilterContext.CONSTANT_FALSE))},
+ {FilterContext.forNot(FilterContext.forNot(vectorFilter))}
+ };
+ }
+
@DataProvider(name = "nestedVectorSubtreeShapes")
public Object[][] nestedVectorSubtreeShapes() {
return new Object[][]{{"AND"}, {"OR"}, {"NOT_OR"}};
}
+ @DataProvider(name = "nonVectorFilterShapes")
+ public Object[][] nonVectorFilterShapes() {
+ return new Object[][]{{FilterContext.CONSTANT_TRUE}, {null}};
+ }
+
+ @Test(dataProvider = "nonVectorFilterShapes")
+ public void testNonVectorPlansApplyTheSnapshotUnchanged(FilterContext
filter) {
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(4);
+ when(metadata.isMutableSegment()).thenReturn(true);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ QueryContext queryContext = mock(QueryContext.class);
+ when(queryContext.getFilter()).thenReturn(filter);
+ MutableRoaringBitmap snapshot = bitmapOf(1, 3);
+ SegmentContext segmentContext = new SegmentContext(segment);
+ segmentContext.setDocIdsSnapshot(snapshot);
+
+ Assert.assertEquals(getFilteredDocIds(new FilterPlanNode(segmentContext,
queryContext).run()), bitmapOf(1, 3));
+ verify(metadata, never()).isMutableSegment();
+ }
+
@Test(dataProvider = "nestedVectorSubtreeShapes")
public void
testNestedVectorSimilaritySubtreeIsNeverMaterializedAsMetadata(String shape) {
FilterAwareVectorIndexReader directVectorReader =
mock(FilterAwareVectorIndexReader.class);
@@ -138,6 +179,146 @@ public class FilterPlanNodeTest {
any(ImmutableRoaringBitmap.class));
}
+ @Test
+ public void testNestedNotVectorIsNotPushedIntoSiblingCandidateScope() {
+ float[] directQuery = {1.0f, 0.0f};
+ float[] nestedQuery = {0.0f, 1.0f};
+ FilterAwareVectorIndexReader directVectorReader =
mock(FilterAwareVectorIndexReader.class);
+ FilterAwareVectorIndexReader nestedVectorReader =
mock(FilterAwareVectorIndexReader.class);
+ when(directVectorReader.supportsPreFilter()).thenReturn(true);
+ when(nestedVectorReader.supportsPreFilter()).thenReturn(true);
+ when(directVectorReader.getDocIds(eq(directQuery), eq(1),
any(ImmutableRoaringBitmap.class)))
+ .thenReturn(bitmapOf(0));
+ when(nestedVectorReader.getDocIds(eq(nestedQuery), eq(1),
any(ImmutableRoaringBitmap.class)))
+ .thenReturn(bitmapOf(0));
+ DataSource directDataSource = mock(DataSource.class);
+ DataSource nestedDataSource = mock(DataSource.class);
+ when(directDataSource.getVectorIndex()).thenReturn(directVectorReader);
+ when(nestedDataSource.getVectorIndex()).thenReturn(nestedVectorReader);
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(2);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ when(segment.getDataSource("embeddingA",
null)).thenReturn(directDataSource);
+ when(segment.getDataSource("embeddingB",
null)).thenReturn(nestedDataSource);
+ FilterContext directVector = FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embeddingA"), directQuery, 1));
+ FilterContext nestedVector = FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embeddingB"), nestedQuery, 1));
+ QueryContext queryContext = mock(QueryContext.class);
+ when(queryContext.getFilter()).thenReturn(
+ FilterContext.forAnd(List.of(directVector,
FilterContext.forNot(nestedVector))));
+ SegmentContext segmentContext = new SegmentContext(segment);
+ segmentContext.setDocIdsSnapshot(bitmapOf(0, 1));
+
+ Assert.assertTrue(getFilteredDocIds(new FilterPlanNode(segmentContext,
queryContext).run()).isEmpty());
+ ArgumentCaptor<ImmutableRoaringBitmap> directScope =
ArgumentCaptor.forClass(ImmutableRoaringBitmap.class);
+ verify(directVectorReader).getDocIds(eq(directQuery), eq(1),
directScope.capture());
+ Assert.assertEquals(directScope.getValue(), bitmapOf(0, 1),
+ "NOT(vector) is a result predicate, not metadata that may narrow its
sibling's top-K corpus");
+ }
+
+ @Test(dataProvider = "vectorFilterShapes")
+ public void
testRequiredDocIdSnapshotReachesVectorCandidatesForAllBooleanShapes(FilterContext
filter) {
+ float[] queryVector = {1.0f, 0.0f};
+ FilterAwareVectorIndexReader vectorReader =
mock(FilterAwareVectorIndexReader.class);
+ when(vectorReader.supportsPreFilter()).thenReturn(true);
+ when(vectorReader.getDocIds(queryVector, 2)).thenReturn(bitmapOf(0, 1));
+ when(vectorReader.getDocIds(eq(queryVector), eq(2),
any(ImmutableRoaringBitmap.class)))
+ .thenReturn(bitmapOf(2, 3));
+
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getVectorIndex()).thenReturn(vectorReader);
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(4);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ when(segment.getDataSource("embedding", null)).thenReturn(dataSource);
+
+ QueryContext queryContext = mock(QueryContext.class);
+ when(queryContext.getFilter()).thenReturn(filter);
+ SegmentContext segmentContext = new SegmentContext(segment);
+ MutableRoaringBitmap snapshot = bitmapOf(2, 3);
+ segmentContext.setDocIdsSnapshot(snapshot);
+
+ BaseFilterOperator operator = new FilterPlanNode(segmentContext,
queryContext).run();
+ Assert.assertEquals(getFilteredDocIds(operator), bitmapOf(2, 3));
+
+ ArgumentCaptor<ImmutableRoaringBitmap> filterCaptor =
ArgumentCaptor.forClass(ImmutableRoaringBitmap.class);
+ verify(vectorReader).getDocIds(eq(queryVector), eq(2),
filterCaptor.capture());
+ Assert.assertEquals(filterCaptor.getValue(), bitmapOf(2, 3));
+ verify(vectorReader, never()).getDocIds(queryVector, 2);
+ }
+
+ @Test
+ public void testEmptyDocIdSnapshotSkipsVectorReader() {
+ FilterAwareVectorIndexReader vectorReader =
mock(FilterAwareVectorIndexReader.class);
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getVectorIndex()).thenReturn(vectorReader);
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(4);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ when(segment.getDataSource("embedding", null)).thenReturn(dataSource);
+
+ QueryContext queryContext = mock(QueryContext.class);
+ when(queryContext.getFilter()).thenReturn(FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), new float[]{1.0f, 0.0f},
2)));
+ SegmentContext segmentContext = new SegmentContext(segment);
+ segmentContext.setDocIdsSnapshot(new MutableRoaringBitmap());
+
+ Assert.assertTrue(getFilteredDocIds(new FilterPlanNode(segmentContext,
queryContext).run()).isEmpty());
+ verifyNoInteractions(vectorReader);
+ }
+
+ @Test
+ public void testNullDocIdSnapshotPreservesUnfilteredVectorSearch() {
+ float[] queryVector = {1.0f, 0.0f};
+ FilterAwareVectorIndexReader vectorReader =
mock(FilterAwareVectorIndexReader.class);
+ when(vectorReader.getDocIds(queryVector, 2)).thenReturn(bitmapOf(0, 1));
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getVectorIndex()).thenReturn(vectorReader);
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(4);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ when(segment.getDataSource("embedding", null)).thenReturn(dataSource);
+
+ QueryContext queryContext = mock(QueryContext.class);
+ when(queryContext.getFilter()).thenReturn(FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), queryVector, 2)));
+
+ Assert.assertEquals(getFilteredDocIds(new FilterPlanNode(new
SegmentContext(segment), queryContext).run()),
+ bitmapOf(0, 1));
+ verify(vectorReader).getDocIds(queryVector, 2);
+ verify(vectorReader, never()).getDocIds(eq(queryVector), eq(2),
any(ImmutableRoaringBitmap.class));
+ }
+
+ @Test
+ public void testRequiredDocIdSnapshotPreservesNotSemantics() {
+ float[] queryVector = {1.0f, 0.0f};
+ FilterAwareVectorIndexReader vectorReader =
mock(FilterAwareVectorIndexReader.class);
+ when(vectorReader.supportsPreFilter()).thenReturn(true);
+ when(vectorReader.getDocIds(eq(queryVector), eq(2),
any(ImmutableRoaringBitmap.class)))
+ .thenReturn(bitmapOf(2));
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getVectorIndex()).thenReturn(vectorReader);
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(4);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ when(segment.getDataSource("embedding", null)).thenReturn(dataSource);
+ QueryContext queryContext = mock(QueryContext.class);
+ FilterContext vectorFilter = FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), queryVector, 2));
+
when(queryContext.getFilter()).thenReturn(FilterContext.forNot(vectorFilter));
+ SegmentContext segmentContext = new SegmentContext(segment);
+ segmentContext.setDocIdsSnapshot(bitmapOf(2, 3));
+
+ Assert.assertEquals(getFilteredDocIds(new FilterPlanNode(segmentContext,
queryContext).run()), bitmapOf(3));
+ verify(vectorReader, never()).getDocIds(queryVector, 2);
+ }
+
@Test
public void
testNonRestrictedMetadataScopePreservesAdaptivePostFilterBehavior() {
float[] queryVector = {1.0f, 0.0f};
@@ -171,6 +352,64 @@ public class FilterPlanNodeTest {
verify(vectorReader, never()).getDocIds(eq(queryVector), eq(2),
any(ImmutableRoaringBitmap.class));
}
+ @Test
+ public void
testPlannerSelectsExactAllowedDocFallbackForNonFilterAwareReader() {
+ float[] queryVector = {1.0f, 0.0f};
+ VectorIndexReader vectorReader = mock(VectorIndexReader.class);
+ ForwardIndexReader<?> forwardIndexReader =
createMockForwardIndexReader(new float[][]{
+ {1.0f, 0.0f}, {1.0f, 0.0f}, {0.0f, 1.0f}, {0.0f, -1.0f}
+ });
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getVectorIndex()).thenReturn(vectorReader);
+ Mockito.doReturn(forwardIndexReader).when(dataSource).getForwardIndex();
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(4);
+ when(metadata.isMutableSegment()).thenReturn(true);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ when(segment.getDataSource("embedding", null)).thenReturn(dataSource);
+ QueryContext queryContext = mock(QueryContext.class);
+ when(queryContext.getFilter()).thenReturn(FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), queryVector, 2)));
+ SegmentContext segmentContext = new SegmentContext(segment);
+ segmentContext.setDocIdsSnapshot(bitmapOf(2, 3));
+
+ BaseFilterOperator operator = new FilterPlanNode(segmentContext,
queryContext).run();
+ Assert.assertEquals(getFilteredDocIds(operator), bitmapOf(2, 3));
+ verify(vectorReader, never()).getDocIds(any(float[].class), eq(2));
+ verify(forwardIndexReader, never()).getFloatMV(eq(0), any());
+ verify(forwardIndexReader, never()).getFloatMV(eq(1), any());
+ verify(forwardIndexReader).getFloatMV(eq(2), any());
+ verify(forwardIndexReader).getFloatMV(eq(3), any());
+ ExactVectorScanFilterOperator exactScan = findOperator(operator,
ExactVectorScanFilterOperator.class);
+ Assert.assertNotNull(exactScan,
+ "Planner must select the exact scan operator when the reader cannot
restrict its search");
+ String explain = exactScan.toExplainString();
+
Assert.assertTrue(explain.contains("fallbackReason:mutable_vector_index_not_filter_aware"),
explain);
+ Assert.assertTrue(explain.contains("requiredDocIdFilterApplied:true"),
explain);
+ Assert.assertTrue(explain.contains("requiredDocIdFilterCardinality:2"),
explain);
+ }
+
+ @Test(expectedExceptions = IllegalStateException.class,
+ expectedExceptionsMessageRegExp = ".*required candidate doc IDs.*no
forward index.*")
+ public void testPlannerFailsWhenRequiredFilterCannotBeHonored() {
+ VectorIndexReader vectorReader = mock(VectorIndexReader.class);
+ DataSource dataSource = mock(DataSource.class);
+ when(dataSource.getVectorIndex()).thenReturn(vectorReader);
+ IndexSegment segment = mock(IndexSegment.class);
+ SegmentMetadata metadata = mock(SegmentMetadata.class);
+ when(metadata.getTotalDocs()).thenReturn(4);
+ when(segment.getSegmentMetadata()).thenReturn(metadata);
+ when(segment.getDataSource("embedding", null)).thenReturn(dataSource);
+ QueryContext queryContext = mock(QueryContext.class);
+ when(queryContext.getFilter()).thenReturn(FilterContext.forPredicate(new
VectorSimilarityPredicate(
+ ExpressionContext.forIdentifier("embedding"), new float[]{1.0f, 0.0f},
2)));
+ SegmentContext segmentContext = new SegmentContext(segment);
+ segmentContext.setDocIdsSnapshot(bitmapOf(2, 3));
+
+ new FilterPlanNode(segmentContext, queryContext).run();
+ }
+
@Test
public void testConsistentSnapshot()
throws Exception {
@@ -273,12 +512,44 @@ public class FilterPlanNodeTest {
return docIds;
}
+ /// Finds the first operator of the given type in the plan tree, or null
when the plan contains none.
+ @Nullable
+ private static <T> T findOperator(Operator operator, Class<T> operatorClass)
{
+ if (operatorClass.isInstance(operator)) {
+ return operatorClass.cast(operator);
+ }
+ List<? extends Operator> children = operator.getChildOperators();
+ if (children != null) {
+ for (Operator child : children) {
+ if (child != null) {
+ T found = findOperator(child, operatorClass);
+ if (found != null) {
+ return found;
+ }
+ }
+ }
+ }
+ return null;
+ }
+
private static MutableRoaringBitmap bitmapOf(int... docIds) {
MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
bitmap.add(docIds);
return bitmap;
}
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static ForwardIndexReader<?> createMockForwardIndexReader(float[][]
vectors) {
+ ForwardIndexReader mockReader = mock(ForwardIndexReader.class);
+ ForwardIndexReaderContext context = mock(ForwardIndexReaderContext.class);
+ when(mockReader.createContext()).thenReturn(context);
+ for (int i = 0; i < vectors.length; i++) {
+ when(mockReader.getFloatMV(Mockito.eq(i),
Mockito.any())).thenReturn(vectors[i]);
+ }
+ return mockReader;
+ }
+
@Test
public void regexpLikeUsesIFSTEvaluatorWhenIFSTAndInvertedAvailable()
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
