This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6669f065c1a Upserts Pruning Optimization - Do not clone
MutableRoaringBitmap for each segment (#17920)
6669f065c1a is described below
commit 6669f065c1a1ab816109a7cb864e8622d2540aeb
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Mon Mar 23 11:30:53 2026 -0700
Upserts Pruning Optimization - Do not clone MutableRoaringBitmap for each
segment (#17920)
---
.../pinot/core/query/pruner/SegmentPrunerService.java | 18 ++++++------------
.../apache/pinot/segment/local/upsert/UpsertUtils.java | 12 ++++++++++++
.../index/mutable/ThreadSafeMutableRoaringBitmap.java | 4 ++++
3 files changed, 22 insertions(+), 12 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/pruner/SegmentPrunerService.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/pruner/SegmentPrunerService.java
index 084a00f9351..5707ef698a0 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/pruner/SegmentPrunerService.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/pruner/SegmentPrunerService.java
@@ -32,7 +32,6 @@ import org.apache.pinot.segment.local.upsert.UpsertUtils;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.spi.trace.InvocationScope;
import org.apache.pinot.spi.trace.Tracing;
-import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -142,27 +141,22 @@ public class SegmentPrunerService {
* emptiness is determined from them; otherwise {@link
IndexSegment#getValidDocIds()} is used.
* @return the new list with filtered elements. This is the list that have
to be used.
*/
- private static List<IndexSegment> removeEmptySegments(List<IndexSegment>
segments, @Nullable QueryContext query) {
+ private static List<IndexSegment> removeEmptySegments(List<IndexSegment>
segments, QueryContext query) {
int selected = 0;
+ boolean skipUpsert =
QueryOptionsUtils.isSkipUpsert(query.getQueryOptions());
for (IndexSegment segment : segments) {
- if (!isEmptySegment(segment, query)) {
+ if (!isEmptySegment(segment, skipUpsert)) {
segments.set(selected++, segment);
}
}
return segments.subList(0, selected);
}
- private static boolean isEmptySegment(IndexSegment segment, @Nullable
QueryContext query) {
+ private static boolean isEmptySegment(IndexSegment segment, boolean
skipUpsert) {
if (segment.getSegmentMetadata().getTotalDocs() == 0) {
return true;
}
- // For upsert tables, treat segments with 0 docs to query as empty only
when the query does not skip upsert.
- // When skipUpsert=true, the query returns all docs (including replaced),
so the segment contributes rows.
- // Use query options map directly: _skipUpsert is only set on the server;
the broker has options in the map.
- if (query != null && query.getQueryOptions() != null &&
QueryOptionsUtils.isSkipUpsert(query.getQueryOptions())) {
- return false;
- }
- MutableRoaringBitmap queryableDocIds =
UpsertUtils.getQueryableDocIdsSnapshotFromSegment(segment);
- return queryableDocIds != null && queryableDocIds.isEmpty();
+ // Check if the segment has 0 queryable docIds while skipUpsert=false
+ return !skipUpsert && UpsertUtils.hasNoQueryableDocs(segment);
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/UpsertUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/UpsertUtils.java
index b508a02aeb0..990dadb9074 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/UpsertUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/UpsertUtils.java
@@ -57,6 +57,18 @@ public class UpsertUtils {
: (useEmptyForNull ? new MutableRoaringBitmap() : null);
}
+ /**
+ * Returns whether the segment has no queryable documents, when no delete
record column we look into validDocIds
+ * for an upsert table
+ */
+ public static boolean hasNoQueryableDocs(IndexSegment segment) {
+ ThreadSafeMutableRoaringBitmap queryableDocIds =
segment.getQueryableDocIds();
+ if (queryableDocIds != null) {
+ return queryableDocIds.isEmpty();
+ }
+ ThreadSafeMutableRoaringBitmap validDocIds = segment.getValidDocIds();
+ return validDocIds != null && validDocIds.isEmpty();
+ }
public static void doReplaceDocId(ThreadSafeMutableRoaringBitmap validDocIds,
@Nullable ThreadSafeMutableRoaringBitmap queryableDocIds, int oldDocId,
int newDocId, RecordInfo recordInfo) {
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/ThreadSafeMutableRoaringBitmap.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/ThreadSafeMutableRoaringBitmap.java
index 05ab86628d5..470913fd4ce 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/ThreadSafeMutableRoaringBitmap.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/mutable/ThreadSafeMutableRoaringBitmap.java
@@ -60,4 +60,8 @@ public class ThreadSafeMutableRoaringBitmap {
public synchronized MutableRoaringBitmap getMutableRoaringBitmap() {
return _mutableRoaringBitmap.clone();
}
+
+ public synchronized boolean isEmpty() {
+ return _mutableRoaringBitmap.isEmpty();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
