This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fd1d3f7b590 Name upsert validDocIds bitmaps read from or written to
disk *OnDiskBitmap (#20152)
fd1d3f7b590 is described below
commit fd1d3f7b590799e1461d3f193e92980c9c4ad0d9
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Wed Nov 4 12:44:52 2026 -0800
Name upsert validDocIds bitmaps read from or written to disk *OnDiskBitmap
(#20152)
* Name upsert validDocIds bitmaps read from or written to disk *OnDiskBitmap
"Snapshot" in upsert code means both the bitmap persisted next to a
segment and the in-memory query-time view (getValidDocIdsSnapshot,
getQueryableDocIdsSnapshot). Variables holding the persisted bitmap
become validDocIdsOnDiskBitmap / queryableDocIdsOnDiskBitmap.
Variable names only.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
* Rename the persisted docIds bitmap file-name constants
*_ON_DISK_BITMAP_FILE_NAME
VALID_DOC_IDS_SNAPSHOT_FILE_NAME and QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME
become VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME and
QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME. The file names on disk are
unchanged. The old constants stay as deprecated aliases so plugins built
against them keep compiling.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
* Address review: rename the preload and reload on-disk bitmaps, align an
argument
doPreloadSegment's validDocIds and the reload backup variables
(restoreValidDocIdsSnapshot, backupValidDocIdsSnapshotFile) hold the
on-disk bitmap, so they take the OnDiskBitmap name too. A wrapped
argument in BaseTableDataManager now lines up with its sibling.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../core/data/manager/BaseTableDataManager.java | 15 +--
.../UpsertSnapshotRestartIntegrationTest.java | 7 +-
.../UpsertTableSegmentPreloadIntegrationTest.java | 2 +-
.../upsert/BasePartitionUpsertMetadataManager.java | 82 +++++++------
.../segment/local/utils/SegmentPreloadUtils.java | 3 +-
.../BasePartitionUpsertMetadataManagerTest.java | 136 ++++++++++++---------
...ertMetadataManagerForConsistentDeletesTest.java | 62 +++++-----
...rrentMapPartitionUpsertMetadataManagerTest.java | 107 ++++++++--------
.../local/upsert/UpsertSnapshotRestartTest.java | 6 +-
.../local/utils/SegmentPreloadUtilsTest.java | 2 +-
.../org/apache/pinot/segment/spi/V1Constants.java | 13 +-
.../pinot/server/api/resources/TablesResource.java | 10 +-
.../pinot/server/api/TablesResourceTest.java | 30 ++---
13 files changed, 257 insertions(+), 218 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
index b863759a7cb..9ecf978f003 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java
@@ -1320,9 +1320,9 @@ public abstract class BaseTableDataManager implements
TableDataManager {
// CRC-mismatched normal reload keeps the local segment
(shouldDownload is false), so the local snapshot still
// maps and the reload proceeds.
UpsertContext upsertContext = isUpsertEnabled() ?
_tableUpsertMetadataManager.getContext() : null;
- boolean restoreValidDocIdsSnapshot = upsertContext != null &&
upsertContext.isSnapshotEnabled()
+ boolean restoreValidDocIdsOnDiskBitmap = upsertContext != null &&
upsertContext.isSnapshotEnabled()
&& (upsertContext.getMetadataTTL() > 0 ||
upsertContext.getDeletedKeysTTL() > 0);
- if (restoreValidDocIdsSnapshot && !crcMatch && shouldDownload) {
+ if (restoreValidDocIdsOnDiskBitmap && !crcMatch && shouldDownload) {
throw new IllegalStateException(String.format(
"Failing reload for segment: %s of upsert table with metadata
TTL: %s because the segment CRC has "
+ "changed from: %s to: %s and the docId-based validDocIds
snapshot cannot be guaranteed to map to "
@@ -1336,10 +1336,10 @@ public abstract class BaseTableDataManager implements
TableDataManager {
// The validDocIds snapshot is a server-local file absent from the
deep-store copy. createBackup moved it into
// the backup dir; capture that path so it can be copied into the
freshly downloaded segment dir below (the
// non-download copyTo path already restores it via copyDirectory).
- File backupValidDocIdsSnapshotFile = restoreValidDocIdsSnapshot ?
new File(
+ File backupValidDocIdsOnDiskBitmapFile =
restoreValidDocIdsOnDiskBitmap ? new File(
SegmentDirectoryPaths.findSegmentDirectory(new
File(indexDir.getParentFile(),
indexDir.getName() +
CommonConstants.Segment.SEGMENT_BACKUP_DIR_SUFFIX)),
- V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME) : null;
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME) : null;
if (forceDownload) {
_logger.info("Force downloading segment: {}", segmentName);
} else {
@@ -1347,9 +1347,10 @@ public abstract class BaseTableDataManager implements
TableDataManager {
localMetadata.getCrc(), zkMetadata.getCrc());
}
indexDir = downloadSegment(zkMetadata);
- if (backupValidDocIdsSnapshotFile != null &&
backupValidDocIdsSnapshotFile.exists()) {
- FileUtils.copyFile(backupValidDocIdsSnapshotFile, new File(
- SegmentDirectoryPaths.findSegmentDirectory(indexDir),
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ if (backupValidDocIdsOnDiskBitmapFile != null &&
backupValidDocIdsOnDiskBitmapFile.exists()) {
+ FileUtils.copyFile(backupValidDocIdsOnDiskBitmapFile, new File(
+ SegmentDirectoryPaths.findSegmentDirectory(indexDir),
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
}
} else {
_logger.info("Reloading existing segment: {} on tier: {}",
segmentName,
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertSnapshotRestartIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertSnapshotRestartIntegrationTest.java
index 8ae14427f3c..9acaaa28c09 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertSnapshotRestartIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertSnapshotRestartIntegrationTest.java
@@ -184,13 +184,13 @@ public class UpsertSnapshotRestartIntegrationTest extends
BaseClusterIntegration
private static File findSnapshot(File segmentDir) {
File v3 = new File(segmentDir, "v3");
- File snap = new File(v3, V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
+ File snap = new File(v3,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
if (snap.exists()) {
return snap;
}
File[] found = new File[1];
FileUtils.listFiles(segmentDir, null, true).forEach(f -> {
- if (f.getName().equals(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)) {
+ if
(f.getName().equals(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)) {
found[0] = f;
}
});
@@ -233,7 +233,8 @@ public class UpsertSnapshotRestartIntegrationTest extends
BaseClusterIntegration
assertNotNull(segmentDataManager, "segment " + segmentName + " not found");
try {
ImmutableSegmentImpl segment = (ImmutableSegmentImpl)
segmentDataManager.getSegment();
- MutableRoaringBitmap snapshot =
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
+ MutableRoaringBitmap snapshot = segment.loadDocIdsFromSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
return snapshot != null &&
snapshot.equals(segment.getValidDocIds().getMutableRoaringBitmap());
} finally {
tableDataManager.releaseSegment(segmentDataManager);
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertTableSegmentPreloadIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertTableSegmentPreloadIntegrationTest.java
index 19f1daca0cf..8f5a8536acb 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertTableSegmentPreloadIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertTableSegmentPreloadIntegrationTest.java
@@ -231,7 +231,7 @@ public class UpsertTableSegmentPreloadIntegrationTest
extends BaseClusterIntegra
assertNotNull(files);
assertEquals(files.length, 3);
for (File file : files) {
- if (!new File(new File(file, "v3"),
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).exists()) {
+ if (!new File(new File(file, "v3"),
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).exists()) {
return false;
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
index 79d7662bec9..9aedfb8019d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManager.java
@@ -383,11 +383,11 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
protected boolean skipAddSegmentOutOfTTL(ImmutableSegmentImpl segment) {
String segmentName = segment.getSegmentName();
_logger.info("Skip adding segment: {} because it's out of TTL",
segmentName);
- MutableRoaringBitmap validDocIdsSnapshot =
-
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
- if (validDocIdsSnapshot != null) {
- MutableRoaringBitmap queryableDocIds = getQueryableDocIds(segment,
validDocIdsSnapshot);
- segment.enableUpsert(this, new
ThreadSafeMutableRoaringBitmap(validDocIdsSnapshot),
+ MutableRoaringBitmap validDocIdsOnDiskBitmap =
+
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
+ if (validDocIdsOnDiskBitmap != null) {
+ MutableRoaringBitmap queryableDocIds = getQueryableDocIds(segment,
validDocIdsOnDiskBitmap);
+ segment.enableUpsert(this, new
ThreadSafeMutableRoaringBitmap(validDocIdsOnDiskBitmap),
queryableDocIds != null ? new
ThreadSafeMutableRoaringBitmap(queryableDocIds) : null);
} else {
_logger.warn("Failed to find validDocIds snapshot to add segment: {} out
of TTL, treating all docs as valid",
@@ -397,11 +397,12 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
return true;
}
- protected boolean skipPreloadSegmentOutOfTTL(ImmutableSegmentImpl segment,
MutableRoaringBitmap validDocIdsSnapshot) {
+ protected boolean skipPreloadSegmentOutOfTTL(ImmutableSegmentImpl segment,
+ MutableRoaringBitmap validDocIdsOnDiskBitmap) {
String segmentName = segment.getSegmentName();
_logger.info("Skip preloading segment: {} because it's out of TTL",
segmentName);
- MutableRoaringBitmap queryableDocIds = getQueryableDocIds(segment,
validDocIdsSnapshot);
- segment.enableUpsert(this, new
ThreadSafeMutableRoaringBitmap(validDocIdsSnapshot),
+ MutableRoaringBitmap queryableDocIds = getQueryableDocIds(segment,
validDocIdsOnDiskBitmap);
+ segment.enableUpsert(this, new
ThreadSafeMutableRoaringBitmap(validDocIdsOnDiskBitmap),
queryableDocIds != null ? new
ThreadSafeMutableRoaringBitmap(queryableDocIds) : null);
// Return true if segment is skipped. This boolean value allows subclass
to decide whether to skip.
return true;
@@ -487,13 +488,14 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
_logger.info("Preloading segment: {}, current primary key count: {}",
segmentName, getNumPrimaryKeys());
long startTimeMs = System.currentTimeMillis();
- MutableRoaringBitmap validDocIds =
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
- Preconditions.checkState(validDocIds != null,
+ MutableRoaringBitmap validDocIdsOnDiskBitmap =
segment.loadDocIdsFromSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
+ Preconditions.checkState(validDocIdsOnDiskBitmap != null,
"Snapshot of validDocIds is required to preload segment: %s, table:
%s", segmentName, _tableNameWithType);
// Cache the maximum even when the snapshot has no valid docs:
preloadSegment() still tracks this segment for
// snapshots, and another thread may destroy it while an updated marker is
being read.
double maxComparisonValue = isTTLEnabled() ?
getMaxComparisonValue(segment) : TTL_WATERMARK_NOT_SET;
- if (validDocIds.isEmpty()) {
+ if (validDocIdsOnDiskBitmap.isEmpty()) {
_logger.info("Skip preloading segment: {} without valid doc, current
primary key count: {}",
segment.getSegmentName(), getNumPrimaryKeys());
ThreadSafeMutableRoaringBitmap queryableDocIds =
@@ -503,13 +505,14 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
}
// Bump watermark after rows are added; see doAddSegment.
if (isTTLEnabled() && canSkipSegmentOutOfTTL(maxComparisonValue)
- && skipPreloadSegmentOutOfTTL(segment, validDocIds)) {
+ && skipPreloadSegmentOutOfTTL(segment, validDocIdsOnDiskBitmap)) {
updateLargestSeenComparisonValue(maxComparisonValue);
return;
}
try (UpsertUtils.RecordInfoReader recordInfoReader = new
UpsertUtils.RecordInfoReader(segment, _primaryKeyColumns,
_comparisonColumns, _deleteRecordColumn)) {
- doPreloadSegment(segment, null, null,
UpsertUtils.getRecordInfoIterator(recordInfoReader, validDocIds));
+ doPreloadSegment(segment, null, null,
+ UpsertUtils.getRecordInfoIterator(recordInfoReader,
validDocIdsOnDiskBitmap));
} catch (Exception e) {
throw new RuntimeException(
String.format("Caught exception while preloading segment: %s, table:
%s", segmentName, _tableNameWithType),
@@ -669,18 +672,20 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
// the reload flow placed it there (see
BaseTableDataManager.reloadSegment); segment commits and uploads always
// build a fresh segment without one and fall through to the full scan
below, unaffected. Rebuilding from just the
// snapshot's valid docs avoids resurrecting primary keys already
expired/deleted by TTL.
- MutableRoaringBitmap validDocIdsSnapshot = null;
+ MutableRoaringBitmap validDocIdsOnDiskBitmap = null;
if (isTTLEnabled() && segment instanceof ImmutableSegmentImpl
immutableSegment) {
- validDocIdsSnapshot =
immutableSegment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
+ validDocIdsOnDiskBitmap = immutableSegment.loadDocIdsFromSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
}
Iterator<RecordInfo> recordInfoIterator;
- if (validDocIdsSnapshot != null) {
+ if (validDocIdsOnDiskBitmap != null) {
// Rebuild from the snapshot's docIds as-is. On a forceDownload reload
with a CRC mismatch the snapshot may have
// been taken on a different copy of the segment, so its docIds are
not guaranteed to map to the same rows here;
// the operator accepted that risk by requesting forceDownload.
_logger.info("Replacing segment: {} on reload using validDocIds
snapshot with {} valid docs (snapshot docIds "
- + "are used as-is and assumed to map to this segment)",
segmentName, validDocIdsSnapshot.getCardinality());
- recordInfoIterator =
UpsertUtils.getRecordInfoIterator(recordInfoReader, validDocIdsSnapshot);
+ + "are used as-is and assumed to map to this segment)",
segmentName,
+ validDocIdsOnDiskBitmap.getCardinality());
+ recordInfoIterator =
UpsertUtils.getRecordInfoIterator(recordInfoReader, validDocIdsOnDiskBitmap);
} else {
// No validDocIds snapshot on the incoming segment: rebuild by
scanning all docs in the segment.
_logger.info("Replacing segment: {} using all {} docs (no validDocIds
snapshot available)", segmentName,
@@ -1114,19 +1119,21 @@ public abstract class
BasePartitionUpsertMetadataManager implements PartitionUps
ThreadSafeMutableRoaringBitmap validDocIds = segment.getValidDocIds();
// NOTE: Segment out of TTL without snapshot might have null
validDocIds
if (validDocIds != null) {
- ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
validDocIdsSnapshot = validDocIds.getBytesAndCardinality();
-
segment.persistDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME,
validDocIdsSnapshot,
- DocIdsSnapshot.metadata(validDocIdsSnapshot,
DocIdsType.VALID_DOC_IDS, trigger));
- numPrimaryKeysInSnapshot += validDocIdsSnapshot.getCardinality();
+ ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
validDocIdsOnDiskBitmap =
+ validDocIds.getBytesAndCardinality();
+
segment.persistDocIdsSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME,
validDocIdsOnDiskBitmap,
+ DocIdsSnapshot.metadata(validDocIdsOnDiskBitmap,
DocIdsType.VALID_DOC_IDS, trigger));
+ numPrimaryKeysInSnapshot += validDocIdsOnDiskBitmap.getCardinality();
}
if (_deleteRecordColumn != null) {
ThreadSafeMutableRoaringBitmap queryableDocIds =
segment.getQueryableDocIds();
if (queryableDocIds != null) {
- ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
queryableDocIdsSnapshot =
+ ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
queryableDocIdsOnDiskBitmap =
queryableDocIds.getBytesAndCardinality();
-
segment.persistDocIdsSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME,
queryableDocIdsSnapshot,
- DocIdsSnapshot.metadata(queryableDocIdsSnapshot,
DocIdsType.QUERYABLE_DOC_IDS, trigger));
- numQueryableDocIdsInSnapshot +=
queryableDocIdsSnapshot.getCardinality();
+ segment.persistDocIdsSnapshot(
+ V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME,
queryableDocIdsOnDiskBitmap,
+ DocIdsSnapshot.metadata(queryableDocIdsOnDiskBitmap,
DocIdsType.QUERYABLE_DOC_IDS, trigger));
+ numQueryableDocIdsInSnapshot +=
queryableDocIdsOnDiskBitmap.getCardinality();
}
}
_updatedSegmentsSinceLastSnapshot.remove(segment);
@@ -1165,23 +1172,24 @@ public abstract class
BasePartitionUpsertMetadataManager implements PartitionUps
ThreadSafeMutableRoaringBitmap validDocIds =
segment.getValidDocIds();
// NOTE: Segment out of TTL without snapshot might have null
validDocIds
if (validDocIds != null) {
- ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
validDocIdsSnapshot =
+ ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
validDocIdsOnDiskBitmap =
validDocIds.getBytesAndCardinality();
-
segment.persistDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME,
validDocIdsSnapshot,
- DocIdsSnapshot.metadata(validDocIdsSnapshot,
DocIdsType.VALID_DOC_IDS, trigger));
+
segment.persistDocIdsSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME,
validDocIdsOnDiskBitmap,
+ DocIdsSnapshot.metadata(validDocIdsOnDiskBitmap,
DocIdsType.VALID_DOC_IDS, trigger));
// The segment has its validDocIds snapshot file on disk now, so
handle it as a segment with snapshot
// from now on, even if persisting the queryableDocIds snapshot
below fails.
_segmentsWithSnapshot.add(segment);
- numPrimaryKeysInSnapshot += validDocIdsSnapshot.getCardinality();
+ numPrimaryKeysInSnapshot +=
validDocIdsOnDiskBitmap.getCardinality();
}
if (_deleteRecordColumn != null) {
ThreadSafeMutableRoaringBitmap queryableDocIds =
segment.getQueryableDocIds();
if (queryableDocIds != null) {
- ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
queryableDocIdsSnapshot =
+ ThreadSafeMutableRoaringBitmap.CardinalityAndBytes
queryableDocIdsOnDiskBitmap =
queryableDocIds.getBytesAndCardinality();
-
segment.persistDocIdsSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME,
queryableDocIdsSnapshot,
- DocIdsSnapshot.metadata(queryableDocIdsSnapshot,
DocIdsType.QUERYABLE_DOC_IDS, trigger));
- numQueryableDocIdsInSnapshot +=
queryableDocIdsSnapshot.getCardinality();
+
segment.persistDocIdsSnapshot(V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME,
+ queryableDocIdsOnDiskBitmap,
+ DocIdsSnapshot.metadata(queryableDocIdsOnDiskBitmap,
DocIdsType.QUERYABLE_DOC_IDS, trigger));
+ numQueryableDocIdsInSnapshot +=
queryableDocIdsOnDiskBitmap.getCardinality();
}
}
_updatedSegmentsSinceLastSnapshot.remove(segment);
@@ -1284,9 +1292,9 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
}
protected void deleteSnapshot(ImmutableSegmentImpl segment) {
- segment.deleteSnapshotFile(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
+
segment.deleteSnapshotFile(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
if (_deleteRecordColumn != null) {
-
segment.deleteSnapshotFile(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME);
+
segment.deleteSnapshotFile(V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
}
}
@@ -1495,7 +1503,7 @@ public abstract class BasePartitionUpsertMetadataManager
implements PartitionUps
/// together, so a missing queryableDocIds snapshot file gets recreated the
next time the segment's snapshot is
/// taken.
protected boolean hasValidDocIdsSnapshotFile(ImmutableSegmentImpl segment) {
- return
segment.hasSnapshotFile(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
+ return
segment.hasSnapshotFile(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
}
protected void doClose()
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtils.java
index 2d3b453824b..40ef3583147 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtils.java
@@ -136,7 +136,8 @@ public class SegmentPreloadUtils {
try {
File indexDir = tableDataManager.getSegmentDataDir(segmentName,
segmentTier, tableConfig);
File snapshotFile =
- new File(SegmentDirectoryPaths.findSegmentDirectory(indexDir),
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
+ new File(SegmentDirectoryPaths.findSegmentDirectory(indexDir),
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
return snapshotFile.exists();
} catch (Exception e) {
return false;
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManagerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManagerTest.java
index 22d86e33915..dc1b1c4a544 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManagerTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/BasePartitionUpsertMetadataManagerTest.java
@@ -218,8 +218,8 @@ public class BasePartitionUpsertMetadataManagerTest {
manager.addRecord(mock(MutableSegmentImpl.class), mock(RecordInfo.class));
manager.takeSnapshot("myTable__0__2__0", "100");
- DocIdsSnapshot first =
segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
- DocIdsSnapshot queryable =
segment.loadDocIdsSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME);
+ DocIdsSnapshot first =
segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
+ DocIdsSnapshot queryable =
segment.loadDocIdsSnapshot(V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
assertEquals(first.metadata().snapshotConsumedUpToOffset(), "100");
assertEquals(queryable.metadata().snapshotConsumedUpToOffset(), "100");
assertEquals(first.metadata().docIdsType(),
DocIdsSnapshot.DocIdsType.VALID_DOC_IDS);
@@ -229,17 +229,19 @@ public class BasePartitionUpsertMetadataManagerTest {
// An unchanged segment keeps its saved timestamp and trigger, even if a
newer consumer triggers a snapshot round.
manager.takeSnapshot("myTable__0__3__0", "200");
-
assertEquals(segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).metadata(),
first.metadata());
+ assertEquals(segment.loadDocIdsSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).metadata(),
first.metadata());
valid.remove(2);
manager.markSegmentAsUpdated(segment);
when(segmentLock.tryLock()).thenReturn(false);
manager.takeSnapshot("myTable__0__3__0", "200");
-
assertEquals(segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).metadata(),
first.metadata());
+ assertEquals(segment.loadDocIdsSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).metadata(),
first.metadata());
// Retrying publishes the new bitmap and its own trigger together.
when(segmentLock.tryLock()).thenReturn(true);
manager.takeSnapshot("myTable__0__4__0", "300");
- DocIdsSnapshot retried =
segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
+ DocIdsSnapshot retried =
segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
assertEquals(retried.metadata().snapshotConsumedUpToOffset(), "300");
assertEquals(retried.docIds(), valid.getMutableRoaringBitmap());
assertNotEquals(retried.metadata().docIdsCrc(),
first.metadata().docIdsCrc());
@@ -247,7 +249,7 @@ public class BasePartitionUpsertMetadataManagerTest {
// Shutdown or explicit snapshots without a consumer must not inherit an
earlier startup's offset.
manager.markSegmentAsUpdated(segment);
manager.takeSnapshot();
-
assertNull(segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)
+
assertNull(segment.loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)
.metadata().snapshotConsumedUpToOffset());
}
@@ -269,13 +271,13 @@ public class BasePartitionUpsertMetadataManagerTest {
seg01.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3), null);
upsertMetadataManager.addSegment(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
seg02.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3, 4, 5),
null);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.addSegment(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -297,11 +299,11 @@ public class BasePartitionUpsertMetadataManagerTest {
assertEquals(TEMP_DIR.list().length, 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
4);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
6);
+
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
6);
assertTrue(segDir03.exists());
-
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
3);
}
@Test
@@ -330,13 +332,13 @@ public class BasePartitionUpsertMetadataManagerTest {
seg01.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3), null);
upsertMetadataManager.addSegment(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
seg02.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3, 4, 5),
null);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.addSegment(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -377,11 +379,14 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.doTakeSnapshot();
assertEquals(segmentsTakenSnapshot.size(), 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+ assertEquals(seg01.loadDocIdsFromSnapshot(
+
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 4);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
6);
+ assertEquals(seg02.loadDocIdsFromSnapshot(
+
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 6);
assertTrue(segDir03.exists());
-
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+ assertEquals(seg03.loadDocIdsFromSnapshot(
+
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 3);
} finally {
executor.shutdownNow();
}
@@ -424,7 +429,7 @@ public class BasePartitionUpsertMetadataManagerTest {
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
seg02.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3, 4, 5),
null);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.addSegment(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -437,9 +442,9 @@ public class BasePartitionUpsertMetadataManagerTest {
assertEquals(segmentsTakenSnapshot.size(), 2);
assertEquals(segmentsTakenSnapshot.get(0), "seg02");
assertTrue(segmentsTakenSnapshot.contains("seg03"));
- assertFalse(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
6);
-
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+ assertFalse(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).exists());
+
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
6);
+
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
3);
}
@Test
@@ -460,13 +465,13 @@ public class BasePartitionUpsertMetadataManagerTest {
seg01.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3), null);
upsertMetadataManager.addSegment(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
seg02.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3, 4, 5),
null);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.addSegment(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -489,11 +494,11 @@ public class BasePartitionUpsertMetadataManagerTest {
assertEquals(TEMP_DIR.list().length, 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
4);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
6);
+
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
6);
assertTrue(segDir03.exists());
-
assertNull(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+
assertNull(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
}
// Losing segment is the one that lost all comparisons with docs from other
segments, thus becomes empty of valid
@@ -517,13 +522,13 @@ public class BasePartitionUpsertMetadataManagerTest {
// addSegment() would track seg, and mark it as updated for snapshotting.
upsertMetadataManager.addSegment(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
seg02.enableUpsert(upsertMetadataManager, createDocIds(0, 1, 2, 3, 4, 5),
null);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.addSegment(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -545,11 +550,11 @@ public class BasePartitionUpsertMetadataManagerTest {
assertEquals(TEMP_DIR.list().length, 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
4);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
6);
+
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
6);
assertTrue(segDir03.exists());
-
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(),
3);
}
@Test
@@ -575,8 +580,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg01);
upsertMetadataManager.markSegmentAsUpdated(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
- FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 =
@@ -586,8 +591,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg02);
upsertMetadataManager.markSegmentAsUpdated(seg02);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
- FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.markSegmentWithSnapshot(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -615,11 +620,14 @@ public class BasePartitionUpsertMetadataManagerTest {
assertEquals(TEMP_DIR.list().length, 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+ assertEquals(seg01.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 3);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+ assertEquals(seg02.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 4);
assertTrue(segDir03.exists());
-
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
2);
+ assertEquals(seg03.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 2);
}
@Test
@@ -651,8 +659,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg01);
upsertMetadataManager.markSegmentAsUpdated(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
- FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
@@ -661,8 +669,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg02);
upsertMetadataManager.markSegmentAsUpdated(seg02);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
- FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.markSegmentWithSnapshot(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -705,11 +713,14 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.doTakeSnapshot();
assertEquals(segmentsTakenSnapshot.size(), 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+ assertEquals(seg01.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 3);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+ assertEquals(seg02.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 4);
assertTrue(segDir03.exists());
-
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
2);
+ assertEquals(seg03.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 2);
} finally {
executor.shutdownNow();
}
@@ -736,8 +747,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg01);
upsertMetadataManager.markSegmentAsUpdated(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
- FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
@@ -746,8 +757,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg02);
upsertMetadataManager.markSegmentAsUpdated(seg02);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
- FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.markSegmentWithSnapshot(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -771,11 +782,13 @@ public class BasePartitionUpsertMetadataManagerTest {
assertEquals(TEMP_DIR.list().length, 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+ assertEquals(seg01.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 3);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+ assertEquals(seg02.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 4);
assertTrue(segDir03.exists());
-
assertNull(seg03.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME));
+
assertNull(seg03.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
}
@Test
@@ -800,8 +813,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg01);
upsertMetadataManager.markSegmentAsUpdated(seg01);
// seg01 has a tmp snapshot file, but no snapshot file
- FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
- FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
+ FileUtils.touch(new File(segDir01,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME + "_tmp"));
File segDir02 = new File(TEMP_DIR, "seg02");
ImmutableSegmentImpl seg02 = createImmutableSegment("seg02", segDir02,
segmentsTakenSnapshot, null);
@@ -810,8 +823,8 @@ public class BasePartitionUpsertMetadataManagerTest {
upsertMetadataManager.trackSegment(seg02);
upsertMetadataManager.markSegmentAsUpdated(seg02);
// seg02 has snapshot file, so its snapshot is taken first.
- FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
- FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
+ FileUtils.touch(new File(segDir02,
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
upsertMetadataManager.markSegmentWithSnapshot(seg02);
File segDir03 = new File(TEMP_DIR, "seg03");
@@ -836,12 +849,15 @@ public class BasePartitionUpsertMetadataManagerTest {
assertEquals(TEMP_DIR.list().length, 3);
assertTrue(segDir01.exists());
-
assertEquals(seg01.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
3);
+ assertEquals(seg01.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 3);
assertTrue(segDir02.exists());
-
assertEquals(seg02.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
4);
+ assertEquals(seg02.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 4);
assertTrue(segDir03.exists());
// seg03 should have empty queryableDocIds snapshot due to all documents
being deleted
-
assertEquals(seg03.loadDocIdsFromSnapshot(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME).getCardinality(),
0);
+ assertEquals(seg03.loadDocIdsFromSnapshot(
+
V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME).getCardinality(), 0);
}
@Test
@@ -1341,9 +1357,9 @@ public class BasePartitionUpsertMetadataManagerTest {
ThreadSafeMutableRoaringBitmap.CardinalityAndBytes docIdsSnapshot,
@Nullable DocIdsSnapshot.Metadata metadata)
throws IOException {
onPersist.run();
- if (V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME.equals(fileName)) {
+ if
(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME.equals(fileName)) {
segmentsTakenSnapshot.add(segName);
- } else if
(V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME.equals(fileName)
+ } else if
(V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME.equals(fileName)
&& queryableDocIdsSegmentsTaken != null) {
queryableDocIdsSegmentsTaken.add(segName);
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest.java
index 672ef5e3f48..5e1f8e8cbfd 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest.java
@@ -166,7 +166,7 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
@Nullable List<PrimaryKey> primaryKeys, SegmentMetadataImpl
segmentMetadata, MutableRoaringBitmap snapshot) {
ImmutableSegmentImpl segment = mockImmutableSegment(sequenceNumber,
validDocIds, queryableDocIds, primaryKeys);
when(segment.getSegmentMetadata()).thenReturn(segmentMetadata);
-
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)).thenReturn(snapshot);
+
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)).thenReturn(snapshot);
return segment;
}
@@ -404,37 +404,37 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
public void testGetQueryableDocIds() {
boolean[] deleteFlags1 = new boolean[]{false, false, false, true, true,
false};
int[] docIds1 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot1 = new MutableRoaringBitmap();
- validDocIdsSnapshot1.add(docIds1);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap1 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap1.add(docIds1);
MutableRoaringBitmap queryableDocIds1 = new MutableRoaringBitmap();
queryableDocIds1.add(new int[]{2, 5});
- verifyGetQueryableDocIds(false, deleteFlags1, validDocIdsSnapshot1,
queryableDocIds1);
+ verifyGetQueryableDocIds(false, deleteFlags1, validDocIdsOnDiskBitmap1,
queryableDocIds1);
// all records are not deleted
boolean[] deleteFlags2 = new boolean[]{false, false, false, false, false,
false};
int[] docIds2 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
- validDocIdsSnapshot2.add(docIds2);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap2.add(docIds2);
MutableRoaringBitmap queryableDocIds2 = new MutableRoaringBitmap();
queryableDocIds2.add(docIds2);
- verifyGetQueryableDocIds(false, deleteFlags2, validDocIdsSnapshot2,
queryableDocIds2);
+ verifyGetQueryableDocIds(false, deleteFlags2, validDocIdsOnDiskBitmap2,
queryableDocIds2);
// delete column has null values
boolean[] deleteFlags3 = new boolean[]{false, false, false, false, false,
false};
int[] docIds3 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot3 = new MutableRoaringBitmap();
- validDocIdsSnapshot3.add(docIds3);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap3 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap3.add(docIds3);
MutableRoaringBitmap queryableDocIds3 = new MutableRoaringBitmap();
queryableDocIds3.add(docIds3);
- verifyGetQueryableDocIds(true, deleteFlags3, validDocIdsSnapshot3,
queryableDocIds3);
+ verifyGetQueryableDocIds(true, deleteFlags3, validDocIdsOnDiskBitmap3,
queryableDocIds3);
// All records are deleted record.
boolean[] deleteFlags4 = new boolean[]{true, true, true, true, true, true};
int[] docIds4 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot4 = new MutableRoaringBitmap();
- validDocIdsSnapshot4.add(docIds4);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap4 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap4.add(docIds4);
MutableRoaringBitmap queryableDocIds4 = new MutableRoaringBitmap();
- verifyGetQueryableDocIds(false, deleteFlags4, validDocIdsSnapshot4,
queryableDocIds4);
+ verifyGetQueryableDocIds(false, deleteFlags4, validDocIdsOnDiskBitmap4,
queryableDocIds4);
}
private void verifyAddReplaceRemoveSegment(HashFunction hashFunction)
@@ -458,11 +458,11 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
ImmutableSegmentImpl segment1 =
mockImmutableSegmentWithSegmentMetadata(1, validDocIds1, null,
primaryKeys1, segmentMetadata1, null);
List<RecordInfo> recordInfoList1;
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment1 snapshot: 0 -> {5, 100}, 1 -> {4, 120}, 2 -> {2, 100}
int[] docIds1 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot1 = new MutableRoaringBitmap();
- validDocIdsSnapshot1.add(docIds1);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap1 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap1.add(docIds1);
recordInfoList1 = getRecordInfoList(primaryKeys.length, primaryKeys,
timestamps, null);
upsertMetadataManager.addSegment(segment1, validDocIds1, null,
recordInfoList1.iterator());
trackedSegments.add(segment1);
@@ -485,11 +485,11 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
mockImmutableSegmentWithSegmentMetadata(2, validDocIds2, null,
getPrimaryKeyList(numRecords, primaryKeys),
segmentMetadata2, null);
List<RecordInfo> recordInfoList2;
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment2 snapshot: 0 -> {0, 100}, 2 -> {2, 120}, 3 -> {3, 80}
// segment1 snapshot: 1 -> {4, 120}
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
- validDocIdsSnapshot2.add(0, 2, 3);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap2.add(0, 2, 3);
recordInfoList2 = getRecordInfoList(primaryKeys.length, primaryKeys,
timestamps, null);
upsertMetadataManager.addSegment(segment2, validDocIds2, null,
recordInfoList2.iterator());
trackedSegments.add(segment2);
@@ -754,11 +754,11 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
List<PrimaryKey> primaryKeys1 = getPrimaryKeyList(numRecords, primaryKeys);
ImmutableSegmentImpl segment1 = mockImmutableSegment(1, validDocIds1,
queryableDocIds1, primaryKeys1);
List<RecordInfo> recordInfoList1;
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment1 snapshot: 0 -> {5, 100}, 1 -> {4, 120}, 2 -> {2, 100}
int[] docIds1 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot1 = new MutableRoaringBitmap();
- validDocIdsSnapshot1.add(docIds1);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap1 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap1.add(docIds1);
recordInfoList1 = getRecordInfoList(primaryKeys.length, primaryKeys,
timestamps, deleteFlags);
upsertMetadataManager.addSegment(segment1, validDocIds1, queryableDocIds1,
recordInfoList1.iterator());
trackedSegments.add(segment1);
@@ -783,11 +783,11 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
ImmutableSegmentImpl segment2 = mockImmutableSegmentWithSegmentMetadata(2,
validDocIds2, queryableDocIds2,
getPrimaryKeyList(numRecords, primaryKeys), segmentMetadata2, null);
List<RecordInfo> recordInfoList2;
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment2 snapshot: 0 -> {0, 100}, 2 -> {2, 120}, 3 -> {3, 80}
// segment1 snapshot: 1 -> {4, 120}
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
- validDocIdsSnapshot2.add(0, 2, 3);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap2.add(0, 2, 3);
recordInfoList2 = getRecordInfoList(primaryKeys.length, primaryKeys,
timestamps, deleteFlags);
upsertMetadataManager.addSegment(segment2, validDocIds2, queryableDocIds2,
recordInfoList2.iterator());
trackedSegments.add(segment2);
@@ -922,11 +922,11 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
return recordInfoList;
}
- /// Get recordInfo from validDocIdsSnapshot (enabledSnapshot = True).
- private List<RecordInfo> getRecordInfoList(MutableRoaringBitmap
validDocIdsSnapshot, int[] primaryKeys,
+ /// Get recordInfo from validDocIdsOnDiskBitmap (enabledSnapshot = True).
+ private List<RecordInfo> getRecordInfoList(MutableRoaringBitmap
validDocIdsOnDiskBitmap, int[] primaryKeys,
int[] timestamps, @Nullable boolean[] deleteRecordFlags) {
List<RecordInfo> recordInfoList = new ArrayList<>();
- Iterator<java.lang.Integer> validDocIdsIterator =
validDocIdsSnapshot.iterator();
+ Iterator<java.lang.Integer> validDocIdsIterator =
validDocIdsOnDiskBitmap.iterator();
validDocIdsIterator.forEachRemaining((docId) -> recordInfoList.add(
new RecordInfo(makePrimaryKey(primaryKeys[docId]), docId,
timestamps[docId],
deleteRecordFlags != null && deleteRecordFlags[docId])));
@@ -1309,7 +1309,7 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
}
public void verifyGetQueryableDocIds(boolean isDeleteColumnNull, boolean[]
deleteFlags,
- MutableRoaringBitmap validDocIdsSnapshot, MutableRoaringBitmap
queryableDocIds) {
+ MutableRoaringBitmap validDocIdsOnDiskBitmap, MutableRoaringBitmap
queryableDocIds) {
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes
upsertMetadataManager =
new
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes(REALTIME_TABLE_NAME,
0,
_contextBuilder.build());
@@ -1332,8 +1332,8 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletesTest
ImmutableSegmentImpl segment =
mockImmutableSegmentWithSegmentMetadata(1, new
ThreadSafeMutableRoaringBitmap(), null, null, segmentMetadata,
- validDocIdsSnapshot);
- assertEquals(upsertMetadataManager.getQueryableDocIds(segment,
validDocIdsSnapshot), queryableDocIds);
+ validDocIdsOnDiskBitmap);
+ assertEquals(upsertMetadataManager.getQueryableDocIds(segment,
validDocIdsOnDiskBitmap), queryableDocIds);
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java
index d4dc3ab6aec..89a07c0d65d 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java
@@ -358,7 +358,8 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
ThreadSafeMutableRoaringBitmap snapshot = new
ThreadSafeMutableRoaringBitmap();
snapshot.add(0);
snapshot.add(2);
-
segment2.persistDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME,
snapshot.getBytesAndCardinality(),
+ segment2.persistDocIdsSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME,
snapshot.getBytesAndCardinality(),
null);
upsertMetadataManager.replaceSegment(segment2, segment1);
@@ -382,37 +383,37 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
boolean[] deleteFlags1 = new boolean[]{false, false, false, true, true,
false};
int[] docIds1 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot1 = new MutableRoaringBitmap();
- validDocIdsSnapshot1.add(docIds1);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap1 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap1.add(docIds1);
MutableRoaringBitmap queryableDocIds1 = new MutableRoaringBitmap();
queryableDocIds1.add(new int[]{2, 5});
- verifyGetQueryableDocIds(false, deleteFlags1, validDocIdsSnapshot1,
queryableDocIds1);
+ verifyGetQueryableDocIds(false, deleteFlags1, validDocIdsOnDiskBitmap1,
queryableDocIds1);
// all records are not deleted
boolean[] deleteFlags2 = new boolean[]{false, false, false, false, false,
false};
int[] docIds2 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
- validDocIdsSnapshot2.add(docIds2);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap2.add(docIds2);
MutableRoaringBitmap queryableDocIds2 = new MutableRoaringBitmap();
queryableDocIds2.add(docIds2);
- verifyGetQueryableDocIds(false, deleteFlags2, validDocIdsSnapshot2,
queryableDocIds2);
+ verifyGetQueryableDocIds(false, deleteFlags2, validDocIdsOnDiskBitmap2,
queryableDocIds2);
// delete column has null values
boolean[] deleteFlags3 = new boolean[]{false, false, false, false, false,
false};
int[] docIds3 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot3 = new MutableRoaringBitmap();
- validDocIdsSnapshot3.add(docIds3);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap3 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap3.add(docIds3);
MutableRoaringBitmap queryableDocIds3 = new MutableRoaringBitmap();
queryableDocIds3.add(docIds3);
- verifyGetQueryableDocIds(true, deleteFlags3, validDocIdsSnapshot3,
queryableDocIds3);
+ verifyGetQueryableDocIds(true, deleteFlags3, validDocIdsOnDiskBitmap3,
queryableDocIds3);
// All records are deleted record.
boolean[] deleteFlags4 = new boolean[]{true, true, true, true, true, true};
int[] docIds4 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot4 = new MutableRoaringBitmap();
- validDocIdsSnapshot4.add(docIds4);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap4 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap4.add(docIds4);
MutableRoaringBitmap queryableDocIds4 = new MutableRoaringBitmap();
- verifyGetQueryableDocIds(false, deleteFlags4, validDocIdsSnapshot4,
queryableDocIds4);
+ verifyGetQueryableDocIds(false, deleteFlags4, validDocIdsOnDiskBitmap4,
queryableDocIds4);
}
private void verifyAddReplaceRemoveSegment(HashFunction hashFunction,
boolean enableSnapshot)
@@ -432,12 +433,12 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
ImmutableSegmentImpl segment1 = mockImmutableSegment(1, validDocIds1,
null, primaryKeys1);
List<RecordInfo> recordInfoList1;
if (enableSnapshot) {
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment1 snapshot: 0 -> {5, 100}, 1 -> {4, 120}, 2 -> {2, 100}
int[] docIds1 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot1 = new MutableRoaringBitmap();
- validDocIdsSnapshot1.add(docIds1);
- recordInfoList1 = getRecordInfoList(validDocIdsSnapshot1, primaryKeys,
timestamps, null);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap1 = new
MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap1.add(docIds1);
+ recordInfoList1 = getRecordInfoList(validDocIdsOnDiskBitmap1,
primaryKeys, timestamps, null);
} else {
// get recordInfo by iterating all records.
recordInfoList1 = getRecordInfoList(numRecords, primaryKeys, timestamps,
null);
@@ -460,12 +461,12 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
mockImmutableSegment(2, validDocIds2, null,
getPrimaryKeyList(numRecords, primaryKeys));
List<RecordInfo> recordInfoList2;
if (enableSnapshot) {
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment2 snapshot: 0 -> {0, 100}, 2 -> {2, 120}, 3 -> {3, 80}
// segment1 snapshot: 1 -> {4, 120}
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
- validDocIdsSnapshot2.add(0, 2, 3);
- recordInfoList2 = getRecordInfoList(validDocIdsSnapshot2, primaryKeys,
timestamps, null);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new
MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap2.add(0, 2, 3);
+ recordInfoList2 = getRecordInfoList(validDocIdsOnDiskBitmap2,
primaryKeys, timestamps, null);
} else {
// get recordInfo by iterating all records.
recordInfoList2 = getRecordInfoList(numRecords, primaryKeys, timestamps,
null);
@@ -732,12 +733,12 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
ImmutableSegmentImpl segment1 = mockImmutableSegment(1, validDocIds1,
queryableDocIds1, primaryKeys1);
List<RecordInfo> recordInfoList1;
if (enableSnapshot) {
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment1 snapshot: 0 -> {5, 100}, 1 -> {4, 120}, 2 -> {2, 100}
int[] docIds1 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot1 = new MutableRoaringBitmap();
- validDocIdsSnapshot1.add(docIds1);
- recordInfoList1 = getRecordInfoList(validDocIdsSnapshot1, primaryKeys,
timestamps, deleteFlags);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap1 = new
MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap1.add(docIds1);
+ recordInfoList1 = getRecordInfoList(validDocIdsOnDiskBitmap1,
primaryKeys, timestamps, deleteFlags);
} else {
// get recordInfo by iterating all records.
recordInfoList1 = getRecordInfoList(numRecords, primaryKeys, timestamps,
deleteFlags);
@@ -763,12 +764,12 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
mockImmutableSegment(2, validDocIds2, queryableDocIds2,
getPrimaryKeyList(numRecords, primaryKeys));
List<RecordInfo> recordInfoList2;
if (enableSnapshot) {
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment2 snapshot: 0 -> {0, 100}, 2 -> {2, 120}, 3 -> {3, 80}
// segment1 snapshot: 1 -> {4, 120}
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
- validDocIdsSnapshot2.add(0, 2, 3);
- recordInfoList2 = getRecordInfoList(validDocIdsSnapshot2, primaryKeys,
timestamps, deleteFlags);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new
MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap2.add(0, 2, 3);
+ recordInfoList2 = getRecordInfoList(validDocIdsOnDiskBitmap2,
primaryKeys, timestamps, deleteFlags);
} else {
// get recordInfo by iterating all records.
recordInfoList2 = getRecordInfoList(numRecords, primaryKeys, timestamps,
deleteFlags);
@@ -916,11 +917,11 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
return recordInfoList;
}
- /// Get recordInfo from validDocIdsSnapshot (enabledSnapshot = True).
- private List<RecordInfo> getRecordInfoList(MutableRoaringBitmap
validDocIdsSnapshot, int[] primaryKeys,
+ /// Get recordInfo from validDocIdsOnDiskBitmap (enabledSnapshot = True).
+ private List<RecordInfo> getRecordInfoList(MutableRoaringBitmap
validDocIdsOnDiskBitmap, int[] primaryKeys,
int[] timestamps, @Nullable boolean[] deleteRecordFlags) {
List<RecordInfo> recordInfoList = new ArrayList<>();
- Iterator<Integer> validDocIdsIterator = validDocIdsSnapshot.iterator();
+ Iterator<Integer> validDocIdsIterator = validDocIdsOnDiskBitmap.iterator();
validDocIdsIterator.forEachRemaining((docId) -> recordInfoList.add(
new RecordInfo(makePrimaryKey(primaryKeys[docId]), docId, new
IntWrapper(timestamps[docId]),
deleteRecordFlags != null && deleteRecordFlags[docId])));
@@ -1109,9 +1110,9 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
when(segmentMetadata.getColumnMetadataFor(comparisonColumns.get(0))).thenReturn(columnMetadata);
doReturn(endTime).when(columnMetadata).getMaxValue();
if (snapshot != null) {
-
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)).thenReturn(snapshot);
+
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)).thenReturn(snapshot);
} else {
-
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)).thenReturn(
+
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)).thenReturn(
validDocIds.getMutableRoaringBitmap());
}
return segment;
@@ -1122,7 +1123,7 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
List<PrimaryKey> primaryKeys, SegmentMetadataImpl segmentMetadata,
MutableRoaringBitmap snapshot) {
ImmutableSegmentImpl segment = mockImmutableSegment(sequenceNumber,
validDocIds, queryableDocIds, primaryKeys);
when(segment.getSegmentMetadata()).thenReturn(segmentMetadata);
-
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)).thenReturn(snapshot);
+
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)).thenReturn(snapshot);
return segment;
}
@@ -1471,7 +1472,7 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
ImmutableSegmentImpl segment = mock(ImmutableSegmentImpl.class);
when(segment.getSegmentName()).thenReturn("emptyValidDocIdsSegment");
-
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)).thenReturn(
+
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)).thenReturn(
new MutableRoaringBitmap());
upsertMetadataManager.preloadSegment(segment);
@@ -1499,7 +1500,7 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
ImmutableSegmentImpl segment = mock(ImmutableSegmentImpl.class);
when(segment.getSegmentName()).thenReturn("emptyValidDocIdsSegment");
-
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)).thenReturn(
+
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)).thenReturn(
new MutableRoaringBitmap());
upsertMetadataManager.preloadSegment(segment);
@@ -1826,12 +1827,12 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
// Add a segment with segmentEndTime: 80, largest seen timestamp: 120. the
segment will be skipped.
List<PrimaryKey> primaryKeys2 = getPrimaryKeyList(numRecords, new
int[]{100, 101, 102, 103});
int[] docIds2 = new int[]{0, 1};
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
- validDocIdsSnapshot2.add(docIds2);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap2.add(docIds2);
ThreadSafeMutableRoaringBitmap validDocIds2 = new
ThreadSafeMutableRoaringBitmap();
ImmutableSegmentImpl segment2 =
mockImmutableSegmentWithEndTime(1, validDocIds2, null, primaryKeys2,
COMPARISON_COLUMNS, new Double(80),
- validDocIdsSnapshot2);
+ validDocIdsOnDiskBitmap2);
upsertMetadataManager.addSegment(segment2);
// out of ttl segment should not be added to recordLocationMap
assertEquals(recordLocationMap.size(), 5);
@@ -1866,16 +1867,16 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
List<PrimaryKey> primaryKeys1 = getPrimaryKeyList(numRecords, primaryKeys);
int[] docIds1 = new int[]{2, 4, 5};
- MutableRoaringBitmap validDocIdsSnapshot1 = new MutableRoaringBitmap();
- validDocIdsSnapshot1.add(docIds1);
+ MutableRoaringBitmap validDocIdsOnDiskBitmap1 = new MutableRoaringBitmap();
+ validDocIdsOnDiskBitmap1.add(docIds1);
ImmutableSegmentImpl segment1 =
mockImmutableSegmentWithEndTime(1, validDocIds1, queryableDocIds1,
primaryKeys1, COMPARISON_COLUMNS,
- new Double(120), validDocIdsSnapshot1);
+ new Double(120), validDocIdsOnDiskBitmap1);
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment1 snapshot: 0 -> {5, 100}, 1 -> {4, 120}, 2 -> {2, 100}
List<RecordInfo> recordInfoList1;
- recordInfoList1 = getRecordInfoList(validDocIdsSnapshot1, primaryKeys,
timestamps, deleteFlags);
+ recordInfoList1 = getRecordInfoList(validDocIdsOnDiskBitmap1, primaryKeys,
timestamps, deleteFlags);
upsertMetadataManager.addSegment(segment1, validDocIds1, queryableDocIds1,
recordInfoList1.iterator());
trackedSegments.add(segment1);
@@ -1894,19 +1895,19 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
deleteFlags = new boolean[]{false, false, true, false, true};
ThreadSafeMutableRoaringBitmap validDocIds2 = new
ThreadSafeMutableRoaringBitmap();
ThreadSafeMutableRoaringBitmap queryableDocIds2 = new
ThreadSafeMutableRoaringBitmap();
- MutableRoaringBitmap validDocIdsSnapshot2 = new MutableRoaringBitmap();
+ MutableRoaringBitmap validDocIdsOnDiskBitmap2 = new MutableRoaringBitmap();
int[] docIds2 = new int[]{3, 4};
- validDocIdsSnapshot2.add(docIds2);
+ validDocIdsOnDiskBitmap2.add(docIds2);
ImmutableSegmentImpl segment2 =
mockImmutableSegmentWithEndTime(2, validDocIds2, queryableDocIds2,
getPrimaryKeyList(numRecords, primaryKeys),
- COMPARISON_COLUMNS, new Double(40), validDocIdsSnapshot2);
+ COMPARISON_COLUMNS, new Double(40), validDocIdsOnDiskBitmap2);
- // get recordInfo from validDocIdSnapshot.
+ // get recordInfo from validDocIdsOnDiskBitmap.
// segment2 snapshot: 3 -> {3, 40}, 4 -> {4, 40}
// segment1 snapshot: 0 -> {5, 100}, 1 -> {4, 120}, 2 -> {2, 100}
List<RecordInfo> recordInfoList2;
- recordInfoList2 = getRecordInfoList(validDocIdsSnapshot2, primaryKeys,
timestamps, deleteFlags);
+ recordInfoList2 = getRecordInfoList(validDocIdsOnDiskBitmap2, primaryKeys,
timestamps, deleteFlags);
upsertMetadataManager.addSegment(segment2, validDocIds2, queryableDocIds2,
recordInfoList2.iterator());
trackedSegments.add(segment2);
@@ -1932,7 +1933,7 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
}
public void verifyGetQueryableDocIds(boolean isDeleteColumnNull, boolean[]
deleteFlags,
- MutableRoaringBitmap validDocIdsSnapshot, MutableRoaringBitmap
queryableDocIds) {
+ MutableRoaringBitmap validDocIdsOnDiskBitmap, MutableRoaringBitmap
queryableDocIds) {
ConcurrentMapPartitionUpsertMetadataManager upsertMetadataManager =
new ConcurrentMapPartitionUpsertMetadataManager(REALTIME_TABLE_NAME,
0, _contextBuilder.build());
@@ -1954,8 +1955,8 @@ public class
ConcurrentMapPartitionUpsertMetadataManagerTest {
ImmutableSegmentImpl segment =
mockImmutableSegmentWithSegmentMetadata(1, new
ThreadSafeMutableRoaringBitmap(), null, null, segmentMetadata,
- validDocIdsSnapshot);
- assertEquals(upsertMetadataManager.getQueryableDocIds(segment,
validDocIdsSnapshot), queryableDocIds);
+ validDocIdsOnDiskBitmap);
+ assertEquals(upsertMetadataManager.getQueryableDocIds(segment,
validDocIdsOnDiskBitmap), queryableDocIds);
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/UpsertSnapshotRestartTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/UpsertSnapshotRestartTest.java
index dc1723d5f58..c2c5b9b238a 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/UpsertSnapshotRestartTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/UpsertSnapshotRestartTest.java
@@ -279,7 +279,7 @@ public class UpsertSnapshotRestartTest {
throws Exception {
ImmutableSegmentImpl segment = load(segmentName);
try {
- return
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME) !=
null;
+ return
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)
!= null;
} finally {
segment.destroy();
}
@@ -289,7 +289,7 @@ public class UpsertSnapshotRestartTest {
throws Exception {
ImmutableSegmentImpl segment = load(segmentName);
try {
- return
String.valueOf(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ return
String.valueOf(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
} finally {
segment.destroy();
}
@@ -324,7 +324,7 @@ public class UpsertSnapshotRestartTest {
List<ImmutableSegmentImpl> segments = addSegBFirst ? List.of(segB, segA) :
List.of(segA, segB);
List<ImmutableSegmentImpl> segmentsToAdd = new ArrayList<>();
for (ImmutableSegmentImpl segment : segments) {
- if (preload &&
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME) !=
null) {
+ if (preload &&
segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME)
!= null) {
manager.preloadSegment(segment);
} else {
segmentsToAdd.add(segment);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtilsTest.java
index e1561916091..93e5793c8d0 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreloadUtilsTest.java
@@ -131,7 +131,7 @@ public class SegmentPreloadUtilsTest {
File seg02IdxDir = new File(tableDataDir, seg02Name);
FileUtils.forceMkdir(seg02IdxDir);
- FileUtils.touch(new File(new File(seg02IdxDir, "v3"),
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ FileUtils.touch(new File(new File(seg02IdxDir, "v3"),
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
when(tableDataManager.getSegmentDataDir(seg02Name, null,
tableConfig)).thenReturn(seg02IdxDir);
try {
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
index c0fdf9a3717..39d12a733ea 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/V1Constants.java
@@ -25,8 +25,17 @@ public class V1Constants {
public static final String SEGMENT_CREATION_META = "creation.meta";
public static final String INDEX_MAP_FILE_NAME = "index_map";
public static final String INDEX_FILE_NAME = "columns.psf";
- public static final String VALID_DOC_IDS_SNAPSHOT_FILE_NAME =
"validdocids.bitmap.snapshot";
- public static final String QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME =
"queryabledocids.bitmap.snapshot";
+ /// The upsert validDocIds bitmap persisted next to the segment. The file
name is unchanged, so existing files load.
+ public static final String VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME =
"validdocids.bitmap.snapshot";
+ /// The upsert queryableDocIds bitmap persisted next to the segment. The
file name is unchanged, so existing files
+ /// load.
+ public static final String QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME =
"queryabledocids.bitmap.snapshot";
+ /// @deprecated Use [#VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME]. Kept so
plugins built against it still compile.
+ @Deprecated
+ public static final String VALID_DOC_IDS_SNAPSHOT_FILE_NAME =
VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME;
+ /// @deprecated Use [#QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME]. Kept so
plugins built against it still compile.
+ @Deprecated
+ public static final String QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME =
QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME;
public static final String TTL_WATERMARK_TABLE_PARTITION =
"ttl.watermark.partition.";
public static class Str {
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 16dcf0f1fad..1a0288d406c 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -841,17 +841,18 @@ public class TablesResource {
if (validDocIdsTypeStr == null) {
// By default, we read the valid doc ids from snapshot.
return Pair.of(ValidDocIdsType.SNAPSHOT,
- ((ImmutableSegmentImpl)
indexSegment).loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ ((ImmutableSegmentImpl)
indexSegment).loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
}
ValidDocIdsType validDocIdsType =
ValidDocIdsType.valueOf(validDocIdsTypeStr.toUpperCase());
switch (validDocIdsType) {
case SNAPSHOT:
return Pair.of(validDocIdsType,
- ((ImmutableSegmentImpl)
indexSegment).loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ ((ImmutableSegmentImpl) indexSegment).loadDocIdsSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
case SNAPSHOT_WITH_DELETE:
return Pair.of(validDocIdsType,
((ImmutableSegmentImpl) indexSegment).loadDocIdsSnapshot(
- V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME));
+ V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
case IN_MEMORY:
return Pair.of(validDocIdsType,
new
DocIdsSnapshot(indexSegment.getValidDocIds().getMutableRoaringBitmap(), null));
@@ -863,7 +864,8 @@ public class TablesResource {
LOGGER.warn("Invalid validDocIdsType: {}. Using default
validDocIdsType: {}", validDocIdsType,
ValidDocIdsType.SNAPSHOT);
return Pair.of(ValidDocIdsType.SNAPSHOT,
- ((ImmutableSegmentImpl)
indexSegment).loadDocIdsSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME));
+ ((ImmutableSegmentImpl) indexSegment).loadDocIdsSnapshot(
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME));
}
}
diff --git
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index 750eff6aab3..87e8c831c5d 100644
---
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -493,8 +493,8 @@ public class TablesResourceTest extends BaseResourceTest {
// Upgrade both legacy fixtures to real persisted captures, without
changing the live bitmaps.
DocIdsSnapshot.Trigger trigger = new
DocIdsSnapshot.Trigger("testTable__0__2__0", "123");
Map<String, DocIdsSnapshot.DocIdsType> fileTypes =
- Map.of(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME,
DocIdsSnapshot.DocIdsType.VALID_DOC_IDS,
- V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME,
DocIdsSnapshot.DocIdsType.QUERYABLE_DOC_IDS);
+ Map.of(V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME,
DocIdsSnapshot.DocIdsType.VALID_DOC_IDS,
+ V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME,
DocIdsSnapshot.DocIdsType.QUERYABLE_DOC_IDS);
for (Map.Entry<String, DocIdsSnapshot.DocIdsType> fileType :
fileTypes.entrySet()) {
ThreadSafeMutableRoaringBitmap bitmap =
new
ThreadSafeMutableRoaringBitmap(segment.loadDocIdsFromSnapshot(fileType.getKey()));
@@ -516,8 +516,8 @@ public class TablesResourceTest extends BaseResourceTest {
return;
}
JsonNode diagnostics = metadata.get("diagnostics");
- String file = expectedType.equals("SNAPSHOT") ?
V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME
- : V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME;
+ String file = expectedType.equals("SNAPSHOT") ?
V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME
+ : V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME;
long capturedAt =
segment.loadDocIdsSnapshot(file).metadata().snapshotCapturedAtMs();
assertEquals(diagnostics.get("snapshotCapturedAtMs").asLong(), capturedAt);
assertTrue(diagnostics.get("snapshotAgeMs").asLong() >= beforeRequest -
capturedAt);
@@ -585,26 +585,26 @@ public class TablesResourceTest extends BaseResourceTest {
when(upsertMetadataManager.getLastSnapshotPass()).thenReturn(new
SnapshotPass(1234L, 2, true, 5678L));
ThreadSafeMutableRoaringBitmap validDocIds = new
ThreadSafeMutableRoaringBitmap();
ThreadSafeMutableRoaringBitmap queryableDocIds = new
ThreadSafeMutableRoaringBitmap();
- ThreadSafeMutableRoaringBitmap validDocIdsSnapshot = new
ThreadSafeMutableRoaringBitmap();
+ ThreadSafeMutableRoaringBitmap validDocIdsOnDiskBitmap = new
ThreadSafeMutableRoaringBitmap();
int[] docIds = new int[]{1, 4, 6, 10, 15, 17, 18, 20};
for (int docId : docIds) {
validDocIds.add(docId);
queryableDocIds.add(docId + 1);
- validDocIdsSnapshot.add(docId + 2);
+ validDocIdsOnDiskBitmap.add(docId + 2);
}
segment.enableUpsert(upsertMetadataManager, validDocIds, queryableDocIds);
- File validDocIdsSnapshotFile =
+ File validDocIdsOnDiskBitmapFile =
new
File(SegmentDirectoryPaths.findSegmentDirectory(segment.getSegmentMetadata().getIndexDir()),
- V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME);
- FileUtils.writeByteArrayToFile(validDocIdsSnapshotFile,
-
RoaringBitmapUtils.serialize(validDocIdsSnapshot.getMutableRoaringBitmap()));
+ V1Constants.VALID_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
+ FileUtils.writeByteArrayToFile(validDocIdsOnDiskBitmapFile,
+
RoaringBitmapUtils.serialize(validDocIdsOnDiskBitmap.getMutableRoaringBitmap()));
// Create the queryableDocIds snapshot file needed for SNAPSHOT_WITH_DELETE
- File queryableDocIdsSnapshotFile =
+ File queryableDocIdsOnDiskBitmapFile =
new
File(SegmentDirectoryPaths.findSegmentDirectory(segment.getSegmentMetadata().getIndexDir()),
- V1Constants.QUERYABLE_DOC_IDS_SNAPSHOT_FILE_NAME);
- FileUtils.writeByteArrayToFile(queryableDocIdsSnapshotFile,
+ V1Constants.QUERYABLE_DOC_IDS_ON_DISK_BITMAP_FILE_NAME);
+ FileUtils.writeByteArrayToFile(queryableDocIdsOnDiskBitmapFile,
RoaringBitmapUtils.serialize(queryableDocIds.getMutableRoaringBitmap()));
// Check no type (default should be validDocIdsSnapshot)
@@ -615,7 +615,7 @@ public class TablesResourceTest extends BaseResourceTest {
byte[] validDocIdsSnapshotBitmap = response.getBitmap();
assertNotNull(validDocIdsSnapshotBitmap);
assertEquals(new
ImmutableRoaringBitmap(ByteBuffer.wrap(validDocIdsSnapshotBitmap)).toMutableRoaringBitmap(),
- validDocIdsSnapshot.getMutableRoaringBitmap());
+ validDocIdsOnDiskBitmap.getMutableRoaringBitmap());
// Check snapshot type
response = _webTarget.path(snapshotPath)
@@ -628,7 +628,7 @@ public class TablesResourceTest extends BaseResourceTest {
validDocIdsSnapshotBitmap = response.getBitmap();
assertNotNull(validDocIdsSnapshotBitmap);
assertEquals(new
ImmutableRoaringBitmap(ByteBuffer.wrap(validDocIdsSnapshotBitmap)).toMutableRoaringBitmap(),
- validDocIdsSnapshot.getMutableRoaringBitmap());
+ validDocIdsOnDiskBitmap.getMutableRoaringBitmap());
// Check onHeap type
response = _webTarget.path(snapshotPath)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
