This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a60eae69f06 Compute star-tree record offsets arithmetically for
fixed-size records (#19318)
a60eae69f06 is described below
commit a60eae69f065ebadba2db75f090de4649f5ccd00
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Aug 20 09:20:08 2026 -0700
Compute star-tree record offsets arithmetically for fixed-size records
(#19318)
---
.../v2/builder/OffHeapSingleTreeBuilder.java | 83 +++++++++++++++++++---
.../v2/builder/OffHeapSingleTreeBuilderTest.java | 18 ++++-
2 files changed, 89 insertions(+), 12 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
index 390216e4cc7..f6ab03fa580 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
@@ -47,7 +47,7 @@ public class OffHeapSingleTreeBuilder extends
BaseSingleTreeBuilder {
private final File _segmentRecordFile;
private final File _starTreeRecordFile;
private final BufferedOutputStream _starTreeRecordOutputStream;
- private final RecordOffsets _starTreeRecordOffsets = new RecordOffsets();
+ private final RecordOffsets _starTreeRecordOffsets;
private PinotDataBuffer _starTreeRecordBuffer;
private int _numReadableStarTreeRecords;
@@ -63,6 +63,27 @@ public class OffHeapSingleTreeBuilder extends
BaseSingleTreeBuilder {
Preconditions.checkState(!_starTreeRecordFile.exists(), "Star-tree record
file: %s already exists",
_starTreeRecordFile);
_starTreeRecordOutputStream = new BufferedOutputStream(new
FileOutputStream(_starTreeRecordFile));
+ _starTreeRecordOffsets = createRecordOffsets();
+ }
+
+ /// Returns [FixedSizeRecordOffsets] when all metrics are serialized with a
fixed size (see
+ /// [#serializeStarTreeRecord]), where the record start offsets can be
computed arithmetically without being stored;
+ /// otherwise returns [VariableSizeRecordOffsets].
+ private RecordOffsets createRecordOffsets() {
+ int recordSize = _numDimensions * Integer.BYTES;
+ for (int i = 0; i < _numMetrics; i++) {
+ switch (_valueAggregators[i].getAggregatedValueType()) {
+ case LONG:
+ recordSize += Long.BYTES;
+ break;
+ case DOUBLE:
+ recordSize += Double.BYTES;
+ break;
+ default:
+ return new VariableSizeRecordOffsets();
+ }
+ }
+ return new FixedSizeRecordOffsets(recordSize);
}
@SuppressWarnings("unchecked")
@@ -339,18 +360,58 @@ public class OffHeapSingleTreeBuilder extends
BaseSingleTreeBuilder {
FileUtils.forceDelete(_starTreeRecordFile);
}
- /// Memory-efficient list of record offsets within the star-tree record
file, tracked as a prefix sum of the appended
- /// record lengths. Start offsets are stored as `int` (4 bytes per record)
until the first record starting beyond
- /// `Integer.MAX_VALUE`, and as `long` (8 bytes per record) afterwards. The
number of star-tree records can go into
- /// the hundreds of millions for large segments, where a boxed `List<Long>`
(~28 bytes per record) would dominate the
- /// heap.
+ /// Per-record offsets within the star-tree record file. [#addRecord] is
invoked once per appended record with the
+ /// serialized record length; [#getStartOffset] and [#getEndOffset] return
absolute offsets within the file.
+ @VisibleForTesting
+ interface RecordOffsets {
+
+ void addRecord(int numBytes);
+
+ long getStartOffset(int index);
+
+ long getEndOffset();
+ }
+
+ /// [RecordOffsets] for fixed-size records (all metrics serialized with a
fixed size), where the offsets are
+ /// computed arithmetically without being stored.
+ @VisibleForTesting
+ static class FixedSizeRecordOffsets implements RecordOffsets {
+ private final int _recordSize;
+ private int _numRecords;
+
+ FixedSizeRecordOffsets(int recordSize) {
+ _recordSize = recordSize;
+ }
+
+ @Override
+ public void addRecord(int numBytes) {
+ assert numBytes == _recordSize;
+ _numRecords++;
+ }
+
+ @Override
+ public long getStartOffset(int index) {
+ return (long) index * _recordSize;
+ }
+
+ @Override
+ public long getEndOffset() {
+ return (long) _numRecords * _recordSize;
+ }
+ }
+
+ /// [RecordOffsets] for variable-size records, tracked as a prefix sum of
the appended record lengths. Start offsets
+ /// are stored as `int` (4 bytes per record) until the first record starting
beyond `Integer.MAX_VALUE`, and as
+ /// `long` (8 bytes per record) afterwards. The number of star-tree records
can go into the hundreds of millions for
+ /// large segments, where a boxed `List<Long>` (~28 bytes per record) would
dominate the heap.
@VisibleForTesting
- static class RecordOffsets {
+ static class VariableSizeRecordOffsets implements RecordOffsets {
private final IntArrayList _intOffsets = new IntArrayList();
private final LongArrayList _longOffsets = new LongArrayList();
private long _endOffset;
- void addRecord(int numBytes) {
+ @Override
+ public void addRecord(int numBytes) {
if (_endOffset <= Integer.MAX_VALUE) {
_intOffsets.add((int) _endOffset);
} else {
@@ -359,12 +420,14 @@ public class OffHeapSingleTreeBuilder extends
BaseSingleTreeBuilder {
_endOffset += numBytes;
}
- long getStartOffset(int index) {
+ @Override
+ public long getStartOffset(int index) {
int numIntOffsets = _intOffsets.size();
return index < numIntOffsets ? _intOffsets.getInt(index) :
_longOffsets.getLong(index - numIntOffsets);
}
- long getEndOffset() {
+ @Override
+ public long getEndOffset() {
return _endOffset;
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilderTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilderTest.java
index 4b76d6f81e5..19e2cba5e31 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilderTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilderTest.java
@@ -18,7 +18,9 @@
*/
package org.apache.pinot.segment.local.startree.v2.builder;
+import
org.apache.pinot.segment.local.startree.v2.builder.OffHeapSingleTreeBuilder.FixedSizeRecordOffsets;
import
org.apache.pinot.segment.local.startree.v2.builder.OffHeapSingleTreeBuilder.RecordOffsets;
+import
org.apache.pinot.segment.local.startree.v2.builder.OffHeapSingleTreeBuilder.VariableSizeRecordOffsets;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@@ -27,8 +29,20 @@ import static org.testng.Assert.assertEquals;
public class OffHeapSingleTreeBuilderTest {
@Test
- public void testRecordOffsets() {
- RecordOffsets offsets = new RecordOffsets();
+ public void testFixedSizeRecordOffsets() {
+ RecordOffsets offsets = new FixedSizeRecordOffsets(1 << 30);
+ for (int i = 0; i < 4; i++) {
+ offsets.addRecord(1 << 30);
+ }
+ assertEquals(offsets.getStartOffset(0), 0L);
+ assertEquals(offsets.getStartOffset(1), 1L << 30);
+ assertEquals(offsets.getStartOffset(3), 3L << 30);
+ assertEquals(offsets.getEndOffset(), 1L << 32);
+ }
+
+ @Test
+ public void testVariableSizeRecordOffsets() {
+ RecordOffsets offsets = new VariableSizeRecordOffsets();
offsets.addRecord(123);
offsets.addRecord(Integer.MAX_VALUE - 123);
offsets.addRecord(456);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
