This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0c1e9bd3564 Skip stats collection for raw constant columns (#19164)
0c1e9bd3564 is described below
commit 0c1e9bd356444c4f7687269b4f8f2058e412def4
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Wed Aug 5 17:24:56 2026 -0700
Skip stats collection for raw constant columns (#19164)
---
.../stats/CompactedNoDictColumnStatistics.java | 24 ++++-
.../stats/MutableNoDictColumnStatistics.java | 7 ++
.../stats/CompactedNoDictColumnStatisticsTest.java | 114 ++++++++++++++++++++-
.../stats/MutableNoDictColumnStatisticsTest.java | 41 ++++++++
4 files changed, 180 insertions(+), 6 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
index c93205b3768..6d01827bfae 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
@@ -74,13 +74,32 @@ public class CompactedNoDictColumnStatistics extends
MutableNoDictColumnStatisti
int maxRowLength = 0;
if (isSingleValue) {
- if (_sortedDocIds != null) {
+ totalEntries = _totalDocs;
+
+ // Min and max are tracked per raw value during ingestion, so equal
bounds identify a constant column without
+ // any scan. They are left null when aggregated metrics are enabled, in
which case the scan below still runs.
+ // The bounds span the whole segment, so this is sufficient but not
necessary: a column that is constant only
+ // among the valid documents is still scanned.
+ Comparable segmentMinValue = _dataSourceMetadata.getMinValue();
+ if (segmentMinValue != null &&
segmentMinValue.equals(_dataSourceMetadata.getMaxValue())) {
+ // Every document holds the same value, so every stat collected below
collapses to that one value
+ minValue = segmentMinValue;
+ maxValue = segmentMinValue;
+ if (isVariableWidth) {
+ int length = getElementLength(segmentMinValue, storedType);
+ minElementLength = length;
+ maxElementLength = length;
+ if (isAscii) {
+ isAscii = length == ((String) segmentMinValue).length();
+ }
+ }
+ isSorted = true;
+ } else if (_sortedDocIds != null) {
// Iterate in sorted doc order, filtered to valid docs, to track
sortedness inline
for (int docId : _sortedDocIds) {
if (!validDocIds.contains(docId)) {
continue;
}
- totalEntries++;
Comparable value = readValue(docId, storedType);
if (minValue == null || value.compareTo(minValue) < 0) {
minValue = value;
@@ -107,7 +126,6 @@ public class CompactedNoDictColumnStatistics extends
MutableNoDictColumnStatisti
IntIterator iterator = validDocIds.getIntIterator();
while (iterator.hasNext()) {
int docId = iterator.next();
- totalEntries++;
Comparable value = readValue(docId, storedType);
if (minValue == null || value.compareTo(minValue) < 0) {
minValue = value;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java
index 91766579579..7896c13f96d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java
@@ -129,6 +129,13 @@ public class MutableNoDictColumnStatistics implements
ColumnStatistics, CLPStats
return false;
}
+ // A single distinct value is always sorted — no scan needed. Min and max
are tracked per raw value during
+ // ingestion, but are left null when aggregated metrics are enabled, so
fall back to the scan when unavailable.
+ Comparable<?> minValue = getMinValue();
+ if (minValue != null && minValue.equals(getMaxValue())) {
+ return true;
+ }
+
int numDocs = _dataSourceMetadata.getNumDocs();
// Verify that values are non-decreasing when iterated in the given order.
The BYTES path uses
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatisticsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatisticsTest.java
index b956fd4ce80..51dd1a8abf7 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatisticsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatisticsTest.java
@@ -25,13 +25,12 @@ import
org.apache.pinot.segment.spi.index.mutable.MutableForwardIndex;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.roaringbitmap.RoaringBitmap;
import org.testng.annotations.Test;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
@@ -42,6 +41,115 @@ import static org.testng.Assert.assertTrue;
/// doc sets, and edge cases such as empty bitmaps and the `isSortedColumn`
flag.
public class CompactedNoDictColumnStatisticsTest {
+ // ======== Constant value ========
+
+ @Test
+ public void testConstantValueSkipsScan() {
+ MutableForwardIndex forwardIndex = mockForwardIndex(DataType.INT, true);
+
+ DataSource dataSource = mockDataSource(forwardIndex);
+ DataSourceMetadata metadata = dataSource.getDataSourceMetadata();
+ when(metadata.getMinValue()).thenReturn(42);
+ when(metadata.getMaxValue()).thenReturn(42);
+
+ RoaringBitmap validDocIds = RoaringBitmap.bitmapOf(0, 1, 2);
+ CompactedNoDictColumnStatistics stats =
+ new CompactedNoDictColumnStatistics(dataSource, null, false,
validDocIds);
+
+ assertEquals(stats.getMinValue(), 42);
+ assertEquals(stats.getMaxValue(), 42);
+ assertTrue(stats.isSorted());
+ assertEquals(stats.getTotalNumberOfEntries(), 3);
+ // Every document holds the same value, so the forward index is never read
+ verify(forwardIndex, never()).getInt(anyInt());
+ }
+
+ @Test
+ public void testConstantBigDecimalDerivesLengthWithoutScan() {
+ BigDecimal value = new BigDecimal("10.5");
+ MutableForwardIndex forwardIndex = mockForwardIndex(DataType.BIG_DECIMAL,
true);
+
+ DataSource dataSource = mockDataSource(forwardIndex);
+ DataSourceMetadata metadata = dataSource.getDataSourceMetadata();
+ when(metadata.getMinValue()).thenReturn(value);
+ when(metadata.getMaxValue()).thenReturn(value);
+
+ RoaringBitmap validDocIds = RoaringBitmap.bitmapOf(0, 1);
+ CompactedNoDictColumnStatistics stats =
+ new CompactedNoDictColumnStatistics(dataSource, null, false,
validDocIds);
+
+ assertEquals(stats.getMinValue(), value);
+ assertEquals(stats.getMaxValue(), value);
+ assertEquals(stats.getLengthOfShortestElement(),
BigDecimalUtils.byteSize(value));
+ assertEquals(stats.getLengthOfLongestElement(),
BigDecimalUtils.byteSize(value));
+ assertTrue(stats.isSorted());
+ verify(forwardIndex, never()).getBigDecimal(anyInt());
+ }
+
+ @Test
+ public void testConstantAsciiStringDerivesLengthWithoutScan() {
+ MutableForwardIndex forwardIndex = mockForwardIndex(DataType.STRING, true);
+
+ DataSource dataSource = mockDataSource(forwardIndex);
+ DataSourceMetadata metadata = dataSource.getDataSourceMetadata();
+ when(metadata.getMinValue()).thenReturn("abc");
+ when(metadata.getMaxValue()).thenReturn("abc");
+
+ RoaringBitmap validDocIds = RoaringBitmap.bitmapOf(0, 1);
+ CompactedNoDictColumnStatistics stats =
+ new CompactedNoDictColumnStatistics(dataSource, null, false,
validDocIds);
+
+ assertEquals(stats.getLengthOfShortestElement(), 3);
+ assertEquals(stats.getLengthOfLongestElement(), 3);
+ assertTrue(stats.isAscii());
+ assertTrue(stats.isSorted());
+ verify(forwardIndex, never()).getString(anyInt());
+ }
+
+ @Test
+ public void testConstantNonAsciiStringDerivesLengthWithoutScan() {
+ MutableForwardIndex forwardIndex = mockForwardIndex(DataType.STRING, true);
+
+ DataSource dataSource = mockDataSource(forwardIndex);
+ DataSourceMetadata metadata = dataSource.getDataSourceMetadata();
+ when(metadata.getMinValue()).thenReturn("é");
+ when(metadata.getMaxValue()).thenReturn("é");
+
+ RoaringBitmap validDocIds = RoaringBitmap.bitmapOf(0, 1);
+ CompactedNoDictColumnStatistics stats =
+ new CompactedNoDictColumnStatistics(dataSource, null, false,
validDocIds);
+
+ // Two UTF-8 bytes for one char -- the derived length must be the encoded
length, not the char count
+ assertEquals(stats.getLengthOfShortestElement(), 2);
+ assertEquals(stats.getLengthOfLongestElement(), 2);
+ assertFalse(stats.isAscii());
+ verify(forwardIndex, never()).getString(anyInt());
+ }
+
+ @Test
+ public void testConstantBytesDerivesLengthWithoutScan() {
+ byte[] value = new byte[]{1, 2, 3};
+ MutableForwardIndex forwardIndex = mockForwardIndex(DataType.BYTES, true);
+
+ DataSource dataSource = mockDataSource(forwardIndex);
+ DataSourceMetadata metadata = dataSource.getDataSourceMetadata();
+ // Min and max are tracked as ByteArray, which is the form
getElementLength expects
+ when(metadata.getMinValue()).thenReturn(new ByteArray(value));
+ when(metadata.getMaxValue()).thenReturn(new ByteArray(value));
+
+ RoaringBitmap validDocIds = RoaringBitmap.bitmapOf(0, 1);
+ CompactedNoDictColumnStatistics stats =
+ new CompactedNoDictColumnStatistics(dataSource, null, false,
validDocIds);
+
+ assertEquals(stats.getMinValue(), new ByteArray(value));
+ assertEquals(stats.getMaxValue(), new ByteArray(value));
+ assertEquals(stats.getLengthOfShortestElement(), 3);
+ assertEquals(stats.getLengthOfLongestElement(), 3);
+ assertTrue(stats.isSorted());
+ assertEquals(stats.getTotalNumberOfEntries(), 2);
+ verify(forwardIndex, never()).getBytes(anyInt());
+ }
+
// ======== INT SV ========
@Test
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
index 7f5da00a68e..7131513310b 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatisticsTest.java
@@ -109,6 +109,47 @@ public class MutableNoDictColumnStatisticsTest {
verify(forwardIndex, times(numDocs)).getInt(anyInt());
}
+ // ======== Constant value ========
+
+ @Test
+ public void testConstantValueSkipsScan() {
+ FieldSpec fieldSpec = new DimensionFieldSpec("col", DataType.INT, true);
+
+ DataSourceMetadata metadata = mockMetadata(fieldSpec, 3);
+ when(metadata.getMinValue()).thenReturn(42);
+ when(metadata.getMaxValue()).thenReturn(42);
+
+ MutableForwardIndex forwardIndex = mock(MutableForwardIndex.class);
+ when(forwardIndex.isSingleValue()).thenReturn(true);
+
+ MutableNoDictColumnStatistics stats =
+ new MutableNoDictColumnStatistics(mockNoDictDataSource(metadata,
forwardIndex), null, false);
+
+ assertTrue(stats.isSorted());
+ // A single distinct value is sorted by construction, so the forward index
is never read
+ verify(forwardIndex, never()).getInt(anyInt());
+ }
+
+ @Test
+ public void testUntrackedMinMaxFallsBackToScan() {
+ int numDocs = 3;
+ FieldSpec fieldSpec = new DimensionFieldSpec("col", DataType.INT, true);
+ Comparable[] values = fixedWidthValues(DataType.INT);
+
+ // Min and max are left null when aggregated metrics are enabled, so
sortedness must be scanned for
+ DataSourceMetadata metadata = mockMetadata(fieldSpec, numDocs);
+
+ MutableForwardIndex forwardIndex = mock(MutableForwardIndex.class);
+ when(forwardIndex.isSingleValue()).thenReturn(true);
+ stubForwardIndexReads(forwardIndex, DataType.INT, values);
+
+ MutableNoDictColumnStatistics stats =
+ new MutableNoDictColumnStatistics(mockNoDictDataSource(metadata,
forwardIndex), null, false);
+
+ assertTrue(stats.isSorted());
+ verify(forwardIndex, times(numDocs)).getInt(anyInt());
+ }
+
// ======== BigDecimal SV ========
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
