This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5ebd445336f Fill BigDecimal typed-adder gaps in the IndexCreator
family (#18451)
5ebd445336f is described below
commit 5ebd445336ff295c231b02afa9f202418e296546
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Sun May 10 03:21:01 2026 -0700
Fill BigDecimal typed-adder gaps in the IndexCreator family (#18451)
---
.../invertedindex/DictionaryBasedIndexBuilder.java | 159 +++++++++++----------
.../pinot/segment/spi/index/IndexCreator.java | 5 +
.../creator/CombinedInvertedIndexCreator.java | 19 +++
.../DictionaryBasedInvertedIndexCreator.java | 13 ++
.../spi/index/creator/ForwardIndexCreator.java | 51 +++----
5 files changed, 146 insertions(+), 101 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/DictionaryBasedIndexBuilder.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/DictionaryBasedIndexBuilder.java
index cc323735f1f..1fe56e2dbc9 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/DictionaryBasedIndexBuilder.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/DictionaryBasedIndexBuilder.java
@@ -24,6 +24,7 @@ import
org.apache.pinot.segment.spi.index.creator.DictionaryBasedInvertedIndexCr
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.ByteArray;
@@ -48,111 +49,107 @@ public final class DictionaryBasedIndexBuilder {
/// the current SPI; eliminating either one would require a length-aware
`addXxxMV(values, dictIds, length)` SPI
/// method.
///
- /// @throws IllegalStateException if the column data type is unsupported, or
if the column is multi-value with a
- /// data type that does not support MV (e.g. BIG_DECIMAL).
- @SuppressWarnings("rawtypes")
+ /// @throws IllegalStateException if the column data type is unsupported.
+ @SuppressWarnings({"rawtypes", "unchecked"})
public static void
addRawValuesViaDictionary(DictionaryBasedInvertedIndexCreator creator,
ForwardIndexReader forwardIndexReader, ForwardIndexReaderContext
readerContext, Dictionary dictionary,
ColumnMetadata columnMetadata, int numDocs) {
- String columnName = columnMetadata.getColumnName();
- boolean singleValue = columnMetadata.isSingleValue();
- switch (columnMetadata.getDataType().getStoredType()) {
- case INT:
- if (singleValue) {
+ DataType dataType = columnMetadata.getDataType();
+ DataType storedType = dataType.getStoredType();
+ if (columnMetadata.isSingleValue()) {
+ switch (storedType) {
+ case INT:
for (int i = 0; i < numDocs; i++) {
int value = forwardIndexReader.getInt(i, readerContext);
creator.addInt(value, dictionary.indexOf(value));
}
- } else {
- for (int i = 0; i < numDocs; i++) {
- int[] values = forwardIndexReader.getIntMV(i, readerContext);
- creator.addIntMV(values, lookupDictIds(dictionary, values));
- }
- }
- return;
- case LONG:
- if (singleValue) {
+ break;
+ case LONG:
for (int i = 0; i < numDocs; i++) {
long value = forwardIndexReader.getLong(i, readerContext);
creator.addLong(value, dictionary.indexOf(value));
}
- } else {
- for (int i = 0; i < numDocs; i++) {
- long[] values = forwardIndexReader.getLongMV(i, readerContext);
- creator.addLongMV(values, lookupDictIds(dictionary, values));
- }
- }
- return;
- case FLOAT:
- if (singleValue) {
+ break;
+ case FLOAT:
for (int i = 0; i < numDocs; i++) {
float value = forwardIndexReader.getFloat(i, readerContext);
creator.addFloat(value, dictionary.indexOf(value));
}
- } else {
- for (int i = 0; i < numDocs; i++) {
- float[] values = forwardIndexReader.getFloatMV(i, readerContext);
- creator.addFloatMV(values, lookupDictIds(dictionary, values));
- }
- }
- return;
- case DOUBLE:
- if (singleValue) {
+ break;
+ case DOUBLE:
for (int i = 0; i < numDocs; i++) {
double value = forwardIndexReader.getDouble(i, readerContext);
creator.addDouble(value, dictionary.indexOf(value));
}
- } else {
+ break;
+ case BIG_DECIMAL:
for (int i = 0; i < numDocs; i++) {
- double[] values = forwardIndexReader.getDoubleMV(i, readerContext);
- creator.addDoubleMV(values, lookupDictIds(dictionary, values));
+ BigDecimal value = forwardIndexReader.getBigDecimal(i,
readerContext);
+ creator.addBigDecimal(value, dictionary.indexOf(value));
}
- }
- return;
- case BIG_DECIMAL:
- if (!singleValue) {
- throw new IllegalStateException(
- "Dictionary-based index over raw values not supported for
multi-value BIG_DECIMAL column: " + columnName);
- }
- for (int i = 0; i < numDocs; i++) {
- BigDecimal value = forwardIndexReader.getBigDecimal(i,
readerContext);
- creator.add(value, dictionary.indexOf(value));
- }
- return;
- case STRING:
- if (singleValue) {
+ break;
+ case STRING:
for (int i = 0; i < numDocs; i++) {
String value = forwardIndexReader.getString(i, readerContext);
creator.addString(value, dictionary.indexOf(value));
}
- } else {
- for (int i = 0; i < numDocs; i++) {
- String[] values = forwardIndexReader.getStringMV(i, readerContext);
- creator.addStringMV(values, lookupDictIds(dictionary, values));
- }
- }
- return;
- case BYTES:
- if (singleValue) {
+ break;
+ case BYTES:
for (int i = 0; i < numDocs; i++) {
byte[] value = forwardIndexReader.getBytes(i, readerContext);
creator.addBytes(value, dictionary.indexOf(new ByteArray(value)));
}
- } else {
+ break;
+ default:
+ throw new IllegalStateException("Unsupported SV data type: " +
dataType);
+ }
+ } else {
+ switch (storedType) {
+ case INT:
+ for (int i = 0; i < numDocs; i++) {
+ int[] values = forwardIndexReader.getIntMV(i, readerContext);
+ creator.addIntMV(values, lookupDictIds(dictionary, values));
+ }
+ break;
+ case LONG:
+ for (int i = 0; i < numDocs; i++) {
+ long[] values = forwardIndexReader.getLongMV(i, readerContext);
+ creator.addLongMV(values, lookupDictIds(dictionary, values));
+ }
+ break;
+ case FLOAT:
+ for (int i = 0; i < numDocs; i++) {
+ float[] values = forwardIndexReader.getFloatMV(i, readerContext);
+ creator.addFloatMV(values, lookupDictIds(dictionary, values));
+ }
+ break;
+ case DOUBLE:
+ for (int i = 0; i < numDocs; i++) {
+ double[] values = forwardIndexReader.getDoubleMV(i, readerContext);
+ creator.addDoubleMV(values, lookupDictIds(dictionary, values));
+ }
+ break;
+ case BIG_DECIMAL:
+ for (int i = 0; i < numDocs; i++) {
+ BigDecimal[] values = forwardIndexReader.getBigDecimalMV(i,
readerContext);
+ creator.addBigDecimalMV(values, lookupDictIds(dictionary, values));
+ }
+ break;
+ case STRING:
+ for (int i = 0; i < numDocs; i++) {
+ String[] values = forwardIndexReader.getStringMV(i, readerContext);
+ creator.addStringMV(values, lookupDictIds(dictionary, values));
+ }
+ break;
+ case BYTES:
for (int i = 0; i < numDocs; i++) {
byte[][] values = forwardIndexReader.getBytesMV(i, readerContext);
- int[] dictIds = new int[values.length];
- for (int j = 0; j < values.length; j++) {
- dictIds[j] = dictionary.indexOf(new ByteArray(values[j]));
- }
- creator.addBytesMV(values, dictIds);
+ creator.addBytesMV(values, lookupDictIds(dictionary, values));
}
- }
- return;
- default:
- throw new IllegalStateException(
- "Unsupported data type for dictionary-based index over raw values:
" + columnMetadata.getDataType()
- + " (column: " + columnName + ")");
+ break;
+ default:
+ throw new IllegalStateException("Unsupported MV data type: " +
dataType);
+ }
}
}
@@ -188,6 +185,14 @@ public final class DictionaryBasedIndexBuilder {
return dictIds;
}
+ private static int[] lookupDictIds(Dictionary dictionary, BigDecimal[]
values) {
+ int[] dictIds = new int[values.length];
+ for (int j = 0; j < values.length; j++) {
+ dictIds[j] = dictionary.indexOf(values[j]);
+ }
+ return dictIds;
+ }
+
private static int[] lookupDictIds(Dictionary dictionary, String[] values) {
int[] dictIds = new int[values.length];
for (int j = 0; j < values.length; j++) {
@@ -195,4 +200,12 @@ public final class DictionaryBasedIndexBuilder {
}
return dictIds;
}
+
+ private static int[] lookupDictIds(Dictionary dictionary, byte[][] values) {
+ int[] dictIds = new int[values.length];
+ for (int j = 0; j < values.length; j++) {
+ dictIds[j] = dictionary.indexOf(new ByteArray(values[j]));
+ }
+ return dictIds;
+ }
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
index 97bc18e985f..d725501d71e 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexCreator.java
@@ -89,6 +89,11 @@ public interface IndexCreator extends Closeable {
add(value, dictId);
}
+ default void addBigDecimal(BigDecimal value, int dictId)
+ throws IOException {
+ add(value, dictId);
+ }
+
default void addString(String value, int dictId)
throws IOException {
add(value, dictId);
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/CombinedInvertedIndexCreator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/CombinedInvertedIndexCreator.java
index 9dc8e366543..61bf4d57a13 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/CombinedInvertedIndexCreator.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/CombinedInvertedIndexCreator.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.segment.spi.index.creator;
+import java.math.BigDecimal;
import javax.annotation.Nullable;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -145,6 +146,15 @@ public interface CombinedInvertedIndexCreator
}
}
+ @Override
+ default void addBigDecimal(BigDecimal value, int dictId) {
+ if (dictId >= 0) {
+ add(dictId);
+ } else {
+ throw new RuntimeException("BigDecimal not supported for range index");
+ }
+ }
+
@Override
default void addString(String value, int dictId) {
if (dictId >= 0) {
@@ -199,6 +209,15 @@ public interface CombinedInvertedIndexCreator
}
}
+ @Override
+ default void addBigDecimalMV(BigDecimal[] values, @Nullable int[] dictIds) {
+ if (dictIds != null) {
+ add(dictIds, dictIds.length);
+ } else {
+ throw new RuntimeException("BigDecimal MV not supported for range
index");
+ }
+ }
+
@Override
default void addStringMV(String[] values, @Nullable int[] dictIds) {
if (dictIds != null) {
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/DictionaryBasedInvertedIndexCreator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/DictionaryBasedInvertedIndexCreator.java
index 2f73fcc0b6e..fc7d13a82b6 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/DictionaryBasedInvertedIndexCreator.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/DictionaryBasedInvertedIndexCreator.java
@@ -18,6 +18,9 @@
*/
package org.apache.pinot.segment.spi.index.creator;
+import java.math.BigDecimal;
+
+
/**
* Support for RoaringBitmap inverted index:
* <pre>
@@ -91,6 +94,11 @@ public interface DictionaryBasedInvertedIndexCreator extends
InvertedIndexCreato
add(dictId);
}
+ @Override
+ default void addBigDecimal(BigDecimal value, int dictId) {
+ add(dictId);
+ }
+
@Override
default void addString(String value, int dictId) {
add(dictId);
@@ -121,6 +129,11 @@ public interface DictionaryBasedInvertedIndexCreator
extends InvertedIndexCreato
add(dictIds, dictIds.length);
}
+ @Override
+ default void addBigDecimalMV(BigDecimal[] values, int[] dictIds) {
+ add(dictIds, dictIds.length);
+ }
+
@Override
default void addStringMV(String[] values, int[] dictIds) {
add(dictIds, dictIds.length);
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java
index 6c79a6a5e98..afb52dfeac4 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/ForwardIndexCreator.java
@@ -84,8 +84,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void add(Object[] cellValues, @Nullable int[] dictIds)
- throws IOException {
+ default void add(Object[] cellValues, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
@@ -168,8 +167,7 @@ public interface ForwardIndexCreator extends IndexCreator {
*/
@Override
- default void addInt(int value, int dictId)
- throws IOException {
+ default void addInt(int value, int dictId) {
if (isDictionaryEncoded()) {
putDictId(dictId);
} else {
@@ -178,8 +176,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addLong(long value, int dictId)
- throws IOException {
+ default void addLong(long value, int dictId) {
if (isDictionaryEncoded()) {
putDictId(dictId);
} else {
@@ -188,8 +185,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addFloat(float value, int dictId)
- throws IOException {
+ default void addFloat(float value, int dictId) {
if (isDictionaryEncoded()) {
putDictId(dictId);
} else {
@@ -198,8 +194,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addDouble(double value, int dictId)
- throws IOException {
+ default void addDouble(double value, int dictId) {
if (isDictionaryEncoded()) {
putDictId(dictId);
} else {
@@ -208,8 +203,16 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addString(String value, int dictId)
- throws IOException {
+ default void addBigDecimal(BigDecimal value, int dictId) {
+ if (isDictionaryEncoded()) {
+ putDictId(dictId);
+ } else {
+ putBigDecimal(value);
+ }
+ }
+
+ @Override
+ default void addString(String value, int dictId) {
if (isDictionaryEncoded()) {
putDictId(dictId);
} else {
@@ -218,8 +221,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addBytes(byte[] value, int dictId)
- throws IOException {
+ default void addBytes(byte[] value, int dictId) {
if (isDictionaryEncoded()) {
putDictId(dictId);
} else {
@@ -228,8 +230,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addIntMV(int[] values, @Nullable int[] dictIds)
- throws IOException {
+ default void addIntMV(int[] values, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
@@ -238,8 +239,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addLongMV(long[] values, @Nullable int[] dictIds)
- throws IOException {
+ default void addLongMV(long[] values, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
@@ -248,8 +248,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addFloatMV(float[] values, @Nullable int[] dictIds)
- throws IOException {
+ default void addFloatMV(float[] values, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
@@ -258,8 +257,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addDoubleMV(double[] values, @Nullable int[] dictIds)
- throws IOException {
+ default void addDoubleMV(double[] values, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
@@ -268,8 +266,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addBigDecimalMV(BigDecimal[] values, @Nullable int[] dictIds)
- throws IOException {
+ default void addBigDecimalMV(BigDecimal[] values, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
@@ -278,8 +275,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addStringMV(String[] values, @Nullable int[] dictIds)
- throws IOException {
+ default void addStringMV(String[] values, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
@@ -288,8 +284,7 @@ public interface ForwardIndexCreator extends IndexCreator {
}
@Override
- default void addBytesMV(byte[][] values, @Nullable int[] dictIds)
- throws IOException {
+ default void addBytesMV(byte[][] values, @Nullable int[] dictIds) {
if (isDictionaryEncoded()) {
putDictIdMV(dictIds);
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
