This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e7b57c06ec7 Add MV BYTES and BIG_DECIMAL support to GenericRow
serializer/deserializer (#18969)
e7b57c06ec7 is described below
commit e7b57c06ec7d1225c6a1489017d31fffe8b1ef32
Author: Xiang Fu <[email protected]>
AuthorDate: Fri Jul 10 23:16:56 2026 +0200
Add MV BYTES and BIG_DECIMAL support to GenericRow serializer/deserializer
(#18969)
GenericRowSerializer/GenericRowDeserializer supported single-value BYTES and
BIG_DECIMAL, but the multi-value switch branches only handled
INT/LONG/FLOAT/
DOUBLE/STRING. Serializing a row with an MV BYTES or MV BIG_DECIMAL column
threw IllegalStateException("Unsupported MV stored type") in the segment
processing framework.
Add the missing MV BYTES and BIG_DECIMAL cases to all four switch blocks
(serialize size pass, serialize write pass, deserialize, compare), mirroring
the existing SV cases and the MV STRING length-prefixed layout. Extend
GenericRowSerDeTest with both MV columns and add a negative compare test
that
exercises the non-zero and differing-cardinality branches.
---
.../genericrow/GenericRowDeserializer.java | 59 ++++++++++++++++++++++
.../genericrow/GenericRowSerializer.java | 30 +++++++++++
.../processing/genericrow/GenericRowSerDeTest.java | 44 ++++++++++++++++
3 files changed, 133 insertions(+)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
index 693c47a5bc2..be84cffb20f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.segment.processing.genericrow;
+import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
@@ -164,6 +165,26 @@ public class GenericRowDeserializer {
multiValue[j] = new String(stringBytes, UTF_8);
}
break;
+ case BYTES:
+ for (int j = 0; j < numValues; j++) {
+ int numBytes = _dataBuffer.getInt(offset);
+ offset += Integer.BYTES;
+ byte[] bytes = new byte[numBytes];
+ _dataBuffer.copyTo(offset, bytes);
+ offset += numBytes;
+ multiValue[j] = bytes;
+ }
+ break;
+ case BIG_DECIMAL:
+ for (int j = 0; j < numValues; j++) {
+ int numBytes = _dataBuffer.getInt(offset);
+ offset += Integer.BYTES;
+ byte[] bigDecimalBytes = new byte[numBytes];
+ _dataBuffer.copyTo(offset, bigDecimalBytes);
+ offset += numBytes;
+ multiValue[j] = BigDecimalUtils.deserialize(bigDecimalBytes);
+ }
+ break;
default:
throw new IllegalStateException("Unsupported MV stored type: " +
_storedTypes[i]);
}
@@ -366,6 +387,44 @@ public class GenericRowDeserializer {
offset2 += numBytes2;
}
break;
+ case BYTES:
+ for (int j = 0; j < numValues; j++) {
+ int numBytes1 = _dataBuffer.getInt(offset1);
+ offset1 += Integer.BYTES;
+ byte[] bytes1 = new byte[numBytes1];
+ _dataBuffer.copyTo(offset1, bytes1);
+ int numBytes2 = _dataBuffer.getInt(offset2);
+ offset2 += Integer.BYTES;
+ byte[] bytes2 = new byte[numBytes2];
+ _dataBuffer.copyTo(offset2, bytes2);
+ int result = ByteArray.compare(bytes1, bytes2);
+ if (result != 0) {
+ return result;
+ }
+ offset1 += numBytes1;
+ offset2 += numBytes2;
+ }
+ break;
+ case BIG_DECIMAL:
+ for (int j = 0; j < numValues; j++) {
+ int numBytes1 = _dataBuffer.getInt(offset1);
+ offset1 += Integer.BYTES;
+ byte[] bigDecimalBytes1 = new byte[numBytes1];
+ _dataBuffer.copyTo(offset1, bigDecimalBytes1);
+ int numBytes2 = _dataBuffer.getInt(offset2);
+ offset2 += Integer.BYTES;
+ byte[] bigDecimalBytes2 = new byte[numBytes2];
+ _dataBuffer.copyTo(offset2, bigDecimalBytes2);
+ BigDecimal bigDecimal1 =
BigDecimalUtils.deserialize(bigDecimalBytes1);
+ BigDecimal bigDecimal2 =
BigDecimalUtils.deserialize(bigDecimalBytes2);
+ int result = bigDecimal1.compareTo(bigDecimal2);
+ if (result != 0) {
+ return result;
+ }
+ offset1 += numBytes1;
+ offset2 += numBytes2;
+ }
+ break;
default:
throw new IllegalStateException("Unsupported MV stored type: " +
_storedTypes[i]);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
index d612eb3e31d..d188be72c57 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
@@ -148,6 +148,22 @@ public class GenericRowSerializer {
}
_objectBytes[i] = stringBytesArray;
break;
+ case BYTES:
+ numBytes += Integer.BYTES * numValues;
+ for (Object element : multiValue) {
+ numBytes += ((byte[]) element).length;
+ }
+ break;
+ case BIG_DECIMAL:
+ numBytes += Integer.BYTES * numValues;
+ byte[][] bigDecimalBytesArray = new byte[numValues][];
+ for (int j = 0; j < numValues; j++) {
+ byte[] bigDecimalBytes = BigDecimalUtils.serialize((BigDecimal)
multiValue[j]);
+ numBytes += bigDecimalBytes.length;
+ bigDecimalBytesArray[j] = bigDecimalBytes;
+ }
+ _objectBytes[i] = bigDecimalBytesArray;
+ break;
default:
throw new IllegalStateException("Unsupported MV stored type: " +
_storedTypes[i]);
}
@@ -240,6 +256,20 @@ public class GenericRowSerializer {
byteBuffer.put(stringBytes);
}
break;
+ case BYTES:
+ for (Object element : multiValue) {
+ byte[] bytes = (byte[]) element;
+ byteBuffer.putInt(bytes.length);
+ byteBuffer.put(bytes);
+ }
+ break;
+ case BIG_DECIMAL:
+ byte[][] bigDecimalBytesArray = (byte[][]) _objectBytes[i];
+ for (byte[] bigDecimalBytes : bigDecimalBytesArray) {
+ byteBuffer.putInt(bigDecimalBytes.length);
+ byteBuffer.put(bigDecimalBytes);
+ }
+ break;
default:
throw new IllegalStateException("Unsupported MV stored type: " +
_storedTypes[i]);
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
index c938009633e..2cd5891bb54 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
@@ -55,6 +55,8 @@ public class GenericRowSerDeTest {
new DimensionFieldSpec("floatMV", DataType.FLOAT, false),
new DimensionFieldSpec("doubleMV", DataType.DOUBLE, false),
new DimensionFieldSpec("stringMV", DataType.STRING, false),
+ new DimensionFieldSpec("bytesMV", DataType.BYTES, false),
+ new DimensionFieldSpec("bigDecimalMV", DataType.BIG_DECIMAL, false),
new DimensionFieldSpec("nullMV", DataType.LONG, false));
_row = new GenericRow();
@@ -78,6 +80,8 @@ public class GenericRowSerDeTest {
_row.putValue("floatMV", new Object[]{123.0f, 456.0f});
_row.putValue("doubleMV", new Object[]{123.0, 456.0});
_row.putValue("stringMV", new Object[]{"123", "456"});
+ _row.putValue("bytesMV", new Object[]{new byte[]{1, 2, 3}, new byte[]{4,
5}});
+ _row.putValue("bigDecimalMV", new Object[]{new BigDecimal("122333"), new
BigDecimal("-4.5")});
_row.putDefaultNullValue("nullMV", new Object[]{Long.MIN_VALUE});
}
@@ -146,6 +150,46 @@ public class GenericRowSerDeTest {
}
}
+ @Test
+ public void testMvBytesAndBigDecimalCompare() {
+ // Exercises the non-zero branch of the MV BYTES / BIG_DECIMAL compare
paths (differing element and differing
+ // cardinality), which the byte-identical rows in testCompare never reach.
+ List<FieldSpec> fieldSpecs = Arrays.asList(new
DimensionFieldSpec("bytesMV", DataType.BYTES, false),
+ new DimensionFieldSpec("bigDecimalMV", DataType.BIG_DECIMAL, false));
+
+ GenericRow low = new GenericRow();
+ low.putValue("bytesMV", new Object[]{new byte[]{1, 2}, new byte[]{3}});
+ low.putValue("bigDecimalMV", new Object[]{new BigDecimal("1.5"), new
BigDecimal("2.5")});
+
+ // Differs from `low` only in the last element of each MV column, with a
larger value.
+ GenericRow high = new GenericRow();
+ high.putValue("bytesMV", new Object[]{new byte[]{1, 2}, new byte[]{4}});
+ high.putValue("bigDecimalMV", new Object[]{new BigDecimal("1.5"), new
BigDecimal("9.9")});
+
+ // Same last element but fewer values, to exercise the
differing-cardinality early return.
+ GenericRow shorter = new GenericRow();
+ shorter.putValue("bytesMV", new Object[]{new byte[]{1, 2}});
+ shorter.putValue("bigDecimalMV", new Object[]{new BigDecimal("1.5")});
+
+ for (int field = 0; field < fieldSpecs.size(); field++) {
+ assertTrue(compareRows(fieldSpecs, low, high, field + 1) < 0);
+ assertTrue(compareRows(fieldSpecs, high, low, field + 1) > 0);
+ assertTrue(compareRows(fieldSpecs, shorter, low, field + 1) < 0);
+ assertTrue(compareRows(fieldSpecs, low, shorter, field + 1) > 0);
+ }
+ }
+
+ private static int compareRows(List<FieldSpec> fieldSpecs, GenericRow row1,
GenericRow row2, int numFieldsToCompare) {
+ GenericRowSerializer serializer = new GenericRowSerializer(fieldSpecs,
false);
+ byte[] bytes1 = serializer.serialize(row1);
+ byte[] bytes2 = serializer.serialize(row2);
+ long numBytes = Math.max(bytes1.length, bytes2.length);
+ PinotDataBuffer dataBuffer = PinotDataBuffer.allocateDirect(numBytes * 2,
PinotDataBuffer.NATIVE_ORDER, null);
+ dataBuffer.readFrom(0L, bytes1);
+ dataBuffer.readFrom(numBytes, bytes2);
+ return new GenericRowDeserializer(dataBuffer, fieldSpecs,
false).compare(0L, numBytes, numFieldsToCompare);
+ }
+
@Test
public void testMapSerDe() {
List<FieldSpec> mapFieldSpecs = Arrays.asList(new
ComplexFieldSpec("mapSV", DataType.MAP, true, new HashMap<>()));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
