This is an automated email from the ASF dual-hosted git repository.
amoghj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new e23cf65e9e API, Core: Fix byte buffer conversion for Variant in bounds
(#14362)
e23cf65e9e is described below
commit e23cf65e9ee0c4f8c5d5677a7ab95eaff2f295c5
Author: Amogh Jahagirdar <[email protected]>
AuthorDate: Sat Oct 18 12:04:36 2025 -0600
API, Core: Fix byte buffer conversion for Variant in bounds (#14362)
---
.../java/org/apache/iceberg/types/Conversions.java | 16 ++++++++++++++++
.../iceberg/variants/TestPrimitiveWrapper.java | 14 ++++++++++++++
.../iceberg/variants/TestShreddedObject.java | 22 ++++++++++++++++++++++
.../apache/iceberg/variants/TestValueArray.java | 14 ++++++++++++++
4 files changed, 66 insertions(+)
diff --git a/api/src/main/java/org/apache/iceberg/types/Conversions.java
b/api/src/main/java/org/apache/iceberg/types/Conversions.java
index e18c7b4362..3b840ba6df 100644
--- a/api/src/main/java/org/apache/iceberg/types/Conversions.java
+++ b/api/src/main/java/org/apache/iceberg/types/Conversions.java
@@ -32,6 +32,9 @@ import java.util.UUID;
import org.apache.iceberg.exceptions.RuntimeIOException;
import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.util.UUIDUtil;
+import org.apache.iceberg.variants.Variant;
+import org.apache.iceberg.variants.VariantMetadata;
+import org.apache.iceberg.variants.VariantValue;
public class Conversions {
@@ -117,6 +120,17 @@ public class Conversions {
return (ByteBuffer) value;
case DECIMAL:
return ByteBuffer.wrap(((BigDecimal)
value).unscaledValue().toByteArray());
+ case VARIANT:
+ // Produce a concatenated buffer of metadata and value
+ Variant variant = (Variant) value;
+ VariantMetadata variantMetadata = variant.metadata();
+ VariantValue variantValue = variant.value();
+ ByteBuffer variantBuffer =
+ ByteBuffer.allocate(variantMetadata.sizeInBytes() +
variantValue.sizeInBytes())
+ .order(ByteOrder.LITTLE_ENDIAN);
+ variantMetadata.writeTo(variantBuffer, 0);
+ variantValue.writeTo(variantBuffer, variantMetadata.sizeInBytes());
+ return variantBuffer;
default:
throw new UnsupportedOperationException("Cannot serialize type: " +
typeId);
}
@@ -177,6 +191,8 @@ public class Conversions {
byte[] unscaledBytes = new byte[buffer.remaining()];
tmp.get(unscaledBytes);
return new BigDecimal(new BigInteger(unscaledBytes), decimal.scale());
+ case VARIANT:
+ return Variant.from(tmp);
default:
throw new UnsupportedOperationException("Cannot deserialize type: " +
type);
}
diff --git
a/core/src/test/java/org/apache/iceberg/variants/TestPrimitiveWrapper.java
b/core/src/test/java/org/apache/iceberg/variants/TestPrimitiveWrapper.java
index 5af3dbfb5f..c131ad0d50 100644
--- a/core/src/test/java/org/apache/iceberg/variants/TestPrimitiveWrapper.java
+++ b/core/src/test/java/org/apache/iceberg/variants/TestPrimitiveWrapper.java
@@ -24,6 +24,8 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Random;
+import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.RandomUtil;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.FieldSource;
@@ -84,4 +86,16 @@ public class TestPrimitiveWrapper {
assertThat(actual).isInstanceOf(VariantPrimitive.class);
assertThat(actual.asPrimitive().get()).isEqualTo(primitive.get());
}
+
+ @ParameterizedTest
+ @FieldSource("PRIMITIVES")
+ public void testByteBufferConversion(VariantPrimitive<?> primitive) {
+ VariantMetadata primitiveMetadata = Variants.metadata("$['primitive']");
+ Variant expectedVariant = Variant.of(primitiveMetadata, primitive);
+ ByteBuffer convertedValue =
Conversions.toByteBuffer(Types.VariantType.get(), expectedVariant);
+ Variant readValue = Conversions.fromByteBuffer(Types.VariantType.get(),
convertedValue);
+
+ VariantTestUtil.assertEqual(expectedVariant.metadata(),
readValue.metadata());
+ VariantTestUtil.assertEqual(expectedVariant.value(), readValue.value());
+ }
}
diff --git
a/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
b/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
index 431f38cb19..1ebb433a54 100644
--- a/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
+++ b/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
@@ -30,6 +30,8 @@ import java.util.Set;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.DateTimeUtil;
import org.apache.iceberg.util.RandomUtil;
import org.junit.jupiter.api.Test;
@@ -60,6 +62,26 @@ public class TestShreddedObject {
assertThat(object.get("c").asPrimitive().get()).isEqualTo(new
BigDecimal("12.21"));
}
+ @Test
+ public void testByteBufferConversion() {
+ Map<String, VariantValue> pathNormalizedFields =
+ ImmutableMap.of(
+ "$['a']",
+ Variants.of(34),
+ "$['b']",
+ Variants.of("iceberg"),
+ "$['c']",
+ Variants.of(new BigDecimal("12.21")));
+ ShreddedObject object = createShreddedObject(pathNormalizedFields);
+ VariantMetadata metadata = Variants.metadata("$['a']", "$['b']", "$['c']");
+ Variant expectedVariant = Variant.of(metadata, object);
+ ByteBuffer convertedValue =
Conversions.toByteBuffer(Types.VariantType.get(), expectedVariant);
+ Variant readValue = Conversions.fromByteBuffer(Types.VariantType.get(),
convertedValue);
+
+ VariantTestUtil.assertEqual(expectedVariant.metadata(),
readValue.metadata());
+ VariantTestUtil.assertEqual(expectedVariant.value(), readValue.value());
+ }
+
@Test
public void testShreddedSerializationMinimalBuffer() {
ShreddedObject object = createShreddedObject(FIELDS);
diff --git a/core/src/test/java/org/apache/iceberg/variants/TestValueArray.java
b/core/src/test/java/org/apache/iceberg/variants/TestValueArray.java
index d281d8a381..7a8b3c533d 100644
--- a/core/src/test/java/org/apache/iceberg/variants/TestValueArray.java
+++ b/core/src/test/java/org/apache/iceberg/variants/TestValueArray.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Random;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.RandomUtil;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -87,6 +89,18 @@ public class TestValueArray {
assertThat(actual.get(2).asPrimitive().get()).isEqualTo(new
BigDecimal("12.21"));
}
+ @Test
+ public void testByteBufferConversion() {
+ ValueArray arr = createArray(ELEMENTS);
+ VariantMetadata metadata = Variants.metadata("$['arr']");
+ Variant expectedVariant = Variant.of(metadata, arr);
+ ByteBuffer convertedValue =
Conversions.toByteBuffer(Types.VariantType.get(), expectedVariant);
+ Variant readValue = Conversions.fromByteBuffer(Types.VariantType.get(),
convertedValue);
+
+ VariantTestUtil.assertEqual(expectedVariant.metadata(),
readValue.metadata());
+ VariantTestUtil.assertEqual(expectedVariant.value(), readValue.value());
+ }
+
@ParameterizedTest
@ValueSource(ints = {300, 70_000, 16_777_300})
public void testMultiByteOffsets(int len) {
