This is an automated email from the ASF dual-hosted git repository. FANNG1 pushed a commit to branch fix/lance-arrow-field-metadata in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit a021fe4a535efa4517dd22fc059aca6c1cccd25b Author: fanng <[email protected]> AuthorDate: Tue Sep 22 10:25:56 2026 +0900 [#13338] fix(lance): preserve Arrow field metadata --- .../ops/gravitino/LanceDataTypeConverter.java | 8 ++++ .../ops/gravitino/TestLanceDataTypeConverter.java | 47 ++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/LanceDataTypeConverter.java b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/LanceDataTypeConverter.java index 52d52d38fb..f098038733 100644 --- a/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/LanceDataTypeConverter.java +++ b/lance/lance-common/src/main/java/org/apache/gravitino/lance/common/ops/gravitino/LanceDataTypeConverter.java @@ -203,6 +203,10 @@ public class LanceDataTypeConverter implements DataTypeConverter<ArrowType, Fiel @Override public Type toGravitino(Field arrowField) { + if (arrowField.getMetadata() != null && !arrowField.getMetadata().isEmpty()) { + return toExternalType(arrowField); + } + FieldType fieldType = arrowField.getFieldType(); switch (fieldType.getType().getTypeID()) { case Map: @@ -315,6 +319,10 @@ public class LanceDataTypeConverter implements DataTypeConverter<ArrowType, Fiel // fallthrough } + return toExternalType(arrowField); + } + + private Type toExternalType(Field arrowField) { String typeString; try { typeString = mapper.writeValueAsString(arrowField); diff --git a/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestLanceDataTypeConverter.java b/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestLanceDataTypeConverter.java index 9908f8feff..12731edf8c 100644 --- a/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestLanceDataTypeConverter.java +++ b/lance/lance-common/src/test/java/org/apache/gravitino/lance/common/ops/gravitino/TestLanceDataTypeConverter.java @@ -26,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.Collections; +import java.util.Map; import java.util.function.Consumer; import java.util.stream.Stream; import org.apache.arrow.vector.complex.MapVector; @@ -257,6 +258,52 @@ public class TestLanceDataTypeConverter { assertEquals(10, ((ArrowType.FixedSizeList) arrowField.getFieldType().getType()).getListSize()); } + @Test + void testFieldWithMetadataConvertsToExternalTypeAndRoundTrips() { + Field blobField = + new Field( + "blob", + new FieldType( + true, + ArrowType.Struct.INSTANCE, + null, + Map.of( + "ARROW:extension:name", "lance.blob.v2", + "lance-encoding:blob-inline-size-threshold", "4096", + "lance-encoding:blob-dedicated-size-threshold", "1048576")), + Arrays.asList( + new Field("data", new FieldType(true, ArrowType.LargeBinary.INSTANCE, null), null), + new Field("uri", new FieldType(true, ArrowType.Utf8.INSTANCE, null), null), + new Field( + "position", new FieldType(true, new ArrowType.Int(64, false), null), null), + new Field("size", new FieldType(true, new ArrowType.Int(64, false), null), null))); + + Type type = CONVERTER.toGravitino(blobField); + + assertInstanceOf(Types.ExternalType.class, type); + assertEquals(blobField, CONVERTER.toArrowField("blob", type, true)); + } + + @Test + void testNestedFieldWithMetadataRoundTrips() { + Field nestedField = + new Field( + "record", + new FieldType(true, ArrowType.Struct.INSTANCE, null), + Collections.singletonList( + new Field( + "tagged_value", + new FieldType( + true, ArrowType.Utf8.INSTANCE, null, Map.of("vendor:type", "tagged")), + null))); + + Type type = CONVERTER.toGravitino(nestedField); + + assertInstanceOf(Types.StructType.class, type); + assertInstanceOf(Types.ExternalType.class, ((Types.StructType) type).fields()[0].type()); + assertEquals(nestedField, CONVERTER.toArrowField("record", type, true)); + } + @ParameterizedTest(name = "[{index}] name={0}, type={1}, nullable={2}") @MethodSource("toArrowFieldArguments") @DisplayName("Test toArrowField for various types")
