This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 1b1f82f5 fix(parquet/pqarrow): Change interface name for arrow type on
read for extension types (#1028)
1b1f82f5 is described below
commit 1b1f82f5848cb84226e9bc502bb8ae764014add2
Author: Colton Loftus <[email protected]>
AuthorDate: Mon Jul 27 15:17:39 2026 -0400
fix(parquet/pqarrow): Change interface name for arrow type on read for
extension types (#1028)
### Rationale for this change
#969 added an interface for a custom arrow type on read for arrow
extension types. This allows a type like geometry to be resolved to the
underlying arrow physical type like byte array.
However, I went to implement this in geoarrow-go today and upon further
investigation I think the interface name is slightly confusing. The
interface `ExtensionParquetLogicalType` doesn't actually include the
`ParquetLogicalType` method but rather it intended to be used to specify
the arrow type from parquet on read. As such I have renamed
`ExtensionParquetLogicalType` to something more distinct.
I figure it makes sense to solidify this before creating a new arrow-go
release and adding it to iceberg-go whereupon it will be much harder to
change.
I changed the interface name to indicate that it is one side of the
read/write path for extension types. I think this is a bit more clear
but if I am misunderstanding and it might be used in both read and
write, then maybe it doesn't make sense.
### What changes are included in this PR?
Comments and a change in the name of an interface. This interface is not
yet in a published release but is on main. Since its only been a few
days, I think it is reasonable to change this.
### Are these changes tested?
Yes
### Are there any user-facing changes?
Yes. Change in interface name. But no changes in behavior.
---
parquet/pqarrow/schema.go | 20 ++++++++++++--------
parquet/pqarrow/schema_test.go | 8 ++++----
2 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/parquet/pqarrow/schema.go b/parquet/pqarrow/schema.go
index 81582303..2957f28e 100644
--- a/parquet/pqarrow/schema.go
+++ b/parquet/pqarrow/schema.go
@@ -121,7 +121,7 @@ func (sm *SchemaManifest) GetFieldIndices(indices []int)
([]int, error) {
}
// ExtensionCustomParquetType is an interface that Arrow ExtensionTypes may
implement
-// to specify the target LogicalType to use when converting to Parquet.
+// to specify the target LogicalType to use when converting to Parquet on
write.
//
// The PrimitiveType is not configurable, and is determined by a fixed mapping
from
// the extension's StorageType to a Parquet type (see getParquetType in
pqarrow source).
@@ -129,16 +129,18 @@ type ExtensionCustomParquetType interface {
ParquetLogicalType() schema.LogicalType
}
-// ExtensionParquetLogicalType is an interface that Arrow ExtensionTypes may
+// ExtensionCustomArrowReadType is an interface that Arrow ExtensionTypes may
// implement to specify how a Parquet LogicalType maps back to an Arrow
-// ExtensionType when converting a Parquet schema to an Arrow schema.
+// ExtensionType when converting a Parquet schema to an Arrow schema on read.
+// The receiver is the registered extension type being asked whether it can
+// represent the Parquet logical type with the given storage type; callers must
+// use the returned extension type because it may be a distinct instance
carrying
+// per-column parameters derived from the Parquet logical type.
//
// ArrowTypeFromParquet should return (nil, nil) if the logical type does not
// map to the extension type. It should return (nil, err) if the logical type
is
-// recognized but cannot be converted into a valid extension type. If a
-// non-nil extension type is returned, that type is used and any previous
-// conversion errors from other extension types are ignored.
-type ExtensionParquetLogicalType interface {
+// recognized but there was an issue converting into a valid extension type.
+type ExtensionCustomArrowReadType interface {
ArrowTypeFromParquet(logical schema.LogicalType, storageType
arrow.DataType) (arrow.ExtensionType, error)
}
@@ -560,13 +562,15 @@ func arrowFromByteArray(logical schema.LogicalType)
(arrow.DataType, error) {
// arrowExtensionFromParquetLogicalType asks registered extension types whether
// they can represent the provided Parquet logical type with the given storage
// type, falling back to the storage type when none opt in.
+// As long as one non-nil extension type is returned and maps without an
error, that
+// type is used and any previous conversion errors from other extension types
are ignored.
func arrowExtensionFromParquetLogicalType(logical schema.LogicalType,
storageType arrow.DataType) (arrow.DataType, error) {
var (
typ arrow.ExtensionType
typeLookupErr error
)
matchedType := arrow.FindRegisteredExtensionType(func(extType
arrow.ExtensionType) bool {
- converter, ok := extType.(ExtensionParquetLogicalType)
+ converter, ok := extType.(ExtensionCustomArrowReadType)
if !ok {
return false
}
diff --git a/parquet/pqarrow/schema_test.go b/parquet/pqarrow/schema_test.go
index 11f969e2..41103b85 100644
--- a/parquet/pqarrow/schema_test.go
+++ b/parquet/pqarrow/schema_test.go
@@ -128,9 +128,9 @@ func (t *testGeometryType) ParquetLogicalType()
schema.LogicalType {
}
var (
- _ pqarrow.ExtensionCustomParquetType = (*testGeometryType)(nil)
- _ pqarrow.ExtensionParquetLogicalType = (*testGeometryType)(nil)
- _ pqarrow.ExtensionParquetLogicalType = (*testFailingGeometryType)(nil)
+ _ pqarrow.ExtensionCustomParquetType = (*testGeometryType)(nil)
+ _ pqarrow.ExtensionCustomArrowReadType = (*testGeometryType)(nil)
+ _ pqarrow.ExtensionCustomArrowReadType = (*testFailingGeometryType)(nil)
)
func TestGetOriginSchemaBase64(t *testing.T) {
@@ -291,7 +291,7 @@ func
TestFromParquetGeospatialRegisteredExtensionReturnsConverterErrorWhenNoMatc
// TestReadWriteGeospatialRegisteredExtensionWithoutStoredSchema verifies that
a
// registered extension type can round trip using only the Parquet logical
type.
// Keeping StoreSchema disabled ensures the reader reconstructs the extension
via
-// ExtensionParquetLogicalType interface logic instead of restoring it from
+// ExtensionCustomArrowReadType interface logic instead of restoring it from
// ARROW:schema metadata.
func TestReadWriteGeospatialRegisteredExtensionWithoutStoredSchema(t
*testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
