This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 86a72804 fix(parquet/pqarrow): prefer stored schema extension type on
read (#1051)
86a72804 is described below
commit 86a72804152a703c4b1ece83591f1fc61353d5de
Author: Tobias Pütz <[email protected]>
AuthorDate: Wed Aug 5 19:46:53 2026 +0200
fix(parquet/pqarrow): prefer stored schema extension type on read (#1051)
## Rationale
Since #1028, the reader reconstructs an extension type directly from a
Parquet logical type (via `ArrowTypeFromParquet`), so the *inferred*
field can now be an extension type rather than the bare storage type it
used to be. `applyOriginalStorageMetadata` still assumed the inferred
type was plain storage when reconciling against the stored
`ARROW:schema`, so when the two extension instances differed it fell
into the "wrap the storage type" branch, found an extension instead of
the storage type, and raised a spurious storage-type mismatch.
This surfaces with GeoArrow WKB: an `authority_code` CRS (e.g. the
default `OGC:CRS84`) round-trips through Parquet's opaque CRS string and
comes back tagged `srid`, so the inferred `WKBType` differs from the one
stored in `ARROW:schema`. Reading any such file — even though the stored
schema is authoritative and correct — failed with:
```
invalid: mismatch storage type 'extension_type<storage=binary>' for
extension type 'extension_type<storage=binary>'
```
## Fix
In `applyOriginalStorageMetadata`'s extension branch, when the inferred
type is already an extension of the same name as the origin's, adopt the
origin's (authoritative) parameters instead of requiring the inferred
type to be the bare storage type.
## Testing
Existing `pqarrow` extension/geospatial/store-schema tests pass;
verified end-to-end that a GeoArrow WKB file with default-CRS
(`authority_code`) columns and a stored `ARROW:schema` now reads back
correctly.
---
parquet/pqarrow/schema.go | 10 +++++-
parquet/pqarrow/schema_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 84 insertions(+), 1 deletion(-)
diff --git a/parquet/pqarrow/schema.go b/parquet/pqarrow/schema.go
index c67ae427..bfb76048 100644
--- a/parquet/pqarrow/schema.go
+++ b/parquet/pqarrow/schema.go
@@ -1149,7 +1149,15 @@ func applyOriginalStorageMetadata(origin arrow.Field,
inferred *SchemaField) (mo
}
if modified && !arrow.TypeEqual(extType, inferred.Field.Type) {
- if !arrow.TypeEqual(extType.StorageType(),
inferred.Field.Type) {
+ // The inferred type may itself be an extension (e.g.
recovered from
+ // a Parquet logical type); compare storage types in
that case and
+ // let the stored schema win.
+ inferredStorage := inferred.Field.Type
+ if inf, ok := inferredStorage.(arrow.ExtensionType); ok
&&
+ inf.ExtensionName() == extType.ExtensionName() {
+ inferredStorage = inf.StorageType()
+ }
+ if !arrow.TypeEqual(extType.StorageType(),
inferredStorage) {
return modified, fmt.Errorf("%w: mismatch
storage type '%s' for extension type '%s'",
arrow.ErrInvalid, inferred.Field.Type,
extType)
}
diff --git a/parquet/pqarrow/schema_test.go b/parquet/pqarrow/schema_test.go
index 6dc9eea4..4fdd6b95 100644
--- a/parquet/pqarrow/schema_test.go
+++ b/parquet/pqarrow/schema_test.go
@@ -362,6 +362,81 @@ func
TestReadWriteGeospatialRegisteredExtensionWithoutStoredSchema(t *testing.T)
assert.Truef(t, array.Equal(expected, actual), "expected: %T %s\ngot:
%T %s", expected, expected, actual, actual)
}
+// TestReadWriteGeospatialRegisteredExtensionWithStoredSchema verifies that a
+// file carrying both a stored ARROW:schema and a Parquet logical type reads
+// back successfully when the extension recovered from the logical type differs
+// from the one stored in ARROW:schema. The stored schema is authoritative, so
+// the reader must prefer it instead of reporting a storage type mismatch.
+func TestReadWriteGeospatialRegisteredExtensionWithStoredSchema(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ logical := schema.GeometryLogicalType{Crs: "EPSG:4326"}
+ geoType := newTestGeometryType(logical)
+ require.NoError(t, arrow.RegisterExtensionType(geoType))
+ defer func() {
+ require.NoError(t,
arrow.UnregisterExtensionType(geoType.ExtensionName()))
+ }()
+
+ bldr := array.NewExtensionBuilder(mem, geoType)
+ defer bldr.Release()
+ binaryBldr := bldr.StorageBuilder().(*array.BinaryBuilder)
+ binaryBldr.AppendValues([][]byte{
+ {1, 2, 3},
+ nil,
+ {4, 5, 6, 7},
+ }, []bool{true, false, true})
+
+ arr := bldr.NewArray()
+ defer arr.Release()
+
+ // Field metadata makes the stored schema authoritative for this field,
so the
+ // reader reconciles it against the type inferred from the Parquet
logical type.
+ field := arrow.Field{
+ Name: "geometry", Type: geoType, Nullable: true,
+ Metadata: arrow.NewMetadata([]string{"PARQUET:field_id"},
[]string{"1"}),
+ }
+ col := arrow.NewColumnFromArr(field, arr)
+ defer col.Release()
+ tbl := array.NewTable(arrow.NewSchema([]arrow.Field{field}, nil),
[]arrow.Column{col}, -1)
+ defer tbl.Release()
+
+ var buf bytes.Buffer
+ // WithStoreSchema embeds the Arrow extension type in ARROW:schema. On
read
+ // the extension is also inferred from the Parquet logical type, so the
read
+ // path reconciles two distinct instances of the same extension.
+ arrowProps :=
pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem),
pqarrow.WithStoreSchema())
+ require.NoError(t, pqarrow.WriteTable(tbl, &buf, tbl.NumRows(),
+ parquet.NewWriterProperties(parquet.WithAllocator(mem)),
arrowProps))
+
+ pf, err := file.NewParquetReader(bytes.NewReader(buf.Bytes()),
file.WithReadProps(parquet.NewReaderProperties(mem)))
+ require.NoError(t, err)
+ defer pf.Close()
+
+ require.NotNil(t,
pf.MetaData().KeyValueMetadata().FindValue("ARROW:schema"))
+
+ reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{},
mem)
+ require.NoError(t, err)
+
+ readTbl, err := reader.ReadTable(context.Background())
+ require.NoError(t, err)
+ defer readTbl.Release()
+
+ require.Equal(t, tbl.NumRows(), readTbl.NumRows())
+ require.Equal(t, arrow.EXTENSION, readTbl.Column(0).DataType().ID())
+
+ readType, ok := readTbl.Column(0).DataType().(*testGeometryType)
+ require.True(t, ok)
+ // The stored schema is authoritative: Deserialize yields an empty
logical
+ // type, which must win over the CRS-bearing type inferred from Parquet.
+ assert.True(t, schema.GeometryLogicalType{}.Equals(readType.logical))
+ assert.True(t, arrow.TypeEqual(arrow.BinaryTypes.Binary,
readType.StorageType()))
+
+ expected :=
tbl.Column(0).Data().Chunk(0).(array.ExtensionArray).Storage()
+ actual :=
readTbl.Column(0).Data().Chunk(0).(array.ExtensionArray).Storage()
+ assert.Truef(t, array.Equal(expected, actual), "expected: %T %s\ngot:
%T %s", expected, expected, actual, actual)
+}
+
func TestToParquetWriterConfig(t *testing.T) {
origSc := arrow.NewSchema([]arrow.Field{
{Name: "f1", Type: arrow.BinaryTypes.String},
