This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 7aec6ac fix(parquet/pqarrow): fix propagation of FieldIds for nested
fields (#324)
7aec6ac is described below
commit 7aec6ac4836608b0244e0258b78474d07b1c2683
Author: Matt Topol <[email protected]>
AuthorDate: Mon Mar 24 09:05:54 2025 -0400
fix(parquet/pqarrow): fix propagation of FieldIds for nested fields (#324)
### Rationale for this change
While implementing things for iceberg-go, I found that an Arrow schema
with nested fields (struct/map/list) that contains metadata values for
FieldID is not respected when writing a file using `pqarrow`.
### What changes are included in this PR?
Fixes the propagation of field ids when constructing a Parquet file from
an arrow schema that contains nested fields, while also adding a
`FileMetadata` function to the FileWriter and `pqarrow.FileWriter` so
that you can inspect the metadata of a written file without having to
read it back into memory.
### Are these changes tested?
Yes
### Are there any user-facing changes?
No
---
parquet/file/file_writer.go | 3 +++
parquet/pqarrow/file_writer.go | 3 +++
parquet/pqarrow/schema.go | 9 +++++----
parquet/pqarrow/schema_test.go | 8 ++++----
4 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/parquet/file/file_writer.go b/parquet/file/file_writer.go
index 3c5608a..68b5eee 100644
--- a/parquet/file/file_writer.go
+++ b/parquet/file/file_writer.go
@@ -217,6 +217,9 @@ func (fw *Writer) Close() (err error) {
return nil
}
+// FileMetadata returns the current state of the FileMetadata that would be
written
+// if this file were to be closed. If the file has already been closed, then
this
+// will return the FileMetaData which was written to the file.
func (fw *Writer) FileMetadata() (*metadata.FileMetaData, error) {
return fw.metadata.Snapshot()
}
diff --git a/parquet/pqarrow/file_writer.go b/parquet/pqarrow/file_writer.go
index f3e65ac..e4e9936 100644
--- a/parquet/pqarrow/file_writer.go
+++ b/parquet/pqarrow/file_writer.go
@@ -339,6 +339,9 @@ func (fw *FileWriter) WriteColumnData(data arrow.Array)
error {
return fw.WriteColumnChunked(chunked, 0, int64(data.Len()))
}
+// FileMetadata returns the current state of the FileMetadata that would be
written
+// if this file were to be closed. If the file has already been closed, then
this
+// will return the FileMetaData which was written to the file.
func (fw *FileWriter) FileMetadata() (*metadata.FileMetaData, error) {
return fw.wr.FileMetadata()
}
diff --git a/parquet/pqarrow/schema.go b/parquet/pqarrow/schema.go
index 0342f28..50f817b 100644
--- a/parquet/pqarrow/schema.go
+++ b/parquet/pqarrow/schema.go
@@ -240,9 +240,10 @@ func repFromNullable(isnullable bool) parquet.Repetition {
return parquet.Repetitions.Required
}
-func structToNode(typ *arrow.StructType, name string, nullable bool, fieldID
int32, props *parquet.WriterProperties, arrprops ArrowWriterProperties)
(schema.Node, error) {
+func structToNode(field arrow.Field, props *parquet.WriterProperties, arrprops
ArrowWriterProperties) (schema.Node, error) {
+ typ := field.Type.(*arrow.StructType)
if typ.NumFields() == 0 {
- return nil, fmt.Errorf("cannot write struct type '%s' with no
children field to parquet. Consider adding a dummy child", name)
+ return nil, fmt.Errorf("cannot write struct type '%s' with no
children field to parquet. Consider adding a dummy child", field.Name)
}
children := make(schema.FieldList, 0, typ.NumFields())
@@ -254,7 +255,7 @@ func structToNode(typ *arrow.StructType, name string,
nullable bool, fieldID int
children = append(children, n)
}
- return schema.NewGroupNode(name, repFromNullable(nullable), children,
fieldID)
+ return schema.NewGroupNode(field.Name, repFromNullable(field.Nullable),
children, fieldIDFromMeta(field.Metadata))
}
func fieldToNode(name string, field arrow.Field, props
*parquet.WriterProperties, arrprops ArrowWriterProperties) (schema.Node, error)
{
@@ -267,7 +268,7 @@ func fieldToNode(name string, field arrow.Field, props
*parquet.WriterProperties
return nil, xerrors.New("nulltype arrow field must be
nullable")
}
case arrow.STRUCT:
- return structToNode(field.Type.(*arrow.StructType), field.Name,
field.Nullable, fieldIDFromMeta(field.Metadata), props, arrprops)
+ return structToNode(field, props, arrprops)
case arrow.FIXED_SIZE_LIST, arrow.LIST:
elemField := field.Type.(arrow.ListLikeType).ElemField()
diff --git a/parquet/pqarrow/schema_test.go b/parquet/pqarrow/schema_test.go
index f075b46..cc1538e 100644
--- a/parquet/pqarrow/schema_test.go
+++ b/parquet/pqarrow/schema_test.go
@@ -292,7 +292,7 @@ func TestConvertArrowFloat16(t *testing.T) {
}
}
-func TestCoerceTImestampV1(t *testing.T) {
+func TestCoerceTimestampV1(t *testing.T) {
parquetFields := make(schema.FieldList, 0)
arrowFields := make([]arrow.Field, 0)
@@ -311,7 +311,7 @@ func TestCoerceTImestampV1(t *testing.T) {
}
}
-func TestAutoCoerceTImestampV1(t *testing.T) {
+func TestAutoCoerceTimestampV1(t *testing.T) {
parquetFields := make(schema.FieldList, 0)
arrowFields := make([]arrow.Field, 0)
@@ -402,7 +402,7 @@ func TestListStructBackwardCompatible(t *testing.T) {
schema.StringLogicalType{},
parquet.Types.ByteArray, -1, 3)),
schema.MustPrimitive(schema.NewPrimitiveNodeLogical("class",
parquet.Repetitions.Optional,
schema.StringLogicalType{},
parquet.Types.ByteArray, -1, 4)),
- }, -1)),
+ }, 5)),
}, schema.NewListLogicalType(), 1)),
}, -1)))
@@ -417,7 +417,7 @@ func TestListStructBackwardCompatible(t *testing.T) {
Metadata:
arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"3"})},
arrow.Field{Name: "class", Type:
arrow.BinaryTypes.String, Nullable: true,
Metadata:
arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"4"})},
- ), Nullable: true, Metadata:
arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"-1"})}),
+ ), Nullable: true, Metadata:
arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"5"})}),
Nullable: true, Metadata:
arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"1"})},
}, nil)
