This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 5cfdc159d perf(glue): reuse schema metadata when building Glue columns
(#2113)
5cfdc159d is described below
commit 5cfdc159da378531c388487472644010a0e43cd3
Author: Minh Vu <[email protected]>
AuthorDate: Tue Oct 6 10:27:39 2026 +0200
perf(glue): reuse schema metadata when building Glue columns (#2113)
* perf(glue): reuse schema metadata when building Glue columns
* test(glue): benchmark documented schema columns
* perf(glue): borrow schema fields during column conversion
---
catalog/glue/schema.go | 18 ++++---
catalog/glue/schema_bench_test.go | 111 ++++++++++++++++++++++++++++++++++++++
2 files changed, 121 insertions(+), 8 deletions(-)
diff --git a/catalog/glue/schema.go b/catalog/glue/schema.go
index d46be8a5b..c20e370af 100644
--- a/catalog/glue/schema.go
+++ b/catalog/glue/schema.go
@@ -23,6 +23,7 @@ import (
"strings"
"github.com/apache/iceberg-go"
+ iceberginternal "github.com/apache/iceberg-go/internal"
"github.com/apache/iceberg-go/table"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/glue/types"
@@ -31,6 +32,8 @@ import (
func schemasToGlueColumns(metadata table.Metadata, existingColumns
[]types.Column) []types.Column {
// preserve the current schema's logical column order and then,
// append columns from historical schemas that are not already present
+ currentSchema := metadata.CurrentSchema()
+ schemas := metadata.Schemas()
var columns []types.Column
addedNames := make(map[string]struct{})
@@ -44,12 +47,12 @@ func schemasToGlueColumns(metadata table.Metadata,
existingColumns []types.Colum
addedNames[name] = struct{}{}
}
- for _, field := range schemaToGlueColumns(metadata.CurrentSchema(),
true) {
+ for _, field := range schemaToGlueColumns(currentSchema, true) {
addColumnWithDedupe(field)
}
- for _, schema := range metadata.Schemas() {
- if schema.ID == metadata.CurrentSchema().ID {
+ for _, schema := range schemas {
+ if schema.ID == currentSchema.ID {
continue
}
@@ -72,17 +75,16 @@ func schemasToGlueColumns(metadata table.Metadata,
existingColumns []types.Colum
// Preserve nil for fields that have never had an Iceberg doc, but keep
an
// explicit empty string when a documented field was cleared in a later
schema.
clearedComments := make(map[string]struct{})
- currentSchema := metadata.CurrentSchema()
- for _, field := range currentSchema.Fields() {
+ for _, field := range
currentSchema.FieldsRef(iceberginternal.SchemaRef{}) {
if field.Doc != "" {
continue
}
- for _, schema := range metadata.Schemas() {
+ for _, schema := range schemas {
if schema.ID == currentSchema.ID {
continue
}
- if previous, ok := schema.FindFieldByID(field.ID); ok
&& previous.Doc != "" {
+ if previous, ok := schema.FindFieldByIDRef(field.ID,
iceberginternal.SchemaRef{}); ok && previous.Doc != "" {
clearedComments[strconv.Itoa(field.ID)] =
struct{}{}
break
@@ -112,7 +114,7 @@ func schemasToGlueColumns(metadata table.Metadata,
existingColumns []types.Colum
// schemaToGlueColumns converts an Iceberg schema to a list of Glue columns.
func schemaToGlueColumns(schema *iceberg.Schema, isCurrent bool)
[]types.Column {
var columns []types.Column
- for _, field := range schema.Fields() {
+ for _, field := range schema.FieldsRef(iceberginternal.SchemaRef{}) {
columns = append(columns, fieldToGlueColumn(field, isCurrent))
}
diff --git a/catalog/glue/schema_bench_test.go
b/catalog/glue/schema_bench_test.go
new file mode 100644
index 000000000..43a146c68
--- /dev/null
+++ b/catalog/glue/schema_bench_test.go
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package glue
+
+import (
+ "fmt"
+ "slices"
+ "testing"
+
+ "github.com/apache/iceberg-go"
+ "github.com/apache/iceberg-go/table"
+ "github.com/aws/aws-sdk-go-v2/service/glue/types"
+)
+
+var schemasToGlueColumnsBenchmarkSink []types.Column
+
+func BenchmarkSchemasToGlueColumns(b *testing.B) {
+ for _, tc := range []struct {
+ schemaCount int
+ fieldCount int
+ }{
+ {schemaCount: 1, fieldCount: 16},
+ {schemaCount: 10, fieldCount: 128},
+ {schemaCount: 100, fieldCount: 128},
+ } {
+ b.Run(fmt.Sprintf("schemas=%d/fields=%d", tc.schemaCount,
tc.fieldCount), func(b *testing.B) {
+ metadata := schemaHistoryBenchmarkMetadata(b,
tc.schemaCount, tc.fieldCount, false)
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for range b.N {
+ schemasToGlueColumnsBenchmarkSink =
schemasToGlueColumns(metadata, nil)
+ }
+ })
+ }
+}
+
+func schemaHistoryBenchmarkMetadata(b *testing.B, schemaCount, fieldCount int,
documented bool) table.Metadata {
+ b.Helper()
+
+ fields := make([]iceberg.NestedField, fieldCount)
+ for i := range fields {
+ fields[i] = iceberg.NestedField{
+ ID: i + 1,
+ Name: fmt.Sprintf("field_%d", i+1),
+ Type: iceberg.PrimitiveTypes.String,
+ }
+ if documented {
+ fields[i].Doc = "field documentation"
+ }
+ }
+
+ base, err := table.NewMetadata(iceberg.NewSchema(0, fields...), nil,
+ table.SortOrder{}, "s3://glue-schema-benchmark", nil)
+ if err != nil {
+ b.Fatal(err)
+ }
+ builder, err := table.MetadataBuilderFromBase(base, "")
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ for schemaID := 1; schemaID < schemaCount; schemaID++ {
+ schemaFields := slices.Clone(fields)
+ if schemaID == schemaCount-1 {
+ schemaFields[0].Name = "current_field"
+ } else {
+ schemaFields[0].Name =
fmt.Sprintf("historical_field_%d", schemaID)
+ }
+ if err := builder.AddSchema(iceberg.NewSchema(schemaID,
schemaFields...)); err != nil {
+ b.Fatal(err)
+ }
+ }
+ if err := builder.SetCurrentSchemaID(schemaCount - 1); err != nil {
+ b.Fatal(err)
+ }
+
+ metadata, err := builder.Build()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ return metadata
+}
+
+func BenchmarkSchemasToGlueColumnsDocumented(b *testing.B) {
+ for _, schemaCount := range []int{1, 100} {
+ b.Run(fmt.Sprintf("schemas=%d/fields=128", schemaCount), func(b
*testing.B) {
+ metadata := schemaHistoryBenchmarkMetadata(b,
schemaCount, 128, true)
+ b.ReportAllocs()
+ for b.Loop() {
+ schemasToGlueColumnsBenchmarkSink =
schemasToGlueColumns(metadata, nil)
+ }
+ })
+ }
+}
