This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 0a16b24c perf(arrow/array): optimize bulk appends for struct builders
(#1189)
0a16b24c is described below
commit 0a16b24c97965de8a2ab71fe57c8832a95396e5b
Author: Minh Vu <[email protected]>
AuthorDate: Thu Aug 27 23:57:08 2026 +0200
perf(arrow/array): optimize bulk appends for struct builders (#1189)
## Summary
- reserve the struct and its children once for the full batch
- update the parent validity bitmap and counters once
- dispatch one bulk null or empty append to each child builder
- preserve the existing Go behavior where null structs append null child
values
- add mixed nested-struct coverage and width-scaled benchmarks
This uses the same bulk-dispatch orientation as [Arrow C++
StructBuilder](https://github.com/apache/arrow/blob/485499fd02ea2b0c323d67871fbe96aae4232504/cpp/src/arrow/array/builder_nested.h#L774-L807),
while keeping Go child-null semantics unchanged.
## Benchmark
65,536 appended structs, representative medians on Apple M1 Pro:
| Fields | Operation | Before | After | Change |
| ---: | --- | ---: | ---: | ---: |
| 1 | nulls | 457 us | 245 us | -46% |
| 1 | empty | 639 us | 301 us | -53% |
| 4 | nulls | 1.40 ms | 0.82 ms | -41% |
| 4 | empty | 1.93 ms | 1.07 ms | -44% |
| 16 | nulls | 5.27 ms | 2.86 ms | -46% |
| 16 | empty | 6.96 ms | 3.86 ms | -45% |
| 64 | nulls | 17.37 ms | 11.20 ms | -36% |
| 64 | empty | 27.31 ms | 15.25 ms | -44% |
Allocations per build also drop:
- 1 field: 48 to 23
- 4 fields: 132 to 53
- 16 fields: 470 to 175
- 64 fields: 1,846 to about 687
Single-value performance stays about the same.
## Tests
- `go test -p 1 ./...`
- `go test -race ./arrow/array -run
TestStructBuilderBulkAppendNullsAndEmptyValues -count=1`
- `go vet ./arrow/array`
Co-authored-by: Matt Topol <[email protected]>
---
arrow/array/struct.go | 22 +++++--
arrow/array/struct_bulk_test.go | 126 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 144 insertions(+), 4 deletions(-)
diff --git a/arrow/array/struct.go b/arrow/array/struct.go
index 49bcc570..a9d80835 100644
--- a/arrow/array/struct.go
+++ b/arrow/array/struct.go
@@ -407,8 +407,16 @@ func (b *StructBuilder) AppendValues(valids []bool) {
func (b *StructBuilder) AppendNull() { b.Append(false) }
func (b *StructBuilder) AppendNulls(n int) {
- for i := 0; i < n; i++ {
- b.AppendNull()
+ if n <= 0 {
+ return
+ }
+
+ b.Reserve(n)
+ bitutil.SetBitsTo(b.nullBitmap.Bytes(), int64(b.length), int64(n),
false)
+ b.length += n
+ b.nulls += n
+ for _, f := range b.fields {
+ f.AppendNulls(n)
}
}
@@ -420,8 +428,14 @@ func (b *StructBuilder) AppendEmptyValue() {
}
func (b *StructBuilder) AppendEmptyValues(n int) {
- for i := 0; i < n; i++ {
- b.AppendEmptyValue()
+ if n <= 0 {
+ return
+ }
+
+ b.Reserve(n)
+ b.unsafeAppendBoolsToBitmap(nil, n)
+ for _, f := range b.fields {
+ f.AppendEmptyValues(n)
}
}
diff --git a/arrow/array/struct_bulk_test.go b/arrow/array/struct_bulk_test.go
new file mode 100644
index 00000000..3e7f8f7a
--- /dev/null
+++ b/arrow/array/struct_bulk_test.go
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package array_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestStructBuilderBulkAppendNullsAndEmptyValues(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ nestedType := arrow.StructOf(arrow.Field{Name: "value", Type:
arrow.PrimitiveTypes.Int32})
+ dtype := arrow.StructOf(
+ arrow.Field{Name: "integer", Type: arrow.PrimitiveTypes.Int64},
+ arrow.Field{Name: "string", Type: arrow.BinaryTypes.String},
+ arrow.Field{Name: "boolean", Type:
arrow.FixedWidthTypes.Boolean},
+ arrow.Field{Name: "nested", Type: nestedType},
+ )
+ builder := array.NewStructBuilder(mem, dtype)
+ defer builder.Release()
+
+ appendStructBuilderValue(builder, 10, "a", true, 100)
+ builder.AppendNulls(5)
+ builder.AppendEmptyValues(4)
+ appendStructBuilderValue(builder, 20, "b", false, 200)
+
+ arr := builder.NewStructArray()
+ defer arr.Release()
+ require.NoError(t, arr.ValidateFull())
+ require.Equal(t, 11, arr.Len())
+ require.Equal(t, 5, arr.NullN())
+
+ for i := range arr.Len() {
+ wantValid := i == 0 || i >= 6
+ assert.Equal(t, wantValid, arr.IsValid(i), "parent value %d", i)
+ for field := 0; field < arr.NumField(); field++ {
+ assert.Equal(t, wantValid, arr.Field(field).IsValid(i),
"field %d value %d", field, i)
+ }
+ }
+
+ assert.Equal(t, []int64{10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20},
arr.Field(0).(*array.Int64).Int64Values())
+ assert.Equal(t, "a", arr.Field(1).(*array.String).Value(0))
+ assert.Equal(t, "b", arr.Field(1).(*array.String).Value(10))
+ for i := 1; i < 10; i++ {
+ assert.Empty(t, arr.Field(1).(*array.String).Value(i))
+ }
+
+ nested := arr.Field(3).(*array.Struct)
+ assert.Equal(t, 5, nested.NullN())
+ nestedValues := nested.Field(0).(*array.Int32)
+ assert.Equal(t, []int32{100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200},
nestedValues.Int32Values())
+ for i := range nestedValues.Len() {
+ assert.Equal(t, i == 0 || i >= 6, nestedValues.IsValid(i),
"nested value %d", i)
+ }
+}
+
+func appendStructBuilderValue(builder *array.StructBuilder, integer int64, str
string, boolean bool, nestedValue int32) {
+ builder.Append(true)
+ builder.FieldBuilder(0).(*array.Int64Builder).Append(integer)
+ builder.FieldBuilder(1).(*array.StringBuilder).Append(str)
+ builder.FieldBuilder(2).(*array.BooleanBuilder).Append(boolean)
+ nested := builder.FieldBuilder(3).(*array.StructBuilder)
+ nested.Append(true)
+ nested.FieldBuilder(0).(*array.Int32Builder).Append(nestedValue)
+}
+
+func BenchmarkStructBuilderBulkAppend(b *testing.B) {
+ for _, fields := range []int{1, 4, 16, 64} {
+ b.Run(fmt.Sprintf("fields_%d", fields), func(b *testing.B) {
+ for _, rows := range []int{1, 1024, 65536} {
+ b.Run(fmt.Sprintf("rows_%d", rows), func(b
*testing.B) {
+ b.Run("nulls", func(b *testing.B) {
+
benchmarkStructBuilderBulkAppend(b, fields, rows, false)
+ })
+ b.Run("empty", func(b *testing.B) {
+
benchmarkStructBuilderBulkAppend(b, fields, rows, true)
+ })
+ })
+ }
+ })
+ }
+}
+
+func benchmarkStructBuilderBulkAppend(b *testing.B, fields, rows int, empty
bool) {
+ structFields := make([]arrow.Field, fields)
+ for i := range structFields {
+ structFields[i] = arrow.Field{Name: fmt.Sprintf("field_%d", i),
Type: arrow.PrimitiveTypes.Int64}
+ }
+ builder := array.NewStructBuilder(memory.DefaultAllocator,
arrow.StructOf(structFields...))
+ defer builder.Release()
+ b.ReportAllocs()
+
+ for b.Loop() {
+ if empty {
+ builder.AppendEmptyValues(rows)
+ } else {
+ builder.AppendNulls(rows)
+ }
+
+ arr := builder.NewStructArray()
+ arr.Release()
+ }
+}
