This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 2120f2e4 perf(arrow/array): use unsafe writes for binary bulk appends
(#1240)
2120f2e4 is described below
commit 2120f2e4aae438105c86a2430cfef1af493094c8
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 23:38:47 2026 +0200
perf(arrow/array): use unsafe writes for binary bulk appends (#1240)
## What
- Use unchecked offset writes after `Reserve` in
`BinaryBuilder.AppendValues`.
- Use unchecked payload copies after `ReserveData`.
- Apply the same path to `AppendStringValues`.
- Keep scalar appends unchanged.
- Make `Reserve` cover the terminal offset when the builder has already
grown its minimum capacity.
- Keep the current physical payload layout for null entries.
- Add mixed validity coverage and benchmarks for Binary, LargeBinary,
String, and LargeString.
## Benchmark
Apple M1 Pro, 65,536 values x 4 bytes, representative run:
| Builder | Before | After | Change |
| --- | ---: | ---: | ---: |
| Binary | 545 us | 473 us | -13% |
| LargeBinary | 584 us | 535 us | -8% |
Allocations stayed at 9 per operation.
## Tests
- `PARQUET_TEST_DATA=parquet-testing/data go test ./...`
- `go test -race ./arrow/array -count=1`
- `go test -tags=assert ./arrow/array -count=1`
- `go vet ./arrow/array`
---
arrow/array/binarybuilder.go | 62 +++++---
arrow/array/binarybuilder_bulk_test.go | 276 +++++++++++++++++++++++++++++++++
arrow/array/bufferbuilder_unsafe.go | 34 ++++
3 files changed, 351 insertions(+), 21 deletions(-)
diff --git a/arrow/array/binarybuilder.go b/arrow/array/binarybuilder.go
index 857e7fb5..166bf6ac 100644
--- a/arrow/array/binarybuilder.go
+++ b/arrow/array/binarybuilder.go
@@ -39,26 +39,29 @@ type BinaryBuilder struct {
offsets bufBuilder
values *byteBufferBuilder
- appendOffsetVal func(int)
- getOffsetVal func(int) int
- maxCapacity uint64
- offsetByteWidth int
+ appendOffsetVal func(int)
+ unsafeAppendOffsetVal func(int)
+ getOffsetVal func(int) int
+ maxCapacity uint64
+ offsetByteWidth int
}
// NewBinaryBuilder can be used for any of the variable length binary types,
// Binary, LargeBinary, String, LargeString by passing the appropriate data
type
func NewBinaryBuilder(mem memory.Allocator, dtype arrow.BinaryDataType)
*BinaryBuilder {
var (
- offsets bufBuilder
- offsetValFn func(int)
- maxCapacity uint64
- offsetByteWidth int
- getOffsetVal func(int) int
+ offsets bufBuilder
+ offsetValFn func(int)
+ unsafeOffsetValFn func(int)
+ maxCapacity uint64
+ offsetByteWidth int
+ getOffsetVal func(int) int
)
switch dtype.Layout().Buffers[1].ByteWidth {
case 4:
b := newInt32BufferBuilder(mem)
offsetValFn = func(v int) { b.AppendValue(int32(v)) }
+ unsafeOffsetValFn = func(v int) { b.unsafeAppendValue(v) }
getOffsetVal = func(i int) int { return int(b.Value(i)) }
offsets = b
maxCapacity = math.MaxInt32
@@ -66,6 +69,7 @@ func NewBinaryBuilder(mem memory.Allocator, dtype
arrow.BinaryDataType) *BinaryB
case 8:
b := newInt64BufferBuilder(mem)
offsetValFn = func(v int) { b.AppendValue(int64(v)) }
+ unsafeOffsetValFn = func(v int) { b.unsafeAppendValue(v) }
getOffsetVal = func(i int) int { return int(b.Value(i)) }
offsets = b
maxCapacity = math.MaxInt64
@@ -73,14 +77,15 @@ func NewBinaryBuilder(mem memory.Allocator, dtype
arrow.BinaryDataType) *BinaryB
}
bb := &BinaryBuilder{
- builder: builder{mem: mem},
- dtype: dtype,
- offsets: offsets,
- values: newByteBufferBuilder(mem),
- appendOffsetVal: offsetValFn,
- maxCapacity: maxCapacity,
- offsetByteWidth: offsetByteWidth,
- getOffsetVal: getOffsetVal,
+ builder: builder{mem: mem},
+ dtype: dtype,
+ offsets: offsets,
+ values: newByteBufferBuilder(mem),
+ appendOffsetVal: offsetValFn,
+ unsafeAppendOffsetVal: unsafeOffsetValFn,
+ maxCapacity: maxCapacity,
+ offsetByteWidth: offsetByteWidth,
+ getOffsetVal: getOffsetVal,
}
bb.refCount.Add(1)
return bb
@@ -168,6 +173,7 @@ func (b *BinaryBuilder) AppendValues(v [][]byte, valid
[]bool) {
}
b.Reserve(len(v))
+ b.reserveOffsetCapacity(len(v))
// Pre-calculate total data size to minimize allocations
totalDataSize := 0
@@ -177,8 +183,8 @@ func (b *BinaryBuilder) AppendValues(v [][]byte, valid
[]bool) {
b.ReserveData(totalDataSize)
for _, vv := range v {
- b.appendNextOffset()
- b.values.Append(vv)
+ b.unsafeAppendNextOffset()
+ b.values.unsafeAppend(vv)
}
b.unsafeAppendBoolsToBitmap(valid, len(v))
@@ -197,6 +203,7 @@ func (b *BinaryBuilder) AppendStringValues(v []string,
valid []bool) {
}
b.Reserve(len(v))
+ b.reserveOffsetCapacity(len(v))
// Pre-calculate total data size to minimize allocations
totalDataSize := 0
@@ -206,8 +213,8 @@ func (b *BinaryBuilder) AppendStringValues(v []string,
valid []bool) {
b.ReserveData(totalDataSize)
for _, vv := range v {
- b.appendNextOffset()
- b.values.Append([]byte(vv))
+ b.unsafeAppendNextOffset()
+ b.values.unsafeAppend([]byte(vv))
}
b.unsafeAppendBoolsToBitmap(valid, len(v))
@@ -265,6 +272,13 @@ func (b *BinaryBuilder) Reserve(n int) {
b.reserve(n, b.Resize)
}
+func (b *BinaryBuilder) reserveOffsetCapacity(n int) {
+ offsetBytes := (b.length + n + 1) * b.offsetByteWidth
+ if b.offsets.Cap() < offsetBytes {
+ b.offsets.resize(offsetBytes)
+ }
+}
+
// ReserveData ensures there is enough space for appending n bytes
// by checking the capacity and resizing the data buffer if necessary.
func (b *BinaryBuilder) ReserveData(n int) {
@@ -359,6 +373,12 @@ func (b *BinaryBuilder) appendNextOffset() {
b.appendOffsetVal(numBytes)
}
+func (b *BinaryBuilder) unsafeAppendNextOffset() {
+ numBytes := b.values.Len()
+ debug.Assert(uint64(numBytes) <= b.maxCapacity, "exceeded maximum
capacity of binary array")
+ b.unsafeAppendOffsetVal(numBytes)
+}
+
func (b *BinaryBuilder) appendCurrentOffsets(n int) {
numBytes := b.values.Len()
debug.Assert(uint64(numBytes) <= b.maxCapacity, "exceeded maximum
capacity of binary array")
diff --git a/arrow/array/binarybuilder_bulk_test.go
b/arrow/array/binarybuilder_bulk_test.go
index 8448cb30..53ad842f 100644
--- a/arrow/array/binarybuilder_bulk_test.go
+++ b/arrow/array/binarybuilder_bulk_test.go
@@ -198,6 +198,69 @@ func assertBinaryBuilderArrayParity(t *testing.T, bulk,
scalar array.Builder) {
assert.True(t, array.Equal(bulkArray, scalarArray))
}
+func TestBinaryBuilderBulkAppendValuesPreservesNullPayload(t *testing.T) {
+ values := []string{"", "one", "世界", "", "five", "six"}
+ valid := []bool{true, false, true, true, false, true}
+ suffix := []string{"tail", ""}
+
+ for _, factory := range binaryBuilderFactories {
+ t.Run(factory.name, func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ builder := factory.new(mem)
+ defer builder.Release()
+ appendBinaryBuilderValue(builder, "prefix")
+ builder.AppendNull()
+ builder.AppendEmptyValue()
+ appendBinaryBuilderValues(builder, values, valid)
+ appendBinaryBuilderValues(builder, suffix, nil)
+
+ arr := builder.NewArray().(array.BinaryLike)
+ defer arr.Release()
+ require.NoError(t, arr.(interface{ ValidateFull() error
}).ValidateFull())
+
+ expectedValues := append([]string{"prefix", "", ""},
values...)
+ expectedValues = append(expectedValues, suffix...)
+ expectedValid := append([]bool{true, false, true},
valid...)
+ expectedValid = append(expectedValid, true, true)
+ expectedData := make([]byte, 0)
+ for _, value := range expectedValues {
+ expectedData = append(expectedData,
[]byte(value)...)
+ }
+
+ assert.Equal(t, len(expectedValues), arr.Len())
+ assert.Equal(t, 3, arr.NullN())
+ assert.Equal(t, expectedData, arr.ValueBytes())
+
+ dataOffset := int64(0)
+ for i, value := range expectedValues {
+ assert.Equal(t, expectedValid[i],
arr.IsValid(i), "value %d", i)
+ assert.Equal(t, dataOffset,
arr.ValueOffset64(i), "value %d offset", i)
+ assert.Equal(t, len(value), arr.ValueLen(i),
"value %d length", i)
+ dataOffset += int64(len(value))
+ }
+ })
+ }
+}
+
+func appendBinaryBuilderValues(builder array.Builder, values []string, valid
[]bool) {
+ switch builder := builder.(type) {
+ case *array.BinaryBuilder:
+ binaryValues := make([][]byte, len(values))
+ for i, value := range values {
+ binaryValues[i] = []byte(value)
+ }
+ builder.AppendValues(binaryValues, valid)
+ case *array.StringBuilder:
+ builder.AppendValues(values, valid)
+ case *array.LargeStringBuilder:
+ builder.AppendValues(values, valid)
+ default:
+ panic(fmt.Sprintf("unexpected binary builder %T", builder))
+ }
+}
+
func BenchmarkBinaryBuilderBulkAppend(b *testing.B) {
for _, tc := range binaryBuilderFactories {
b.Run(tc.name, func(b *testing.B) {
@@ -231,3 +294,216 @@ func benchmarkBinaryBuilderBulkAppend(b *testing.B,
factory binaryBuilderFactory
arr.Release()
}
}
+
+func BenchmarkBinaryBuilderScalarAppend(b *testing.B) {
+ const rows = 64 * 1024
+
+ b.Run("append", func(b *testing.B) {
+ builder := array.NewBinaryBuilder(memory.DefaultAllocator,
arrow.BinaryTypes.Binary)
+ defer builder.Release()
+ value := []byte("data")
+ builder.Resize(rows)
+ builder.ReserveData(rows * len(value))
+ builder.Resize(0)
+
+ b.SetBytes(int64(rows * len(value)))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ b.StopTimer()
+ builder.Resize(rows)
+ b.StartTimer()
+ for range rows {
+ builder.Append(value)
+ }
+ b.StopTimer()
+ builder.Resize(0)
+ b.StartTimer()
+ }
+ })
+
+ b.Run("append_null", func(b *testing.B) {
+ builder := array.NewBinaryBuilder(memory.DefaultAllocator,
arrow.BinaryTypes.Binary)
+ defer builder.Release()
+ builder.Resize(rows)
+ builder.Resize(0)
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ b.StopTimer()
+ builder.Resize(rows)
+ b.StartTimer()
+ for range rows {
+ builder.AppendNull()
+ }
+ b.StopTimer()
+ builder.Resize(0)
+ b.StartTimer()
+ }
+ })
+
+ b.Run("append_empty_value", func(b *testing.B) {
+ builder := array.NewBinaryBuilder(memory.DefaultAllocator,
arrow.BinaryTypes.Binary)
+ defer builder.Release()
+ builder.Resize(rows)
+ builder.Resize(0)
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ b.StopTimer()
+ builder.Resize(rows)
+ b.StartTimer()
+ for range rows {
+ builder.AppendEmptyValue()
+ }
+ b.StopTimer()
+ builder.Resize(0)
+ b.StartTimer()
+ }
+ })
+}
+
+type binaryBuilderValuesFactory struct {
+ name string
+ new binaryBuilderFactory
+ values func(rows, width int) any
+ appendValues func(array.Builder, any, []bool)
+}
+
+var binaryBuilderValuesFactories = []binaryBuilderValuesFactory{
+ {
+ name: "binary",
+ new: func(mem memory.Allocator) array.Builder {
+ return array.NewBinaryBuilder(mem,
arrow.BinaryTypes.Binary)
+ },
+ values: func(rows, width int) any {
+ value := make([]byte, width)
+ for i := range value {
+ value[i] = byte('a' + i%26)
+ }
+ values := make([][]byte, rows)
+ for i := range values {
+ values[i] = value
+ }
+ return values
+ },
+ appendValues: func(builder array.Builder, values any, valid
[]bool) {
+
builder.(*array.BinaryBuilder).AppendValues(values.([][]byte), valid)
+ },
+ },
+ {
+ name: "large_binary",
+ new: func(mem memory.Allocator) array.Builder {
+ return array.NewBinaryBuilder(mem,
arrow.BinaryTypes.LargeBinary)
+ },
+ values: func(rows, width int) any {
+ value := make([]byte, width)
+ for i := range value {
+ value[i] = byte('a' + i%26)
+ }
+ values := make([][]byte, rows)
+ for i := range values {
+ values[i] = value
+ }
+ return values
+ },
+ appendValues: func(builder array.Builder, values any, valid
[]bool) {
+
builder.(*array.BinaryBuilder).AppendValues(values.([][]byte), valid)
+ },
+ },
+ {
+ name: "string",
+ new: func(mem memory.Allocator) array.Builder {
+ return array.NewStringBuilder(mem)
+ },
+ values: func(rows, width int) any {
+ value := make([]byte, width)
+ for i := range value {
+ value[i] = byte('a' + i%26)
+ }
+ values := make([]string, rows)
+ for i := range values {
+ values[i] = string(value)
+ }
+ return values
+ },
+ appendValues: func(builder array.Builder, values any, valid
[]bool) {
+
builder.(*array.StringBuilder).AppendValues(values.([]string), valid)
+ },
+ },
+ {
+ name: "large_string",
+ new: func(mem memory.Allocator) array.Builder {
+ return array.NewLargeStringBuilder(mem)
+ },
+ values: func(rows, width int) any {
+ value := make([]byte, width)
+ for i := range value {
+ value[i] = byte('a' + i%26)
+ }
+ values := make([]string, rows)
+ for i := range values {
+ values[i] = string(value)
+ }
+ return values
+ },
+ appendValues: func(builder array.Builder, values any, valid
[]bool) {
+
builder.(*array.LargeStringBuilder).AppendValues(values.([]string), valid)
+ },
+ },
+}
+
+func BenchmarkBinaryBuilderAppendValues(b *testing.B) {
+ validityPatterns := []struct {
+ name string
+ valid func(rows int) []bool
+ }{
+ {name: "all_valid", valid: func(int) []bool { return nil }},
+ {
+ name: "10pct_null",
+ valid: func(rows int) []bool {
+ valid := make([]bool, rows)
+ for i := range valid {
+ valid[i] = i%10 != 0
+ }
+ return valid
+ },
+ },
+ {
+ name: "50pct_null",
+ valid: func(rows int) []bool {
+ valid := make([]bool, rows)
+ for i := range valid {
+ valid[i] = i%2 != 0
+ }
+ return valid
+ },
+ },
+ }
+
+ for _, factory := range binaryBuilderValuesFactories {
+ for _, rows := range []int{1024, 65536} {
+ for _, width := range []int{4, 16, 64, 256} {
+ values := factory.values(rows, width)
+ for _, pattern := range validityPatterns {
+ valid := pattern.valid(rows)
+ name :=
fmt.Sprintf("%s/rows_%d/width_%d/%s", factory.name, rows, width, pattern.name)
+ b.Run(name, func(b *testing.B) {
+ builder :=
factory.new(memory.DefaultAllocator)
+ defer builder.Release()
+ b.SetBytes(int64(rows * width))
+ b.ReportAllocs()
+
+ for b.Loop() {
+
factory.appendValues(builder, values, valid)
+ arr :=
builder.NewArray()
+ arr.Release()
+ }
+ })
+ }
+ }
+ }
+ }
+}
diff --git a/arrow/array/bufferbuilder_unsafe.go
b/arrow/array/bufferbuilder_unsafe.go
new file mode 100644
index 00000000..cf2df843
--- /dev/null
+++ b/arrow/array/bufferbuilder_unsafe.go
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package array
+
+import "github.com/apache/arrow-go/v18/arrow"
+
+// unsafeAppendValue appends v without checking the buffer capacity.
+// The caller must reserve enough space before using it.
+func (b *int32BufferBuilder) unsafeAppendValue(v int) {
+ arrow.Int32Traits.PutValue(b.bytes[b.length:], int32(v))
+ b.length += arrow.Int32SizeBytes
+}
+
+// unsafeAppendValue appends v without checking the buffer capacity.
+// The caller must reserve enough space before using it.
+func (b *int64BufferBuilder) unsafeAppendValue(v int) {
+ arrow.Int64Traits.PutValue(b.bytes[b.length:], int64(v))
+ b.length += arrow.Int64SizeBytes
+}
