This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 58995de6 perf(arrow/array): bulk append BinaryView nulls (#1232)
58995de6 is described below
commit 58995de6452b818579e86ec8bed1881e24433a6f
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 23:38:37 2026 +0200
perf(arrow/array): bulk append BinaryView nulls (#1232)
## What
- Replace the per-value validity updates in
`BinaryViewBuilder.AppendNulls` with one bulk bitmap update.
- Update the length and null count once per batch.
- Treat zero and negative counts as no-ops.
- Add parity coverage for aligned and unaligned starting positions.
## Benchmark
Apple M1 Pro, Go 1.26.3. The builder is preallocated and reused, so this
measures the bitmap append itself. Medians from 5 runs at offset 0:
Command: `go test -vet=off ./arrow/array -run "^$" -bench
"^BenchmarkBinaryViewBuilderAppendNulls$" -benchmem -benchtime=500ms
-count=5 -cpu=1`
| Values | Before | After |
| ---: | ---: | ---: |
| 1,024 | 764.8 ns/op | 19.66 ns/op |
| 65,536 | 40,266 ns/op | 152.2 ns/op |
Both versions use 0 B/op and 0 allocs/op in this preallocated benchmark.
## Tests
- `go test ./arrow/array -count=1`
- `PARQUET_TEST_DATA=parquet-testing/data go test ./... -count=1`
---
arrow/array/binarybuilder.go | 13 ++-
.../binaryview_builder_bulk_benchmark_test.go | 47 +++++++++
arrow/array/binaryview_builder_bulk_test.go | 111 +++++++++++++++++++++
3 files changed, 168 insertions(+), 3 deletions(-)
diff --git a/arrow/array/binarybuilder.go b/arrow/array/binarybuilder.go
index 8d7c5977..857e7fb5 100644
--- a/arrow/array/binarybuilder.go
+++ b/arrow/array/binarybuilder.go
@@ -606,10 +606,17 @@ func (b *BinaryViewBuilder) AppendNull() {
}
func (b *BinaryViewBuilder) AppendNulls(n int) {
- b.Reserve(n)
- for i := 0; i < n; i++ {
- b.UnsafeAppendBoolToBitmap(false)
+ if n <= 0 {
+ return
}
+ if n > math.MaxInt-b.length {
+ panic("arrow/array: builder length overflow")
+ }
+
+ b.Reserve(n)
+ bitutil.SetBitsTo(b.nullBitmap.Bytes(), int64(b.length), int64(n),
false)
+ b.length += n
+ b.nulls += n
}
func (b *BinaryViewBuilder) AppendEmptyValue() {
diff --git a/arrow/array/binaryview_builder_bulk_benchmark_test.go
b/arrow/array/binaryview_builder_bulk_benchmark_test.go
new file mode 100644
index 00000000..ff1043c4
--- /dev/null
+++ b/arrow/array/binaryview_builder_bulk_benchmark_test.go
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package array
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow/memory"
+)
+
+func BenchmarkBinaryViewBuilderAppendNulls(b *testing.B) {
+ for _, count := range []int{1024, 65536} {
+ for _, offset := range []int{0, 1, 7} {
+ b.Run(fmt.Sprintf("count_%d/offset_%d", count, offset),
func(b *testing.B) {
+ builder :=
NewBinaryViewBuilder(memory.DefaultAllocator)
+ builder.Resize(count + offset)
+ builder.length = offset
+ defer builder.Release()
+
+ b.ReportAllocs()
+ b.SetBytes(int64(count))
+ b.ResetTimer()
+ for b.Loop() {
+ builder.AppendNulls(count)
+ builder.length = offset
+ builder.nulls = 0
+ }
+ })
+ }
+ }
+}
diff --git a/arrow/array/binaryview_builder_bulk_test.go
b/arrow/array/binaryview_builder_bulk_test.go
new file mode 100644
index 00000000..f7abd5d0
--- /dev/null
+++ b/arrow/array/binaryview_builder_bulk_test.go
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package array_test
+
+import (
+ "fmt"
+ "math"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestBinaryViewBuilderBulkAppendNulls(t *testing.T) {
+ starts := []int{0, 1, 7, 8, 9, 15, 16, 17}
+ batchSizes := []int{-1, 0, 1, 2, 7, 8, 9, 16, 17}
+
+ for _, start := range starts {
+ for _, batchSize := range batchSizes {
+ t.Run(fmt.Sprintf("start_%d_batch_%d", start,
batchSize), func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ bulk := array.NewBinaryViewBuilder(mem)
+ defer bulk.Release()
+ scalar := array.NewBinaryViewBuilder(mem)
+ defer scalar.Release()
+
+ appendBinaryViewBuilderPrefix(bulk, start)
+ appendBinaryViewBuilderPrefix(scalar, start)
+ bulk.AppendNulls(batchSize)
+ for i := 0; i < batchSize; i++ {
+ scalar.AppendNull()
+ }
+
+ bulk.Append([]byte("tail"))
+ scalar.Append([]byte("tail"))
+
+ assertBinaryViewBuilderArrayParity(t, bulk,
scalar)
+ })
+ }
+ }
+}
+
+func TestBinaryViewBuilderAppendNullsRejectsLengthOverflow(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ builder := array.NewBinaryViewBuilder(mem)
+ defer builder.Release()
+ builder.AppendEmptyValue()
+
+ assert.PanicsWithValue(t, "arrow/array: builder length overflow",
func() {
+ builder.AppendNulls(math.MaxInt)
+ })
+ assert.Equal(t, 1, builder.Len())
+ assert.Equal(t, 0, builder.NullN())
+
+ arr := builder.NewArray().(*array.BinaryView)
+ defer arr.Release()
+ require.NoError(t, arr.ValidateFull())
+ assert.Equal(t, 1, arr.Len())
+ assert.Equal(t, 0, arr.NullN())
+ assert.True(t, arr.IsValid(0))
+}
+
+func appendBinaryViewBuilderPrefix(builder *array.BinaryViewBuilder, n int) {
+ for i := 0; i < n; i++ {
+ switch i % 3 {
+ case 0:
+ builder.Append([]byte(fmt.Sprintf("value-%d", i)))
+ case 1:
+ builder.AppendNull()
+ case 2:
+ builder.AppendEmptyValue()
+ }
+ }
+}
+
+func assertBinaryViewBuilderArrayParity(t *testing.T, bulk, scalar
*array.BinaryViewBuilder) {
+ t.Helper()
+
+ assert.Equal(t, scalar.Len(), bulk.Len())
+ assert.Equal(t, scalar.NullN(), bulk.NullN())
+
+ bulkArray := bulk.NewArray()
+ defer bulkArray.Release()
+ scalarArray := scalar.NewArray()
+ defer scalarArray.Release()
+
+ require.NoError(t, bulkArray.(interface{ ValidateFull() error
}).ValidateFull())
+ require.NoError(t, scalarArray.(interface{ ValidateFull() error
}).ValidateFull())
+ assert.True(t, array.Equal(bulkArray, scalarArray))
+}
