This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 2c703211 perf(arrow/array): write bulk binary offsets and payloads
directly (#1413)
2c703211 is described below
commit 2c703211a38567399ba1bd0491a82e4c0b6b2c73
Author: Minh Vu <[email protected]>
AuthorDate: Tue Oct 27 17:15:44 2026 +0100
perf(arrow/array): write bulk binary offsets and payloads directly (#1413)
### Rationale for this change
`AppendValues`/`AppendStringValues` on the Binary, LargeBinary, String
and LargeString builders still wrote each offset and payload separately
after reserving space.
### What changes are included in this PR?
- A shared generic helper, `appendBinaryValues[T ~string | ~[]byte]`,
writes the offsets directly into the reserved offset buffer and copies
the payloads in a single pass, for both 32- and 64-bit offsets.
- Nulls keep their payload bytes, and the validity bitmap and terminal
offset are handled as before.
### Are these changes tested?
Yes: `TestBinaryBuilderDirectBulkOffsetsAcrossBatches` (in
`binarybuilder_bulk_test.go`) covers unaligned prefixes, repeated
batches, nulls, empty values and mixing with scalar appends for all four
builder types.
`BenchmarkBinaryBuilderAppendValues` on amd64 (pinned, GOMAXPROCS=1,
`main` → this PR) is about 20–37% faster on the stable cases, e.g.
string 1024 rows width 4: 8.74 → 5.50 µs; large_string 65536 rows width
4: 782 → 574 µs.
### Are there any user-facing changes?
No API changes; faster bulk appends to binary and string builders.
---------
Co-authored-by: Matt Topol <[email protected]>
---
arrow/array/binarybuilder.go | 75 +++++++++++++++++----------------
arrow/array/binarybuilder_bulk_test.go | 76 ++++++++++++++++++++++++++++++++++
2 files changed, 115 insertions(+), 36 deletions(-)
diff --git a/arrow/array/binarybuilder.go b/arrow/array/binarybuilder.go
index 166bf6ac..761ab3e2 100644
--- a/arrow/array/binarybuilder.go
+++ b/arrow/array/binarybuilder.go
@@ -164,60 +164,63 @@ func (b *BinaryBuilder) AppendEmptyValues(n int) {
// in v are valid (not null). The valid slice must either be empty or be equal
in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendValues(v [][]byte, valid []bool) {
- if len(v) != len(valid) && len(valid) != 0 {
- panic("len(v) != len(valid) && len(valid) != 0")
- }
-
- if len(v) == 0 {
- return
- }
-
- b.Reserve(len(v))
- b.reserveOffsetCapacity(len(v))
-
- // Pre-calculate total data size to minimize allocations
- totalDataSize := 0
- for _, vv := range v {
- totalDataSize += len(vv)
- }
- b.ReserveData(totalDataSize)
-
- for _, vv := range v {
- b.unsafeAppendNextOffset()
- b.values.unsafeAppend(vv)
- }
-
- b.unsafeAppendBoolsToBitmap(valid, len(v))
+ appendBinaryValues(b, v, valid)
}
// AppendStringValues will append the values in the v slice. The valid slice
determines which values
// in v are valid (not null). The valid slice must either be empty or be equal
in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendStringValues(v []string, valid []bool) {
- if len(v) != len(valid) && len(valid) != 0 {
+ appendBinaryValues(b, v, valid)
+}
+
+// appendBinaryValues writes offsets and payloads in one pass after reserving
+// both buffers. The terminal offset is appended by newData, as for scalar
+// appends. Invalid slots still retain their payload and offsets.
+func appendBinaryValues[T ~string | ~[]byte](b *BinaryBuilder, values []T,
valid []bool) {
+ if len(values) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
-
- if len(v) == 0 {
+ if len(values) == 0 {
return
}
- b.Reserve(len(v))
- b.reserveOffsetCapacity(len(v))
+ b.Reserve(len(values))
+ b.reserveOffsetCapacity(len(values))
- // Pre-calculate total data size to minimize allocations
totalDataSize := 0
- for _, vv := range v {
- totalDataSize += len(vv)
+ for _, v := range values {
+ totalDataSize += len(v)
}
b.ReserveData(totalDataSize)
- for _, vv := range v {
- b.unsafeAppendNextOffset()
- b.values.unsafeAppend([]byte(vv))
+ pos := b.values.length
+ debug.Assert(uint64(pos+totalDataSize) <= b.maxCapacity, "exceeded
maximum capacity of binary array")
+ data := b.values.bytes
+
+ switch b.offsetByteWidth {
+ case arrow.Int32SizeBytes:
+ offsets := b.offsets.(*int32BufferBuilder)
+ end := offsets.length + len(values)*arrow.Int32SizeBytes
+ dst :=
arrow.Int32Traits.CastFromBytes(offsets.bytes[offsets.length:end])
+ for i, v := range values {
+ dst[i] = int32(pos)
+ pos += copy(data[pos:], v)
+ }
+ offsets.length = end
+ case arrow.Int64SizeBytes:
+ offsets := b.offsets.(*int64BufferBuilder)
+ end := offsets.length + len(values)*arrow.Int64SizeBytes
+ dst :=
arrow.Int64Traits.CastFromBytes(offsets.bytes[offsets.length:end])
+ for i, v := range values {
+ dst[i] = int64(pos)
+ pos += copy(data[pos:], v)
+ }
+ offsets.length = end
}
- b.unsafeAppendBoolsToBitmap(valid, len(v))
+ b.values.length = pos
+ b.unsafeAppendBoolsToBitmap(valid, len(values))
}
func (b *BinaryBuilder) UnsafeAppend(v []byte) {
diff --git a/arrow/array/binarybuilder_bulk_test.go
b/arrow/array/binarybuilder_bulk_test.go
index 53ad842f..bcc8a637 100644
--- a/arrow/array/binarybuilder_bulk_test.go
+++ b/arrow/array/binarybuilder_bulk_test.go
@@ -18,7 +18,9 @@
package array_test
import (
+ "bytes"
"fmt"
+ "strings"
"testing"
"github.com/apache/arrow-go/v18/arrow"
@@ -48,6 +50,80 @@ var binaryBuilderFactories = []struct {
}},
}
+func TestBinaryBuilderDirectBulkOffsetsAcrossBatches(t *testing.T) {
+ for _, factory := range binaryBuilderFactories {
+ t.Run(factory.name, func(t *testing.T) {
+ for _, prefix := range []int{0, 1, 7, 8, 9, 31, 64} {
+ t.Run(fmt.Sprintf("prefix_%d", prefix), func(t
*testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+ builder := factory.new(mem)
+ defer builder.Release()
+
+ var expectedPayload []byte
+ var expectedOffsets []int64
+ var expectedValid []bool
+ appendExpected := func(s string, valid
bool) {
+ expectedOffsets =
append(expectedOffsets, int64(len(expectedPayload)))
+ expectedPayload =
append(expectedPayload, s...)
+ expectedValid =
append(expectedValid, valid)
+ }
+ for i := 0; i < prefix; i++ {
+ s := fmt.Sprintf("prefix_%d", i)
+
appendBinaryBuilderValue(builder, s)
+ appendExpected(s, true)
+ }
+
+ for batch := 0; batch < 5; batch++ {
+ vals := make([]string, 17+batch)
+ valid := make([]bool, len(vals))
+ for i := range vals {
+ if i%7 != 0 {
+ vals[i] =
strings.Repeat("a", (i+batch*3)%71)
+ }
+ valid[i] = (i+batch)%4
!= 0
+ appendExpected(vals[i],
valid[i])
+ }
+
appendBinaryBuilderValues(builder, vals, valid)
+ }
+
+ builder.AppendNulls(3)
+ for range 3 {
+ appendExpected("", false)
+ }
+ builder.AppendEmptyValues(2)
+ for range 2 {
+ appendExpected("", true)
+ }
+ appendBinaryBuilderValues(builder,
[]string{"tail", ""}, nil)
+ appendExpected("tail", true)
+ appendExpected("", true)
+
+ arr :=
builder.NewArray().(array.BinaryLike)
+ defer arr.Release()
+ require.NoError(t, arr.(interface{
ValidateFull() error }).ValidateFull())
+ require.Equal(t, len(expectedOffsets),
arr.Len())
+ require.True(t,
bytes.Equal(expectedPayload, arr.ValueBytes()))
+ nulls := 0
+ for i := range expectedOffsets {
+ require.Equal(t,
expectedOffsets[i], arr.ValueOffset64(i), "offset %d", i)
+ require.Equal(t,
expectedValid[i], arr.IsValid(i), "valid %d", i)
+ if !expectedValid[i] {
+ nulls++
+ }
+ end := len(expectedPayload)
+ if i+1 < len(expectedOffsets) {
+ end =
int(expectedOffsets[i+1])
+ }
+ require.Equal(t,
end-int(expectedOffsets[i]), arr.ValueLen(i), "length %d", i)
+ }
+ require.Equal(t, nulls, arr.NullN())
+ })
+ }
+ })
+ }
+}
+
func TestBinaryBuilderBulkAppendNullsAndEmptyValues(t *testing.T) {
for _, tc := range binaryBuilderFactories {
t.Run(tc.name, func(t *testing.T) {
