This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 9c86a0cf perf(arrow/array): copy view buffers directly (#1324)
9c86a0cf is described below
commit 9c86a0cf736b1eb7642d2bec6507a19c5b4e9806
Author: Minh Vu <[email protected]>
AuthorDate: Fri Sep 25 18:01:49 2026 +0200
perf(arrow/array): copy view buffers directly (#1324)
**What**
- Copy BinaryView headers and ListView sizes directly from `ArrayData`
during concatenation.
**Why**
- Avoid one temporary `memory.Buffer` wrapper per input chunk.
- Reduce allocations for fragmented view arrays.
**Implementation**
- Use the existing `concatFixedWidthBuffers` helper for BinaryView
headers and ListView/LargeListView sizes.
- Added a benchmark for 64, 1,024, and 8,192 chunks.
- Existing concatenate tests cover sliced and nullable view arrays.
Benchmark on an Apple M1 Pro:
| Case | Runtime before -> after | Allocations before -> after |
| --- | ---: | ---: |
| BinaryView, 1,024 chunks | 1.02ms -> 0.93ms | 1,050 -> 25 |
| BinaryView, 8,192 chunks | roughly flat | 8,227 -> 34 |
| ListView, 8,192 chunks | 2.27ms -> 1.92ms | 26,477 -> 18,388 |
| LargeListView, 8,192 chunks | 2.28ms -> 1.95ms | 26,124 -> 17,304 |
Tests:
- `go test ./arrow/array -count=1`
- `go vet ./arrow/array`
- `git diff --check`
---
arrow/array/concat.go | 12 +++-
arrow/array/concat_test.go | 102 ++++++++++++++++++++++++++++++++
arrow/array/concat_view_failure_test.go | 59 ++++++++++++++++++
3 files changed, 170 insertions(+), 3 deletions(-)
diff --git a/arrow/array/concat.go b/arrow/array/concat.go
index a138717e..4079208c 100644
--- a/arrow/array/concat.go
+++ b/arrow/array/concat.go
@@ -154,6 +154,12 @@ func concatBuffers(bufs []*memory.Buffer, mem
memory.Allocator) *memory.Buffer {
func concatFixedWidthBuffers(data []arrow.ArrayData, idx, byteWidth, length
int, mem memory.Allocator) *memory.Buffer {
out := memory.NewResizableBuffer(mem)
+ success := false
+ defer func() {
+ if !success {
+ out.Release()
+ }
+ }()
out.Resize(length * byteWidth)
dst := out.Bytes()
for _, d := range data {
@@ -167,6 +173,7 @@ func concatFixedWidthBuffers(data []arrow.ArrayData, idx,
byteWidth, length int,
copy(dst, buf.Bytes()[begin:begin+nbytes])
dst = dst[nbytes:]
}
+ success = true
return out
}
@@ -611,8 +618,7 @@ func concatListView(data []arrow.ArrayData, offsetType
arrow.FixedWidthDataType,
}
// Concatenate the sizes
- sizeBuffers := gatherBuffersFixedWidthType(data, 2, offsetType)
- sizeBuffer := concatBuffers(sizeBuffers, mem)
+ sizeBuffer := concatFixedWidthBuffers(data, 2, offsetType.Bytes(),
out.length, mem)
out.childData = []arrow.ArrayData{values}
out.buffers[1] = offsetBuffer
@@ -720,7 +726,7 @@ func concat(data []arrow.ArrayData, mem memory.Allocator)
(arr arrow.ArrayData,
}
}
- out.buffers[1] = concatBuffers(gatherFixedBuffers(data, 1,
arrow.ViewHeaderSizeBytes), mem)
+ out.buffers[1] = concatFixedWidthBuffers(data, 1,
arrow.ViewHeaderSizeBytes, out.length, mem)
var (
s =
arrow.ViewHeaderTraits.CastFromBytes(out.buffers[1].Bytes())
diff --git a/arrow/array/concat_test.go b/arrow/array/concat_test.go
index 1eabbac7..e1831e26 100644
--- a/arrow/array/concat_test.go
+++ b/arrow/array/concat_test.go
@@ -141,6 +141,108 @@ func BenchmarkConcatenateFixedWidth(b *testing.B) {
}
}
+func BenchmarkConcatenateViews(b *testing.B) {
+ const totalValues = 1 << 16
+
+ makeBinaryView := func(mem memory.Allocator) arrow.Array {
+ builder := array.NewBinaryViewBuilder(mem)
+ value := []byte("01234567890123456789012345678901")
+ builder.Reserve(totalValues)
+ for i := 0; i < totalValues; i++ {
+ builder.Append(value)
+ }
+ result := builder.NewArray()
+ builder.Release()
+ return result
+ }
+
+ makeListView := func(mem memory.Allocator, large bool) arrow.Array {
+ valid := make([]bool, totalValues)
+ values := make([]int8, totalValues)
+ for i := range valid {
+ valid[i] = true
+ }
+
+ var builder array.VarLenListLikeBuilder
+ if large {
+ builder = array.NewLargeListViewBuilder(mem,
arrow.PrimitiveTypes.Int8)
+ offsets := make([]int64, totalValues)
+ sizes := make([]int64, totalValues)
+ for i := range offsets {
+ offsets[i] = int64(i)
+ sizes[i] = 1
+ }
+
builder.(*array.LargeListViewBuilder).AppendValuesWithSizes(offsets, sizes,
valid)
+ } else {
+ builder = array.NewListViewBuilder(mem,
arrow.PrimitiveTypes.Int8)
+ offsets := make([]int32, totalValues)
+ sizes := make([]int32, totalValues)
+ for i := range offsets {
+ offsets[i] = int32(i)
+ sizes[i] = 1
+ }
+
builder.(*array.ListViewBuilder).AppendValuesWithSizes(offsets, sizes, valid)
+ }
+
builder.ValueBuilder().(*array.Int8Builder).AppendValues(values, nil)
+ result := builder.NewArray()
+ builder.Release()
+ return result
+ }
+
+ tests := []struct {
+ name string
+ make func(memory.Allocator) arrow.Array
+ }{
+ {"binary_view", makeBinaryView},
+ {"list_view", func(mem memory.Allocator) arrow.Array {
+ return makeListView(mem, false)
+ }},
+ {"large_list_view", func(mem memory.Allocator) arrow.Array {
+ return makeListView(mem, true)
+ }},
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ b.Run(tt.name, func(b *testing.B) {
+ mem := memory.NewGoAllocator()
+ backing := tt.make(mem)
+ defer backing.Release()
+
+ for _, chunkCount := range []int{64, 1024, 8192} {
+ chunkCount := chunkCount
+ b.Run(fmt.Sprintf("chunks=%d", chunkCount),
func(b *testing.B) {
+ chunkSize := totalValues / chunkCount
+ inputs := make([]arrow.Array,
chunkCount)
+ for i := range inputs {
+ begin := int64(i * chunkSize)
+ inputs[i] =
array.NewSlice(backing, begin, begin+int64(chunkSize))
+ }
+ defer func() {
+ for _, input := range inputs {
+ input.Release()
+ }
+ }()
+
+ b.SetBytes(int64(totalValues))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ result, err :=
array.Concatenate(inputs, mem)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if result.Len() != totalValues {
+ b.Fatalf("result length
= %d, want %d", result.Len(), totalValues)
+ }
+ result.Release()
+ }
+ })
+ }
+ })
+ }
+}
+
func TestConcatenateFixedWidthSlices(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
diff --git a/arrow/array/concat_view_failure_test.go
b/arrow/array/concat_view_failure_test.go
new file mode 100644
index 00000000..1858209e
--- /dev/null
+++ b/arrow/array/concat_view_failure_test.go
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/stretchr/testify/require"
+)
+
+func TestConcatViewShortHeaderReleasesDestination(t *testing.T) {
+ for _, dtype := range []arrow.DataType{&arrow.BinaryViewType{},
&arrow.StringViewType{}} {
+ for _, tc := range []struct {
+ headerBytes int
+ offset int
+ }{
+ {0, 0},
+ {1, 0},
+ {arrow.ViewHeaderSizeBytes - 1, 0},
+ {arrow.ViewHeaderSizeBytes, 1},
+ } {
+ t.Run(fmt.Sprintf("%s/bytes=%d/offset=%d",
dtype.Name(), tc.headerBytes, tc.offset), func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+ header := memory.NewBufferBytes(make([]byte,
tc.headerBytes))
+ defer header.Release()
+ variadic := memory.NewResizableBuffer(mem)
+ variadic.Resize(32)
+ defer variadic.Release()
+ data := NewData(dtype, 1, []*memory.Buffer{nil,
header, variadic}, nil, 0, tc.offset)
+ defer data.Release()
+
+ result, err := concat([]arrow.ArrayData{data},
mem)
+ if result != nil {
+ defer result.Release()
+ }
+ require.Error(t, err)
+ require.Nil(t, result)
+ })
+ }
+ }
+}
