This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new d70b0f05 perf(arrow/ipc): slice already-zero-based offset buffers
(#1191)
d70b0f05 is described below
commit d70b0f050f02010e4339ddedd293d24ab537e713
Author: Minh Vu <[email protected]>
AuthorDate: Wed Aug 26 19:53:35 2026 +0200
perf(arrow/ipc): slice already-zero-based offset buffers (#1191)
### Rationale for this change
IPC currently allocates a new offsets buffer whenever a variable-length
array has unused trailing offsets. For a prefix slice, the first logical
offset is already zero, so the loop only copies every offset and
subtracts zero.
Arrow C++ handles this case by slicing the existing offsets buffer.
### What changes are included in this PR?
- Read the first logical offset instead of treating every sliced array
as needing rebasing.
- Rebase only when that offset is non-zero.
- Use `memory.SliceBuffer` when the required offsets are already
zero-based.
- Add coverage for int32 and int64 offsets, middle slices, and slices
with a non-zero array offset whose first logical value offset is zero.
The benchmark uses a 524,288-value slice from a 1,048,576-value backing
array. Medians from 6 runs on an Apple M1 Pro were:
| prefix slice | main | this PR |
|---|---:|---:|
| int32 time/op | 238 us | 40.7 ns |
| int32 B/op | 2,106,000 | 80 |
| int64 time/op | 283 us | 40.7 ns |
| int64 B/op | 4,203,153 | 80 |
| allocs/op | 8 | 1 |
Unsliced buffers remain allocation-free. Middle slices that need
rebasing are unchanged.
### Are these changes tested?
- `go test ./arrow/ipc -count=1`
- `go test ./arrow/... -count=1`
- `go test ./arrow/ipc -run "^$" -bench
"^BenchmarkGetZeroBasedValueOffsets$" -benchmem -count=6`
### Are there any user-facing changes?
No. The IPC output and public API are unchanged.
---
arrow/ipc/writer.go | 23 +++-------
arrow/ipc/writer_test.go | 115 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 120 insertions(+), 18 deletions(-)
diff --git a/arrow/ipc/writer.go b/arrow/ipc/writer.go
index 24340f44..a771849f 100644
--- a/arrow/ipc/writer.go
+++ b/arrow/ipc/writer.go
@@ -922,29 +922,14 @@ func (w *recordEncoder) getZeroBasedValueOffsets(arr
arrow.Array) *memory.Buffer
dataTypeWidth := arr.DataType().Layout().Buffers[1].ByteWidth
- // if we have a non-zero offset, then the value offsets do not start at
- // zero. we must a) create a new offsets array with shifted offsets and
- // b) slice the values array accordingly
- hasNonZeroOffset := data.Offset() != 0
-
- // or if there are more value offsets than values (the array has been
sliced)
- // we need to trim off the trailing offsets
- hasMoreOffsetsThanValues := offsetBytesNeeded < voffsets.Len()
-
- // or if the offsets do not start from the zero index, we need to shift
them
- // and slice the values array
var firstOffset int64
if dataTypeWidth == 8 {
- firstOffset =
arrow.Int64Traits.CastFromBytes(voffsets.Bytes())[0]
+ firstOffset =
arrow.Int64Traits.CastFromBytes(voffsets.Bytes())[data.Offset()]
} else {
- firstOffset =
int64(arrow.Int32Traits.CastFromBytes(voffsets.Bytes())[0])
+ firstOffset =
int64(arrow.Int32Traits.CastFromBytes(voffsets.Bytes())[data.Offset()])
}
- offsetsDoNotStartFromZero := firstOffset != 0
-
- // determine whether the offsets array should be shifted
- needsTruncateAndShift := hasNonZeroOffset || hasMoreOffsetsThanValues
|| offsetsDoNotStartFromZero
- if needsTruncateAndShift {
+ if firstOffset != 0 {
shiftedOffsets := memory.NewResizableBuffer(w.mem)
shiftedOffsets.Resize(offsetBytesNeeded)
@@ -970,6 +955,8 @@ func (w *recordEncoder) getZeroBasedValueOffsets(arr
arrow.Array) *memory.Buffer
}
voffsets = shiftedOffsets
+ } else if data.Offset() != 0 || offsetBytesNeeded < voffsets.Len() {
+ voffsets = memory.SliceBuffer(voffsets,
data.Offset()*dataTypeWidth, offsetBytesNeeded)
} else {
voffsets.Retain()
}
diff --git a/arrow/ipc/writer_test.go b/arrow/ipc/writer_test.go
index 6d18f5f0..e49c4f54 100644
--- a/arrow/ipc/writer_test.go
+++ b/arrow/ipc/writer_test.go
@@ -308,6 +308,7 @@ func TestGetZeroBasedValueOffsets(t *testing.T) {
offsets := env.getZeroBasedValueOffsets(arr)
defer offsets.Release()
assert.Equal(t, 44, offsets.Len(), "include all offsets if array is not
sliced")
+ assert.Same(t, arr.Data().Buffers()[1], offsets)
sl := array.NewSlice(arr, 0, 4)
defer sl.Release()
@@ -315,6 +316,120 @@ func TestGetZeroBasedValueOffsets(t *testing.T) {
offsets = env.getZeroBasedValueOffsets(sl)
defer offsets.Release()
assert.Equal(t, 20, offsets.Len(), "trim trailing offsets after slice")
+ assert.Same(t, sl.Data().Buffers()[1], offsets.Parent())
+
+ sl = array.NewSlice(arr, 2, 6)
+ defer sl.Release()
+
+ offsets = env.getZeroBasedValueOffsets(sl)
+ defer offsets.Release()
+ assert.Nil(t, offsets.Parent(), "rebase offsets when the first logical
offset is non-zero")
+ assert.Equal(t, []int32{0, 1, 2, 3, 4},
arrow.Int32Traits.CastFromBytes(offsets.Bytes()))
+
+ emptyPrefixBuilder := array.NewStringBuilder(alloc)
+ emptyPrefixBuilder.AppendValues([]string{"", "", "a"}, nil)
+ emptyPrefix := emptyPrefixBuilder.NewArray()
+ emptyPrefixBuilder.Release()
+ defer emptyPrefix.Release()
+
+ sl = array.NewSlice(emptyPrefix, 1, 3)
+ defer sl.Release()
+
+ offsets = env.getZeroBasedValueOffsets(sl)
+ defer offsets.Release()
+ assert.Same(t, sl.Data().Buffers()[1], offsets.Parent())
+ assert.Equal(t, []int32{0, 0, 1},
arrow.Int32Traits.CastFromBytes(offsets.Bytes()))
+
+ largeBuilder := array.NewLargeStringBuilder(alloc)
+ largeBuilder.AppendValues(vals, nil)
+ large := largeBuilder.NewArray()
+ largeBuilder.Release()
+ defer large.Release()
+
+ sl = array.NewSlice(large, 0, 4)
+ defer sl.Release()
+
+ offsets = env.getZeroBasedValueOffsets(sl)
+ defer offsets.Release()
+ assert.Same(t, sl.Data().Buffers()[1], offsets.Parent())
+ assert.Equal(t, []int64{0, 1, 2, 3, 4},
arrow.Int64Traits.CastFromBytes(offsets.Bytes()))
+
+ sl = array.NewSlice(large, 2, 6)
+ defer sl.Release()
+
+ offsets = env.getZeroBasedValueOffsets(sl)
+ defer offsets.Release()
+ assert.Nil(t, offsets.Parent(), "rebase int64 offsets when the first
logical offset is non-zero")
+ assert.Equal(t, []int64{0, 1, 2, 3, 4},
arrow.Int64Traits.CastFromBytes(offsets.Bytes()))
+
+ largeEmptyPrefixBuilder := array.NewLargeStringBuilder(alloc)
+ largeEmptyPrefixBuilder.AppendValues([]string{"", "", "a"}, nil)
+ largeEmptyPrefix := largeEmptyPrefixBuilder.NewArray()
+ largeEmptyPrefixBuilder.Release()
+ defer largeEmptyPrefix.Release()
+
+ sl = array.NewSlice(largeEmptyPrefix, 1, 3)
+ defer sl.Release()
+
+ offsets = env.getZeroBasedValueOffsets(sl)
+ defer offsets.Release()
+ assert.Same(t, sl.Data().Buffers()[1], offsets.Parent())
+ assert.Equal(t, []int64{0, 0, 1},
arrow.Int64Traits.CastFromBytes(offsets.Bytes()))
+}
+
+func BenchmarkGetZeroBasedValueOffsets(b *testing.B) {
+ alloc := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer alloc.AssertSize(b, 0)
+
+ const n = 1 << 20
+ values := make([]string, n)
+ for i := range values {
+ values[i] = "x"
+ }
+
+ stringBuilder := array.NewStringBuilder(alloc)
+ stringBuilder.AppendValues(values, nil)
+ strings := stringBuilder.NewArray()
+ stringBuilder.Release()
+ defer strings.Release()
+
+ largeStringBuilder := array.NewLargeStringBuilder(alloc)
+ largeStringBuilder.AppendValues(values, nil)
+ largeStrings := largeStringBuilder.NewArray()
+ largeStringBuilder.Release()
+ defer largeStrings.Release()
+
+ env := &recordEncoder{mem: alloc}
+ for _, tc := range []struct {
+ name string
+ arr arrow.Array
+ }{
+ {name: "int32", arr: strings},
+ {name: "int64", arr: largeStrings},
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ for _, slice := range []struct {
+ name string
+ begin, end int64
+ }{
+ {name: "full", begin: 0, end: n},
+ {name: "prefix", begin: 0, end: n / 2},
+ {name: "middle", begin: n / 4, end: 3 * n / 4},
+ } {
+ b.Run(slice.name, func(b *testing.B) {
+ arr := array.NewSlice(tc.arr,
slice.begin, slice.end)
+ defer arr.Release()
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ offsets :=
env.getZeroBasedValueOffsets(arr)
+ offsets.Release()
+ }
+ })
+ }
+ })
+ }
}
func TestWriterCatchPanic(t *testing.T) {
