This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new a958ec39 perf(parquet): reuse delta PutSpaced scratch (#1230)
a958ec39 is described below
commit a958ec3963508b0fc4e56d152bab3ec2d45f333a
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 19:43:17 2026 +0200
perf(parquet): reuse delta PutSpaced scratch (#1230)
## What
- Reuse an allocator-backed scratch buffer in
`deltaBitPackEncoder.PutSpaced`.
- Keep the buffer capacity between calls and release it with the
encoder.
- Keep the existing compaction path unchanged.
## Benchmark
Apple M1 Pro, Go 1.26.3. Steady-state repeated `PutSpaced` plus
`FlushValues` calls on the same encoder, with 10% nulls. Scratch growth
is done before timing.
Command: `go test -vet=off ./parquet/internal/encoding -run "^$" -bench
"^BenchmarkDeltaBinaryPackedPutSpaced(Int32|Int64)$" -benchmem
-benchtime=300ms -count=3 -cpu=1`
| Type | Length | Before | After |
| --- | ---: | ---: | ---: |
| INT32 | 1,024 | 5,333 B/op, 36 allocs/op | 469 B/op, 35 allocs/op |
| INT32 | 65,536 | 282,188 B/op, 1,851 allocs/op | 11,836 B/op, 1,850
allocs/op |
| INT64 | 1,024 | 9,941 B/op, 36 allocs/op | 469 B/op, 35 allocs/op |
| INT64 | 65,536 | 544,346 B/op, 1,851 allocs/op | 11,836 B/op, 1,850
allocs/op |
## Tests
- `go test ./parquet/internal/encoding -count=1`
- `PARQUET_TEST_DATA=parquet-testing/data go test ./... -count=1`
---
parquet/internal/encoding/delta_bit_packing.go | 18 ++++-
.../encoding/delta_bit_packing_benchmark_test.go | 93 ++++++++++++++++++++++
.../encoding/delta_bit_packing_validation_test.go | 9 +++
3 files changed, 116 insertions(+), 4 deletions(-)
diff --git a/parquet/internal/encoding/delta_bit_packing.go
b/parquet/internal/encoding/delta_bit_packing.go
index e8b3a30f..e0fbcbdd 100644
--- a/parquet/internal/encoding/delta_bit_packing.go
+++ b/parquet/internal/encoding/delta_bit_packing.go
@@ -371,6 +371,7 @@ type deltaBitPackEncoder[T int32 | int64] struct {
miniBlockSize uint64
numMiniBlocks uint64
deltas []int64
+ spacedScratch *memory.Buffer
}
// flushBlock flushes out a finished block for writing to the underlying
encoder
@@ -502,15 +503,24 @@ func (enc *deltaBitPackEncoder[T])
EstimatedDataEncodedSize() int64 {
return int64(enc.bitWriter.Written())
}
+func (enc *deltaBitPackEncoder[T]) Release() {
+ enc.encoder.Release()
+ if enc.spacedScratch != nil {
+ enc.spacedScratch.Release()
+ enc.spacedScratch = nil
+ }
+}
+
// PutSpaced takes a slice of values along with a bitmap that describes the
nulls and an offset into the bitmap
// in order to write spaced data to the encoder.
func (enc *deltaBitPackEncoder[T]) PutSpaced(in []T, validBits []byte,
validBitsOffset int64) {
- buffer := memory.NewResizableBuffer(enc.mem)
+ if enc.spacedScratch == nil {
+ enc.spacedScratch = memory.NewResizableBuffer(enc.mem)
+ }
dt := arrow.GetDataType[T]().(arrow.FixedWidthDataType)
- buffer.Reserve(dt.Bytes() * len(in))
- defer buffer.Release()
+ enc.spacedScratch.ResizeNoShrink(dt.Bytes() * len(in))
- data := arrow.GetData[T](buffer.Buf())
+ data := arrow.GetData[T](enc.spacedScratch.Buf())
nvalid := spacedCompress(in, data, validBits, validBitsOffset)
enc.Put(data[:nvalid])
}
diff --git a/parquet/internal/encoding/delta_bit_packing_benchmark_test.go
b/parquet/internal/encoding/delta_bit_packing_benchmark_test.go
new file mode 100644
index 00000000..04c0a65f
--- /dev/null
+++ b/parquet/internal/encoding/delta_bit_packing_benchmark_test.go
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encoding_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/bitutil"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/internal/encoding"
+)
+
+func BenchmarkDeltaBinaryPackedPutSpacedInt32(b *testing.B) {
+ benchmarkDeltaBinaryPackedPutSpaced[int32](b, parquet.Types.Int32,
arrow.Int32SizeBytes)
+}
+
+func BenchmarkDeltaBinaryPackedPutSpacedInt64(b *testing.B) {
+ benchmarkDeltaBinaryPackedPutSpaced[int64](b, parquet.Types.Int64,
arrow.Int64SizeBytes)
+}
+
+func benchmarkDeltaBinaryPackedPutSpaced[T int32 | int64](b *testing.B, typ
parquet.Type, bytesPerValue int) {
+ patterns := []struct {
+ name string
+ valid func(int) bool
+ }{
+ {name: "all_valid", valid: func(int) bool { return true }},
+ {name: "ten_percent_null", valid: func(i int) bool { return
i%10 != 0 }},
+ {name: "fifty_percent_null", valid: func(i int) bool { return
i%2 != 0 }},
+ {name: "ninety_percent_null", valid: func(i int) bool { return
i%10 == 0 }},
+ }
+
+ for _, length := range []int{1024, 64 * 1024} {
+ b.Run(fmt.Sprintf("length_%d", length), func(b *testing.B) {
+ values := make([]T, length)
+ for i := range values {
+ values[i] = T(i)
+ }
+
+ for _, pattern := range patterns {
+ b.Run(pattern.name, func(b *testing.B) {
+ validBits := make([]byte,
bitutil.BytesForBits(int64(length)))
+ for i := range length {
+ if pattern.valid(i) {
+
bitutil.SetBit(validBits, i)
+ }
+ }
+
+ encoder := encoding.NewEncoder(
+ typ,
parquet.Encodings.DeltaBinaryPacked,
+ false, nil,
memory.DefaultAllocator,
+ ).(encoding.Encoder[T])
+ defer encoder.Release()
+
+ encoder.PutSpaced(values, validBits, 0)
+ buf, err := encoder.FlushValues()
+ if err != nil {
+ b.Fatal(err)
+ }
+ buf.Release()
+
+ b.ReportAllocs()
+ b.SetBytes(int64(length *
bytesPerValue))
+ b.ResetTimer()
+ for b.Loop() {
+ encoder.PutSpaced(values,
validBits, 0)
+ buf, err :=
encoder.FlushValues()
+ if err != nil {
+ b.Fatal(err)
+ }
+ buf.Release()
+ }
+ })
+ }
+ })
+ }
+}
diff --git a/parquet/internal/encoding/delta_bit_packing_validation_test.go
b/parquet/internal/encoding/delta_bit_packing_validation_test.go
index 030aa96a..0f71f804 100644
--- a/parquet/internal/encoding/delta_bit_packing_validation_test.go
+++ b/parquet/internal/encoding/delta_bit_packing_validation_test.go
@@ -117,3 +117,12 @@ func TestDeltaBitPackDecoderBoundsPackedScratch(t
*testing.T) {
require.Error(t, err)
require.LessOrEqual(t, cap(dec.deltaBuf), deltaBitPackScratchSize)
}
+
+func TestDeltaBitPackEncoderReleasesSpacedScratch(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ enc := NewEncoder(parquet.Types.Int32,
parquet.Encodings.DeltaBinaryPacked, false, nil,
mem).(*deltaBitPackEncoder[int32])
+ enc.PutSpaced([]int32{1, 2, 3, 4}, []byte{0x0f}, 0)
+ enc.Release()
+}
