This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 77d27ce5 perf(parquet): reuse dictionary decoder scratch (#1222)
77d27ce5 is described below
commit 77d27ce5ecf819dd4350f59c3ecdf187a1b01e80
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 21:51:28 2026 +0200
perf(parquet): reuse dictionary decoder scratch (#1222)
## Summary
- Reuse the `[]int` conversion scratch in dictionary index decoding.
- Reuse the `[]bool` validity scratch in spaced dictionary decoding.
- Clear reused validity entries before applying the next bitmap.
- Add repeated dense and nullable decode tests.
- Add a benchmark for 1, 64, 4096, and 65536 values.
## Benchmark
**Command**
```text
go test ./parquet/internal/encoding -run '^$' -bench
'^BenchmarkDictByteArrayDecoderDecodeIndices$' -benchmem -benchtime=100ms
-count=3
```
**Machine:** Apple M1 Pro, arm64
**Go:** 1.26.3
The benchmark warms up one decode, then reuses the same decoder and
dictionary builder for repeated batches. The spaced cases use 25% null
values.
| Case | Before | After |
| --- | ---: | ---: |
| Dense, 1 value | 4,944 B/op, 3 allocs/op | 4,944 B/op, 3 allocs/op |
| Dense, 64 values | 5,712 B/op, 5 allocs/op | 5,200 B/op, 4 allocs/op |
| Dense, 4096 values | 54,096 B/op, 5 allocs/op | 21,328 B/op, 4
allocs/op |
| Dense, 65536 values | 791,378 B/op, 5 allocs/op | 267,089 B/op, 4
allocs/op |
| Spaced, 64 values | 9,936 B/op, 8 allocs/op | 9,360 B/op, 6 allocs/op
|
| Spaced, 4096 values | 58,192 B/op, 6 allocs/op | 21,328 B/op, 4
allocs/op |
| Spaced, 65536 values | 856,912 B/op, 6 allocs/op | 267,088 B/op, 4
allocs/op |
## Tests
- `go test ./parquet/internal/encoding`
- `go test ./parquet/file`
---
parquet/internal/encoding/decoder.go | 37 ++--
.../internal/encoding/decoder_benchmark_test.go | 198 +++++++++++++++++++++
2 files changed, 224 insertions(+), 11 deletions(-)
diff --git a/parquet/internal/encoding/decoder.go
b/parquet/internal/encoding/decoder.go
index 9c124da2..90ab6cbd 100644
--- a/parquet/internal/encoding/decoder.go
+++ b/parquet/internal/encoding/decoder.go
@@ -108,7 +108,9 @@ type dictDecoder[T parquet.ColumnTypes] struct {
dictValueDecoder utils.DictionaryConverter[T]
idxDecoder *utils.TypedRleDecoder[T]
- idxScratchSpace []uint64
+ idxScratchSpace []uint64
+ idxAppendScratch []int
+ validScratch []bool
}
// SetDict sets a decoder that can be used to decode the dictionary that is
@@ -169,11 +171,15 @@ func (d *dictDecoder[T]) DecodeIndices(numValues int,
bldr array.Builder) (int,
n, err := d.idxDecoder.GetBatch(d.idxScratchSpace)
- toAppend := make([]int, n)
- for i, v := range d.idxScratchSpace {
- toAppend[i] = int(v)
+ if cap(d.idxAppendScratch) < n {
+ d.idxAppendScratch = make([]int, n, bitutil.NextPowerOf2(n))
+ } else {
+ d.idxAppendScratch = d.idxAppendScratch[:n]
+ }
+ for i, v := range d.idxScratchSpace[:n] {
+ d.idxAppendScratch[i] = int(v)
}
- bldr.(*array.BinaryDictionaryBuilder).AppendIndices(toAppend, nil)
+ bldr.(*array.BinaryDictionaryBuilder).AppendIndices(d.idxAppendScratch,
nil)
d.nvals -= n
return n, err
}
@@ -190,15 +196,24 @@ func (d *dictDecoder[T]) DecodeIndicesSpaced(numValues,
nullCount int, validBits
return n, err
}
- valid := make([]bool, n)
+ if cap(d.validScratch) < n {
+ d.validScratch = make([]bool, n, bitutil.NextPowerOf2(n))
+ } else {
+ d.validScratch = d.validScratch[:n]
+ }
+ clear(d.validScratch)
bitutils.VisitBitBlocks(validBits, offset, int64(n),
- func(pos int64) { valid[pos] = true }, func() {})
+ func(pos int64) { d.validScratch[pos] = true }, func() {})
- toAppend := make([]int, n)
- for i, v := range d.idxScratchSpace {
- toAppend[i] = int(v)
+ if cap(d.idxAppendScratch) < n {
+ d.idxAppendScratch = make([]int, n, bitutil.NextPowerOf2(n))
+ } else {
+ d.idxAppendScratch = d.idxAppendScratch[:n]
+ }
+ for i, v := range d.idxScratchSpace[:n] {
+ d.idxAppendScratch[i] = int(v)
}
- bldr.(*array.BinaryDictionaryBuilder).AppendIndices(toAppend, valid)
+ bldr.(*array.BinaryDictionaryBuilder).AppendIndices(d.idxAppendScratch,
d.validScratch)
d.nvals -= n - nullCount
return n, nil
}
diff --git a/parquet/internal/encoding/decoder_benchmark_test.go
b/parquet/internal/encoding/decoder_benchmark_test.go
new file mode 100644
index 00000000..6b9e0a14
--- /dev/null
+++ b/parquet/internal/encoding/decoder_benchmark_test.go
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encoding_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/bitutil"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/internal/encoding"
+ "github.com/stretchr/testify/require"
+)
+
+func encodeDictIndices(t testing.TB, values []parquet.ByteArray)
encoding.Buffer {
+ t.Helper()
+
+ enc := encoding.NewEncoder(parquet.Types.ByteArray,
parquet.Encodings.PlainDict, true, nil,
memory.DefaultAllocator).(*encoding.DictByteArrayEncoder)
+ defer enc.Release()
+
+ enc.Put(values)
+ buf, err := enc.FlushValues()
+ if err != nil {
+ t.Fatalf("could not encode dictionary indices: %v", err)
+ }
+ return buf
+}
+
+func newBinaryDictionaryBuilder(mem memory.Allocator)
*array.BinaryDictionaryBuilder {
+ dictBuilder := array.NewStringBuilder(mem)
+ dictBuilder.Append("one")
+ dictBuilder.Append("two")
+ dictBuilder.Append("three")
+ dict := dictBuilder.NewArray()
+ dictBuilder.Release()
+
+ dictType := &arrow.DictionaryType{
+ IndexType: arrow.PrimitiveTypes.Int32,
+ ValueType: arrow.BinaryTypes.String,
+ }
+ bldr := array.NewDictionaryBuilderWithDict(mem, dictType,
dict).(*array.BinaryDictionaryBuilder)
+ dict.Release()
+ return bldr
+}
+
+func TestDictByteArrayDecoderDecodeIndices(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ values := []parquet.ByteArray{
+ []byte("one"), []byte("two"), []byte("one"), []byte("three"),
+ []byte("two"), []byte("one"),
+ }
+ indices := encodeDictIndices(t, values)
+ defer indices.Release()
+
+ decoder := encoding.NewDictDecoder(parquet.Types.ByteArray, nil,
mem).(*encoding.DictByteArrayDecoder)
+ bldr := newBinaryDictionaryBuilder(mem)
+ defer bldr.Release()
+
+ require.NoError(t, decoder.SetData(len(values), indices.Bytes()))
+ n, err := decoder.DecodeIndices(len(values), bldr)
+ require.NoError(t, err)
+ require.Equal(t, len(values), n)
+
+ arr := bldr.NewDictionaryArray()
+ for i, want := range []int{0, 1, 0, 2, 1, 0} {
+ require.False(t, arr.IsNull(i))
+ require.Equal(t, want, arr.GetValueIndex(i))
+ }
+ arr.Release()
+
+ require.NoError(t, decoder.SetData(len(values), indices.Bytes()))
+ n, err = decoder.DecodeIndices(len(values), bldr)
+ require.NoError(t, err)
+ require.Equal(t, len(values), n)
+}
+
+func TestDictByteArrayDecoderDecodeIndicesSpaced(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ values := []parquet.ByteArray{
+ []byte("one"), []byte("two"), []byte("three"), []byte("one"),
+ }
+ indices := encodeDictIndices(t, values)
+ defer indices.Release()
+
+ decoder := encoding.NewDictDecoder(parquet.Types.ByteArray, nil,
mem).(*encoding.DictByteArrayDecoder)
+ bldr := newBinaryDictionaryBuilder(mem)
+ defer bldr.Release()
+
+ validBits := []byte{0x0f}
+ require.NoError(t, decoder.SetData(len(values), indices.Bytes()))
+ n, err := decoder.DecodeIndicesSpaced(8, 4, validBits, 0, bldr)
+ require.NoError(t, err)
+ require.Equal(t, 8, n)
+
+ arr := bldr.NewDictionaryArray()
+ for i := 0; i < 8; i++ {
+ require.Equal(t, i < 4, !arr.IsNull(i))
+ }
+ arr.Release()
+
+ validBits = []byte{0xf0}
+ require.NoError(t, decoder.SetData(len(values), indices.Bytes()))
+ n, err = decoder.DecodeIndicesSpaced(8, 4, validBits, 0, bldr)
+ require.NoError(t, err)
+ require.Equal(t, 8, n)
+
+ arr = bldr.NewDictionaryArray()
+ defer arr.Release()
+ for i := 0; i < 8; i++ {
+ require.Equal(t, i >= 4, !arr.IsNull(i))
+ }
+}
+
+func BenchmarkDictByteArrayDecoderDecodeIndices(b *testing.B) {
+ for _, nvalues := range []int{1, 64, 4096, 65536} {
+ b.Run(fmt.Sprintf("dense/%d", nvalues), func(b *testing.B) {
+ benchmarkDictByteArrayDecoderDecodeIndices(b, nvalues,
0)
+ })
+ }
+ for _, nvalues := range []int{64, 4096, 65536} {
+ b.Run(fmt.Sprintf("spaced/%d/25pct-null", nvalues), func(b
*testing.B) {
+ benchmarkDictByteArrayDecoderDecodeIndices(b, nvalues,
nvalues/4)
+ })
+ }
+}
+
+func benchmarkDictByteArrayDecoderDecodeIndices(b *testing.B, nvalues,
nullCount int) {
+ values := make([]parquet.ByteArray, nvalues-nullCount)
+ for i := range values {
+ values[i] = []byte("one")
+ }
+ indices := encodeDictIndices(b, values)
+ defer indices.Release()
+
+ decoder := encoding.NewDictDecoder(parquet.Types.ByteArray, nil,
memory.DefaultAllocator).(*encoding.DictByteArrayDecoder)
+ bldr := newBinaryDictionaryBuilder(memory.DefaultAllocator)
+ defer bldr.Release()
+
+ var validBits []byte
+ if nullCount > 0 {
+ validBits = make([]byte, bitutil.BytesForBits(int64(nvalues)))
+ for i := nullCount; i < nvalues; i++ {
+ bitutil.SetBit(validBits, i)
+ }
+ }
+
+ decode := func() {
+ bldr.Resize(0)
+ if err := decoder.SetData(len(values), indices.Bytes()); err !=
nil {
+ b.Fatal(err)
+ }
+ var (
+ n int
+ err error
+ )
+ if nullCount == 0 {
+ n, err = decoder.DecodeIndices(nvalues, bldr)
+ } else {
+ n, err = decoder.DecodeIndicesSpaced(nvalues,
nullCount, validBits, 0, bldr)
+ }
+ if err != nil {
+ b.Fatal(err)
+ }
+ if n != nvalues {
+ b.Fatalf("decoded %d values, want %d", n, nvalues)
+ }
+ }
+
+ decode()
+ b.ReportAllocs()
+ b.SetBytes(int64(nvalues))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ decode()
+ }
+ b.StopTimer()
+}
