This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new ab36c9aa perf(parquet): reuse fixed-length zero scratch (#1236)
ab36c9aa is described below
commit ab36c9aada052e8ba05c146cf12b4ca24f4780ed
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 17:59:07 2026 +0200
perf(parquet): reuse fixed-length zero scratch (#1236)
## Summary
- Reuse a zero-value buffer in `PlainFixedLenByteArrayEncoder`.
- Allocate it lazily only when `Put` sees a nil value.
- Clear the buffer reference when the encoder is released.
- Add coverage for lazy creation and reuse.
- Add benchmarks for widths 4, 16, and 32 with repeated small `Put`
calls.
## Benchmark
Apple M1 Pro. Median of 3 runs. All-valid input.
| width | rows per put | puts | upstream main | this PR | change |
| ---: | ---: | ---: | ---: | ---: | ---: |
| 4 | 1 | 64 | 489 ns | 470 ns | -4% |
| 16 | 1 | 64 | 516 ns | 470 ns | -9% |
| 32 | 1 | 64 | 498 ns | 471 ns | -5% |
The benchmark also covers 1, 16, and 1024 rows with 1, 64, and 1024
repeated puts, plus sparse nil input.
```text
go test ./parquet/internal/encoding -run '^$' -bench
'BenchmarkPlainEncodingFixedLenByteArray' -benchmem -benchtime=100ms -count=3
```
## Tests
- `go test ./parquet/internal/encoding -count=1`
- `PARQUET_TEST_DATA=/path/to/parquet-testing/data go test
./parquet/file ./parquet/pqarrow -count=1`
- `PARQUET_TEST_DATA=/path/to/parquet-testing/data go test -race
./parquet/internal/encoding ./parquet/file ./parquet/pqarrow -count=1`
- `go vet ./parquet/internal/encoding ./parquet/file ./parquet/pqarrow`
---
.../encoding/fixed_len_byte_array_encoder.go | 13 +++-
.../fixed_len_byte_array_encoder_benchmark_test.go | 80 ++++++++++++++++++++++
.../encoding/fixed_len_byte_array_encoder_test.go | 24 +++++++
3 files changed, 114 insertions(+), 3 deletions(-)
diff --git a/parquet/internal/encoding/fixed_len_byte_array_encoder.go
b/parquet/internal/encoding/fixed_len_byte_array_encoder.go
index 854b8b8a..190802ee 100644
--- a/parquet/internal/encoding/fixed_len_byte_array_encoder.go
+++ b/parquet/internal/encoding/fixed_len_byte_array_encoder.go
@@ -31,6 +31,7 @@ type PlainFixedLenByteArrayEncoder struct {
encoder
bitSetReader bitutils.SetBitRunReader
+ zeroValue []byte
}
// Put writes the provided values to the encoder
@@ -43,17 +44,23 @@ func (enc *PlainFixedLenByteArrayEncoder) Put(in
[]parquet.FixedLenByteArray) {
bytesNeeded := len(in) * typeLen
enc.sink.Reserve(bytesNeeded)
- emptyValue := make([]byte, typeLen)
-
for _, val := range in {
if val == nil {
- enc.sink.UnsafeWrite(emptyValue)
+ if len(enc.zeroValue) != typeLen {
+ enc.zeroValue = make([]byte, typeLen)
+ }
+ enc.sink.UnsafeWrite(enc.zeroValue)
} else {
enc.sink.UnsafeWrite(val[:typeLen])
}
}
}
+func (enc *PlainFixedLenByteArrayEncoder) Release() {
+ enc.encoder.Release()
+ enc.zeroValue = nil
+}
+
// PutSpaced is like Put but works with data that is spaced out according to
the passed in bitmap
func (enc *PlainFixedLenByteArrayEncoder) PutSpaced(in
[]parquet.FixedLenByteArray, validBits []byte, validBitsOffset int64) {
if validBits != nil {
diff --git
a/parquet/internal/encoding/fixed_len_byte_array_encoder_benchmark_test.go
b/parquet/internal/encoding/fixed_len_byte_array_encoder_benchmark_test.go
new file mode 100644
index 00000000..d4ea9186
--- /dev/null
+++ b/parquet/internal/encoding/fixed_len_byte_array_encoder_benchmark_test.go
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encoding_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/internal/encoding"
+ "github.com/apache/arrow-go/v18/parquet/schema"
+)
+
+func BenchmarkPlainEncodingFixedLenByteArray(b *testing.B) {
+ for _, width := range []int{4, 16, 32} {
+ for _, nvalues := range []int{1, 16, 1024} {
+ for _, puts := range []int{1, 64, 1024} {
+ for _, withNulls := range []bool{false, true} {
+ validity := "all-valid"
+ if withNulls {
+ validity = "sparse-nil"
+ }
+ name :=
fmt.Sprintf("width=%d/rows=%d/puts=%d/validity=%s", width, nvalues, puts,
validity)
+ values :=
makeFixedLenByteArrayValues(nvalues, width, withNulls)
+
+ b.Run(name, func(b *testing.B) {
+ col :=
schema.NewColumn(schema.NewFixedLenByteArrayNode("fixedlenbytearray",
parquet.Repetitions.Required, int32(width), -1), 0, 0)
+ encoder :=
encoding.NewEncoder(parquet.Types.FixedLenByteArray, parquet.Encodings.Plain,
+ false, col,
memory.DefaultAllocator).(encoding.FixedLenByteArrayEncoder)
+ defer encoder.Release()
+
+ b.SetBytes(int64(width *
nvalues * puts))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ for j := 0; j < puts;
j++ {
+
encoder.Put(values)
+ }
+ buf, err :=
encoder.FlushValues()
+ if err != nil {
+ b.Fatal(err)
+ }
+ buf.Release()
+ }
+ })
+ }
+ }
+ }
+ }
+}
+
+func makeFixedLenByteArrayValues(nvalues, width int, withNulls bool)
[]parquet.FixedLenByteArray {
+ values := make([]parquet.FixedLenByteArray, nvalues)
+ for i := range values {
+ if withNulls && i%8 == 0 {
+ continue
+ }
+
+ values[i] = make(parquet.FixedLenByteArray, width)
+ for j := range values[i] {
+ values[i][j] = byte(j)
+ }
+ }
+ return values
+}
diff --git a/parquet/internal/encoding/fixed_len_byte_array_encoder_test.go
b/parquet/internal/encoding/fixed_len_byte_array_encoder_test.go
index 67e83b02..1edee315 100644
--- a/parquet/internal/encoding/fixed_len_byte_array_encoder_test.go
+++ b/parquet/internal/encoding/fixed_len_byte_array_encoder_test.go
@@ -78,3 +78,27 @@ func TestPlainFixedLenByteArrayEncoder_Put(t *testing.T) {
})
}
}
+
+func TestPlainFixedLenByteArrayEncoder_ReusesZeroValue(t *testing.T) {
+ sink := NewPooledBufferWriter(0)
+ elem := schema.NewFixedLenByteArrayNode("test",
parquet.Repetitions.Required, 4, 0)
+ descr := schema.NewColumn(elem, 0, 0)
+ encoder := &PlainFixedLenByteArrayEncoder{
+ encoder: encoder{
+ descr: descr,
+ sink: sink,
+ },
+ }
+ defer encoder.Release()
+
+ encoder.Put([]parquet.FixedLenByteArray{[]byte("abcd")})
+ require.Nil(t, encoder.zeroValue)
+ sink.Reset(0)
+ encoder.Put([]parquet.FixedLenByteArray{nil})
+ zeroValue := encoder.zeroValue
+ require.Equal(t, []byte{0, 0, 0, 0}, zeroValue)
+
+ sink.Reset(0)
+ encoder.Put([]parquet.FixedLenByteArray{nil})
+ require.Same(t, &zeroValue[0], &encoder.zeroValue[0])
+}
