This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new abc96fbe perf(parquet): batch plain BYTE_ARRAY encoding (#1193)
abc96fbe is described below
commit abc96fbe17485f9978ae4f9f2e26e46e39a992ee
Author: Minh Vu <[email protected]>
AuthorDate: Mon Aug 17 22:35:59 2026 +0200
perf(parquet): batch plain BYTE_ARRAY encoding (#1193)
### Rationale for this change
`PlainByteArrayEncoder.Put` currently calls `PutByteArray` for every
value. Each call checks sink capacity, writes the 4-byte length, then
writes the payload. Large batches repeat that sink work for every value.
### What changes are included in this PR?
- Calculate the encoded batch size in one pass.
- Reserve the sink capacity once.
- Write lengths and payloads directly into the reserved region.
- Add benchmarks for `Put` and all-valid `PutSpaced` with 1,048,576
values at three widths.
The single-value `PutByteArray` path is unchanged. `PutSpaced` benefits
through its existing valid-run batching.
Medians from 6 runs on an Apple M1 Pro were:
| method | value width | main | this PR | change |
|---|---:|---:|---:|---:|
| Put | 4 bytes | 10.25 ms | 3.85 ms | -62.4% |
| Put | 16 bytes | 11.00 ms | 4.38 ms | -60.2% |
| Put | 64 bytes | 13.77 ms | 7.00 ms | -49.2% |
| PutSpaced | 4 bytes | 10.33 ms | 3.95 ms | -61.8% |
| PutSpaced | 16 bytes | 11.13 ms | 4.44 ms | -60.1% |
| PutSpaced | 64 bytes | 13.54 ms | 7.45 ms | -45.0% |
Allocation counts are unchanged.
### Are these changes tested?
- `go test ./parquet/... -count=1`
- `go test ./parquet/internal/encoding -run "^$" -bench
"^BenchmarkEncodePlainByteArray$" -benchmem -count=6`
The existing encoding suite covers plain BYTE_ARRAY round trips and
spaced validity patterns.
### Are there any user-facing changes?
No. The encoded Parquet bytes and public API are unchanged.
---
parquet/internal/encoding/byte_array_encoder.go | 12 +++++++-
.../internal/encoding/encoding_benchmarks_test.go | 35 ++++++++++++++++++++++
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git a/parquet/internal/encoding/byte_array_encoder.go
b/parquet/internal/encoding/byte_array_encoder.go
index 397a18a7..cb80fbab 100644
--- a/parquet/internal/encoding/byte_array_encoder.go
+++ b/parquet/internal/encoding/byte_array_encoder.go
@@ -48,9 +48,19 @@ func (enc *PlainByteArrayEncoder) PutByteArray(val
parquet.ByteArray) {
// Put writes out all of the values in this slice to the encoding sink
func (enc *PlainByteArrayEncoder) Put(in []parquet.ByteArray) {
+ encodedSize := 0
for _, val := range in {
- enc.PutByteArray(val)
+ encodedSize += val.Len() + arrow.Uint32SizeBytes
+ }
+ enc.sink.Reserve(encodedSize)
+
+ out := enc.sink.buf.Buf()[enc.sink.pos : enc.sink.pos+encodedSize]
+ for _, val := range in {
+ binary.LittleEndian.PutUint32(out, uint32(val.Len()))
+ copy(out[arrow.Uint32SizeBytes:], val)
+ out = out[arrow.Uint32SizeBytes+val.Len():]
}
+ enc.sink.pos += encodedSize
}
// PutSpaced uses the bitmap of validBits to leave out anything that is null
according
diff --git a/parquet/internal/encoding/encoding_benchmarks_test.go
b/parquet/internal/encoding/encoding_benchmarks_test.go
index 4a9c3ae5..09c176d9 100644
--- a/parquet/internal/encoding/encoding_benchmarks_test.go
+++ b/parquet/internal/encoding/encoding_benchmarks_test.go
@@ -17,6 +17,7 @@
package encoding_test
import (
+ "bytes"
"fmt"
"math"
"testing"
@@ -467,6 +468,40 @@ func BenchmarkEncodeDictNumeric(b *testing.B) {
})
}
+func BenchmarkEncodePlainByteArray(b *testing.B) {
+ const nvalues = 1 << 20
+ validBits := bytes.Repeat([]byte{0xff}, nvalues/8)
+
+ for _, width := range []int{4, 16, 64} {
+ value := bytes.Repeat([]byte{'a'}, width)
+ values := make([]parquet.ByteArray, nvalues)
+ for i := range values {
+ values[i] = value
+ }
+
+ for _, tc := range []struct {
+ name string
+ put func(encoding.ByteArrayEncoder)
+ }{
+ {name: "Put", put: func(enc encoding.ByteArrayEncoder)
{ enc.Put(values) }},
+ {name: "PutSpaced", put: func(enc
encoding.ByteArrayEncoder) { enc.PutSpaced(values, validBits, 0) }},
+ } {
+ b.Run(fmt.Sprintf("width=%d/%s", width, tc.name),
func(b *testing.B) {
+ b.SetBytes(int64(nvalues * (width +
arrow.Uint32SizeBytes)))
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ enc :=
encoding.NewEncoder(parquet.Types.ByteArray, parquet.Encodings.Plain, false,
nil, memory.DefaultAllocator).(encoding.ByteArrayEncoder)
+ tc.put(enc)
+ if got, want :=
enc.EstimatedDataEncodedSize(), int64(nvalues*(width+arrow.Uint32SizeBytes));
got != want {
+ b.Fatalf("encoded size = %d,
want %d", got, want)
+ }
+ enc.Release()
+ }
+ })
+ }
+ }
+}
+
func BenchmarkDecodeDictByteArray(b *testing.B) {
const (
nunique = 100
