This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 846fda47 perf(parquet/variant): skip metadata construction in Encode
(#1217)
846fda47 is described below
commit 846fda471c4fafc991492af3d374775aacedcbbc
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 22:32:12 2026 +0200
perf(parquet/variant): skip metadata construction in Encode (#1217)
## Summary
- **Skip metadata construction in `Encode`** for primitive values.
- Keep `Of` unchanged for callers that need a complete `Value`.
- Add byte-for-byte parity coverage for primitive types, boundaries, and
options.
- Add benchmarks for the encoded and metadata-building paths.
## Benchmark
Apple M1 Pro, darwin/arm64. Times are rounded medians from 5 runs. `Of`
is the previous metadata-building path.
| Input | `Encode` | `Of` |
| --- | ---: | ---: |
| bool | 84 ns, 176 B, 3 allocs | 103 ns, 179 B, 4 allocs |
| int64 max | 101 ns, 192 B, 5 allocs | 126 ns, 200 B, 6 allocs |
| short string | 96 ns, 192 B, 4 allocs | 121 ns, 195 B, 5 allocs |
| long string | 154 ns, 276 B, 6 allocs | 182 ns, 280 B, 7 allocs |
| decimal128 | 148 ns, 216 B, 6 allocs | 167 ns, 224 B, 7 allocs |
## Tests
- `go test ./parquet/variant -count=1`
- `go test -race ./parquet/variant -run '^TestEncodeMatchesOf$'
-count=1`
- `go vet ./parquet/variant`
- `PARQUET_TEST_DATA="$PWD/parquet-testing/data" go test -count=1 ./...`
---
parquet/variant/builder.go | 11 +-
parquet/variant/encode_test.go | 287 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 292 insertions(+), 6 deletions(-)
diff --git a/parquet/variant/builder.go b/parquet/variant/builder.go
index 68fc178d..cf6a6ce8 100644
--- a/parquet/variant/builder.go
+++ b/parquet/variant/builder.go
@@ -892,14 +892,13 @@ type variantPrimitiveType interface {
}
// Encode is a convenience function that produces the encoded bytes for a
primitive
-// variant value. At the moment this is just delegating to the
[Builder.Append] method,
-// but in the future it will be optimized to avoid the extra overhead and
reduce allocations.
+// variant value. It does not construct metadata since primitive values do not
use it.
func Encode[T variantPrimitiveType](v T, opt ...AppendOpt) ([]byte, error) {
- out, err := Of(v, opt...)
- if err != nil {
- return nil, fmt.Errorf("failed to encode variant value: %w",
err)
+ var b Builder
+ if err := b.Append(v, opt...); err != nil {
+ return nil, fmt.Errorf("failed to encode variant value: failed
to append value: %w", err)
}
- return out.value, nil
+ return b.BuildWithoutMeta(), nil
}
func Of[T variantPrimitiveType](v T, opt ...AppendOpt) (Value, error) {
diff --git a/parquet/variant/encode_test.go b/parquet/variant/encode_test.go
new file mode 100644
index 00000000..407cac44
--- /dev/null
+++ b/parquet/variant/encode_test.go
@@ -0,0 +1,287 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package variant_test
+
+import (
+ "math"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/decimal"
+ "github.com/apache/arrow-go/v18/arrow/decimal128"
+ "github.com/apache/arrow-go/v18/parquet/variant"
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+type encodeTestCase struct {
+ name string
+ encode func() ([]byte, error)
+ of func() (variant.Value, error)
+}
+
+func encodeTestCases() []encodeTestCase {
+ testTime := time.Date(2023, 5, 15, 14, 30, 0, 123456789,
time.FixedZone("test", 2*60*60))
+ shortString := strings.Repeat("a", 63)
+ longString := shortString + "b"
+ binaryValue := []byte{0, 1, 2, 0xff}
+ emptyBinary := []byte{}
+ uuidValue := uuid.MustParse("00112233-4455-6677-8899-aabbccddeeff")
+ decimal4Value := variant.DecimalValue[decimal.Decimal32]{Scale: 2,
Value: decimal.Decimal32(1234)}
+ decimal8Value := variant.DecimalValue[decimal.Decimal64]{Scale: 2,
Value: decimal.Decimal64(1234567890)}
+ decimal16Value := variant.DecimalValue[decimal.Decimal128]{
+ Scale: 2, Value: decimal128.FromU64(1234567891234567890),
+ }
+
+ return []encodeTestCase{
+ {
+ name: "bool_true",
+ encode: func() ([]byte, error) { return
variant.Encode(true) },
+ of: func() (variant.Value, error) { return
variant.Of(true) },
+ },
+ {
+ name: "bool_false",
+ encode: func() ([]byte, error) { return
variant.Encode(false) },
+ of: func() (variant.Value, error) { return
variant.Of(false) },
+ },
+ {
+ name: "int8_min",
+ encode: func() ([]byte, error) { return
variant.Encode(int8(-128)) },
+ of: func() (variant.Value, error) { return
variant.Of(int8(-128)) },
+ },
+ {
+ name: "uint8_max",
+ encode: func() ([]byte, error) { return
variant.Encode(uint8(255)) },
+ of: func() (variant.Value, error) { return
variant.Of(uint8(255)) },
+ },
+ {
+ name: "int16_min",
+ encode: func() ([]byte, error) { return
variant.Encode(int16(-32768)) },
+ of: func() (variant.Value, error) { return
variant.Of(int16(-32768)) },
+ },
+ {
+ name: "uint16_max",
+ encode: func() ([]byte, error) { return
variant.Encode(uint16(65535)) },
+ of: func() (variant.Value, error) { return
variant.Of(uint16(65535)) },
+ },
+ {
+ name: "int32_min",
+ encode: func() ([]byte, error) { return
variant.Encode(int32(-2147483648)) },
+ of: func() (variant.Value, error) { return
variant.Of(int32(-2147483648)) },
+ },
+ {
+ name: "uint32_max",
+ encode: func() ([]byte, error) { return
variant.Encode(uint32(1<<32 - 1)) },
+ of: func() (variant.Value, error) { return
variant.Of(uint32(1<<32 - 1)) },
+ },
+ {
+ name: "int64_min",
+ encode: func() ([]byte, error) { return
variant.Encode(int64(-1 << 63)) },
+ of: func() (variant.Value, error) { return
variant.Of(int64(-1 << 63)) },
+ },
+ {
+ name: "int64_max",
+ encode: func() ([]byte, error) { return
variant.Encode(int64(1<<63 - 1)) },
+ of: func() (variant.Value, error) { return
variant.Of(int64(1<<63 - 1)) },
+ },
+ {
+ name: "int",
+ encode: func() ([]byte, error) { return
variant.Encode(int(123456)) },
+ of: func() (variant.Value, error) { return
variant.Of(int(123456)) },
+ },
+ {
+ name: "uint",
+ encode: func() ([]byte, error) { return
variant.Encode(uint(123456)) },
+ of: func() (variant.Value, error) { return
variant.Of(uint(123456)) },
+ },
+ {
+ name: "float32",
+ encode: func() ([]byte, error) { return
variant.Encode(float32(math.MaxFloat32)) },
+ of: func() (variant.Value, error) { return
variant.Of(float32(math.MaxFloat32)) },
+ },
+ {
+ name: "float64",
+ encode: func() ([]byte, error) { return
variant.Encode(math.Copysign(0, -1)) },
+ of: func() (variant.Value, error) { return
variant.Of(math.Copysign(0, -1)) },
+ },
+ {
+ name: "date",
+ encode: func() ([]byte, error) { return
variant.Encode(arrow.Date32(-2147483648)) },
+ of: func() (variant.Value, error) { return
variant.Of(arrow.Date32(-2147483648)) },
+ },
+ {
+ name: "time",
+ encode: func() ([]byte, error) { return
variant.Encode(arrow.Time64(-123456789)) },
+ of: func() (variant.Value, error) { return
variant.Of(arrow.Time64(-123456789)) },
+ },
+ {
+ name: "timestamp",
+ encode: func() ([]byte, error) {
+ return
variant.Encode(arrow.Timestamp(-123456789), variant.OptTimestampNano,
variant.OptTimestampUTC)
+ },
+ of: func() (variant.Value, error) {
+ return variant.Of(arrow.Timestamp(-123456789),
variant.OptTimestampNano, variant.OptTimestampUTC)
+ },
+ },
+ {
+ name: "short_string",
+ encode: func() ([]byte, error) { return
variant.Encode(shortString) },
+ of: func() (variant.Value, error) { return
variant.Of(shortString) },
+ },
+ {
+ name: "empty_string",
+ encode: func() ([]byte, error) { return
variant.Encode("") },
+ of: func() (variant.Value, error) { return
variant.Of("") },
+ },
+ {
+ name: "long_string",
+ encode: func() ([]byte, error) { return
variant.Encode(longString) },
+ of: func() (variant.Value, error) { return
variant.Of(longString) },
+ },
+ {
+ name: "binary",
+ encode: func() ([]byte, error) { return
variant.Encode(binaryValue) },
+ of: func() (variant.Value, error) { return
variant.Of(binaryValue) },
+ },
+ {
+ name: "empty_binary",
+ encode: func() ([]byte, error) { return
variant.Encode(emptyBinary) },
+ of: func() (variant.Value, error) { return
variant.Of(emptyBinary) },
+ },
+ {
+ name: "time_default",
+ encode: func() ([]byte, error) {
+ return variant.Encode(testTime)
+ },
+ of: func() (variant.Value, error) {
+ return variant.Of(testTime)
+ },
+ },
+ {
+ name: "time_nanos_utc",
+ encode: func() ([]byte, error) {
+ return variant.Encode(testTime,
variant.OptTimestampNano, variant.OptTimestampUTC)
+ },
+ of: func() (variant.Value, error) {
+ return variant.Of(testTime,
variant.OptTimestampNano, variant.OptTimestampUTC)
+ },
+ },
+ {
+ name: "time_as_date",
+ encode: func() ([]byte, error) {
+ return variant.Encode(testTime,
variant.OptTimeAsDate)
+ },
+ of: func() (variant.Value, error) {
+ return variant.Of(testTime,
variant.OptTimeAsDate)
+ },
+ },
+ {
+ name: "time_as_time",
+ encode: func() ([]byte, error) {
+ return variant.Encode(testTime,
variant.OptTimeAsTime)
+ },
+ of: func() (variant.Value, error) {
+ return variant.Of(testTime,
variant.OptTimeAsTime)
+ },
+ },
+ {
+ name: "uuid",
+ encode: func() ([]byte, error) { return
variant.Encode(uuidValue) },
+ of: func() (variant.Value, error) { return
variant.Of(uuidValue) },
+ },
+ {
+ name: "decimal4",
+ encode: func() ([]byte, error) { return
variant.Encode(decimal4Value) },
+ of: func() (variant.Value, error) { return
variant.Of(decimal4Value) },
+ },
+ {
+ name: "decimal8",
+ encode: func() ([]byte, error) { return
variant.Encode(decimal8Value) },
+ of: func() (variant.Value, error) { return
variant.Of(decimal8Value) },
+ },
+ {
+ name: "decimal16",
+ encode: func() ([]byte, error) { return
variant.Encode(decimal16Value) },
+ of: func() (variant.Value, error) { return
variant.Of(decimal16Value) },
+ },
+ }
+}
+
+func TestEncodeMatchesOf(t *testing.T) {
+ for _, tc := range encodeTestCases() {
+ t.Run(tc.name, func(t *testing.T) {
+ encoded, err := tc.encode()
+ require.NoError(t, err)
+
+ value, err := tc.of()
+ require.NoError(t, err)
+
+ assert.Equal(t, value.Bytes(), encoded)
+ })
+ }
+}
+
+var encodeBenchmarkSink []byte
+
+func benchmarkEncode(b *testing.B, encode func() ([]byte, error)) {
+ b.Helper()
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for range b.N {
+ encoded, err := encode()
+ if err != nil {
+ b.Fatal(err)
+ }
+ encodeBenchmarkSink = encoded
+ }
+}
+
+func benchmarkOf(b *testing.B, of func() (variant.Value, error)) {
+ b.Helper()
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for range b.N {
+ value, err := of()
+ if err != nil {
+ b.Fatal(err)
+ }
+ encodeBenchmarkSink = value.Bytes()
+ }
+}
+
+func BenchmarkEncode(b *testing.B) {
+ for _, tc := range encodeTestCases() {
+ tc := tc
+ b.Run(tc.name, func(b *testing.B) {
+ benchmarkEncode(b, tc.encode)
+ })
+ }
+}
+
+func BenchmarkOf(b *testing.B) {
+ for _, tc := range encodeTestCases() {
+ tc := tc
+ b.Run(tc.name, func(b *testing.B) {
+ benchmarkOf(b, tc.of)
+ })
+ }
+}
