This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 382eb552 fix(parquet/encoding): bound lazy dictionary growth (#1011)
382eb552 is described below
commit 382eb55255ecb52cf14f66ea657fab738e7a3a14
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 19:22:05 2026 +0200
fix(parquet/encoding): bound lazy dictionary growth (#1011)
### Rationale for this change
The lazy dictionary converter sizes its backing slice from the requested
dictionary index before checking whether that index exists. A bad index
can therefore trigger a much larger allocation than the dictionary
itself. Removing the initial capacity also needs to preserve the
declared dictionary length for Arrow dictionary insertion.
### What changes are included in this PR?
- Store the declared dictionary length separately from the lazy backing
slice.
- Reject out-of-range indexes before allocating or decoding.
- Grow only by the values required for a valid index and reject short
dictionary streams.
- Materialize the full declared dictionary when inserting it into an
Arrow builder.
### Are these changes tested?
Yes. Focused tests cover bounded growth and declared length tracking.
The encoding and pqarrow suites, including dictionary insertion paths,
pass.
---
parquet/internal/encoding/byte_array_decoder.go | 8 +++--
.../internal/encoding/dictionary_converter_test.go | 40 ++++++++++++++++++++++
parquet/internal/encoding/typed_encoder.go | 28 +++++++--------
3 files changed, 59 insertions(+), 17 deletions(-)
diff --git a/parquet/internal/encoding/byte_array_decoder.go
b/parquet/internal/encoding/byte_array_decoder.go
index 10112b07..5089a4f7 100644
--- a/parquet/internal/encoding/byte_array_decoder.go
+++ b/parquet/internal/encoding/byte_array_decoder.go
@@ -190,8 +190,12 @@ func (pbad *PlainByteArrayDecoder) DecodeSpaced(out
[]parquet.ByteArray, nullCou
func (d *DictByteArrayDecoder) InsertDictionary(bldr array.Builder) error {
conv := d.dictValueDecoder.(*dictConverter[parquet.ByteArray])
- dictLength := cap(conv.dict)
- conv.ensure(pqutils.IndexType(dictLength))
+ dictLength := conv.dictLen
+ if dictLength > 0 {
+ if err := conv.ensure(pqutils.IndexType(dictLength - 1)); err
!= nil {
+ return err
+ }
+ }
byteArrayData := memory.NewResizableBuffer(d.mem)
defer byteArrayData.Release()
diff --git a/parquet/internal/encoding/dictionary_converter_test.go
b/parquet/internal/encoding/dictionary_converter_test.go
new file mode 100644
index 00000000..9e318d90
--- /dev/null
+++ b/parquet/internal/encoding/dictionary_converter_test.go
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encoding
+
+import (
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/internal/utils"
+ "github.com/stretchr/testify/require"
+)
+
+func TestDictionaryConverterBoundsGrowthByValuesLeft(t *testing.T) {
+ decoder := NewDecoder(parquet.Types.Int32, parquet.Encodings.Plain,
nil, memory.DefaultAllocator)
+ require.NoError(t, decoder.SetData(1, []byte{42, 0, 0, 0}))
+ converter := NewDictConverter[int32](decoder).(*dictConverter[int32])
+
+ require.Zero(t, cap(converter.dict))
+ require.Equal(t, 1, converter.dictLen)
+ require.Error(t, converter.ensure(utils.IndexType(1<<30)))
+ require.Zero(t, cap(converter.dict))
+
+ require.NoError(t, converter.ensure(0))
+ require.Equal(t, []int32{42}, converter.dict)
+}
diff --git a/parquet/internal/encoding/typed_encoder.go
b/parquet/internal/encoding/typed_encoder.go
index 82bb6fc9..273052aa 100644
--- a/parquet/internal/encoding/typed_encoder.go
+++ b/parquet/internal/encoding/typed_encoder.go
@@ -315,27 +315,25 @@ func (d *typedDictDecoder[T]) DecodeSpaced(out []T,
nullCount int, validBits []b
type dictConverter[T parquet.ColumnTypes] struct {
valueDecoder Decoder[T]
+ dictLen int
dict []T
zeroVal T
}
func (dc *dictConverter[T]) ensure(idx utils.IndexType) error {
+ if idx < 0 || int64(idx) >= int64(dc.dictLen) {
+ return fmt.Errorf("parquet: dictionary index %d is out of
range", idx)
+ }
if len(dc.dict) <= int(idx) {
- if cap(dc.dict) <= int(idx) {
- val := make([]T, int(idx+1)-len(dc.dict))
- n, err := dc.valueDecoder.Decode(val)
- if err != nil {
- return err
- }
- dc.dict = append(dc.dict, val[:n]...)
- } else {
- cur := len(dc.dict)
- n, err := dc.valueDecoder.Decode(dc.dict[cur : idx+1])
- if err != nil {
- return err
- }
- dc.dict = dc.dict[:cur+n]
+ val := make([]T, int(idx)+1-len(dc.dict))
+ n, err := dc.valueDecoder.Decode(val)
+ if err != nil {
+ return err
+ }
+ if n != len(val) {
+ return errors.New("parquet: dictionary contains fewer
values than declared")
}
+ dc.dict = append(dc.dict, val[:n]...)
}
return nil
}
@@ -717,7 +715,7 @@ func (enc *DictFixedLenByteArrayEncoder) Type()
parquet.Type {
// NewDictConverter creates a dict converter of the appropriate type, using
the passed in
// decoder as the decoder to decode the dictionary index.
func NewDictConverter[T parquet.ColumnTypes](dict TypedDecoder)
utils.DictionaryConverter[T] {
- return &dictConverter[T]{valueDecoder: dict.(Decoder[T]), dict:
make([]T, 0, dict.ValuesLeft())}
+ return &dictConverter[T]{valueDecoder: dict.(Decoder[T]), dictLen:
dict.ValuesLeft()}
}
var (
