This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 601311b Fix: Handle null values in PlainFixedLenByteArrayEncoder
gracefully (#320)
601311b is described below
commit 601311bdb15c4c2612779a361acf0f66f9f5b754
Author: Saurabh Singh <[email protected]>
AuthorDate: Mon Mar 24 18:37:47 2025 +0530
Fix: Handle null values in PlainFixedLenByteArrayEncoder gracefully (#320)
Fixes: #71
### Rationale for this change
Ensures `PlainFixedLenByteArrayEncoder` handles null values gracefully
in non-nullable fields by writing zero-filled bytes instead of
panicking, maintaining consistency with `DictFixedLenByteArrayEncoder`.
### What changes are included in this PR?
- Replaces panics on null values with writing zero-filled bytes to
maintain byte alignment.
- Ensures encoding consistency between dictionary and plain encoding.
- Prevents unexpected failures when handling schema inconsistencies.
### Are these changes tested?
Yes:
- Passed existing test suite.
- Verified with local Arrow-Go modifications.
### Are there any user-facing changes?
Not Sure (Need possible guidance if required)
---------
Signed-off-by: Saurabh Kumar Singh <[email protected]>
---
.../encoding/fixed_len_byte_array_encoder.go | 8 ++-
.../encoding/fixed_len_byte_array_encoder_test.go | 80 ++++++++++++++++++++++
2 files changed, 86 insertions(+), 2 deletions(-)
diff --git a/parquet/internal/encoding/fixed_len_byte_array_encoder.go
b/parquet/internal/encoding/fixed_len_byte_array_encoder.go
index 3f2a097..56cf242 100644
--- a/parquet/internal/encoding/fixed_len_byte_array_encoder.go
+++ b/parquet/internal/encoding/fixed_len_byte_array_encoder.go
@@ -41,11 +41,15 @@ func (enc *PlainFixedLenByteArrayEncoder) Put(in
[]parquet.FixedLenByteArray) {
bytesNeeded := len(in) * typeLen
enc.sink.Reserve(bytesNeeded)
+
+ emptyValue := make([]byte, typeLen)
+
for _, val := range in {
if val == nil {
- panic("value cannot be nil")
+ enc.sink.UnsafeWrite(emptyValue)
+ } else {
+ enc.sink.UnsafeWrite(val[:typeLen])
}
- enc.sink.UnsafeWrite(val[:typeLen])
}
}
diff --git a/parquet/internal/encoding/fixed_len_byte_array_encoder_test.go
b/parquet/internal/encoding/fixed_len_byte_array_encoder_test.go
new file mode 100644
index 0000000..67e83b0
--- /dev/null
+++ b/parquet/internal/encoding/fixed_len_byte_array_encoder_test.go
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encoding
+
+import (
+ "testing"
+
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/schema"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPlainFixedLenByteArrayEncoder_Put(t *testing.T) {
+ sink := NewPooledBufferWriter(0)
+ elem := schema.NewFixedLenByteArrayNode("test",
parquet.Repetitions.Required, 4, 0)
+ descr := schema.NewColumn(elem, 0, 0)
+ encoder := &PlainFixedLenByteArrayEncoder{
+ encoder: encoder{
+ descr: descr,
+ sink: sink,
+ },
+ }
+
+ tests := []struct {
+ name string
+ input []parquet.FixedLenByteArray
+ expected []byte
+ }{
+ {
+ name: "Normal input",
+ input: []parquet.FixedLenByteArray{
+ []byte("abcd"),
+ []byte("efgh"),
+ []byte("ijkl"),
+ },
+ expected: []byte("abcdefghijkl"),
+ },
+ {
+ name: "Input with nil values",
+ input: []parquet.FixedLenByteArray{
+ []byte("abcd"),
+ nil,
+ []byte("ijkl"),
+ },
+ expected: []byte("abcd\x00\x00\x00\x00ijkl"), // Nil
replaced with zero bytes
+ },
+ {
+ name: "Empty input",
+ input: []parquet.FixedLenByteArray{},
+ expected: []byte{},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Reset the sink before each test
+ sink.Reset(0)
+
+ // Perform the encoding
+ encoder.Put(tt.input)
+
+ // Assert the result
+ require.Equal(t, tt.expected, sink.Bytes(), "Encoded
bytes should match expected output")
+ })
+ }
+}
