This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new caec5052 fix(arrow/array): make dictionary suffix null indexes
relative (#1064)
caec5052 is described below
commit caec50527ba7d3ad375635efde10dfe89a0e794f
Author: Minh Vu <[email protected]>
AuthorDate: Wed Aug 5 19:46:32 2026 +0200
fix(arrow/array): make dictionary suffix null indexes relative (#1064)
## What changed
Clear the validity bit using the null index relative to `startOffset`
when exporting a dictionary suffix.
## Why
`GetDictArrayData` builds a new dictionary whose bitmap starts at the
requested offset, but it previously cleared the absolute memo-table
index. For nonzero offsets this could mark the wrong dictionary entry
valid while still reporting one null.
The regression test exports a suffix containing a null and verifies that
the corresponding relative entry is null.
## Testing
- `go test ./arrow/array`
---
arrow/array/util.go | 2 +-
arrow/array/util_test.go | 26 ++++++++++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/arrow/array/util.go b/arrow/array/util.go
index e369f788..1b303bf6 100644
--- a/arrow/array/util.go
+++ b/arrow/array/util.go
@@ -370,7 +370,7 @@ func GetDictArrayData(mem memory.Allocator, valueType
arrow.DataType, memoTable
nullcount = 1
buffers[0].Resize(int(bitutil.BytesForBits(int64(dictLen))))
memory.Set(buffers[0].Bytes(), 0xFF)
- bitutil.ClearBit(buffers[0].Bytes(), idx)
+ bitutil.ClearBit(buffers[0].Bytes(), idx-startOffset)
}
return NewData(valueType, dictLen, buffers, nil, nullcount, 0), nil
diff --git a/arrow/array/util_test.go b/arrow/array/util_test.go
index 8fd36be8..6870cf8b 100644
--- a/arrow/array/util_test.go
+++ b/arrow/array/util_test.go
@@ -32,11 +32,37 @@ import (
"github.com/apache/arrow-go/v18/arrow/decimal256"
"github.com/apache/arrow-go/v18/arrow/internal/arrdata"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/internal/hashing"
"github.com/apache/arrow-go/v18/internal/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
+func TestGetDictArrayDataNullInSuffix(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ memo := hashing.NewMemoTable[int32](0)
+ for _, value := range []int32{10, 20, 30, 40} {
+ _, _, err := memo.GetOrInsert(value)
+ require.NoError(t, err)
+ }
+ nullIndex, found := memo.GetOrInsertNull()
+ require.False(t, found)
+ require.Equal(t, 4, nullIndex)
+
+ data, err := array.GetDictArrayData(mem, arrow.PrimitiveTypes.Int32,
memo, 3)
+ require.NoError(t, err)
+ defer data.Release()
+
+ dict := array.MakeFromData(data)
+ defer dict.Release()
+ require.Equal(t, 2, dict.Len())
+ require.Equal(t, 1, dict.NullN())
+ assert.False(t, dict.IsNull(0))
+ assert.True(t, dict.IsNull(1))
+}
+
var typemap = map[arrow.DataType]reflect.Type{
arrow.PrimitiveTypes.Int8: reflect.TypeOf(int8(0)),
arrow.PrimitiveTypes.Uint8: reflect.TypeOf(uint8(0)),
