This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 24148019 fix(array): dictionary null index zeroing (#871)
24148019 is described below
commit 241480198dd8105596ee3627fc9dfcf6c3308401
Author: Minh Vu <[email protected]>
AuthorDate: Wed Jul 1 00:44:35 2026 +0200
fix(array): dictionary null index zeroing (#871)
### What changed
Dictionary concatenation now converts the element position to a byte
offset before zeroing null index runs in concatDictIndices.
### Why
concatDictIndices tracks pos in elements, but the null-run branch used
it as a byte offset. For dictionary index widths larger than one byte,
that could zero the wrong byte range, corrupting adjacent valid indices
while leaving the null slot's bytes stale.
### Validation
- go test ./arrow/array -run
TestConcatDictionaryNullIndexRunsPreserveWideIndices -count=1
- go test ./arrow/array -count=1
-
PARQUET_TEST_DATA=/Users/hoangvu/Code/OSS/arrow-go/parquet-testing/data
ARROW_TEST_DATA=/Users/hoangvu/Code/OSS/arrow-go/arrow-testing/data go
test ./... -count=1
---
arrow/array/concat.go | 3 ++-
arrow/array/concat_test.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 56 insertions(+), 1 deletion(-)
diff --git a/arrow/array/concat.go b/arrow/array/concat.go
index beec182d..974df418 100644
--- a/arrow/array/concat.go
+++ b/arrow/array/concat.go
@@ -282,7 +282,8 @@ func concatDictIndices(mem memory.Allocator, data
[]arrow.ArrayData, idxType arr
return
}
} else {
-
memory.Set(outData[pos:pos+(int(run.Len)*idxWidth)], 0x00)
+ bytePos := pos * idxWidth
+
memory.Set(outData[bytePos:bytePos+(int(run.Len)*idxWidth)], 0x00)
}
pos += int(run.Len)
diff --git a/arrow/array/concat_test.go b/arrow/array/concat_test.go
index 7dd0e0c0..5b595e7c 100644
--- a/arrow/array/concat_test.go
+++ b/arrow/array/concat_test.go
@@ -594,6 +594,60 @@ func TestConcatDictionaryNullSlots(t *testing.T) {
assert.Truef(t, array.Equal(actual, expected), "got: %s, expected: %s",
actual, expected)
}
+func TestConcatDictionaryNullIndexRunsPreserveWideIndices(t *testing.T) {
+ tests := []arrow.DataType{
+ arrow.PrimitiveTypes.Int16,
+ arrow.PrimitiveTypes.Uint16,
+ arrow.PrimitiveTypes.Int32,
+ arrow.PrimitiveTypes.Uint32,
+ arrow.PrimitiveTypes.Int64,
+ arrow.PrimitiveTypes.Uint64,
+ }
+
+ for _, indexType := range tests {
+ t.Run(indexType.Name(), func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dt := &arrow.DictionaryType{IndexType: indexType,
ValueType: arrow.BinaryTypes.String}
+
+ indices1, _, err := array.FromJSON(mem, indexType,
strings.NewReader(`[256, null, 1]`))
+ require.NoError(t, err)
+ defer indices1.Release()
+ indices2, _, err := array.FromJSON(mem, indexType,
strings.NewReader(`[0]`))
+ require.NoError(t, err)
+ defer indices2.Release()
+
+ dictBuilder := array.NewStringBuilder(mem)
+ defer dictBuilder.Release()
+ for i := 0; i <= 256; i++ {
+ dictBuilder.Append(fmt.Sprintf("v%03d", i))
+ }
+ dict1 := dictBuilder.NewStringArray()
+ defer dict1.Release()
+
+ dictBuilder.Append("other")
+ dict2 := dictBuilder.NewStringArray()
+ defer dict2.Release()
+
+ arr1 := array.NewDictionaryArray(dt, indices1, dict1)
+ defer arr1.Release()
+ arr2 := array.NewDictionaryArray(dt, indices2, dict2)
+ defer arr2.Release()
+
+ actual, err := array.Concatenate([]arrow.Array{arr1,
arr2}, mem)
+ require.NoError(t, err)
+ defer actual.Release()
+
+ dict := actual.(*array.Dictionary)
+ assert.Equal(t, "v256", dict.ValueStr(0))
+ assert.True(t, dict.IsNull(1))
+ assert.Equal(t, "v001", dict.ValueStr(2))
+ assert.Equal(t, "other", dict.ValueStr(3))
+ })
+ }
+}
+
func TestConcatRunEndEncoded(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
