This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 05a1ee07 fix(array): reject mismatched known null counts during full
validation (#993)
05a1ee07 is described below
commit 05a1ee07871928f5b80135a87d39f046f672e665
Author: Minh Vu <[email protected]>
AuthorDate: Fri Jul 24 18:05:00 2026 +0200
fix(array): reject mismatched known null counts during full validation
(#993)
## Summary
- validate known null counts against the validity bitmap during
ValidateFull
- add a regression test for arrays whose metadata null count disagrees
with the bitmap
## Why
ValidateFull currently bounds-checks a known null count but does not
verify that it matches the actual bitmap. That lets malformed arrays
pass validation even though downstream code can observe contradictory
null metadata.
## Validation
- go test ./arrow/array
- go test ./arrow/compute/exec
---
arrow/array/validate.go | 28 ++++++++++++++++++++++++++++
arrow/array/validate_test.go | 26 ++++++++++++++++++++++++++
2 files changed, 54 insertions(+)
diff --git a/arrow/array/validate.go b/arrow/array/validate.go
index e33202e3..2a50412a 100644
--- a/arrow/array/validate.go
+++ b/arrow/array/validate.go
@@ -70,6 +70,11 @@ func validateArray(arr arrow.Array, full bool, path string)
error {
if err := validateArrayStructure(data); err != nil {
return validationError(path, err)
}
+ if full {
+ if err := validateNullCount(data); err != nil {
+ return validationError(path, err)
+ }
+ }
if v, ok := arr.(Validator); ok {
var err error
@@ -130,6 +135,29 @@ func validateArray(arr arrow.Array, full bool, path
string) error {
return nil
}
+func validateNullCount(data *Data) error {
+ if data.nulls == UnknownNullCount {
+ return nil
+ }
+
+ var actualNulls int
+ switch data.dtype.ID() {
+ case arrow.NULL:
+ actualNulls = data.length
+ case arrow.SPARSE_UNION, arrow.DENSE_UNION, arrow.RUN_END_ENCODED:
+ actualNulls = 0
+ default:
+ if len(data.buffers) > 0 && data.buffers[0] != nil {
+ actualNulls = data.length -
bitutil.CountSetBits(data.buffers[0].Bytes(), data.offset, data.length)
+ }
+ }
+
+ if actualNulls != data.nulls {
+ return fmt.Errorf("arrow/array: null count value (%d) does not
match actual number of nulls in array (%d)", data.nulls, actualNulls)
+ }
+ return nil
+}
+
func validateArrayData(data *Data) error {
if data == nil || data.dtype == nil {
return fmt.Errorf("arrow/array: array data has no data type")
diff --git a/arrow/array/validate_test.go b/arrow/array/validate_test.go
index 53ad126b..7b83f7a4 100644
--- a/arrow/array/validate_test.go
+++ b/arrow/array/validate_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/bitutil"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -63,6 +64,19 @@ func makeLargeStringArrayRaw(t *testing.T, offsets []int64,
data string, length,
return NewLargeStringData(d)
}
+func makeInt32ArrayRaw(t *testing.T, values []int32, validity []byte, nulls,
length, offset int) *Int32 {
+ t.Helper()
+ valueBuf := memory.NewBufferBytes(arrow.Int32Traits.CastToBytes(values))
+ var validityBuf *memory.Buffer
+ if validity != nil {
+ validityBuf = memory.NewBufferBytes(validity)
+ }
+ data := NewData(arrow.PrimitiveTypes.Int32, length,
[]*memory.Buffer{validityBuf, valueBuf}, nil, nulls, offset)
+ arr := NewInt32Data(data)
+ data.Release()
+ return arr
+}
+
func TestBinaryValidate(t *testing.T) {
t.Run("valid array passes", func(t *testing.T) {
// offsets [0,3,6,9], data "abcdefghi" — 3 elements of 3 bytes
each
@@ -195,6 +209,18 @@ func TestTopLevelValidate(t *testing.T) {
require.Error(t, ValidateFull(arr))
})
+ t.Run("known null count mismatch passes Validate but fails
ValidateFull", func(t *testing.T) {
+ validity := make([]byte, bitutil.BytesForBits(2))
+ bitutil.SetBit(validity, 0)
+ arr := makeInt32ArrayRaw(t, []int32{10, 20}, validity, 0, 2, 0)
+ defer arr.Release()
+
+ assert.NoError(t, Validate(arr))
+ err := ValidateFull(arr)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "does not match actual number
of nulls")
+ })
+
t.Run("Validate returns nil for non-Validator types", func(t
*testing.T) {
// Bool arrays don't implement Validator — should return nil
bldr := NewBooleanBuilder(memory.NewGoAllocator())
