This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 4a540f51 feat(arrow/extensions): add UnshredVariant to collapse a
shredded variant array (#859)
4a540f51 is described below
commit 4a540f5144843a25db8cffe828dd7deeb34260e1
Author: Neelesh Salian <[email protected]>
AuthorDate: Tue Jun 23 09:33:17 2026 -0700
feat(arrow/extensions): add UnshredVariant to collapse a shredded variant
array (#859)
### Rationale for this change
A shredded variant array reads back in the shredded storage layout,
which shares the `parquet.variant` extension name with the non-shredded
layout but has a different storage type. There was no way to collapse it
short of decoding row by row. iceberg-go needs this at scan-projection
time to read a shredded variant column identically to a non-shredded one
(apache/iceberg-go#986) (PR:
https://github.com/apache/iceberg-go/pull/1238).
### What changes are included in this PR?
- Adds `extensions.UnshredVariant(arr, mem)`, returning an equivalent
non-shredded `VariantArray`. An already-non-shredded array is returned
unchanged with an added reference.
- Fixed a typo in `UntypedValues()`'s doc string too
### Are these changes tested?
Yes. `TestUnshredVariant` covers scalar, list, nested object, residual
fields, null vs present variant-null, and passthrough.
### Are there any user-facing changes?
One new exported function. No change to existing behavior.
---
arrow/extensions/variant.go | 41 +++++++++++-
arrow/extensions/variant_test.go | 135 +++++++++++++++++++++++++++++++++++++++
2 files changed, 175 insertions(+), 1 deletion(-)
diff --git a/arrow/extensions/variant.go b/arrow/extensions/variant.go
index 659f571c..379822c4 100644
--- a/arrow/extensions/variant.go
+++ b/arrow/extensions/variant.go
@@ -419,7 +419,7 @@ func (v *VariantArray) Metadata() arrow.TypedArray[[]byte] {
// UntypedValues returns the untyped variant values for each element of the
array,
// if the array is not shredded this will contain the variant bytes for each
value.
// If the array is shredded, this will contain any variant values that are
either
-// partially shredded objects or are not shredded at all (e.g. a value that
doesnt
+// partially shredded objects or are not shredded at all (e.g. a value that
doesn't
// match the types of the shredding).
//
// The shredded array and the untyped values array together are used to encode
a
@@ -458,6 +458,45 @@ func (v *VariantArray) IsShredded() bool {
return v.ExtensionType().(*VariantType).typedValueFieldIdx != -1
}
+// UnshredVariant returns an equivalent VariantArray in the non-shredded layout
+// (a struct of metadata and value), reassembling each row's value from the
+// shredded typed_value and value columns. If the array is already non-shredded
+// it is returned unchanged with an added reference so callers can Release it
the
+// same way.
+//
+// A physically null row becomes null. A present row that holds an encoded
variant
+// null (the 0x00 value) stays present.
+func UnshredVariant(arr *VariantArray, mem memory.Allocator) (*VariantArray,
error) {
+ if !arr.IsShredded() {
+ arr.Retain()
+
+ return arr, nil
+ }
+
+ // Rebuild through the default-type builder rather than reusing the
input's
+ // metadata column: that column may be dictionary- or
large-binary-encoded,
+ // while the non-shredded layout requires plain binary metadata.
+ bldr := NewVariantBuilder(mem, NewDefaultVariantType())
+ defer bldr.Release()
+ bldr.Reserve(arr.Len())
+
+ storage := arr.Storage()
+ for i := 0; i < arr.Len(); i++ {
+ if storage.IsNull(i) {
+ bldr.AppendNull()
+
+ continue
+ }
+ val, err := arr.Value(i)
+ if err != nil {
+ return nil, fmt.Errorf("variant: reassembling shredded
row %d: %w", i, err)
+ }
+ bldr.Append(val)
+ }
+
+ return bldr.NewArray().(*VariantArray), nil
+}
+
// IsNull will also take into account the special case where there is an
// encoded null variant in the untyped values array for this index and return
// appropriately.
diff --git a/arrow/extensions/variant_test.go b/arrow/extensions/variant_test.go
index 925d0621..3580123d 100644
--- a/arrow/extensions/variant_test.go
+++ b/arrow/extensions/variant_test.go
@@ -1637,3 +1637,138 @@ func TestShreddedVariantNested(t *testing.T) {
assert.Truef(t, arrow.TypeEqual(vt.Storage, s), "expected %s, got %s",
s, vt.Storage)
}
+
+func TestUnshredVariant(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ appendVal := func(bldr *extensions.VariantBuilder, raw any) {
+ var b variant.Builder
+ require.NoError(t, b.Append(raw))
+ v, err := b.Build()
+ require.NoError(t, err)
+ bldr.Append(v)
+ }
+
+ rowJSON := func(t *testing.T, a *extensions.VariantArray, i int) string
{
+ t.Helper()
+ v, err := a.Value(i)
+ require.NoError(t, err)
+ j, err := v.MarshalJSON()
+ require.NoError(t, err)
+
+ return string(j)
+ }
+
+ // unshred reassembles a shredded array, checks the result is
non-shredded and
+ // preserves each row's null-ness, and returns it for value assertions.
The
+ // caller releases the result.
+ unshred := func(t *testing.T, shreddedArr *extensions.VariantArray)
*extensions.VariantArray {
+ t.Helper()
+ require.True(t, shreddedArr.IsShredded())
+
+ out, err := extensions.UnshredVariant(shreddedArr, mem)
+ require.NoError(t, err)
+ assert.False(t, out.IsShredded())
+ require.Equal(t, shreddedArr.Len(), out.Len())
+ for i := 0; i < shreddedArr.Len(); i++ {
+ assert.Equalf(t, shreddedArr.Storage().IsNull(i),
out.Storage().IsNull(i), "row %d null-ness", i)
+ }
+
+ return out
+ }
+
+ t.Run("scalar primitive", func(t *testing.T) {
+ bldr := extensions.NewVariantBuilder(mem,
extensions.NewShreddedVariantType(arrow.PrimitiveTypes.Int64))
+ defer bldr.Release()
+ appendVal(bldr, int64(7))
+ bldr.AppendNull()
+ appendVal(bldr, int64(-3))
+ arr := bldr.NewArray().(*extensions.VariantArray)
+ defer arr.Release()
+
+ out := unshred(t, arr)
+ defer out.Release()
+ assert.JSONEq(t, `7`, rowJSON(t, out, 0))
+ assert.True(t, out.Storage().IsNull(1))
+ assert.JSONEq(t, `-3`, rowJSON(t, out, 2))
+ })
+
+ t.Run("list", func(t *testing.T) {
+ bldr := extensions.NewVariantBuilder(mem,
extensions.NewShreddedVariantType(arrow.ListOf(arrow.BinaryTypes.String)))
+ defer bldr.Release()
+ appendVal(bldr, []any{"comedy", "drama"})
+ arr := bldr.NewArray().(*extensions.VariantArray)
+ defer arr.Release()
+
+ out := unshred(t, arr)
+ defer out.Release()
+ assert.JSONEq(t, `["comedy","drama"]`, rowJSON(t, out, 0))
+ })
+
+ t.Run("nested object", func(t *testing.T) {
+ bldr := extensions.NewVariantBuilder(mem,
extensions.NewShreddedVariantType(arrow.StructOf(
+ arrow.Field{Name: "a", Type:
arrow.PrimitiveTypes.Int64},
+ arrow.Field{Name: "c", Type:
arrow.StructOf(arrow.Field{Name: "x", Type: arrow.PrimitiveTypes.Int64})},
+ )))
+ defer bldr.Release()
+ appendVal(bldr, map[string]any{"a": int64(1), "c":
map[string]any{"x": int64(9)}})
+ arr := bldr.NewArray().(*extensions.VariantArray)
+ defer arr.Release()
+
+ out := unshred(t, arr)
+ defer out.Release()
+ assert.JSONEq(t, `{"a":1,"c":{"x":9}}`, rowJSON(t, out, 0))
+ })
+
+ t.Run("scalar falling to residual", func(t *testing.T) {
+ // a string in an int64-shredded column does not match
typed_value, so it
+ // lands entirely in the residual value column with typed_value
null.
+ bldr := extensions.NewVariantBuilder(mem,
extensions.NewShreddedVariantType(arrow.PrimitiveTypes.Int64))
+ defer bldr.Release()
+ appendVal(bldr, "hello")
+ arr := bldr.NewArray().(*extensions.VariantArray)
+ defer arr.Release()
+
+ out := unshred(t, arr)
+ defer out.Release()
+ assert.JSONEq(t, `"hello"`, rowJSON(t, out, 0))
+ })
+
+ t.Run("object with residual and nulls", func(t *testing.T) {
+ bldr := extensions.NewVariantBuilder(mem,
extensions.NewShreddedVariantType(arrow.StructOf(
+ arrow.Field{Name: "a", Type:
arrow.PrimitiveTypes.Int64},
+ )))
+ defer bldr.Release()
+ appendVal(bldr, map[string]any{"a": int64(7), "extra": "x"}) //
extra lands in residual value
+ bldr.AppendNull() //
physical null
+ appendVal(bldr, nil) //
present variant-null
+ arr := bldr.NewArray().(*extensions.VariantArray)
+ defer arr.Release()
+
+ out := unshred(t, arr)
+ defer out.Release()
+ // row 0: the shredded field and the residual field are both
present.
+ assert.JSONEq(t, `{"a":7,"extra":"x"}`, rowJSON(t, out, 0))
+ // row 1 is a physical null; row 2 is a present variant-null,
not collapsed.
+ assert.True(t, out.Storage().IsNull(1))
+ assert.False(t, out.Storage().IsNull(2))
+ v2, err := out.Value(2)
+ require.NoError(t, err)
+ assert.Equal(t, variant.Null, v2.Type())
+ })
+
+ t.Run("already non-shredded returns same array", func(t *testing.T) {
+ bldr := extensions.NewVariantBuilder(mem,
extensions.NewDefaultVariantType())
+ defer bldr.Release()
+ appendVal(bldr, map[string]any{"a": int64(1)})
+ arr := bldr.NewArray().(*extensions.VariantArray)
+ defer arr.Release()
+
+ out, err := extensions.UnshredVariant(arr, mem)
+ require.NoError(t, err)
+ defer out.Release()
+ assert.Same(t, arr, out)
+ assert.False(t, out.IsShredded())
+ })
+}
