This is an automated email from the ASF dual-hosted git repository.
CurtHagenlocher pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-dotnet.git
The following commit(s) were added to refs/heads/main by this push:
new d952f30 fix: Read large_binary and binary_view storage in shredded
variant readers (#449)
d952f30 is described below
commit d952f30b5226dc5929867924793f4ce57f3f0aba
Author: Curt Hagenlocher <[email protected]>
AuthorDate: Sun Sep 27 09:40:33 2026 -0700
fix: Read large_binary and binary_view storage in shredded variant readers
(#449)
## What's Changed
`VariantArray` accepts `binary`, `large_binary` or `binary_view` for
`metadata` and `value` at every nesting level, and
`ShredSchema.FromArrowType` maps `large_utf8` / `large_binary`
`typed_value` columns. The shredded readers (`ShreddedVariant`,
`ShreddedObject`, `ShreddedArray`) cast those columns to `BinaryArray` /
`StringArray`, so any other representation threw `InvalidCastException`.
This even affected `GetLogicalVariantValue` on unshredded `large_binary`
columns.
This replaces the casts with two small helpers in `ShreddingHelpers`,
`GetBytes` and `GetString`, that read from any binary or string
representation. (`IBinaryArray` would have been the natural abstraction,
but it's internal to `Apache.Arrow`.)
The new tests shred a column that has residuals at the top level, in a
partially shredded object, in an object field, and in a list element.
They then convert every binary field, recursively, to `large_binary` or
`binary_view` (and string/binary `typed_value` columns to `large_utf8` /
`large_binary`), and read everything back, both directly and through the
`Shred` / `Reassemble` entry points added in #447. All of these tests
fail without the fix.
Performance: `GetLogicalVariantValue` over 100k rows (BenchmarkDotNet,
medium job, two runs) is indistinguishable from the old cast for both
shredded and unshredded `binary` columns. An inlinable `BinaryArray`
fast path was also tried and measured slower, so it isn't included.
Closes #448.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../Shredding/ShreddedArray.cs | 5 +-
.../Shredding/ShreddedObject.cs | 8 +-
.../Shredding/ShreddedVariant.cs | 10 +-
.../Shredding/ShreddingHelpers.cs | 34 +++
.../Shredding/ShreddedVariantStorageTypeTests.cs | 234 +++++++++++++++++++++
5 files changed, 277 insertions(+), 14 deletions(-)
diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs
b/src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs
index 2dcea00..e1d5956 100644
--- a/src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs
+++ b/src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs
@@ -106,7 +106,7 @@ namespace Apache.Arrow.Operations.Shredding
reader = default;
return false;
}
- ReadOnlySpan<byte> bytes = ((BinaryArray)_residual).GetBytes(_row,
out _);
+ ReadOnlySpan<byte> bytes = ShreddingHelpers.GetBytes(_residual,
_row);
reader = new VariantReader(_metadata, bytes);
return true;
}
@@ -145,8 +145,7 @@ namespace Apache.Arrow.Operations.Shredding
{
return VariantValue.Null;
}
- BinaryArray residualBinary = (BinaryArray)_residual;
- ReadOnlySpan<byte> bytes = residualBinary.GetBytes(_row, out _);
+ ReadOnlySpan<byte> bytes = ShreddingHelpers.GetBytes(_residual,
_row);
return new VariantReader(_metadata, bytes).ToVariantValue();
}
}
diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs
b/src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs
index 07e2381..98a0651 100644
--- a/src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs
+++ b/src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs
@@ -104,7 +104,7 @@ namespace Apache.Arrow.Operations.Shredding
reader = default;
return false;
}
- ReadOnlySpan<byte> bytes =
((BinaryArray)_residual).GetBytes(_index, out _);
+ ReadOnlySpan<byte> bytes = ShreddingHelpers.GetBytes(_residual,
_index);
reader = new VariantReader(_metadata, bytes);
return true;
}
@@ -128,8 +128,7 @@ namespace Apache.Arrow.Operations.Shredding
// No shredded fields at this row — whatever is in the residual IS
the value.
if (!typedPopulated)
{
- BinaryArray binary = (BinaryArray)_residual;
- ReadOnlySpan<byte> bytes = binary.GetBytes(_index, out _);
+ ReadOnlySpan<byte> bytes =
ShreddingHelpers.GetBytes(_residual, _index);
return new VariantReader(_metadata, bytes).ToVariantValue();
}
@@ -151,8 +150,7 @@ namespace Apache.Arrow.Operations.Shredding
// Partially shredded object — merge residual unshredded fields.
if (residualPopulated)
{
- BinaryArray residualBinary = (BinaryArray)_residual;
- ReadOnlySpan<byte> residualBytes =
residualBinary.GetBytes(_index, out _);
+ ReadOnlySpan<byte> residualBytes =
ShreddingHelpers.GetBytes(_residual, _index);
VariantReader residualReader = new VariantReader(_metadata,
residualBytes);
if (!residualReader.IsObject)
{
diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs
b/src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs
index 5ef3575..3b58a75 100644
--- a/src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs
+++ b/src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs
@@ -125,8 +125,7 @@ namespace Apache.Arrow.Operations.Shredding
{
if (HasResidual)
{
- BinaryArray binary = (BinaryArray)_valueArray;
- ReadOnlySpan<byte> bytes = binary.GetBytes(_index, out _);
+ ReadOnlySpan<byte> bytes =
ShreddingHelpers.GetBytes(_valueArray, _index);
reader = new VariantReader(_metadata, bytes);
return true;
}
@@ -168,8 +167,7 @@ namespace Apache.Arrow.Operations.Shredding
{
throw new InvalidOperationException("No residual value to
read.");
}
- BinaryArray binary = (BinaryArray)_valueArray;
- ReadOnlySpan<byte> bytes = binary.GetBytes(_index, out _);
+ ReadOnlySpan<byte> bytes = ShreddingHelpers.GetBytes(_valueArray,
_index);
return new VariantReader(_metadata, bytes).ToVariantValue();
}
@@ -266,10 +264,10 @@ namespace Apache.Arrow.Operations.Shredding
public long GetTimestampNtzNanos() =>
((TimestampArray)RequireTyped(ShredType.TimestampNtzNanos)).GetValue(_index).Value;
/// <summary>Reads the shredded string value at this slot.</summary>
- public string GetString() =>
((StringArray)RequireTyped(ShredType.String)).GetString(_index);
+ public string GetString() =>
ShreddingHelpers.GetString(RequireTyped(ShredType.String), _index);
/// <summary>Reads the shredded binary value at this slot as a byte
span.</summary>
- public ReadOnlySpan<byte> GetBinaryBytes() =>
((BinaryArray)RequireTyped(ShredType.Binary)).GetBytes(_index);
+ public ReadOnlySpan<byte> GetBinaryBytes() =>
ShreddingHelpers.GetBytes(RequireTyped(ShredType.Binary), _index);
/// <summary>Reads the shredded UUID at this slot.</summary>
public Guid GetUuid()
diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs
b/src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs
index f7be85e..36b628e 100644
--- a/src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs
+++ b/src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs
@@ -44,5 +44,39 @@ namespace Apache.Arrow.Operations.Shredding
return new ShreddedVariant(slotSchema, metadata, valueArr,
typedArr, index);
}
+
+ /// <summary>
+ /// Reads the bytes at <paramref name="index"/> from any binary
representation a
+ /// variant column may use (binary, large_binary or binary_view).
+ /// </summary>
+ public static ReadOnlySpan<byte> GetBytes(IArrowArray array, int index)
+ {
+ switch (array)
+ {
+ case BinaryArray binary: return binary.GetBytes(index);
+ case LargeBinaryArray largeBinary: return
largeBinary.GetBytes(index);
+ case BinaryViewArray binaryView: return
binaryView.GetBytes(index);
+ default:
+ throw new InvalidOperationException(
+ $"Cannot read variant bytes from an array of type
{array.Data.DataType.TypeId}.");
+ }
+ }
+
+ /// <summary>
+ /// Reads the string at <paramref name="index"/> from any string
representation
+ /// (utf8, large_utf8 or utf8_view).
+ /// </summary>
+ public static string GetString(IArrowArray array, int index)
+ {
+ switch (array)
+ {
+ case StringArray str: return str.GetString(index);
+ case LargeStringArray largeStr: return
largeStr.GetString(index);
+ case StringViewArray strView: return strView.GetString(index);
+ default:
+ throw new InvalidOperationException(
+ $"Cannot read a string from an array of type
{array.Data.DataType.TypeId}.");
+ }
+ }
}
}
diff --git
a/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantStorageTypeTests.cs
b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantStorageTypeTests.cs
new file mode 100644
index 0000000..0843b78
--- /dev/null
+++
b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantStorageTypeTests.cs
@@ -0,0 +1,234 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using Apache.Arrow;
+using Apache.Arrow.Operations.Shredding;
+using Apache.Arrow.Scalars.Variant;
+using Apache.Arrow.Types;
+using Xunit;
+
+namespace Apache.Arrow.Operations.Tests.Shredding
+{
+ /// <summary>
+ /// The shredded readers must accept every binary representation a variant
column
+ /// may use for its <c>metadata</c> and <c>value</c> fields (binary,
large_binary,
+ /// binary_view), at every nesting level, as well as large_utf8 /
large_binary
+ /// <c>typed_value</c> columns.
+ /// </summary>
+ public class ShreddedVariantStorageTypeTests
+ {
+ public enum Storage { LargeBinary, BinaryView }
+
+ private static readonly ShredSchema Schema = ShredSchema.ForObject(new
Dictionary<string, ShredSchema>
+ {
+ ["a"] = ShredSchema.Primitive(ShredType.Int32),
+ ["s"] = ShredSchema.Primitive(ShredType.String),
+ ["bin"] = ShredSchema.Primitive(ShredType.Binary),
+ ["tags"] =
ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)),
+ });
+
+ private static VariantValue Obj(params (string Name, VariantValue
Value)[] fields)
+ {
+ var dict = new Dictionary<string, VariantValue>();
+ foreach (var (name, value) in fields) dict[name] = value;
+ return VariantValue.FromObject(dict);
+ }
+
+ // Exercises residuals at the top level, inside a partially shredded
object,
+ // inside an object field element group, and inside a list element
group.
+ private static readonly List<VariantValue?> Rows = new
List<VariantValue?>
+ {
+ Obj(("a", VariantValue.FromInt32(1)),
+ ("s", VariantValue.FromString("hello")),
+ ("bin", VariantValue.FromBinary(new byte[] { 1, 2, 3 })),
+ ("tags", VariantValue.FromArray(VariantValue.FromInt32(7),
VariantValue.FromString("residual element"))),
+ ("extra", VariantValue.FromBoolean(true))),
+ null,
+ VariantValue.FromString("not an object"),
+ Obj(("a", VariantValue.FromString("residual field")),
+ ("tags", VariantValue.FromString("not an array"))),
+ };
+
+ private static VariantArray BuildShredded()
+ {
+ (byte[] metadata, IReadOnlyList<ShredResult> rows) =
VariantShredder.Shred(Rows, Schema);
+ return ShreddedVariantArrayBuilder.Build(Schema, metadata, rows);
+ }
+
+ [Theory]
+ [InlineData(Storage.LargeBinary)]
+ [InlineData(Storage.BinaryView)]
+ public void GetLogicalVariantValue_ReadsAlternateStorage(Storage
storage)
+ {
+ VariantArray array = Convert(BuildShredded(), storage);
+
+ Assert.Equal(Rows.Count, array.Length);
+ for (int i = 0; i < Rows.Count; i++)
+ {
+ if (Rows[i].HasValue)
+ Assert.Equal(Rows[i].Value,
array.GetLogicalVariantValue(i));
+ else
+ Assert.True(array.IsNull(i));
+ }
+ }
+
+ [Theory]
+ [InlineData(Storage.LargeBinary)]
+ [InlineData(Storage.BinaryView)]
+ public void
GetLogicalVariantValue_ReadsAlternateStorage_Unshredded(Storage storage)
+ {
+ var builder = new VariantArray.Builder();
+ builder.AppendRange(Rows);
+ VariantArray array = Convert(builder.Build(), storage);
+
+ Assert.Equal(Rows[0].Value, array.GetLogicalVariantValue(0));
+ }
+
+ [Fact]
+ public void TypedAccessors_ReadLargeTypedColumns()
+ {
+ VariantArray array = Convert(BuildShredded(), Storage.LargeBinary);
+ ShreddedObject obj = array.GetShreddedVariant(0).GetObject();
+
+ Assert.True(obj.TryGetField("s", out ShreddedVariant s));
+ Assert.Equal("hello", s.GetString());
+ Assert.True(obj.TryGetField("bin", out ShreddedVariant bin));
+ Assert.Equal(new byte[] { 1, 2, 3 },
bin.GetBinaryBytes().ToArray());
+ }
+
+ [Theory]
+ [InlineData(Storage.LargeBinary, true)]
+ [InlineData(Storage.LargeBinary, false)]
+ [InlineData(Storage.BinaryView, true)]
+ [InlineData(Storage.BinaryView, false)]
+ public void Shred_ReadsAlternateStorage(Storage storage, bool
shreddedInput)
+ {
+ VariantArray input = shreddedInput ? BuildShredded() :
BuildUnshredded();
+
+ AssertRows(Convert(input, storage).Shred(Schema));
+ }
+
+ [Theory]
+ [InlineData(Storage.LargeBinary)]
+ [InlineData(Storage.BinaryView)]
+ public void Reassemble_ReadsAlternateStorage(Storage storage)
+ {
+ VariantArray reassembled = Convert(BuildShredded(),
storage).Reassemble();
+
+ Assert.False(reassembled.IsShredded);
+ AssertRows(reassembled);
+ }
+
+ private static VariantArray BuildUnshredded()
+ {
+ var builder = new VariantArray.Builder();
+ builder.AppendRange(Rows);
+ return builder.Build();
+ }
+
+ private static void AssertRows(VariantArray array)
+ {
+ Assert.Equal(Rows.Count, array.Length);
+ for (int i = 0; i < Rows.Count; i++)
+ {
+ if (Rows[i].HasValue)
+ Assert.Equal(Rows[i].Value,
array.GetLogicalVariantValue(i));
+ else
+ Assert.True(array.IsNull(i));
+ }
+ }
+
+ // ---------------------------------------------------------------
+ // Storage conversion
+ // ---------------------------------------------------------------
+
+ private static VariantArray Convert(VariantArray array, Storage
storage)
+ {
+ return new
VariantArray((StructArray)ConvertArray(array.StorageArray, null, storage));
+ }
+
+ /// <summary>
+ /// Recursively rewrites <c>metadata</c> / <c>value</c> binary fields
to the target
+ /// storage. For <see cref="Storage.LargeBinary"/> it also rewrites
string and binary
+ /// <c>typed_value</c> columns to large_utf8 / large_binary; there is
no view
+ /// counterpart because the shredding schema doesn't map view types
for typed_value.
+ /// </summary>
+ private static IArrowArray ConvertArray(IArrowArray array, string
fieldName, Storage storage)
+ {
+ bool isVariantBinary = fieldName == "metadata" || fieldName ==
"value";
+ switch (array)
+ {
+ case StringArray str when !isVariantBinary && storage ==
Storage.LargeBinary:
+ {
+ var b = new LargeStringArray.Builder();
+ for (int i = 0; i < str.Length; i++)
+ {
+ if (str.IsNull(i)) b.AppendNull(); else
b.Append(str.GetString(i));
+ }
+ return b.Build();
+ }
+ case StringArray str:
+ return str;
+ case BinaryArray bin when isVariantBinary || storage ==
Storage.LargeBinary:
+ return storage == Storage.LargeBinary ? ToLargeBinary(bin)
: (IArrowArray)ToBinaryView(bin);
+ case StructArray st:
+ {
+ var type = (StructType)st.Data.DataType;
+ var fields = new List<Field>();
+ var children = new List<IArrowArray>();
+ for (int f = 0; f < type.Fields.Count; f++)
+ {
+ Field field = type.Fields[f];
+ IArrowArray child = ConvertArray(st.Fields[f],
field.Name, storage);
+ children.Add(child);
+ fields.Add(new Field(field.Name,
child.Data.DataType, field.IsNullable));
+ }
+ return new StructArray(new StructType(fields),
st.Length, children, st.NullBitmapBuffer, st.NullCount);
+ }
+ case ListArray list:
+ {
+ Field element =
((ListType)list.Data.DataType).ValueField;
+ IArrowArray values = ConvertArray(list.Values,
element.Name, storage);
+ var listType = new ListType(new Field(element.Name,
values.Data.DataType, element.IsNullable));
+ return new ListArray(listType, list.Length,
list.ValueOffsetsBuffer, values, list.NullBitmapBuffer, list.NullCount);
+ }
+ default:
+ return array;
+ }
+ }
+
+ private static LargeBinaryArray ToLargeBinary(BinaryArray bin)
+ {
+ var b = new LargeBinaryArray.Builder();
+ for (int i = 0; i < bin.Length; i++)
+ {
+ if (bin.IsNull(i)) b.AppendNull(); else
b.Append(bin.GetBytes(i));
+ }
+ return b.Build();
+ }
+
+ private static BinaryViewArray ToBinaryView(BinaryArray bin)
+ {
+ var b = new BinaryViewArray.Builder();
+ for (int i = 0; i < bin.Length; i++)
+ {
+ if (bin.IsNull(i)) b.AppendNull(); else
b.Append(bin.GetBytes(i));
+ }
+ return b.Build();
+ }
+ }
+}
