This is an automated email from the ASF dual-hosted git repository.
CurtHagenlocher pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-dotnet.git
The following commit(s) were added to refs/heads/main by this push:
new 2d5b4cb feat: Support SQL-NULL rows when shredding variant columns
(#445)
2d5b4cb is described below
commit 2d5b4cbf6652b3305276e02efc4f1fcb0338fcf9
Author: Curt Hagenlocher <[email protected]>
AuthorDate: Sat Sep 26 21:01:39 2026 -0700
feat: Support SQL-NULL rows when shredding variant columns (#445)
## What's Changed
The shredding pipeline had no way to represent a SQL-NULL row: every
entry point took `VariantValue`, and `ShreddedVariantArrayBuilder.Build`
always produced a storage struct with no validity. Callers had to use
`VariantValue.Null` as a placeholder, which is stored as a present
variant null and changes what `IS NULL` means.
This follows the `VariantValue?` convention already used by
`VariantArray.Builder`:
- `ShredSchemaInferer.Infer(IEnumerable<VariantValue?>, ShredOptions)`
ignores null rows, so they don't count toward frequency thresholds. An
all-null column infers as unshredded.
- `VariantShredder.Shred(IEnumerable<VariantValue?>, ShredSchema)`
leaves null rows out of the shared metadata and returns a `null` entry
for each one.
- `ShreddedVariantArrayBuilder.Build` treats a `null` entry in `rows` as
a null element of the storage struct. Its children are emitted as
missing, and `metadata` is still populated because that field is
required. No validity buffer is allocated when there are no nulls.
- `VariantUnshredder.Reconstruct` returns `null` for a `null` result
instead of throwing, so rows from the nullable `Shred` overload can be
unshredded one at a time.
Existing signatures are unchanged. Passing an untyped `null` or
`default` literal for `values` (e.g. `Shred(null, schema)`) is now
ambiguous between the overloads. Such a call can only throw
`ArgumentNullException`, so this is unlikely to matter in practice.
Typed null variables, arrays and collection expressions all still
resolve as before, and `VariantArray.Builder.AppendRange` already uses
the same overload pair.
This lays the groundwork for the array-level entry points proposed in
#399, which can read validity directly from the input `VariantArray`.
Closes #398.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../Shredding/ShredSchemaInferer.cs | 22 +++
.../Shredding/ShreddedVariantArrayBuilder.cs | 35 ++++-
.../Shredding/VariantShredder.cs | 33 +++++
.../Shredding/VariantUnshredder.cs | 12 +-
.../Shredding/ShredNullRowTests.cs | 164 +++++++++++++++++++++
5 files changed, 259 insertions(+), 7 deletions(-)
diff --git a/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs
b/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs
index 4537c3c..ed29532 100644
--- a/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs
+++ b/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs
@@ -53,6 +53,28 @@ namespace Apache.Arrow.Operations.Shredding
return BuildSchema(stats, totalCount, options, 0);
}
+ /// <summary>
+ /// Infers a shredding schema by analyzing the given nullable values.
+ /// <c>null</c> entries are SQL-NULL rows and are ignored; they carry
no type
+ /// information and do not count toward frequency thresholds.
+ /// </summary>
+ /// <param name="values">The variant values to analyze.</param>
+ /// <param name="options">Options controlling depth, frequency, and
type consistency thresholds.</param>
+ /// <returns>An inferred <see cref="ShredSchema"/>.</returns>
+ public ShredSchema Infer(IEnumerable<VariantValue?> values,
ShredOptions options = null)
+ {
+ if (values == null) throw new
ArgumentNullException(nameof(values));
+ return Infer(NonNull(values), options);
+ }
+
+ private static IEnumerable<VariantValue>
NonNull(IEnumerable<VariantValue?> values)
+ {
+ foreach (VariantValue? value in values)
+ {
+ if (value.HasValue) yield return value.Value;
+ }
+ }
+
private void CollectStats(VariantValue value, TypeStats stats, int
depth, int maxDepth)
{
ShredType type = VariantShredder.GetShredType(value);
diff --git
a/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs
b/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs
index 44b6de5..3428fa1 100644
--- a/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs
+++ b/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs
@@ -36,7 +36,10 @@ namespace Apache.Arrow.Operations.Shredding
/// </summary>
/// <param name="schema">The shredding schema applied to each
row.</param>
/// <param name="metadata">The column-level variant metadata (shared
across rows).</param>
- /// <param name="rows">Per-row shred results whose residual bytes
reference <paramref name="metadata"/>.</param>
+ /// <param name="rows">
+ /// Per-row shred results whose residual bytes reference <paramref
name="metadata"/>.
+ /// A <c>null</c> entry produces a null (SQL-NULL) element in the
resulting array.
+ /// </param>
/// <param name="allocator">Arrow memory allocator, or default if
null.</param>
public static VariantArray Build(
ShredSchema schema,
@@ -50,6 +53,33 @@ namespace Apache.Arrow.Operations.Shredding
int rowCount = rows.Count;
+ // Top-level validity. Children of a null row are emitted as
missing
+ // (value and typed_value both null); metadata is still populated
+ // because the metadata field is non-nullable.
+ ArrowBuffer.BitmapBuilder validity = new
ArrowBuffer.BitmapBuilder(rowCount);
+ int nullCount = 0;
+ ShredResult[] childRows = null;
+ for (int i = 0; i < rowCount; i++)
+ {
+ if (rows[i] == null)
+ {
+ if (childRows == null)
+ {
+ childRows = new ShredResult[rowCount];
+ for (int j = 0; j < i; j++) childRows[j] = rows[j];
+ }
+ childRows[i] = ShredResult.Missing;
+ validity.Append(false);
+ nullCount++;
+ }
+ else
+ {
+ if (childRows != null) childRows[i] = rows[i];
+ validity.Append(true);
+ }
+ }
+ if (childRows != null) rows = childRows;
+
// metadata column: emit the shared bytes once per row. (A
dictionary-encoded
// or run-end-encoded representation would compress this;
VariantArray's reader
// already handles those, but for simplicity we emit the plain
binary form.)
@@ -81,8 +111,9 @@ namespace Apache.Arrow.Operations.Shredding
}
StructType structType = new StructType(fields);
+ ArrowBuffer nullBitmap = nullCount > 0 ? validity.Build(allocator)
: ArrowBuffer.Empty;
StructArray structArr = new StructArray(
- structType, rowCount, children, ArrowBuffer.Empty, nullCount:
0);
+ structType, rowCount, children, nullBitmap, nullCount);
// The public VariantArray(IArrowArray) constructor infers the
VariantType
// from the struct's shape (including detecting the shredded
layout).
return new VariantArray(structArr);
diff --git a/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs
b/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs
index ef2ed11..501e028 100644
--- a/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs
+++ b/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs
@@ -66,6 +66,39 @@ namespace Apache.Arrow.Operations.Shredding
return (metadataBytes, results);
}
+ /// <summary>
+ /// Shreds a column of nullable variant values. A <c>null</c> entry is
a
+ /// SQL-NULL row, as opposed to <see cref="VariantValue.Null"/>, which
is a
+ /// present variant null. SQL-NULL rows produce a <c>null</c> entry in
the
+ /// returned rows, which <see
cref="ShreddedVariantArrayBuilder.Build"/> turns
+ /// into a null element of the resulting array.
+ /// </summary>
+ public static (byte[] Metadata, IReadOnlyList<ShredResult> Rows) Shred(
+ IEnumerable<VariantValue?> values,
+ ShredSchema schema)
+ {
+ if (values == null) throw new
ArgumentNullException(nameof(values));
+ if (schema == null) throw new
ArgumentNullException(nameof(schema));
+
+ List<VariantValue?> rows = values as List<VariantValue?> ?? new
List<VariantValue?>(values);
+
+ VariantMetadataBuilder metadata = new VariantMetadataBuilder();
+ foreach (VariantValue? row in rows)
+ {
+ if (row.HasValue) CollectFieldNames(row.Value, metadata);
+ }
+ byte[] metadataBytes = metadata.Build(out int[] idRemap);
+
+ ShredResult[] results = new ShredResult[rows.Count];
+ for (int i = 0; i < rows.Count; i++)
+ {
+ VariantValue? row = rows[i];
+ results[i] = row.HasValue ? Shred(row.Value, schema, metadata,
idRemap) : null;
+ }
+
+ return (metadataBytes, results);
+ }
+
/// <summary>
/// Shreds a single variant value against a caller-managed metadata
dictionary.
/// Use this when combining shredded columns with external metadata,
or when
diff --git a/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs
b/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs
index 346e57d..ecc9e10 100644
--- a/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs
+++ b/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs
@@ -30,19 +30,21 @@ namespace Apache.Arrow.Operations.Shredding
/// <summary>
/// Reconstructs a variant value from a shredded result.
/// </summary>
- /// <param name="shredded">The shredded (value, typed_value)
pair.</param>
+ /// <param name="shredded">
+ /// The shredded (value, typed_value) pair, or null for a SQL-NULL row
as produced by
+ /// <see
cref="VariantShredder.Shred(System.Collections.Generic.IEnumerable{VariantValue?},
ShredSchema)"/>.
+ /// </param>
/// <param name="schema">The shredding schema that was used to produce
the result.</param>
/// <param name="metadata">The column-level variant metadata
bytes.</param>
/// <returns>
- /// The reconstructed <see cref="VariantValue"/>, or null if the field
is missing
- /// (both value and typed_value are null).
+ /// The reconstructed <see cref="VariantValue"/>, or null if <paramref
name="shredded"/>
+ /// is null (a SQL-NULL row) or the field is missing (both value and
typed_value are null).
/// </returns>
public static VariantValue? Reconstruct(ShredResult shredded,
ShredSchema schema, ReadOnlySpan<byte> metadata)
{
- if (shredded == null) throw new
ArgumentNullException(nameof(shredded));
if (schema == null) throw new
ArgumentNullException(nameof(schema));
- if (shredded.IsMissing)
+ if (shredded == null || shredded.IsMissing)
{
return null;
}
diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs
b/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs
new file mode 100644
index 0000000..f275a3f
--- /dev/null
+++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShredNullRowTests.cs
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Collections.Generic;
+using Apache.Arrow;
+using Apache.Arrow.Operations.Shredding;
+using Apache.Arrow.Scalars.Variant;
+using Xunit;
+
+namespace Apache.Arrow.Operations.Tests.Shredding
+{
+ /// <summary>
+ /// Tests for SQL-NULL rows (a null <see cref="VariantValue"/>?) flowing
through
+ /// inference, shredding and array assembly, and staying distinct from a
present
+ /// <see cref="VariantValue.Null"/>.
+ /// </summary>
+ public class ShredNullRowTests
+ {
+ private static VariantValue Obj(int a) => VariantValue.FromObject(
+ new Dictionary<string, VariantValue> { ["a"] =
VariantValue.FromInt32(a) });
+
+ private static VariantArray ShredAndBuild(IReadOnlyList<VariantValue?>
values, ShredSchema schema)
+ {
+ (byte[] metadata, IReadOnlyList<ShredResult> rows) =
VariantShredder.Shred(values, schema);
+ return ShreddedVariantArrayBuilder.Build(schema, metadata, rows);
+ }
+
+ private static void AssertRows(IReadOnlyList<VariantValue?> expected,
VariantArray array)
+ {
+ Assert.Equal(expected.Count, array.Length);
+ for (int i = 0; i < expected.Count; i++)
+ {
+ if (expected[i].HasValue)
+ {
+ Assert.False(array.IsNull(i), $"row {i} should be valid");
+ Assert.Equal(expected[i].Value,
array.GetLogicalVariantValue(i));
+ }
+ else
+ {
+ Assert.True(array.IsNull(i), $"row {i} should be null");
+ }
+ }
+ }
+
+ [Fact]
+ public void SqlNullRow_IsDistinctFromVariantNull()
+ {
+ var values = new List<VariantValue?> { Obj(1), null,
VariantValue.Null, Obj(3) };
+
+ ShredSchema schema = new ShredSchemaInferer().Infer(values,
ShredOptions.Default);
+ VariantArray array = ShredAndBuild(values, schema);
+
+ Assert.Equal(ShredType.Object, schema.TypedValueType);
+ Assert.Equal(1, array.NullCount);
+ Assert.Equal(1, array.StorageArray.NullCount);
+ AssertRows(values, array);
+ Assert.False(array.IsNull(2));
+ Assert.True(array.GetLogicalVariantValue(2).IsNull);
+ }
+
+ [Fact]
+ public void SqlNullRow_Unshredded()
+ {
+ var values = new List<VariantValue?> {
VariantValue.FromString("x"), null, VariantValue.FromInt32(5) };
+ AssertRows(values, ShredAndBuild(values,
ShredSchema.Unshredded()));
+ }
+
+ [Fact]
+ public void SqlNullRow_Primitive()
+ {
+ var values = new List<VariantValue?> { null,
VariantValue.FromInt32(1), VariantValue.FromString("residual"), null };
+ VariantArray array = ShredAndBuild(values,
ShredSchema.Primitive(ShredType.Int32));
+ Assert.Equal(2, array.NullCount);
+ AssertRows(values, array);
+ }
+
+ [Fact]
+ public void SqlNullRow_Array()
+ {
+ var values = new List<VariantValue?>
+ {
+ VariantValue.FromArray(VariantValue.FromInt32(1),
VariantValue.FromInt32(2)),
+ null,
+ VariantValue.FromArray(VariantValue.FromInt32(3)),
+ };
+ AssertRows(values, ShredAndBuild(values,
ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32))));
+ }
+
+ [Fact]
+ public void AllRowsNull()
+ {
+ var values = new List<VariantValue?> { null, null };
+ ShredSchema schema = new ShredSchemaInferer().Infer(values);
+ Assert.Equal(ShredType.None, schema.TypedValueType);
+
+ VariantArray array = ShredAndBuild(values, schema);
+ Assert.Equal(2, array.NullCount);
+ AssertRows(values, array);
+ }
+
+ [Fact]
+ public void NoNullRows_ProducesNoValidityBuffer()
+ {
+ var values = new List<VariantValue?> { Obj(1), Obj(2) };
+ VariantArray array = ShredAndBuild(values, new
ShredSchemaInferer().Infer(values));
+ Assert.Equal(0, array.NullCount);
+ Assert.True(array.StorageArray.NullBitmapBuffer.IsEmpty);
+ }
+
+ [Fact]
+ public void SlicedArray_KeepsValidityAligned()
+ {
+ var values = new List<VariantValue?> { Obj(1), null, Obj(3), null,
Obj(5) };
+ VariantArray array = ShredAndBuild(values, new
ShredSchemaInferer().Infer(values));
+
+ var sliced = (VariantArray)ArrowArrayFactory.Slice(array, 1, 3);
+ AssertRows(new List<VariantValue?> { null, Obj(3), null }, sliced);
+ }
+
+ [Fact]
+ public void Infer_IgnoresNullRows()
+ {
+ // With nulls counted, Int32 would appear in only a third of the
rows.
+ var values = new List<VariantValue?> { VariantValue.FromInt32(1),
null, null, null, VariantValue.FromInt32(2), null };
+ ShredSchema schema = new ShredSchemaInferer().Infer(values);
+ Assert.Equal(ShredType.Int32, schema.TypedValueType);
+ }
+
+ [Fact]
+ public void Shred_ReturnsNullEntryForNullRow()
+ {
+ var values = new List<VariantValue?> { Obj(1), null };
+ (byte[] metadata, IReadOnlyList<ShredResult> rows) =
VariantShredder.Shred(values, ShredSchema.Unshredded());
+ Assert.NotNull(metadata);
+ Assert.NotNull(rows[0]);
+ Assert.Null(rows[1]);
+ }
+
+ [Fact]
+ public void Reconstruct_RoundTripsNullRows()
+ {
+ var values = new List<VariantValue?> { Obj(1), null,
VariantValue.Null, VariantValue.FromString("s") };
+ ShredSchema schema = new ShredSchemaInferer().Infer(values);
+ (byte[] metadata, IReadOnlyList<ShredResult> rows) =
VariantShredder.Shred(values, schema);
+
+ for (int i = 0; i < values.Count; i++)
+ {
+ Assert.Equal(values[i], VariantUnshredder.Reconstruct(rows[i],
schema, metadata));
+ }
+ }
+ }
+}
