This is an automated email from the ASF dual-hosted git repository. quinnj pushed a commit to branch preservation/storage-20261008/julia-dev-Arrow-decimals-f2071e12 in repository https://gitbox.apache.org/repos/asf/arrow-julia.git
commit e532414df057ad8a586a2b0723d5b7b38c816116 Author: Jacob Quinn <[email protected]> AuthorDate: Tue Sep 1 09:34:01 2026 -0600 Widen the decimal typed claim to any isbits value type of the layout width Decimal is the one Core layout with no Julia VALUE type: `juliatype` can only name the 32/64-bit coefficient, and 128/256-bit decimals surface as raw native-endian bytes — one heap-allocated `Vector{UInt8}` per element, and no bulk path at all. Widen the typed claim for `DecimalType` alone: a caller that DOES own a decimal value type may claim any isbits type of exactly the layout width, and Core copies the two's-complement native-endian coefficient verbatim. `juliatype`'s own domain stays claimable, so every existing claim keeps working, and every other layout keeps exact-match-only. This is what lets the facade read all four widths through one bulk byte copy instead of the element loop. Co-Authored-By: Claude Fable 5 <[email protected]> --- docs/dev/core-README.md | 11 ++++++--- src/ArrowCore.jl | 64 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/docs/dev/core-README.md b/docs/dev/core-README.md index c57a614..f045f17 100644 --- a/docs/dev/core-README.md +++ b/docs/dev/core-README.md @@ -180,9 +180,14 @@ conversion; `Missing <: T` admits nulls; `Any` is the dynamic path), composites recurse (List → `Vector{E}`, Struct → `Vector{Pair}` or a NamedTuple with name checks, Dictionary → pool values, run-end encoding transparent), unions refuse every static claim, and claims are checked -against the descriptor before any element is read. Closed fixed-width -claims over Int/Float/temporal/Decimal32/64 columns use one bounds-checked -bulk byte copy plus a bitmap null punch. +against the descriptor before any element is read. Decimal is the sole +widened leaf: Core has no decimal VALUE type (`juliatype` names the 32/64-bit +coefficient and raw bytes for 128/256), so a caller that owns one may claim +ANY isbits type of exactly the layout width and Core copies the +two's-complement native-endian coefficient verbatim — that is how the facade +reads every width as `Decimals.Decimal{P,S,T}`. Closed fixed-width claims +over Int/Float/temporal columns and over decimals of all four widths use one +bounds-checked bulk byte copy plus a bitmap null punch. `validate_full` adds UTF-8 well-formedness for Utf8 and Utf8View, the advisory nullability contract, Date64 day divisibility, time-of-day range, diff --git a/src/ArrowCore.jl b/src/ArrowCore.jl index 68a0841..ef9c4f1 100644 --- a/src/ArrowCore.jl +++ b/src/ArrowCore.jl @@ -2048,6 +2048,17 @@ juliatype(::BinaryType) = Vector{UInt8} juliatype(t::FixedSizeBinaryType) = Vector{UInt8} juliatype(t::ViewType) = t.utf8 ? String : Vector{UInt8} juliatype(t::DecimalType) = t.bits == 32 ? Int32 : t.bits == 64 ? Int64 : Vector{UInt8} + +# Decimal is the one layout Core has no VALUE type for: the coefficient is +# `primwidth(t)` two's-complement native-endian bytes, and `juliatype` can +# only name the 32/64-bit ones (Base has no 128/256-bit decimal, hence the +# byte-vector domain above). So the typed claim widens here alone: a caller +# that DOES own a decimal value type — the facade's `Decimals.Decimal{P,S,T}` +# — may claim any isbits type of exactly the layout width, and Core copies +# the coefficient bytes verbatim. Every other layout keeps exact-match-only. +_decimalclaim(::Type{E}, t::DecimalType) where {E} = + isbitstype(E) && primwidth(t) == Int64(sizeof(E)) + juliatype(t::IntervalType) = t.unit == YEAR_MONTH ? Int32 : t.unit == DAY_TIME ? NamedTuple{(:days, :millis),Tuple{Int32,Int32}} : @@ -2552,7 +2563,11 @@ function _prechargevalue!( ) where {T} isvalid_at(d, i) || return nothing if t.bits > 64 - _reservevector!(estimate, UInt8, primwidth(t), "decimal value") + # An isbits decimal claim stores the coefficient inline in the + # result vector, already charged as a whole; only the byte-vector + # domain allocates per element. + _decimalclaim(Base.nonmissingtype(T), t) || + _reservevector!(estimate, UInt8, primwidth(t), "decimal value") elseif T === Any _reserveobject!(estimate, primwidth(t), "boxed decimal value") end @@ -3009,12 +3024,7 @@ function materialize(::Type{T}, f::Field, d::ArrayData, budget) where {T} # Nullable fixed-width bulk extraction first builds Vector{E}, then its # public Vector{Union{Missing,E}}. Account for that private scratch copy. E = Base.nonmissingtype(T) - if Missing <: T && - isbitstype(E) && - d.type isa - Union{IntType,FloatType,TimestampType,DateType,TimeType,DurationType,DecimalType} && - E === juliatype(d.type) && - primwidth(d.type) == Int64(sizeof(E)) + if Missing <: T && _bulkeligible(E, d.type) _reservevector!(estimate, E, d.len, "typed materialization scratch") end _commitestimate!(estimate, "typed materialized column") @@ -3112,6 +3122,13 @@ function _checkclaim(::Type{T}, f::Field, d::ArrayData)::Nothing where {T} return nothing end E = Base.nonmissingtype(T) + if t isa DecimalType + # The one widened leaf: see `_decimalclaim`. The byte-vector domain + # `juliatype` reports stays claimable for 128/256. + (E === juliatype(t) || _decimalclaim(E, t)) || + _typedrefuse(E, _layoutname(t), f) + return nothing + end E === _juliatype_of(t) || _typedrefuse(E, _layoutname(t), f) return nothing end @@ -3252,7 +3269,6 @@ function _typedvalue( TimeType, DurationType, ViewType, - DecimalType, IntervalType, MapType, }, @@ -3272,6 +3288,16 @@ function _typedvalue( return _value(t, f, d, i)::E end +# Decimal's own leaf: `juliatype`'s domain, or any isbits type of exactly the +# layout width (`_decimalclaim`), whose bytes ARE the coefficient. +function _typedvalue(::Type{T}, t::DecimalType, f::Field, d::ArrayData, i::Int64) where {T} + isvalid_at(d, i) || return _typedmissing(T, f) + E = Base.nonmissingtype(T) + E === juliatype(t) && return _value(t, f, d, i)::E + _decimalclaim(E, t) || _typedrefuse(E, _layoutname(t), f) + return loadat(rolebuffer(d, DATA), E, _slotbyteoff(d, i, primwidth(t)))::E +end + function _typedvalue( ::Type{T}, t::Union{ListType,ListViewType}, @@ -3428,6 +3454,20 @@ end _bulkmaterialize(::Type{T}, ::ArrowType, ::Field, ::ArrayData) where {T} = nothing +# Does the element claim `E` take the bulk byte copy over layout `t`? The +# claim's byte size must equal the DESCRIPTOR's layout width: an invalid +# 24-bit descriptor falls through juliatype's fallback to a 64-bit Julia +# type, and copying at the claim's width would misread — such descriptors +# take the element loop (and validation refuses them). Decimal admits any +# isbits claim of that width (`_decimalclaim`); every other layout demands +# its exact `juliatype`. +_bulkeligible(::Type{E}, t::ArrowType) where {E} = + t isa + Union{IntType,FloatType,TimestampType,DateType,TimeType,DurationType,DecimalType} && + isbitstype(E) && + (E === juliatype(t) || t isa DecimalType) && + primwidth(t) == Int64(sizeof(E)) + function _bulkmaterialize( ::Type{T}, t::Union{IntType,FloatType,TimestampType,DateType,TimeType,DurationType,DecimalType}, @@ -3435,14 +3475,8 @@ function _bulkmaterialize( d::ArrayData, ) where {T} E = Base.nonmissingtype(T) - isbitstype(E) || return nothing - E === juliatype(t) || return nothing + _bulkeligible(E, t) || return nothing w = Int64(sizeof(E)) - # The claim's byte size must equal the DESCRIPTOR's layout width: an - # invalid 24-bit descriptor falls through juliatype's fallback to a - # 64-bit Julia type, and copying at the claim's width would misread — - # such descriptors take the element loop (and validation refuses them). - primwidth(t) == w || return nothing n = d.len # The typed path serves unvalidated data too: subslice re-checks the # extraction window against the buffer's declared bounds.
