ericphanson opened a new issue, #332:
URL: https://github.com/apache/arrow-julia/issues/332
Setup:
```julia
using Arrow
struct A0 end
struct A1
x::Int
end
struct A2
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A0}) = :A0
ArrowTypes.JuliaType(::Val{:A0}) = A0
ArrowTypes.arrowname(::Type{A1}) = :A1
ArrowTypes.JuliaType(::Val{:A1}) = A1
ArrowTypes.arrowname(::Type{A2}) = :A2
ArrowTypes.JuliaType(::Val{:A2}) = A2
struct MyUnion{T <:Tuple}
elts::T
end
ArrowTypes.arrowname(::Type{<:MyUnion}) = :MyUnion
ArrowTypes.JuliaType(::Val{:MyUnion}) = MyUnion
ArrowTypes.ArrowType(::Type{<:MyUnion}) = ArrowTypes.UnionKind()
ArrowTypes.toarrow(u::MyUnion{T}) where {T} =
collect(Union{T.parameters...}, u.elts)
ArrowTypes.fromarrow(::Type{<:MyUnion}, args...) = MyUnion(args)
```
Then:
```julia
julia> u = MyUnion((A0(), A1(1), A2(1, 2.0)))
MyUnion{Tuple{A0, A1, A2}}((A0(), A1(1), A2(1, 2.0)))
julia> ArrowTypes.toarrow(u)
3-element Vector{Union{A0, A1, A2}}:
A0()
A1(1)
A2(1, 2.0)
julia> tbl = (; col = [u]);
julia> Arrow.Table(Arrow.tobuffer(tbl)).col[1]
ERROR: MethodError: no method matching isstringtype(::ArrowTypes.StructKind)
Closest candidates are:
isstringtype(::ArrowTypes.ListKind{stringtype}) where stringtype at
~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:196
isstringtype(::Type{ArrowTypes.ListKind{stringtype}}) where stringtype at
~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:197
Stacktrace:
[1] getindex(l::Arrow.List{MyUnion, Int32, Arrow.DenseUnion{Union{Missing,
A0, A1, A2}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing,
Tuple{Union{Missing, A0, A1}, A2}}, Tuple{Arrow.DenseUnion{Union{Missing, A0,
A1}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing, Tuple{Union{Missing,
A0}, A1}}, Tuple{Arrow.Struct{Union{Missing, A0}, Tuple{}}, Arrow.Struct{A1,
Tuple{Arrow.Primitive{Int64, Vector{Int64}}}}}}, Arrow.Struct{A2,
Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64,
Vector{Float64}}}}}}}, i::Int64)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:52
[2] top-level scope
@ REPL[25]:1
```
I took a guess and added
```julia
ArrowTypes.isstringtype(::ArrowTypes.StructKind) = false
```
which seems to fix it
```julia
julia> Arrow.Table(Arrow.tobuffer(tbl)).col[1]
MyUnion{Tuple{SubArray{Union{Missing, A0, A1, A2}, 1,
Arrow.DenseUnion{Union{Missing, A0, A1, A2},
Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing, Tuple{Union{Missing, A0,
A1}, A2}}, Tuple{Arrow.DenseUnion{Union{Missing, A0, A1},
Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing, Tuple{Union{Missing, A0},
A1}}, Tuple{Arrow.Struct{Union{Missing, A0}, Tuple{}}, Arrow.Struct{A1,
Tuple{Arrow.Primitive{Int64, Vector{Int64}}}}}}, Arrow.Struct{A2,
Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64,
Vector{Float64}}}}}}, Tuple{UnitRange{Int64}}, true}}}((Union{Missing, A0, A1,
A2}[A0(), A1(1), A2(1, 2.0)],))
```
However, in my real code, I was using this macro to define the methods:
```julia
macro arrow_record(T1)
T = esc(T1)
name = :(Symbol("JuliaLang.", string(parentmodule($T)), '.',
string(nameof($T))))
return quote
ArrowTypes.arrowname(::Type{$T}) = $name
ArrowTypes.ArrowType(::Type{$T}) = fieldtypes($T)
ArrowTypes.toarrow(obj::$T) = ntuple(i -> getfield(obj, i),
fieldcount($T))
ArrowTypes.JuliaType(::Val{$name}, ::Any) = $T
ArrowTypes.fromarrow(::Type{$T}, args) = $T(args...)
ArrowTypes.fromarrow(::Type{$T}, arg::$T) = arg
end
end
```
and had a second `A2`-style struct,
```julia
struct A22
x::Int
y::Float64
end
```
If I do
```julia
@arrow_record A2
@arrow_record A22
```
and define
```
julia> u1 = MyUnion((A2(1, 2.0), A22(2, 3.0)))
MyUnion{Tuple{A2, A22}}((A2(1, 2.0), A22(2, 3.0)))
julia> u2 = MyUnion((A22(1, 2.0), A2(2, 3.0)))
MyUnion{Tuple{A22, A2}}((A22(1, 2.0), A2(2, 3.0)))
julia> tbl = (; col = [u1, u2]);
```
Then I get
```julia
julia> Arrow.Table(Arrow.tobuffer(tbl)).col[1]
ERROR: TypeError: in Union, expected Type, got a value of type
Tuple{DataType, DataType}
Stacktrace:
[1] ArrowType(#unused#::Type{Union{Missing, A2}})
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:71
[2] ArrowTypes.ToArrow(x::Vector{Union{Missing, A2}})
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:338
[3] arrowvector(x::Vector{Union{Missing, A2}}, i::Int64, nl::Int64,
fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding},
meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64,
kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol},
NamedTuple{(:denseunions, :lareglists, :compression, :dictencodenested),
Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:61
[4] #56
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:220 [inlined]
[5] iterate
@ ./generator.jl:47 [inlined]
[6]
collect(itr::Base.Generator{Base.Iterators.Enumerate{Tuple{Vector{Union{Missing,
A2}}, Vector{A22}}}, Arrow.var"#56#62"{Base.Pairs{Symbol, Union{Nothing,
Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions,
:lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool,
Nothing, Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}}})
@ Base ./array.jl:787
[7] map(f::Function,
A::Base.Iterators.Enumerate{Tuple{Vector{Union{Missing, A2}}, Vector{A22}}})
@ Base ./abstractarray.jl:2961
[8] arrowvector(::ArrowTypes.UnionKind,
x::Arrow.DenseUnionVector{Arrow.ToList{Union{A2, A22}, false, Vector{Union{A2,
A22}}, Int32}, Tuple{A2, A22}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64,
Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol,
Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth,
:denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64,
Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:220
[9] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[10] arrowvector(x::Arrow.DenseUnionVector{Arrow.ToList{Union{A2, A22},
false, Vector{Union{A2, A22}}, Int32}, Tuple{A2, A22}}, i::Int64, nl::Int64,
fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding},
meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64,
kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol},
NamedTuple{(:denseunions, :lareglists, :compression, :dictencodenested),
Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[11] arrowvector(U::Union, x::Arrow.ToList{Union{A2, A22}, false,
Vector{Union{A2, A22}}, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64,
Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; denseunions::Bool,
kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol},
NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression,
:dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:210
[12] arrowvector(x::Arrow.ToList{Union{A2, A22}, false, Vector{Union{A2,
A22}}, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool,
dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool},
NTuple{4, Symbol}, NamedTuple{(:lareglists, :compression, :denseunions,
:dictencodenested), Tuple{Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[13] arrowvector(::ArrowTypes.ListKind{false},
x::ArrowTypes.ToArrow{Vector{Union{A2, A22}}, Vector{MyUnion}}, i::Int64,
nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding},
meta::Base.ImmutableDict{String, String}; largelists::Bool,
kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol},
NamedTuple{(:dictencode, :maxdepth, :compression, :denseunions,
:dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:198
[14] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[15] arrowvector(x::Vector{MyUnion}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing;
dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol,
Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists,
:denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[16] toarrowvector(x::Vector{MyUnion}, i::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing,
kw::Base.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists,
:denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool,
Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:36
[17] (::Arrow.var"#139#140"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool,
Bool, Int64, Nothing, Vector{Arrow.DictEncoding}, Vector{Type},
Vector{Any}})(col::Vector{MyUnion}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:331
[18] eachcolumn
@ ~/.julia/packages/Tables/PxO1m/src/utils.jl:70 [inlined]
[19] toarrowtable(cols::NamedTuple{(:col,), Tuple{Vector{MyUnion}}},
dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing,
denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64,
meta::Nothing, colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:328
[20] macro expansion
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:196 [inlined]
[21] macro expansion
@ ./task.jl:454 [inlined]
[22] write(writer::Arrow.Writer{IOBuffer}, source::NamedTuple{(:col,),
Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:186
[23] (::Arrow.var"#135#136"{NamedTuple{(:col,),
Tuple{Vector{MyUnion}}}})(writer::Arrow.Writer{IOBuffer})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:284
[24] open(::Arrow.var"#135#136"{NamedTuple{(:col,),
Tuple{Vector{MyUnion}}}}, ::Type, ::Vararg{Any}; kwargs::Base.Pairs{Symbol,
Bool, Tuple{Symbol}, NamedTuple{(:file,), Tuple{Bool}}})
@ Base ./io.jl:384
[25] #write#134
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:283 [inlined]
[26] write
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:282 [inlined]
[27] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}};
kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:193
[28] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:191
[29] top-level scope
@ REPL[87]:1
```
So then I turned off dense unions,
```julia
julia> Arrow.Table(Arrow.tobuffer(tbl; denseunions=false)).col[1]
ERROR: MethodError: no method matching zero(::Type{A2})
Closest candidates are:
zero(::Union{Type{P}, P}) where P<:Dates.Period at
~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/Dates/src/periods.jl:53
zero(::AbstractIrrational) at irrationals.jl:150
zero(::SparseArrays.AbstractSparseArray) at
~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/SparseArrays/src/SparseArrays.jl:42
...
Stacktrace:
[1] default(T::Type)
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:305
[2] getindex(A::Arrow.ToSparseUnion{A2, Arrow.ToList{Union{A2, A22},
false, Vector{Union{A2, A22}}, Int32}}, i::Int64)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:160
...
```
So then I added that,
```julia
julia> ArrowTypes.default(::Type{A2}) = A2(1,2)
julia> ArrowTypes.default(::Type{A22}) = A22(1,2)
julia> Arrow.Table(Arrow.tobuffer(tbl; denseunions=false)).col[1]
MyUnion{Tuple{SubArray{Union{A2, A22}, 1, Arrow.SparseUnion{Union{A2, A22},
Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{A2, A22}},
Tuple{Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64, Vector{Int64}},
Arrow.Primitive{Float64, Vector{Float64}}}}, Arrow.Struct{A22,
Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64,
Vector{Float64}}}}}}, Tuple{UnitRange{Int64}}, true}}}((Union{A2, A22}[A2(1,
2.0), A22(2, 3.0)],))
```
and it seems to work!
Ok great. But then in the real code, I have an abstract supertype: (starting
a new Julia session now!)
```julia
using Arrow
ArrowTypes.isstringtype(::ArrowTypes.StructKind) = false
abstract type A end
struct A0 <: A end
struct A1 <: A
x::Int
end
struct A2 <: A
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A0}) = :A0
ArrowTypes.JuliaType(::Val{:A0}) = A0
ArrowTypes.default(::Type{A0}) = A0()
ArrowTypes.arrowname(::Type{A1}) = :A1
ArrowTypes.JuliaType(::Val{:A1}) = A1
ArrowTypes.default(::Type{A1}) = A1(1)
ArrowTypes.arrowname(::Type{A2}) = :A2
ArrowTypes.JuliaType(::Val{:A2}) = A2
ArrowTypes.default(::Type{A2}) = A2(1, 2)
struct MyUnion{T<:NTuple{N,A} where {N}} <: A
elts::T
end
ArrowTypes.arrowname(::Type{<:MyUnion}) = :MyUnion
ArrowTypes.JuliaType(::Val{:MyUnion}) = MyUnion
ArrowTypes.ArrowType(::Type{<:MyUnion}) = ArrowTypes.UnionKind()
ArrowTypes.toarrow(u::MyUnion{T}) where {T} =
collect(Union{T.parameters...}, u.elts)
ArrowTypes.fromarrow(::Type{<:MyUnion}, args...) = MyUnion(args)
```
and then I get
```julia
julia> tbl = (; col = [MyUnion((A0(), A2(-1, 1))), MyUnion((A2(-2, 2),))])
(col = MyUnion[MyUnion{Tuple{A0, A2}}((A0(), A2(-1, 1.0))),
MyUnion{Tuple{A2}}((A2(-2, 2.0),))],)
julia> Arrow.Table(Arrow.tobuffer(tbl; denseunions=false)).col[1]
ERROR: MethodError: no method matching MyUnion(::Tuple{SubArray{Union{A0,
A2}, 1, Arrow.SparseUnion{Union{A0, A2},
Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{A0, A2}},
Tuple{Arrow.Struct{A0, Tuple{}}, Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64,
Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}},
Tuple{UnitRange{Int64}}, true}})
Closest candidates are:
MyUnion(::T) where T<:(Tuple{Vararg{A, N}} where N) at REPL[40]:2
Stacktrace:
[1] fromarrow(#unused#::Type{MyUnion}, args::SubArray{Union{A0, A2}, 1,
Arrow.SparseUnion{Union{A0, A2}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse,
nothing, Tuple{A0, A2}}, Tuple{Arrow.Struct{A0, Tuple{}}, Arrow.Struct{A2,
Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64,
Vector{Float64}}}}}}, Tuple{UnitRange{Int64}}, true})
@ Main ./REPL[45]:1
[2] getindex(l::Arrow.List{MyUnion, Int32, Arrow.SparseUnion{Union{A0, A2},
Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{A0, A2}},
Tuple{Arrow.Struct{A0, Tuple{}}, Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64,
Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}}}, i::Int64)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:61
[3] top-level scope
@ REPL[46]:1
```
which to me looks like the `fromarrow` for `MyUnion` is getting called
before the arguments are `fromarrow`'d into the right types?
Also, if I add a few more types:
```julia
struct A22 <: A
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A22}) = :A22
ArrowTypes.JuliaType(::Val{:A22}) = A22
ArrowTypes.default(::Type{A22}) = A22(1, 2)
struct A23 <: A
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A23}) = :A23
ArrowTypes.JuliaType(::Val{:A23}) = A23
ArrowTypes.default(::Type{A23}) = A23(1, 2)
tbl = (; col = [MyUnion((A2(1, 1), A22(-2, 2), A0())), MyUnion((A22(0, 0),
A22(1, 1), A23(1, 2)))])
```
I get
```julia
julia> Arrow.tobuffer(tbl; denseunions=false)
ERROR: MethodError: no method matching zero(::Type{Union{A0, A2, A22}})
Closest candidates are:
zero(::Union{Type{P}, P}) where P<:Dates.Period at
~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/Dates/src/periods.jl:53
zero(::AbstractIrrational) at irrationals.jl:150
zero(::SparseArrays.AbstractSparseArray) at
~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/SparseArrays/src/SparseArrays.jl:42
...
Stacktrace:
[1] default(T::Type)
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:305
[2] getindex
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:160 [inlined]
[3] iterate(A::Arrow.ToSparseUnion{Union{A0, A2, A22},
ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector,
Int32}}}, state::Tuple{Base.OneTo{Int64}, Int64})
@ Base ./abstractarray.jl:1167
[4] iterate
@ ./iterators.jl:167 [inlined]
[5]
sparsetypeids(#unused#::Type{Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse,
nothing, Tuple{Union{A0, A2}, A22}}}, x::Arrow.ToSparseUnion{Union{A0, A2,
A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false,
Vector, Int32}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:144
[6] arrowvector(::ArrowTypes.UnionKind,
x::Arrow.SparseUnionVector{Arrow.ToSparseUnion{Union{A0, A2, A22},
ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector,
Int32}}}, Tuple{Union{A0, A2}, A22}}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing;
kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol},
NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression,
:dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:225
[7] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[8] arrowvector(x::Arrow.SparseUnionVector{Arrow.ToSparseUnion{Union{A0,
A2, A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false,
Vector, Int32}}}, Tuple{Union{A0, A2}, A22}}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing;
dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol,
Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists,
:compression, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[9] arrowvector(U::Union, x::Arrow.ToSparseUnion{Union{A0, A2, A22},
ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector,
Int32}}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; denseunions::Bool,
kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol},
NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression,
:dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:210
[10] arrowvector(x::Arrow.ToSparseUnion{Union{A0, A2, A22},
ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector,
Int32}}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool,
dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool},
NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists, :compression,
:dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[11] (::Arrow.var"#59#65"{Base.Pairs{Symbol, Union{Nothing, Integer},
NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions,
:lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool,
Nothing, Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding},
DataType})(j::Int64)
@ Arrow ./none:0
[12] iterate
@ ./generator.jl:47 [inlined]
[13] collect(itr::Base.Generator{UnitRange{Int64},
Arrow.var"#59#65"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6,
Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists,
:compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing,
Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}, DataType}})
@ Base ./array.jl:787
[14] _totuple
@ ./tuple.jl:349 [inlined]
[15] Tuple(itr::Base.Generator{UnitRange{Int64},
Arrow.var"#59#65"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6,
Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists,
:compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing,
Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}, DataType}})
@ Base ./tuple.jl:317
[16] arrowvector(::ArrowTypes.UnionKind,
x::Arrow.SparseUnionVector{ArrowTypes.ToArrow{Union{A0, A2, A22, A23},
Arrow.ToList{Any, false, Vector, Int32}}, Tuple{Union{A0, A2, A22}, A23}},
i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol,
Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth,
:denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64,
Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:226
[17] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[18] arrowvector(x::Arrow.SparseUnionVector{ArrowTypes.ToArrow{Union{A0,
A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}, Tuple{Union{A0, A2,
A22}, A23}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool,
dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool},
NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists, :compression,
:dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[19] arrowvector(U::Union, x::ArrowTypes.ToArrow{Union{A0, A2, A22, A23},
Arrow.ToList{Any, false, Vector, Int32}}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing;
denseunions::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5,
Symbol}, NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression,
:dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:210
[20] arrowvector(x::Arrow.ToList{Any, false, Vector, Int32}, i::Int64,
nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding},
meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64,
kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol},
NamedTuple{(:lareglists, :compression, :denseunions, :dictencodenested),
Tuple{Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[21] arrowvector(::ArrowTypes.ListKind{false},
x::ArrowTypes.ToArrow{Vector, Vector{MyUnion}}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding},
meta::Base.ImmutableDict{String, String}; largelists::Bool,
kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol},
NamedTuple{(:dictencode, :maxdepth, :compression, :denseunions,
:dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:198
[22] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[23] arrowvector(x::Vector{MyUnion}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing;
dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol,
Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists,
:denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[24] toarrowvector(x::Vector{MyUnion}, i::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing,
kw::Base.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists,
:denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool,
Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:36
[25] (::Arrow.var"#139#140"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool,
Bool, Int64, Nothing, Vector{Arrow.DictEncoding}, Vector{Type},
Vector{Any}})(col::Vector{MyUnion}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:331
[26] eachcolumn
@ ~/.julia/packages/Tables/PxO1m/src/utils.jl:70 [inlined]
[27] toarrowtable(cols::NamedTuple{(:col,), Tuple{Vector{MyUnion}}},
dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing,
denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64,
meta::Nothing, colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:328
[28] macro expansion
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:196 [inlined]
[29] macro expansion
@ ./task.jl:454 [inlined]
[30] write(writer::Arrow.Writer{IOBuffer}, source::NamedTuple{(:col,),
Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:186
[31] (::Arrow.var"#135#136"{NamedTuple{(:col,),
Tuple{Vector{MyUnion}}}})(writer::Arrow.Writer{IOBuffer})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:284
[32] open(::Arrow.var"#135#136"{NamedTuple{(:col,),
Tuple{Vector{MyUnion}}}}, ::Type, ::Vararg{Any}; kwargs::Base.Pairs{Symbol,
Bool, Tuple{Symbol, Symbol}, NamedTuple{(:file, :denseunions), Tuple{Bool,
Bool}}})
@ Base ./io.jl:384
[33] #write#134
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:283 [inlined]
[34] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}};
kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:denseunions,),
Tuple{Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:193
[35] top-level scope
@ REPL[53]:1
```
Then if I `@arrow_record` them, I get a different error:
```julia
julia> macro arrow_record(T1)
T = esc(T1)
name = :(Symbol("JuliaLang.", string(parentmodule($T)), '.',
string(nameof($T))))
return quote
ArrowTypes.arrowname(::Type{$T}) = $name
ArrowTypes.ArrowType(::Type{$T}) = fieldtypes($T)
ArrowTypes.toarrow(obj::$T) = ntuple(i -> getfield(obj, i),
fieldcount($T))
ArrowTypes.JuliaType(::Val{$name}, ::Any) = $T
ArrowTypes.fromarrow(::Type{$T}, args) = $T(args...)
ArrowTypes.fromarrow(::Type{$T}, arg::$T) = arg
end
end
@arrow_record (macro with 1 method)
julia> @arrow_record A0
julia> @arrow_record A2
julia> @arrow_record A22
julia> @arrow_record A23
julia> @arrow_record A1
julia> Arrow.tobuffer(tbl; denseunions=false)
ERROR: ArgumentError: type does not have a definite number of fields
Stacktrace:
[1] fieldcount(t::Any)
@ Base ./reflection.jl:804
[2] arrowvector(::ArrowTypes.StructKind,
x::ArrowTypes.ToArrow{Tuple{Vararg{Real}}, Arrow.ToList{Any, false, Vector,
Int32}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol,
Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth,
:lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Int64,
Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/struct.jl:93
[3] arrowvector(::Type{Tuple{Vararg{Real}}},
x::ArrowTypes.ToArrow{Tuple{Vararg{Real}}, Arrow.ToList{Any, false, Vector,
Int32}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol,
Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth,
:lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Int64,
Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90
[4] arrowvector(x::Arrow.ToList{Any, false, Vector, Int32}, i::Int64,
nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding},
meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64,
kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol},
NamedTuple{(:lareglists, :compression, :denseunions, :dictencodenested),
Tuple{Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[5] arrowvector(::ArrowTypes.ListKind{false},
x::ArrowTypes.ToArrow{Vector, Vector{MyUnion}}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding},
meta::Base.ImmutableDict{String, String}; largelists::Bool,
kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol},
NamedTuple{(:dictencode, :maxdepth, :compression, :denseunions,
:dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:198
[6] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[7] arrowvector(x::Vector{MyUnion}, i::Int64, nl::Int64, fi::Int64,
de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing;
dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol,
Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists,
:denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[8] toarrowvector(x::Vector{MyUnion}, i::Int64, de::Dict{Int64, Any},
ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing,
kw::Base.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists,
:denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool,
Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:36
[9] (::Arrow.var"#139#140"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool,
Bool, Int64, Nothing, Vector{Arrow.DictEncoding}, Vector{Type},
Vector{Any}})(col::Vector{MyUnion}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:331
[10] eachcolumn
@ ~/.julia/packages/Tables/PxO1m/src/utils.jl:70 [inlined]
[11] toarrowtable(cols::NamedTuple{(:col,), Tuple{Vector{MyUnion}}},
dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing,
denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64,
meta::Nothing, colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:328
[12] macro expansion
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:196 [inlined]
[13] macro expansion
@ ./task.jl:454 [inlined]
[14] write(writer::Arrow.Writer{IOBuffer}, source::NamedTuple{(:col,),
Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:186
[15] (::Arrow.var"#135#136"{NamedTuple{(:col,),
Tuple{Vector{MyUnion}}}})(writer::Arrow.Writer{IOBuffer})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:284
[16] open(::Arrow.var"#135#136"{NamedTuple{(:col,),
Tuple{Vector{MyUnion}}}}, ::Type, ::Vararg{Any}; kwargs::Base.Pairs{Symbol,
Bool, Tuple{Symbol, Symbol}, NamedTuple{(:file, :denseunions), Tuple{Bool,
Bool}}})
@ Base ./io.jl:384
[17] #write#134
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:283 [inlined]
[18] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}};
kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:denseunions,),
Tuple{Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:193
[19] top-level scope
@ REPL[63]:1
```
(Note that one gives the same error if `denseunions=true`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@arrow.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
