This is an automated email from the ASF dual-hosted git repository. sbinet pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new 462cbe8 ARROW-5592: [Go] implement Duration array 462cbe8 is described below commit 462cbe8715df8b1ad3a3d2c2418de9433d4b00d3 Author: Sebastien Binet <bi...@cern.ch> AuthorDate: Fri Jun 14 18:21:59 2019 +0200 ARROW-5592: [Go] implement Duration array Author: Sebastien Binet <bi...@cern.ch> Closes #4563 from sbinet/issue-5592 and squashes the following commits: de7bbd32e <Sebastien Binet> ARROW-5592: implement Duration array --- go/arrow/array/array.go | 2 +- go/arrow/array/array_test.go | 2 +- go/arrow/array/compare.go | 6 + go/arrow/array/numeric.gen.go | 57 ++ go/arrow/array/numericbuilder.gen.go | 138 +++++ go/arrow/array/numericbuilder.gen_test.go | 815 +++++++++++++++++++++++++ go/arrow/array/numericbuilder.gen_test.go.tmpl | 43 ++ go/arrow/datatype_fixedwidth.go | 77 ++- go/arrow/numeric.tmpldata | 12 + go/arrow/type_traits_numeric.gen.go | 49 ++ 10 files changed, 1171 insertions(+), 30 deletions(-) diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index c13dd07..1912f3e 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -194,7 +194,7 @@ func init() { arrow.MAP: unsupportedArrayType, arrow.EXTENSION: unsupportedArrayType, arrow.FIXED_SIZE_LIST: func(data *Data) Interface { return NewFixedSizeListData(data) }, - arrow.DURATION: unsupportedArrayType, + arrow.DURATION: func(data *Data) Interface { return NewDurationData(data) }, // invalid data types to fill out array size 2⁵-1 31: invalidDataType, diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go index 884bb8d..724f3b4 100644 --- a/go/arrow/array/array_test.go +++ b/go/arrow/array/array_test.go @@ -80,13 +80,13 @@ func TestMakeFromData(t *testing.T) { array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0), array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0), }}, + {name: "duration", d: &testDataType{arrow.DURATION}}, // unsupported types {name: "union", d: &testDataType{arrow.UNION}, expPanic: true, expError: "unsupported data type: UNION"}, {name: "dictionary", d: &testDataType{arrow.DICTIONARY}, expPanic: true, expError: "unsupported data type: DICTIONARY"}, {name: "map", d: &testDataType{arrow.Type(27)}, expPanic: true, expError: "unsupported data type: MAP"}, {name: "extension", d: &testDataType{arrow.Type(28)}, expPanic: true, expError: "unsupported data type: EXTENSION"}, - {name: "duration", d: &testDataType{arrow.Type(30)}, expPanic: true, expError: "unsupported data type: DURATION"}, // invalid types {name: "invalid(-1)", d: &testDataType{arrow.Type(-1)}, expPanic: true, expError: "invalid data type: Type(-1)"}, diff --git a/go/arrow/array/compare.go b/go/arrow/array/compare.go index 0ea0b61..c6665c9 100644 --- a/go/arrow/array/compare.go +++ b/go/arrow/array/compare.go @@ -158,6 +158,9 @@ func ArrayEqual(left, right Interface) bool { case *DayTimeInterval: r := right.(*DayTimeInterval) return arrayEqualDayTimeInterval(l, r) + case *Duration: + r := right.(*Duration) + return arrayEqualDuration(l, r) default: panic(errors.Errorf("arrow/array: unknown array type %T", l)) @@ -341,6 +344,9 @@ func arrayApproxEqual(left, right Interface, opt equalOption) bool { case *DayTimeInterval: r := right.(*DayTimeInterval) return arrayEqualDayTimeInterval(l, r) + case *Duration: + r := right.(*Duration) + return arrayEqualDuration(l, r) default: panic(errors.Errorf("arrow/array: unknown array type %T", l)) diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go index d72d7d0..21c4e4b 100644 --- a/go/arrow/array/numeric.gen.go +++ b/go/arrow/array/numeric.gen.go @@ -879,3 +879,60 @@ func arrayEqualDate64(left, right *Date64) bool { } return true } + +// A type which represents an immutable sequence of arrow.Duration values. +type Duration struct { + array + values []arrow.Duration +} + +func NewDurationData(data *Data) *Duration { + a := &Duration{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Duration) Value(i int) arrow.Duration { return a.values[i] } +func (a *Duration) DurationValues() []arrow.Duration { return a.values } + +func (a *Duration) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Duration) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.DurationTraits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualDuration(left, right *Duration) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go index 3c9421b..b113908 100644 --- a/go/arrow/array/numericbuilder.gen.go +++ b/go/arrow/array/numericbuilder.gen.go @@ -2070,6 +2070,143 @@ func (b *Date64Builder) newData() (data *Data) { return } +type DurationBuilder struct { + builder + + dtype *arrow.DurationType + data *memory.Buffer + rawData []arrow.Duration +} + +func NewDurationBuilder(mem memory.Allocator, dtype *arrow.DurationType) *DurationBuilder { + return &DurationBuilder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *DurationBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *DurationBuilder) Append(v arrow.Duration) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *DurationBuilder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *DurationBuilder) UnsafeAppend(v arrow.Duration) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *DurationBuilder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *DurationBuilder) AppendValues(v []arrow.Duration, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.DurationTraits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *DurationBuilder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.DurationTraits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.DurationTraits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *DurationBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *DurationBuilder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.DurationTraits.BytesRequired(n)) + b.rawData = arrow.DurationTraits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Duration array from the memory buffers used by the builder and resets the DurationBuilder +// so it can be used to build a new array. +func (b *DurationBuilder) NewArray() Interface { + return b.NewDurationArray() +} + +// NewDurationArray creates a Duration array from the memory buffers used by the builder and resets the DurationBuilder +// so it can be used to build a new array. +func (b *DurationBuilder) NewDurationArray() (a *Duration) { + data := b.newData() + a = NewDurationData(data) + data.Release() + return +} + +func (b *DurationBuilder) newData() (data *Data) { + bytesRequired := arrow.DurationTraits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + var ( _ Builder = (*Int64Builder)(nil) _ Builder = (*Uint64Builder)(nil) @@ -2086,4 +2223,5 @@ var ( _ Builder = (*Time64Builder)(nil) _ Builder = (*Date32Builder)(nil) _ Builder = (*Date64Builder)(nil) + _ Builder = (*DurationBuilder)(nil) ) diff --git a/go/arrow/array/numericbuilder.gen_test.go b/go/arrow/array/numericbuilder.gen_test.go index 099629e..d5f6aaa 100644 --- a/go/arrow/array/numericbuilder.gen_test.go +++ b/go/arrow/array/numericbuilder.gen_test.go @@ -34,6 +34,9 @@ func TestNewInt64Builder(t *testing.T) { ab := array.NewInt64Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -74,6 +77,46 @@ func TestNewInt64Builder(t *testing.T) { assert.Len(t, a.Int64Values(), 2) a.Release() + + var ( + want = []int64{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewInt64Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Int64); !ok { + t.Fatalf("could not type-assert to array.Int64") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Int64) + if !ok { + t.Fatalf("could not type-assert to array.Int64") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestInt64Builder_AppendValues(t *testing.T) { @@ -157,6 +200,9 @@ func TestNewUint64Builder(t *testing.T) { ab := array.NewUint64Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -197,6 +243,46 @@ func TestNewUint64Builder(t *testing.T) { assert.Len(t, a.Uint64Values(), 2) a.Release() + + var ( + want = []uint64{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewUint64Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Uint64); !ok { + t.Fatalf("could not type-assert to array.Uint64") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Uint64) + if !ok { + t.Fatalf("could not type-assert to array.Uint64") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestUint64Builder_AppendValues(t *testing.T) { @@ -280,6 +366,9 @@ func TestNewFloat64Builder(t *testing.T) { ab := array.NewFloat64Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -320,6 +409,46 @@ func TestNewFloat64Builder(t *testing.T) { assert.Len(t, a.Float64Values(), 2) a.Release() + + var ( + want = []float64{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewFloat64Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Float64); !ok { + t.Fatalf("could not type-assert to array.Float64") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Float64) + if !ok { + t.Fatalf("could not type-assert to array.Float64") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestFloat64Builder_AppendValues(t *testing.T) { @@ -403,6 +532,9 @@ func TestNewInt32Builder(t *testing.T) { ab := array.NewInt32Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -443,6 +575,46 @@ func TestNewInt32Builder(t *testing.T) { assert.Len(t, a.Int32Values(), 2) a.Release() + + var ( + want = []int32{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewInt32Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Int32); !ok { + t.Fatalf("could not type-assert to array.Int32") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Int32) + if !ok { + t.Fatalf("could not type-assert to array.Int32") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestInt32Builder_AppendValues(t *testing.T) { @@ -526,6 +698,9 @@ func TestNewUint32Builder(t *testing.T) { ab := array.NewUint32Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -566,6 +741,46 @@ func TestNewUint32Builder(t *testing.T) { assert.Len(t, a.Uint32Values(), 2) a.Release() + + var ( + want = []uint32{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewUint32Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Uint32); !ok { + t.Fatalf("could not type-assert to array.Uint32") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Uint32) + if !ok { + t.Fatalf("could not type-assert to array.Uint32") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestUint32Builder_AppendValues(t *testing.T) { @@ -649,6 +864,9 @@ func TestNewFloat32Builder(t *testing.T) { ab := array.NewFloat32Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -689,6 +907,46 @@ func TestNewFloat32Builder(t *testing.T) { assert.Len(t, a.Float32Values(), 2) a.Release() + + var ( + want = []float32{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewFloat32Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Float32); !ok { + t.Fatalf("could not type-assert to array.Float32") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Float32) + if !ok { + t.Fatalf("could not type-assert to array.Float32") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestFloat32Builder_AppendValues(t *testing.T) { @@ -772,6 +1030,9 @@ func TestNewInt16Builder(t *testing.T) { ab := array.NewInt16Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -812,6 +1073,46 @@ func TestNewInt16Builder(t *testing.T) { assert.Len(t, a.Int16Values(), 2) a.Release() + + var ( + want = []int16{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewInt16Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Int16); !ok { + t.Fatalf("could not type-assert to array.Int16") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Int16) + if !ok { + t.Fatalf("could not type-assert to array.Int16") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestInt16Builder_AppendValues(t *testing.T) { @@ -895,6 +1196,9 @@ func TestNewUint16Builder(t *testing.T) { ab := array.NewUint16Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -935,6 +1239,46 @@ func TestNewUint16Builder(t *testing.T) { assert.Len(t, a.Uint16Values(), 2) a.Release() + + var ( + want = []uint16{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewUint16Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Uint16); !ok { + t.Fatalf("could not type-assert to array.Uint16") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Uint16) + if !ok { + t.Fatalf("could not type-assert to array.Uint16") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestUint16Builder_AppendValues(t *testing.T) { @@ -1018,6 +1362,9 @@ func TestNewInt8Builder(t *testing.T) { ab := array.NewInt8Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -1058,6 +1405,46 @@ func TestNewInt8Builder(t *testing.T) { assert.Len(t, a.Int8Values(), 2) a.Release() + + var ( + want = []int8{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewInt8Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Int8); !ok { + t.Fatalf("could not type-assert to array.Int8") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Int8) + if !ok { + t.Fatalf("could not type-assert to array.Int8") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestInt8Builder_AppendValues(t *testing.T) { @@ -1141,6 +1528,9 @@ func TestNewUint8Builder(t *testing.T) { ab := array.NewUint8Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -1181,6 +1571,46 @@ func TestNewUint8Builder(t *testing.T) { assert.Len(t, a.Uint8Values(), 2) a.Release() + + var ( + want = []uint8{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewUint8Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Uint8); !ok { + t.Fatalf("could not type-assert to array.Uint8") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Uint8) + if !ok { + t.Fatalf("could not type-assert to array.Uint8") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestUint8Builder_AppendValues(t *testing.T) { @@ -1265,6 +1695,9 @@ func TestNewTimestampBuilder(t *testing.T) { ab := array.NewTimestampBuilder(mem, dtype) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -1305,6 +1738,46 @@ func TestNewTimestampBuilder(t *testing.T) { assert.Len(t, a.TimestampValues(), 2) a.Release() + + var ( + want = []arrow.Timestamp{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewTimestampArray() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Timestamp); !ok { + t.Fatalf("could not type-assert to array.Timestamp") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Timestamp) + if !ok { + t.Fatalf("could not type-assert to array.Timestamp") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestTimestampBuilder_AppendValues(t *testing.T) { @@ -1392,6 +1865,9 @@ func TestNewTime32Builder(t *testing.T) { ab := array.NewTime32Builder(mem, dtype) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -1432,6 +1908,46 @@ func TestNewTime32Builder(t *testing.T) { assert.Len(t, a.Time32Values(), 2) a.Release() + + var ( + want = []arrow.Time32{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewTime32Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Time32); !ok { + t.Fatalf("could not type-assert to array.Time32") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Time32) + if !ok { + t.Fatalf("could not type-assert to array.Time32") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestTime32Builder_AppendValues(t *testing.T) { @@ -1519,6 +2035,9 @@ func TestNewTime64Builder(t *testing.T) { ab := array.NewTime64Builder(mem, dtype) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -1559,6 +2078,46 @@ func TestNewTime64Builder(t *testing.T) { assert.Len(t, a.Time64Values(), 2) a.Release() + + var ( + want = []arrow.Time64{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewTime64Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Time64); !ok { + t.Fatalf("could not type-assert to array.Time64") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Time64) + if !ok { + t.Fatalf("could not type-assert to array.Time64") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestTime64Builder_AppendValues(t *testing.T) { @@ -1645,6 +2204,9 @@ func TestNewDate32Builder(t *testing.T) { ab := array.NewDate32Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -1685,6 +2247,46 @@ func TestNewDate32Builder(t *testing.T) { assert.Len(t, a.Date32Values(), 2) a.Release() + + var ( + want = []arrow.Date32{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewDate32Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Date32); !ok { + t.Fatalf("could not type-assert to array.Date32") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Date32) + if !ok { + t.Fatalf("could not type-assert to array.Date32") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestDate32Builder_AppendValues(t *testing.T) { @@ -1768,6 +2370,9 @@ func TestNewDate64Builder(t *testing.T) { ab := array.NewDate64Builder(mem) defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -1808,6 +2413,46 @@ func TestNewDate64Builder(t *testing.T) { assert.Len(t, a.Date64Values(), 2) a.Release() + + var ( + want = []arrow.Date64{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewDate64Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Date64); !ok { + t.Fatalf("could not type-assert to array.Date64") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Date64) + if !ok { + t.Fatalf("could not type-assert to array.Date64") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func TestDate64Builder_AppendValues(t *testing.T) { @@ -1883,3 +2528,173 @@ func TestDate64Builder_Resize(t *testing.T) { ab.Resize(32) assert.Equal(t, 5, ab.Len()) } + +func TestNewDurationBuilder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.DurationType{Unit: arrow.Second} + ab := array.NewDurationBuilder(mem, dtype) + defer ab.Release() + + ab.Retain() + ab.Release() + + ab.Append(1) + ab.Append(2) + ab.Append(3) + ab.AppendNull() + ab.Append(5) + ab.Append(6) + ab.AppendNull() + ab.Append(8) + ab.Append(9) + ab.Append(10) + + // check state of builder before NewDurationArray + assert.Equal(t, 10, ab.Len(), "unexpected Len()") + assert.Equal(t, 2, ab.NullN(), "unexpected NullN()") + + a := ab.NewDurationArray() + + // check state of builder after NewDurationArray + assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewDurationArray did not reset state") + assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewDurationArray did not reset state") + assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewDurationArray did not reset state") + + // check state of array + assert.Equal(t, 2, a.NullN(), "unexpected null count") + assert.Equal(t, []arrow.Duration{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.DurationValues(), "unexpected DurationValues") + assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity + assert.Len(t, a.DurationValues(), 10, "unexpected length of DurationValues") + + a.Release() + + ab.Append(7) + ab.Append(8) + + a = ab.NewDurationArray() + + assert.Equal(t, 0, a.NullN()) + assert.Equal(t, []arrow.Duration{7, 8}, a.DurationValues()) + assert.Len(t, a.DurationValues(), 2) + + a.Release() + + var ( + want = []arrow.Duration{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.NewDurationArray() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.Duration); !ok { + t.Fatalf("could not type-assert to array.Duration") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Duration) + if !ok { + t.Fatalf("could not type-assert to array.Duration") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() +} + +func TestDurationBuilder_AppendValues(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.DurationType{Unit: arrow.Second} + ab := array.NewDurationBuilder(mem, dtype) + defer ab.Release() + + exp := []arrow.Duration{0, 1, 2, 3} + ab.AppendValues(exp, nil) + a := ab.NewDurationArray() + assert.Equal(t, exp, a.DurationValues()) + + a.Release() +} + +func TestDurationBuilder_Empty(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.DurationType{Unit: arrow.Second} + ab := array.NewDurationBuilder(mem, dtype) + defer ab.Release() + + exp := []arrow.Duration{0, 1, 2, 3} + + ab.AppendValues([]arrow.Duration{}, nil) + a := ab.NewDurationArray() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues(nil, nil) + a = ab.NewDurationArray() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues([]arrow.Duration{}, nil) + ab.AppendValues(exp, nil) + a = ab.NewDurationArray() + assert.Equal(t, exp, a.DurationValues()) + a.Release() + + ab.AppendValues(exp, nil) + ab.AppendValues([]arrow.Duration{}, nil) + a = ab.NewDurationArray() + assert.Equal(t, exp, a.DurationValues()) + a.Release() +} + +func TestDurationBuilder_Resize(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.DurationType{Unit: arrow.Second} + ab := array.NewDurationBuilder(mem, dtype) + defer ab.Release() + + assert.Equal(t, 0, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + ab.Reserve(63) + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + for i := 0; i < 63; i++ { + ab.Append(0) + } + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 63, ab.Len()) + + ab.Resize(5) + assert.Equal(t, 5, ab.Len()) + + ab.Resize(32) + assert.Equal(t, 5, ab.Len()) +} diff --git a/go/arrow/array/numericbuilder.gen_test.go.tmpl b/go/arrow/array/numericbuilder.gen_test.go.tmpl index ea74536..e4f7810 100644 --- a/go/arrow/array/numericbuilder.gen_test.go.tmpl +++ b/go/arrow/array/numericbuilder.gen_test.go.tmpl @@ -38,6 +38,9 @@ func TestNew{{.Name}}Builder(t *testing.T) { {{end -}} defer ab.Release() + ab.Retain() + ab.Release() + ab.Append(1) ab.Append(2) ab.Append(3) @@ -78,6 +81,46 @@ func TestNew{{.Name}}Builder(t *testing.T) { assert.Len(t, a.{{.Name}}Values(), 2) a.Release() + + var ( + want = []{{or .QualifiedType .Type}}{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.New{{.Name}}Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.{{.Name}}); !ok { + t.Fatalf("could not type-assert to array.{{.Name}}") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.{{.Name}}) + if !ok { + t.Fatalf("could not type-assert to array.{{.Name}}") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() } func Test{{.Name}}Builder_AppendValues(t *testing.T) { diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index 14766fa..8dc9c81 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -46,6 +46,7 @@ type ( TimeUnit int Date32 int32 Date64 int64 + Duration int64 ) const ( @@ -92,36 +93,18 @@ func (*Time64Type) Name() string { return "time64" } func (*Time64Type) BitWidth() int { return 64 } func (t *Time64Type) String() string { return "time64[" + t.Unit.String() + "]" } -var ( - FixedWidthTypes = struct { - Boolean FixedWidthDataType - Date32 FixedWidthDataType - Date64 FixedWidthDataType - DayTimeInterval FixedWidthDataType - Float16 FixedWidthDataType - MonthInterval FixedWidthDataType - Time32s FixedWidthDataType - Time32ms FixedWidthDataType - Time64us FixedWidthDataType - Time64ns FixedWidthDataType - Timestamp FixedWidthDataType - }{ - Boolean: &BooleanType{}, - Date32: &Date32Type{}, - Date64: &Date64Type{}, - DayTimeInterval: &DayTimeIntervalType{}, - Float16: &Float16Type{}, - MonthInterval: &MonthIntervalType{}, - Time32s: &Time32Type{Unit: Second}, - Time32ms: &Time32Type{Unit: Millisecond}, - Time64us: &Time64Type{Unit: Microsecond}, - Time64ns: &Time64Type{Unit: Nanosecond}, - Timestamp: &TimestampType{Unit: Nanosecond, TimeZone: "UTC"}, - } +// DurationType is encoded as a 64-bit signed integer, representing an amount +// of elapsed time without any relation to a calendar artifact. +type DurationType struct { + Unit TimeUnit +} - _ FixedWidthDataType = (*FixedSizeBinaryType)(nil) -) +func (*DurationType) ID() Type { return DURATION } +func (*DurationType) Name() string { return "duration" } +func (*DurationType) BitWidth() int { return 64 } +func (t *DurationType) String() string { return "duration[" + t.Unit.String() + "]" } +// Float16Type represents a floating point value encoded with a 16-bit precision. type Float16Type struct{} func (t *Float16Type) ID() Type { return FLOAT16 } @@ -161,3 +144,41 @@ func (*DayTimeIntervalType) String() string { return "day_time_interval" } // BitWidth returns the number of bits required to store a single element of this data type in memory. func (t *DayTimeIntervalType) BitWidth() int { return 64 } + +var ( + FixedWidthTypes = struct { + Boolean FixedWidthDataType + Date32 FixedWidthDataType + Date64 FixedWidthDataType + DayTimeInterval FixedWidthDataType + Duration_s FixedWidthDataType + Duration_ms FixedWidthDataType + Duration_us FixedWidthDataType + Duration_ns FixedWidthDataType + Float16 FixedWidthDataType + MonthInterval FixedWidthDataType + Time32s FixedWidthDataType + Time32ms FixedWidthDataType + Time64us FixedWidthDataType + Time64ns FixedWidthDataType + Timestamp FixedWidthDataType + }{ + Boolean: &BooleanType{}, + Date32: &Date32Type{}, + Date64: &Date64Type{}, + DayTimeInterval: &DayTimeIntervalType{}, + Duration_s: &DurationType{Unit: Second}, + Duration_ms: &DurationType{Unit: Millisecond}, + Duration_us: &DurationType{Unit: Microsecond}, + Duration_ns: &DurationType{Unit: Nanosecond}, + Float16: &Float16Type{}, + MonthInterval: &MonthIntervalType{}, + Time32s: &Time32Type{Unit: Second}, + Time32ms: &Time32Type{Unit: Millisecond}, + Time64us: &Time64Type{Unit: Microsecond}, + Time64ns: &Time64Type{Unit: Nanosecond}, + Timestamp: &TimestampType{Unit: Nanosecond, TimeZone: "UTC"}, + } + + _ FixedWidthDataType = (*FixedSizeBinaryType)(nil) +) diff --git a/go/arrow/numeric.tmpldata b/go/arrow/numeric.tmpldata index 45452ab..127a5a1 100644 --- a/go/arrow/numeric.tmpldata +++ b/go/arrow/numeric.tmpldata @@ -125,5 +125,17 @@ "InternalType": "int64", "Default": "0", "Size": "8" + }, + { + "Name": "Duration", + "name": "duration", + "Type": "Duration", + "QualifiedType": "arrow.Duration", + "InternalType": "int64", + "Default": "0", + "Size": "8", + "Opt": { + "Parametric": true + } } ] diff --git a/go/arrow/type_traits_numeric.gen.go b/go/arrow/type_traits_numeric.gen.go index 14fafbc..c8c063a 100644 --- a/go/arrow/type_traits_numeric.gen.go +++ b/go/arrow/type_traits_numeric.gen.go @@ -40,6 +40,7 @@ var ( Time64Traits time64Traits Date32Traits date32Traits Date64Traits date64Traits + DurationTraits durationTraits ) // Int64 traits @@ -761,3 +762,51 @@ func (date64Traits) CastToBytes(b []Date64) []byte { // Copy copies src to dst. func (date64Traits) Copy(dst, src []Date64) { copy(dst, src) } + +// Duration traits + +const ( + // DurationSizeBytes specifies the number of bytes required to store a single Duration in memory + DurationSizeBytes = int(unsafe.Sizeof(Duration(0))) +) + +type durationTraits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (durationTraits) BytesRequired(n int) int { return DurationSizeBytes * n } + +// PutValue +func (durationTraits) PutValue(b []byte, v Duration) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Duration. +// +// NOTE: len(b) must be a multiple of DurationSizeBytes. +func (durationTraits) CastFromBytes(b []byte) []Duration { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Duration + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / DurationSizeBytes + s.Cap = h.Cap / DurationSizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (durationTraits) CastToBytes(b []Duration) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * DurationSizeBytes + s.Cap = h.Cap * DurationSizeBytes + + return res +} + +// Copy copies src to dst. +func (durationTraits) Copy(dst, src []Duration) { copy(dst, src) }