This is an automated email from the ASF dual-hosted git repository. wesm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new f406721 ARROW-3676: [Go] implement Decimal128 array f406721 is described below commit f40672143cf9d0989083a654a33b7b7c0725ed75 Author: Sebastien Binet <bi...@cern.ch> AuthorDate: Fri Jun 21 16:59:41 2019 -0500 ARROW-3676: [Go] implement Decimal128 array Author: Sebastien Binet <bi...@cern.ch> Closes #4633 from sbinet/issue-3676 and squashes the following commits: eb62bc712 <Sebastien Binet> go/arrow/array: exercize more of Decimal128 API surface 0ea83919b <Sebastien Binet> go/arrow/array: use Decimal128.Value in Stringer 9b57f7425 <Sebastien Binet> go/arrow/decimal128: test hi/lo bits 7778a48a6 <Sebastien Binet> ARROW-3676: implement Decimal128 array --- go/arrow/array/array.go | 2 +- go/arrow/array/array_test.go | 11 +- go/arrow/array/compare.go | 6 + go/arrow/array/decimal128.go | 235 +++++++++++++++++++++++++++++++++ go/arrow/array/decimal128_test.go | 179 +++++++++++++++++++++++++ go/arrow/datatype_fixedwidth.go | 18 ++- go/arrow/datatype_fixedwidth_test.go | 27 ++++ go/arrow/decimal128/decimal128.go | 73 ++++++++++ go/arrow/decimal128/decimal128_test.go | 94 +++++++++++++ go/arrow/type_traits_decimal128.go | 75 +++++++++++ go/arrow/type_traits_test.go | 45 +++++++ 11 files changed, 759 insertions(+), 6 deletions(-) diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index 1912f3e..d8418f8 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -186,7 +186,7 @@ func init() { arrow.TIME32: func(data *Data) Interface { return NewTime32Data(data) }, arrow.TIME64: func(data *Data) Interface { return NewTime64Data(data) }, arrow.INTERVAL: func(data *Data) Interface { return NewIntervalData(data) }, - arrow.DECIMAL: unsupportedArrayType, + arrow.DECIMAL: func(data *Data) Interface { return NewDecimal128Data(data) }, arrow.LIST: func(data *Data) Interface { return NewListData(data) }, arrow.STRUCT: func(data *Data) Interface { return NewStructData(data) }, arrow.UNION: unsupportedArrayType, diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go index 724f3b4..ba3a961 100644 --- a/go/arrow/array/array_test.go +++ b/go/arrow/array/array_test.go @@ -43,9 +43,6 @@ func TestMakeFromData(t *testing.T) { expPanic bool expError string }{ - // unsupported types - {name: "map", d: &testDataType{arrow.MAP}, expPanic: true, expError: "unsupported data type: MAP"}, - // supported types {name: "null", d: &testDataType{arrow.NULL}}, {name: "bool", d: &testDataType{arrow.BOOL}}, @@ -59,11 +56,17 @@ func TestMakeFromData(t *testing.T) { {name: "int64", d: &testDataType{arrow.INT64}}, {name: "float32", d: &testDataType{arrow.FLOAT32}}, {name: "float64", d: &testDataType{arrow.FLOAT64}}, + {name: "string", d: &testDataType{arrow.STRING}, size: 3}, {name: "binary", d: &testDataType{arrow.BINARY}, size: 3}, + {name: "fixed_size_binary", d: &testDataType{arrow.FIXED_SIZE_BINARY}}, + {name: "date32", d: &testDataType{arrow.DATE32}}, + {name: "date64", d: &testDataType{arrow.DATE64}}, {name: "timestamp", d: &testDataType{arrow.TIMESTAMP}}, {name: "time32", d: &testDataType{arrow.TIME32}}, {name: "time64", d: &testDataType{arrow.TIME64}}, - {name: "fixed_size_binary", d: &testDataType{arrow.FIXED_SIZE_BINARY}}, + {name: "month_interval", d: arrow.FixedWidthTypes.MonthInterval}, + {name: "day_time_interval", d: arrow.FixedWidthTypes.DayTimeInterval}, + {name: "decimal", d: &testDataType{arrow.DECIMAL}}, {name: "list", d: &testDataType{arrow.LIST}, child: []*array.Data{ array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0), diff --git a/go/arrow/array/compare.go b/go/arrow/array/compare.go index c6665c9..17839c6 100644 --- a/go/arrow/array/compare.go +++ b/go/arrow/array/compare.go @@ -128,6 +128,9 @@ func ArrayEqual(left, right Interface) bool { case *Float64: r := right.(*Float64) return arrayEqualFloat64(l, r) + case *Decimal128: + r := right.(*Decimal128) + return arrayEqualDecimal128(l, r) case *Date32: r := right.(*Date32) return arrayEqualDate32(l, r) @@ -314,6 +317,9 @@ func arrayApproxEqual(left, right Interface, opt equalOption) bool { case *Float64: r := right.(*Float64) return arrayApproxEqualFloat64(l, r, opt) + case *Decimal128: + r := right.(*Decimal128) + return arrayEqualDecimal128(l, r) case *Date32: r := right.(*Date32) return arrayEqualDate32(l, r) diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go new file mode 100644 index 0000000..390d87e --- /dev/null +++ b/go/arrow/array/decimal128.go @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array // import "github.com/apache/arrow/go/arrow/array" + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/decimal128" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// A type which represents an immutable sequence of 128-bit decimal values. +type Decimal128 struct { + array + + values []decimal128.Num +} + +func NewDecimal128Data(data *Data) *Decimal128 { + a := &Decimal128{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Decimal128) Value(i int) decimal128.Num { return a.values[i] } + +func (a *Decimal128) Values() []decimal128.Num { return a.values } + +func (a *Decimal128) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Decimal128) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Decimal128Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualDecimal128(left, right *Decimal128) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +type Decimal128Builder struct { + builder + + dtype *arrow.Decimal128Type + data *memory.Buffer + rawData []decimal128.Num +} + +func NewDecimal128Builder(mem memory.Allocator, dtype *arrow.Decimal128Type) *Decimal128Builder { + return &Decimal128Builder{ + builder: builder{refCount: 1, mem: mem}, + dtype: dtype, + } +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Decimal128Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Decimal128Builder) Append(v decimal128.Num) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Decimal128Builder) UnsafeAppend(v decimal128.Num) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Decimal128Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Decimal128Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Decimal128Builder) AppendValues(v []decimal128.Num, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + if len(v) > 0 { + arrow.Decimal128Traits.Copy(b.rawData[b.length:], v) + } + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Decimal128Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Decimal128Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Decimal128Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Decimal128Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Decimal128Traits.BytesRequired(n)) + b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder +// so it can be used to build a new array. +func (b *Decimal128Builder) NewArray() Interface { + return b.NewDecimal128Array() +} + +// NewDecimal128Array creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder +// so it can be used to build a new array. +func (b *Decimal128Builder) NewDecimal128Array() (a *Decimal128) { + data := b.newData() + a = NewDecimal128Data(data) + data.Release() + return +} + +func (b *Decimal128Builder) newData() (data *Data) { + bytesRequired := arrow.Decimal128Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +var ( + _ Interface = (*Decimal128)(nil) + _ Builder = (*Decimal128Builder)(nil) +) diff --git a/go/arrow/array/decimal128_test.go b/go/arrow/array/decimal128_test.go new file mode 100644 index 0000000..5a39d92 --- /dev/null +++ b/go/arrow/array/decimal128_test.go @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array_test + +import ( + "testing" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/decimal128" + "github.com/apache/arrow/go/arrow/memory" + "github.com/stretchr/testify/assert" +) + +func TestNewDecimal128Builder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + ab := array.NewDecimal128Builder(mem, &arrow.Decimal128Type{Precision: 10, Scale: 1}) + defer ab.Release() + + ab.Retain() + ab.Release() + + want := []decimal128.Num{ + decimal128.New(1, 1), + decimal128.New(2, 2), + decimal128.New(3, 3), + {}, + decimal128.FromI64(-5), + decimal128.FromI64(-6), + {}, + decimal128.FromI64(8), + decimal128.FromI64(9), + decimal128.FromI64(10), + } + valids := []bool{true, true, true, false, true, true, false, true, true, true} + + for i, valid := range valids { + switch { + case valid: + ab.Append(want[i]) + default: + ab.AppendNull() + } + } + + // check state of builder before NewDecimal128Array + assert.Equal(t, 10, ab.Len(), "unexpected Len()") + assert.Equal(t, 2, ab.NullN(), "unexpected NullN()") + + a := ab.NewArray().(*array.Decimal128) + a.Retain() + a.Release() + + // check state of builder after NewDecimal128Array + assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewDecimal128Array did not reset state") + assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewDecimal128Array did not reset state") + assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewDecimal128Array did not reset state") + + // check state of array + assert.Equal(t, 2, a.NullN(), "unexpected null count") + + assert.Equal(t, want, a.Values(), "unexpected Decimal128Values") + assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity + assert.Len(t, a.Values(), 10, "unexpected length of Decimal128Values") + + a.Release() + ab.Append(decimal128.FromI64(7)) + ab.Append(decimal128.FromI64(8)) + + a = ab.NewDecimal128Array() + + assert.Equal(t, 0, a.NullN()) + assert.Equal(t, []decimal128.Num{decimal128.FromI64(7), decimal128.FromI64(8)}, a.Values()) + assert.Len(t, a.Values(), 2) + + a.Release() +} + +func TestDecimal128Builder_Empty(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + ab := array.NewDecimal128Builder(mem, &arrow.Decimal128Type{Precision: 10, Scale: 1}) + defer ab.Release() + + want := []decimal128.Num{decimal128.FromI64(3), decimal128.FromI64(4)} + + ab.AppendValues([]decimal128.Num{}, nil) + a := ab.NewDecimal128Array() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues(nil, nil) + a = ab.NewDecimal128Array() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues(want, nil) + a = ab.NewDecimal128Array() + assert.Equal(t, want, a.Values()) + a.Release() + + ab.AppendValues([]decimal128.Num{}, nil) + ab.AppendValues(want, nil) + a = ab.NewDecimal128Array() + assert.Equal(t, want, a.Values()) + a.Release() + + ab.AppendValues(want, nil) + ab.AppendValues([]decimal128.Num{}, nil) + a = ab.NewDecimal128Array() + assert.Equal(t, want, a.Values()) + a.Release() +} + +func TestDecimal128Slice(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Decimal128Type{Precision: 10, Scale: 1} + b := array.NewDecimal128Builder(mem, dtype) + defer b.Release() + + var data = []decimal128.Num{ + decimal128.FromI64(-1), + decimal128.FromI64(+0), + decimal128.FromI64(+1), + decimal128.New(-4, 4), + } + b.AppendValues(data[:2], nil) + b.AppendNull() + b.Append(data[3]) + + arr := b.NewDecimal128Array() + defer arr.Release() + + if got, want := arr.Len(), len(data); got != want { + t.Fatalf("invalid array length: got=%d, want=%d", got, want) + } + + slice := array.NewSliceData(arr.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.Decimal128) + if !ok { + t.Fatalf("could not type-assert to array.String") + } + + if got, want := v.String(), `[(null) {4 -4}]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + if got, want := v.NullN(), 1; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + if got, want := v.Data().Offset(), 2; got != want { + t.Fatalf("invalid offset: got=%d, want=%d", got, want) + } +} diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index 8dc9c81..5f63ea1 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -16,7 +16,10 @@ package arrow -import "strconv" +import ( + "fmt" + "strconv" +) type BooleanType struct{} @@ -114,6 +117,19 @@ func (t *Float16Type) String() string { return "float16" } // BitWidth returns the number of bits required to store a single element of this data type in memory. func (t *Float16Type) BitWidth() int { return 16 } +// Decimal128Type represents a fixed-size 128-bit decimal type. +type Decimal128Type struct { + Precision int32 + Scale int32 +} + +func (*Decimal128Type) ID() Type { return DECIMAL } +func (*Decimal128Type) Name() string { return "decimal" } +func (*Decimal128Type) BitWidth() int { return 16 } +func (t *Decimal128Type) String() string { + return fmt.Sprintf("%s(%d, %d)", t.Name(), t.Precision, t.Scale) +} + // MonthInterval represents a number of months. type MonthInterval int32 diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go index 865f0ae..5fb02e0 100644 --- a/go/arrow/datatype_fixedwidth_test.go +++ b/go/arrow/datatype_fixedwidth_test.go @@ -40,3 +40,30 @@ func TestTimeUnit_String(t *testing.T) { }) } } + +func TestDecimal128Type(t *testing.T) { + for _, tc := range []struct { + precision int32 + scale int32 + want string + }{ + {1, 10, "decimal(1, 10)"}, + {10, 10, "decimal(10, 10)"}, + {10, 1, "decimal(10, 1)"}, + } { + t.Run(tc.want, func(t *testing.T) { + dt := arrow.Decimal128Type{Precision: tc.precision, Scale: tc.scale} + if got, want := dt.BitWidth(), 16; got != want { + t.Fatalf("invalid bitwidth: got=%d, want=%d", got, want) + } + + if got, want := dt.ID(), arrow.DECIMAL; got != want { + t.Fatalf("invalid type ID: got=%v, want=%v", got, want) + } + + if got, want := dt.String(), tc.want; got != want { + t.Fatalf("invalid stringer: got=%q, want=%q", got, want) + } + }) + } +} diff --git a/go/arrow/decimal128/decimal128.go b/go/arrow/decimal128/decimal128.go new file mode 100644 index 0000000..2f1b181 --- /dev/null +++ b/go/arrow/decimal128/decimal128.go @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal128 // import "github.com/apache/arrow/go/arrow/decimal128" + +var ( + MaxDecimal128 = New(542101086242752217, 687399551400673280-1) +) + +// Num represents a signed 128-bit integer in two's complement. +// Calculations wrap around and overflow is ignored. +// +// For a discussion of the algorithms, look at Knuth's volume 2, +// Semi-numerical Algorithms section 4.3.1. +// +// Adapted from the Apache ORC C++ implementation +type Num struct { + lo uint64 // low bits + hi int64 // high bits +} + +// New returns a new signed 128-bit integer value. +func New(hi int64, lo uint64) Num { + return Num{lo: lo, hi: hi} +} + +// FromU64 returns a new signed 128-bit integer value from the provided uint64 one. +func FromU64(v uint64) Num { + return New(0, v) +} + +// FromI64 returns a new signed 128-bit integer value from the provided int64 one. +func FromI64(v int64) Num { + switch { + case v > 0: + return New(0, uint64(v)) + case v < 0: + return New(-1, uint64(v)) + default: + return Num{} + } +} + +// LowBits returns the low bits of the two's complement representation of the number. +func (n Num) LowBits() uint64 { return n.lo } + +// HighBits returns the high bits of the two's complement representation of the number. +func (n Num) HighBits() int64 { return n.hi } + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +func (n Num) Sign() int { + if n == (Num{}) { + return 0 + } + return int(1 | (n.hi >> 63)) +} diff --git a/go/arrow/decimal128/decimal128_test.go b/go/arrow/decimal128/decimal128_test.go new file mode 100644 index 0000000..cf4ebd4 --- /dev/null +++ b/go/arrow/decimal128/decimal128_test.go @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal128 // import "github.com/apache/arrow/go/arrow/decimal128" + +import ( + "fmt" + "math" + "math/big" + "testing" +) + +func TestFromU64(t *testing.T) { + for _, tc := range []struct { + v uint64 + want Num + sign int + }{ + {0, Num{0, 0}, 0}, + {1, Num{1, 0}, +1}, + {2, Num{2, 0}, +1}, + {math.MaxInt64, Num{math.MaxInt64, 0}, +1}, + {math.MaxUint64, Num{math.MaxUint64, 0}, +1}, + } { + t.Run(fmt.Sprintf("%+0#x", tc.v), func(t *testing.T) { + v := FromU64(tc.v) + ref := new(big.Int).SetUint64(tc.v) + if got, want := v, tc.want; got != want { + t.Fatalf("invalid value. got=%+0#x, want=%+0#x (big-int=%+0#x)", got, want, ref) + } + if got, want := v.Sign(), tc.sign; got != want { + t.Fatalf("invalid sign for %+0#x: got=%v, want=%v", v, got, want) + } + if got, want := v.Sign(), ref.Sign(); got != want { + t.Fatalf("invalid sign for %+0#x: got=%v, want=%v", v, got, want) + } + if got, want := v.LowBits(), tc.want.lo; got != want { + t.Fatalf("invalid low-bits: got=%+0#x, want=%+0#x", got, want) + } + if got, want := v.HighBits(), tc.want.hi; got != want { + t.Fatalf("invalid high-bits: got=%+0#x, want=%+0#x", got, want) + } + }) + } +} + +func TestFromI64(t *testing.T) { + for _, tc := range []struct { + v int64 + want Num + sign int + }{ + {0, Num{0, 0}, 0}, + {1, Num{1, 0}, 1}, + {2, Num{2, 0}, 1}, + {math.MaxInt64, Num{math.MaxInt64, 0}, 1}, + {math.MinInt64, Num{u64Cnv(math.MinInt64), -1}, -1}, + } { + t.Run(fmt.Sprintf("%+0#x", tc.v), func(t *testing.T) { + v := FromI64(tc.v) + ref := big.NewInt(tc.v) + if got, want := v, tc.want; got != want { + t.Fatalf("invalid value. got=%+0#x, want=%+0#x (big-int=%+0#x)", got, want, ref) + } + if got, want := v.Sign(), tc.sign; got != want { + t.Fatalf("invalid sign for %+0#x: got=%v, want=%v", v, got, want) + } + if got, want := v.Sign(), ref.Sign(); got != want { + t.Fatalf("invalid sign for %+0#x: got=%v, want=%v", v, got, want) + } + if got, want := v.LowBits(), tc.want.lo; got != want { + t.Fatalf("invalid low-bits: got=%+0#x, want=%+0#x", got, want) + } + if got, want := v.HighBits(), tc.want.hi; got != want { + t.Fatalf("invalid high-bits: got=%+0#x, want=%+0#x", got, want) + } + }) + } +} + +func u64Cnv(i int64) uint64 { return uint64(i) } diff --git a/go/arrow/type_traits_decimal128.go b/go/arrow/type_traits_decimal128.go new file mode 100644 index 0000000..debe6c7 --- /dev/null +++ b/go/arrow/type_traits_decimal128.go @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "encoding/binary" + "reflect" + "unsafe" + + "github.com/apache/arrow/go/arrow/decimal128" +) + +// Decimal128 traits +var Decimal128Traits decimal128Traits + +const ( + // Decimal128SizeBytes specifies the number of bytes required to store a single decimal128 in memory + Decimal128SizeBytes = int(unsafe.Sizeof(decimal128.Num{})) +) + +type decimal128Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (decimal128Traits) BytesRequired(n int) int { return Decimal128SizeBytes * n } + +// PutValue +func (decimal128Traits) PutValue(b []byte, v decimal128.Num) { + binary.LittleEndian.PutUint64(b[:8], uint64(v.LowBits())) + binary.LittleEndian.PutUint64(b[8:], uint64(v.HighBits())) +} + +// CastFromBytes reinterprets the slice b to a slice of type uint16. +// +// NOTE: len(b) must be a multiple of Uint16SizeBytes. +func (decimal128Traits) CastFromBytes(b []byte) []decimal128.Num { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []decimal128.Num + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Decimal128SizeBytes + s.Cap = h.Cap / Decimal128SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (decimal128Traits) CastToBytes(b []decimal128.Num) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Decimal128SizeBytes + s.Cap = h.Cap * Decimal128SizeBytes + + return res +} + +// Copy copies src to dst. +func (decimal128Traits) Copy(dst, src []decimal128.Num) { copy(dst, src) } diff --git a/go/arrow/type_traits_test.go b/go/arrow/type_traits_test.go index f2f1d9a..59ad06e 100644 --- a/go/arrow/type_traits_test.go +++ b/go/arrow/type_traits_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/decimal128" "github.com/apache/arrow/go/arrow/float16" ) @@ -87,6 +88,50 @@ func TestFloat16Traits(t *testing.T) { } } +func TestDecimal128Traits(t *testing.T) { + const N = 10 + nbytes := arrow.Decimal128Traits.BytesRequired(N) + b1 := arrow.Decimal128Traits.CastToBytes([]decimal128.Num{ + decimal128.New(0, 10), + decimal128.New(1, 10), + decimal128.New(2, 10), + decimal128.New(3, 10), + decimal128.New(4, 10), + decimal128.New(5, 10), + decimal128.New(6, 10), + decimal128.New(7, 10), + decimal128.New(8, 10), + decimal128.New(9, 10), + }) + + b2 := make([]byte, nbytes) + for i := 0; i < N; i++ { + beg := i * arrow.Decimal128SizeBytes + end := (i + 1) * arrow.Decimal128SizeBytes + arrow.Decimal128Traits.PutValue(b2[beg:end], decimal128.New(int64(i), 10)) + } + + if !reflect.DeepEqual(b1, b2) { + v1 := arrow.Decimal128Traits.CastFromBytes(b1) + v2 := arrow.Decimal128Traits.CastFromBytes(b2) + t.Fatalf("invalid values:\nb1=%v\nb2=%v\nv1=%v\nv2=%v\n", b1, b2, v1, v2) + } + + v1 := arrow.Decimal128Traits.CastFromBytes(b1) + for i, v := range v1 { + if got, want := v, decimal128.New(int64(i), 10); got != want { + t.Fatalf("invalid value[%d]. got=%v, want=%v", i, got, want) + } + } + + v2 := make([]decimal128.Num, N) + arrow.Decimal128Traits.Copy(v2, v1) + + if !reflect.DeepEqual(v1, v2) { + t.Fatalf("invalid values:\nv1=%v\nv2=%v\n", v1, v2) + } +} + func TestMonthIntervalTraits(t *testing.T) { const N = 10 b1 := arrow.MonthIntervalTraits.CastToBytes([]arrow.MonthInterval{