This is an automated email from the ASF dual-hosted git repository. raulcd pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/arrow-swift.git
commit 6ec33ca7dd5aa91ef3fc4f2cbf6b068f3dd7e37e Author: abandy <[email protected]> AuthorDate: Fri Feb 3 02:10:44 2023 -0500 GH-20484: [Swift] Initial Arrow implementation (#14561) Initial check-in for a swift implementation of Arrow. Based on my understanding of the spec and looking through the C++ and C# current implementations. * Closes: #20484 Lead-authored-by: Alva Bandy <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]> --- Arrow/Package.swift | 40 ++++ Arrow/README.md | 49 ++++ Arrow/Sources/Arrow/ArrowArray.swift | 123 ++++++++++ Arrow/Sources/Arrow/ArrowArrayBuilder.swift | 129 ++++++++++ Arrow/Sources/Arrow/ArrowBuffer.swift | 63 +++++ Arrow/Sources/Arrow/ArrowBufferBuilder.swift | 336 +++++++++++++++++++++++++++ Arrow/Sources/Arrow/ArrowData.swift | 50 ++++ Arrow/Sources/Arrow/ArrowSchema.swift | 69 ++++++ Arrow/Sources/Arrow/ArrowTable.swift | 125 ++++++++++ Arrow/Sources/Arrow/ArrowType.swift | 133 +++++++++++ Arrow/Sources/Arrow/BitUtility.swift | 43 ++++ Arrow/Sources/Arrow/ChunkedArray.swift | 49 ++++ Arrow/Sources/Arrow/MemoryAllocator.swift | 31 +++ Arrow/Tests/ArrowTests/ArrowTests.swift | 166 +++++++++++++ 14 files changed, 1406 insertions(+) diff --git a/Arrow/Package.swift b/Arrow/Package.swift new file mode 100644 index 0000000..d89ef51 --- /dev/null +++ b/Arrow/Package.swift @@ -0,0 +1,40 @@ +// swift-tools-version:5.7 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import PackageDescription + +let package = Package( + name: "Arrow", + products: [ + // Products define the executables and libraries a package produces, and make them visible to other packages. + .library( + name: "Arrow", + targets: ["Arrow"]), + ], + targets: [ + // Targets are the basic building blocks of a package. A target can define a module or a test suite. + // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "Arrow"), + .testTarget( + name: "ArrowTests", + dependencies: ["Arrow"]), + ] +) diff --git a/Arrow/README.md b/Arrow/README.md new file mode 100644 index 0000000..4ddfba1 --- /dev/null +++ b/Arrow/README.md @@ -0,0 +1,49 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# Apache Arrow Swift + +An implementation of Arrow targeting Swift. + +## Status + +## Memory Management + +- Allocations are 64-byte aligned and padded to 8-bytes. +- Allocations are automatically garbage collected + +## Arrays + +### Primitive Types + +- Int8, Int16, Int32, Int64 +- UInt8, UInt16, UInt32, UInt64 +- Float, Double +- String (utf-8) + +### Parametric Types + +- Date32 +- Date64 + +### Type Metadata + +- Data Types +- Fields +- Schema diff --git a/Arrow/Sources/Arrow/ArrowArray.swift b/Arrow/Sources/Arrow/ArrowArray.swift new file mode 100644 index 0000000..47c7335 --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowArray.swift @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class ArrowArray<T> { + public typealias ItemType = T + public let arrowData: ArrowData + public var nullCount : UInt {get{return self.arrowData.nullCount}} + public var length: UInt {get{return self.arrowData.length}} + + public required init(_ arrowData: ArrowData) { + self.arrowData = arrowData + } + + public func isNull(_ at: UInt) throws -> Bool { + if at >= self.length { + throw ValidationError.outOfBounds(index: at) + } + + return self.arrowData.isNull(at) + } + + public subscript(_ index: UInt) -> T? { + get{ + fatalError("subscript() has not been implemented") + } + } +} + +public class FixedArray<T>: ArrowArray<T> { + public override subscript(_ index: UInt) -> T? { + get{ + if self.arrowData.isNull(index) { + return nil + } + + let byteOffset = self.arrowData.stride * Int(index); + return self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(as: T.self) + } + } +} + +public class StringArray: ArrowArray<String> { + public override subscript(_ index: UInt) -> String? { + get{ + let offsetIndex = MemoryLayout<Int32>.stride * Int(index) + if self.arrowData.isNull(index) { + return nil + } + + let offsets = self.arrowData.buffers[1] + let values = self.arrowData.buffers[2] + + var startIndex: Int32 = 0 + if index > 0 { + startIndex = offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self) + } + + let endIndex = offsets.rawPointer.advanced(by: offsetIndex + MemoryLayout<Int32>.stride ).load(as: Int32.self) + let arrayLength = Int(endIndex - startIndex); + let rawPointer = values.rawPointer.advanced(by: Int(startIndex)).bindMemory(to: UInt8.self, capacity: arrayLength) + let buffer = UnsafeBufferPointer<UInt8>(start: rawPointer, count: arrayLength); + let byteArray = Array(buffer) + return String(data: Data(byteArray), encoding: .utf8) + } + } +} + +public class BoolArray: ArrowArray<Bool> { + public override subscript(_ index: UInt) -> Bool? { + get{ + if self.arrowData.isNull(index) { + return nil + } + + let valueBuffer = self.arrowData.buffers[1]; + return BitUtility.isSet(index, buffer: valueBuffer) + } + } +} + +public class Date32Array: ArrowArray<Date> { + public override subscript(_ index: UInt) -> Date? { + get{ + if self.arrowData.isNull(index) { + return nil + } + + let byteOffset = self.arrowData.stride * Int(index); + let milliseconds = self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(as: UInt32.self) + return Date(timeIntervalSince1970: TimeInterval(milliseconds * 86400)) + } + } +} + +public class Date64Array: ArrowArray<Date> { + public override subscript(_ index: UInt) -> Date? { + get{ + if self.arrowData.isNull(index) { + return nil + } + + let byteOffset = self.arrowData.stride * Int(index); + let milliseconds = self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(as: UInt64.self) + return Date(timeIntervalSince1970: TimeInterval(milliseconds / 1000)) + } + } +} diff --git a/Arrow/Sources/Arrow/ArrowArrayBuilder.swift b/Arrow/Sources/Arrow/ArrowArrayBuilder.swift new file mode 100644 index 0000000..8672186 --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowArrayBuilder.swift @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class ArrowArrayBuilder<T: ArrowBufferBuilder, U: ArrowArray<T.ItemType>> { + let type: ArrowType.Info + let bufferBuilder: T + var length: UInt {get{return self.bufferBuilder.length}} + var capacity: UInt {get{return self.bufferBuilder.capacity}} + var nullCount : UInt {get{return self.bufferBuilder.nullCount}} + var offset: UInt {get{return self.bufferBuilder.offset}} + + fileprivate init(_ type: ArrowType.Info) throws { + self.type = type; + self.bufferBuilder = try T() + } + + func append(_ val: T.ItemType?) { + self.bufferBuilder.append(val) + } + + func finish() throws -> ArrowArray<T.ItemType> { + let buffers = self.bufferBuilder.finish(); + let arrowData = try ArrowData(self.type, buffers: buffers, nullCount: self.nullCount, stride: self.getStride()); + return U(arrowData); + } + + func getStride() -> Int { + MemoryLayout<T.ItemType>.stride + } +} + +public class NumberArrayBuilder<T> : ArrowArrayBuilder<FixedBufferBuilder<T>, FixedArray<T>> { + fileprivate convenience init() throws { + try self.init(ArrowType.infoForNumericType(T.self)); + } +} + +public class StringArrayBuilder : ArrowArrayBuilder<VariableBufferBuilder<String>, StringArray> { + fileprivate convenience init() throws { + try self.init(ArrowType.ArrowString); + } +} + +public class BoolArrayBuilder : ArrowArrayBuilder<BoolBufferBuilder, BoolArray> { + fileprivate convenience init() throws { + try self.init(ArrowType.ArrowBool); + } +} + +public class Date32ArrayBuilder : ArrowArrayBuilder<Date32BufferBuilder, Date32Array> { + fileprivate convenience init() throws { + try self.init(ArrowType.ArrowDate32) + } + + override func getStride() -> Int { + MemoryLayout<Int32>.stride + } +} + +public class Date64ArrayBuilder : ArrowArrayBuilder<Date64BufferBuilder, Date64Array> { + fileprivate convenience init() throws { + try self.init(ArrowType.ArrowDate64) + } + + override func getStride() -> Int { + MemoryLayout<Int64>.stride + } +} + +public class ArrowArrayBuilders { + public static func loadNumberArrayBuilder<T>() throws -> NumberArrayBuilder<T> { + let t = T.self + if t == Int8.self { + return try NumberArrayBuilder<T>() + } else if t == Int16.self { + return try NumberArrayBuilder<T>() + } else if t == Int32.self { + return try NumberArrayBuilder<T>() + } else if t == Int64.self { + return try NumberArrayBuilder<T>() + } else if t == UInt8.self { + return try NumberArrayBuilder<T>() + } else if t == UInt16.self { + return try NumberArrayBuilder<T>() + } else if t == UInt32.self { + return try NumberArrayBuilder<T>() + } else if t == UInt64.self { + return try NumberArrayBuilder<T>() + } else if t == Float.self { + return try NumberArrayBuilder<T>() + } else if t == Double.self { + return try NumberArrayBuilder<T>() + } else { + throw ValidationError.unknownType + } + } + + public static func loadStringArrayBuilder() throws -> StringArrayBuilder { + return try StringArrayBuilder() + } + + public static func loadBoolArrayBuilder() throws -> BoolArrayBuilder { + return try BoolArrayBuilder() + } + + public static func loadDate32ArrayBuilder() throws -> Date32ArrayBuilder { + return try Date32ArrayBuilder() + } + + public static func loadDate64ArrayBuilder() throws -> Date64ArrayBuilder { + return try Date64ArrayBuilder() + } +} diff --git a/Arrow/Sources/Arrow/ArrowBuffer.swift b/Arrow/Sources/Arrow/ArrowBuffer.swift new file mode 100644 index 0000000..f2c984e --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowBuffer.swift @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class ArrowBuffer { + static let min_length: UInt = 1 << 5 + static let max_length = UInt.max + fileprivate(set) var length: UInt + let capacity: UInt + let rawPointer: UnsafeMutableRawPointer + + init(length: UInt, capacity: UInt, rawPointer: UnsafeMutableRawPointer) { + self.length = length + self.capacity = capacity + self.rawPointer = rawPointer + } + + deinit { + self.rawPointer.deallocate() + } + + static func createBuffer(_ length: UInt, size: UInt, doAlign: Bool = true) -> ArrowBuffer { + let actualLen = max(length, ArrowBuffer.min_length) + let byteCount = size * actualLen + var capacity = byteCount + if doAlign { + capacity = alignTo64(byteCount) + } + + let memory = MemoryAllocator(64) + let rawPointer = memory.allocateArray(Int(capacity)) + rawPointer.initializeMemory(as: UInt8.self, repeating: 0, count: Int(capacity)) + return ArrowBuffer(length: actualLen, capacity: capacity, rawPointer: rawPointer) + } + + static func copyCurrent(_ from: ArrowBuffer, to: inout ArrowBuffer, len: UInt) { + to.rawPointer.copyMemory(from: from.rawPointer, byteCount: Int(len)) + } + + private static func alignTo64(_ length: UInt) -> UInt { + let buf_alignment = length % 64; + if buf_alignment != 0 { + return length + (64 - buf_alignment) + 8; + } + + return length + 8; + } +} diff --git a/Arrow/Sources/Arrow/ArrowBufferBuilder.swift b/Arrow/Sources/Arrow/ArrowBufferBuilder.swift new file mode 100644 index 0000000..14ca875 --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowBufferBuilder.swift @@ -0,0 +1,336 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public protocol ArrowBufferBuilder { + associatedtype ItemType + var capacity: UInt {get} + var length: UInt {get} + var nullCount : UInt {get} + var offset: UInt {get} + init() throws + func append(_ newValue: ItemType?) + func isNull(_ index: UInt) -> Bool + func resize(_ length: UInt) + func finish() -> [ArrowBuffer] +} + +public class BaseBufferBuilder<T> { + var values: ArrowBuffer + var nulls: ArrowBuffer + var stride: Int; + public var offset: UInt = 0 + public var capacity: UInt {get{return self.values.capacity}} + public var length: UInt = 0 + public var nullCount : UInt = 0 + + init(values: ArrowBuffer, nulls: ArrowBuffer, stride: Int = MemoryLayout<T>.stride) throws { + self.stride = stride + self.values = values + self.nulls = nulls + } + + public func isNull(_ index: UInt) -> Bool { + return self.nulls.length == 0 || BitUtility.isSet(index + self.offset, buffer: self.nulls) + } + + func resizeLength(_ data: ArrowBuffer, len: UInt = 0) -> UInt { + if len == 0 || len < data.length * 2 { + return UInt(data.length * 2); + } + + return UInt(len * 2); + } +} + +public class FixedBufferBuilder<T>: BaseBufferBuilder<T>, ArrowBufferBuilder { + public typealias ItemType = T + private let defaultVal: ItemType + public required init() throws { + self.defaultVal = try FixedBufferBuilder<T>.defaultValueForType() + let values = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<T>.stride)) + let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride)) + try super.init(values: values, nulls: nulls) + } + + public func append(_ newValue: ItemType?) { + let index = UInt(self.length) + let byteIndex = self.stride * Int(index) + self.length += 1 + if length > self.values.length { + self.resize(length) + } + + if let val = newValue { + self.values.rawPointer.advanced(by: byteIndex).storeBytes(of: val, as: T.self) + } else { + self.nullCount += 1 + BitUtility.setBit(index + self.offset, buffer: self.nulls) + self.values.rawPointer.advanced(by: byteIndex).storeBytes(of: defaultVal, as: T.self) + } + } + + public func resize(_ length: UInt) { + if length > self.values.length { + let resizeLength = resizeLength(self.values); + var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<T>.size)) + var nulls = ArrowBuffer.createBuffer(resizeLength/8 + 1, size: UInt(MemoryLayout<UInt8>.size)) + ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity) + ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity) + self.values = values; + self.nulls = nulls; + } + } + + public func finish() -> [ArrowBuffer] { + let length = self.length + var values = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<T>.size)) + var nulls = ArrowBuffer.createBuffer(length/8 + 1, size: UInt(MemoryLayout<UInt8>.size)) + ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity) + ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity) + return [nulls, values] + } + + fileprivate static func defaultValueForType() throws -> T { + let t = T.self + if t == Int8.self { + return Int8(0) as! T + }else if t == Int16.self { + return Int16(0) as! T + }else if t == Int32.self { + return Int32(0) as! T + }else if t == Int64.self { + return Int64(0) as! T + }else if t == UInt8.self { + return UInt8(0) as! T + }else if t == UInt16.self { + return UInt16(0) as! T + }else if t == UInt32.self { + return UInt32(0) as! T + }else if t == UInt64.self { + return UInt64(0) as! T + }else if t == Float.self { + return Float(0) as! T + }else if t == Double.self { + return Double(0) as! T + } + + throw ValidationError.unknownType + } +} + +public class BoolBufferBuilder: BaseBufferBuilder<Bool>, ArrowBufferBuilder { + public typealias ItemType = Bool + public required init() throws { + let values = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride)) + let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride)) + try super.init(values: values, nulls: nulls) + } + + public func append(_ newValue: ItemType?) { + let index = UInt(self.length) + self.length += 1 + if (length/8) > self.values.length { + self.resize(length) + } + + if newValue != nil { + if newValue == true { + BitUtility.setBit(index + self.offset, buffer: self.values) + } else { + BitUtility.clearBit(index + self.offset, buffer: self.values) + } + + } else { + self.nullCount += 1 + BitUtility.setBit(index + self.offset, buffer: self.nulls) + BitUtility.clearBit(index + self.offset, buffer: self.values) + } + } + + public func resize(_ length: UInt) { + if (length/8) > self.values.length { + let resizeLength = resizeLength(self.values) + var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size)) + var nulls = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size)) + ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity) + ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity) + self.values = values; + self.nulls = nulls; + } + } + + public func finish() -> [ArrowBuffer] { + let length = self.length + var values = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<UInt8>.size)) + var nulls = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<UInt8>.size)) + ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity) + ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity) + return [nulls, values] + } +} + +public class VariableBufferBuilder<T>: BaseBufferBuilder<T>, ArrowBufferBuilder { + public typealias ItemType = T + var offsets: ArrowBuffer + let binaryStride = MemoryLayout<UInt8>.stride + public required init() throws { + let values = ArrowBuffer.createBuffer(0, size: UInt(binaryStride)) + let nulls = ArrowBuffer.createBuffer(0, size: UInt(binaryStride)) + self.offsets = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<Int32>.stride)) + try super.init(values: values, nulls: nulls, stride: binaryStride) + } + + public func append(_ newValue: ItemType?) { + let index = UInt(self.length) + self.length += 1 + let offsetIndex = MemoryLayout<Int32>.stride * Int(index) + if self.length >= self.offsets.length { + self.resize(UInt( self.offsets.length + 1)) + } + var binData: Data + var isNull = false + if let val = newValue { + binData = getBytesFor(val)! + }else { + var nullVal = 0 + isNull = true + binData = Data(bytes: &nullVal, count: MemoryLayout<UInt32>.size) + } + + var currentIndex: Int32 = 0; + var currentOffset: Int32 = Int32(binData.count); + if index > 0 { + currentIndex = self.offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self) + currentOffset += currentIndex + if currentOffset > self.values.length { + self.value_resize(UInt(currentOffset)) + } + } + + if isNull { + self.nullCount += 1 + BitUtility.setBit(index + self.offset, buffer: self.nulls) + } + + binData.withUnsafeBytes { bufferPointer in + let rawPointer = bufferPointer.baseAddress! + self.values.rawPointer.advanced(by: Int(currentIndex)).copyMemory(from: rawPointer, byteCount: binData.count) + } + + self.offsets.rawPointer.advanced(by: (offsetIndex + MemoryLayout<Int32>.stride)).storeBytes(of: currentOffset, as: Int32.self) + } + + public func value_resize(_ length: UInt) { + if length > self.values.length { + let resizeLength = resizeLength(self.values, len: length); + var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size)) + ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity) + self.values = values; + } + } + + public func resize(_ length: UInt) { + if length > self.offsets.length { + let resizeLength = resizeLength(self.offsets, len: length); + var nulls = ArrowBuffer.createBuffer(resizeLength/8 + 1, size: UInt(MemoryLayout<UInt8>.size)) + var offsets = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<Int32>.size)) + ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity) + ArrowBuffer.copyCurrent(self.offsets, to: &offsets, len: self.offsets.capacity) + self.nulls = nulls; + self.offsets = offsets; + } + } + + public func finish() -> [ArrowBuffer] { + let length = self.length + var values = ArrowBuffer.createBuffer(self.values.length, size: UInt(MemoryLayout<UInt8>.size)) + var nulls = ArrowBuffer.createBuffer(length/8 + 1, size: UInt(MemoryLayout<UInt8>.size)) + var offsets = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<Int32>.size)) + ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity) + ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity) + ArrowBuffer.copyCurrent(self.offsets, to: &offsets, len: offsets.capacity) + return [nulls, offsets, values] + } +} + +public class Date32BufferBuilder: ArrowBufferBuilder { + public typealias ItemType = Date + public var capacity: UInt {get{return self.bufferBuilder.capacity}} + public var length: UInt {get{return self.bufferBuilder.length}} + public var nullCount : UInt {get{return self.bufferBuilder.nullCount}} + public var offset: UInt {get{return self.bufferBuilder.offset}} + private let bufferBuilder: FixedBufferBuilder<Int32> + public required init() throws { + self.bufferBuilder = try FixedBufferBuilder() + } + + public func append(_ newValue: ItemType?) { + if let val = newValue { + let daysSinceEpoch = Int32(val.timeIntervalSince1970 / 86400) + self.bufferBuilder.append(daysSinceEpoch) + } else { + self.bufferBuilder.append(nil) + } + } + + public func isNull(_ index: UInt) -> Bool { + return self.bufferBuilder.isNull(index) + } + + public func resize(_ length: UInt) { + self.bufferBuilder.resize(length) + } + + public func finish() -> [ArrowBuffer] { + return self.bufferBuilder.finish() + } +} + +public class Date64BufferBuilder: ArrowBufferBuilder { + public typealias ItemType = Date + public var capacity: UInt {get{return self.bufferBuilder.capacity}} + public var length: UInt {get{return self.bufferBuilder.length}} + public var nullCount : UInt {get{return self.bufferBuilder.nullCount}} + public var offset: UInt {get{return self.bufferBuilder.offset}} + private let bufferBuilder: FixedBufferBuilder<Int64> + public required init() throws { + self.bufferBuilder = try FixedBufferBuilder() + } + + public func append(_ newValue: ItemType?) { + if let val = newValue { + let daysSinceEpoch = Int64(val.timeIntervalSince1970 * 1000) + self.bufferBuilder.append(daysSinceEpoch) + } else { + self.bufferBuilder.append(nil) + } + } + + public func isNull(_ index: UInt) -> Bool { + return self.bufferBuilder.isNull(index) + } + + public func resize(_ length: UInt) { + self.bufferBuilder.resize(length) + } + + public func finish() -> [ArrowBuffer] { + return self.bufferBuilder.finish() + } +} diff --git a/Arrow/Sources/Arrow/ArrowData.swift b/Arrow/Sources/Arrow/ArrowData.swift new file mode 100644 index 0000000..cdfe6fd --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowData.swift @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class ArrowData { + let type: ArrowType.Info + let buffers: [ArrowBuffer] + let nullCount: UInt + let length: UInt + let stride: Int + + init(_ type: ArrowType.Info, buffers: [ArrowBuffer], nullCount: UInt, stride: Int) throws { + switch(type) { + case let .PrimitiveInfo(typeId): + if typeId == ArrowTypeId.Unknown { + throw ValidationError.unknownType + } + case let .VariableInfo(typeId): + if typeId == ArrowTypeId.Unknown { + throw ValidationError.unknownType + } + } + + self.type = type + self.buffers = buffers + self.nullCount = nullCount + self.length = buffers[1].length + self.stride = stride + } + + func isNull(_ at: UInt) -> Bool { + let nullBuffer = buffers[0]; + return nullBuffer.length == 0 || BitUtility.isSet(at, buffer: nullBuffer) + } +} diff --git a/Arrow/Sources/Arrow/ArrowSchema.swift b/Arrow/Sources/Arrow/ArrowSchema.swift new file mode 100644 index 0000000..af9564a --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowSchema.swift @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +public class ArrowField { + let type: ArrowType.Info + let name: String + let isNullable: Bool + + init(_ name: String, type: ArrowType.Info, isNullable: Bool) { + self.name = name + self.type = type + self.isNullable = isNullable + } +} + +public class ArrowSchema { + let fields: [ArrowField] + let fieldLookup: [String: Int] + init(_ fields: [ArrowField]) { + var fieldLookup = [String: Int]() + for (index, field) in fields.enumerated() { + fieldLookup[field.name] = index; + } + + self.fields = fields + self.fieldLookup = fieldLookup + } + + func field(_ index: Int) -> ArrowField { + return self.fields[index] + } + + func fieldIndex(_ name: String) -> Int? { + return self.fieldLookup[name] + } + + public class Builder { + private var fields: [ArrowField] = [] + + func addField(_ field: ArrowField) -> Builder { + fields.append(field) + return self + } + + func addField(_ name: String, type: ArrowType.Info, isNullable: Bool) -> Builder { + fields.append(ArrowField(name, type: type, isNullable: isNullable)) + return self + } + + func finish() -> ArrowSchema { + return ArrowSchema(fields) + } + } +} diff --git a/Arrow/Sources/Arrow/ArrowTable.swift b/Arrow/Sources/Arrow/ArrowTable.swift new file mode 100644 index 0000000..25677ba --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowTable.swift @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class ChunkedArrayHolder { + let type: ArrowType.Info + let length: UInt + let nullCount: UInt + let holder: Any + + init<T>(_ chunked: ChunkedArray<T>) { + self.holder = chunked + self.length = chunked.length + self.type = chunked.type + self.nullCount = chunked.nullCount + } +} + +public class ArrowColumn { + let field: ArrowField + private let dataHolder: ChunkedArrayHolder + var type: ArrowType.Info {get{return self.dataHolder.type}} + var length: UInt {get{return self.dataHolder.length}} + var nullCount: UInt {get{return self.dataHolder.nullCount}} + + func data<T>() -> ChunkedArray<T> { + return (self.dataHolder.holder as! ChunkedArray<T>) + } + + var name: String {get{return field.name}} + init<T>(_ field: ArrowField, chunked: ChunkedArray<T>) { + self.field = field + self.dataHolder = ChunkedArrayHolder(chunked) + } +} + +public class ArrowTable { + let schema: ArrowSchema + var columnCount: UInt {get{return UInt(self.columns.count)}} + let rowCount: UInt + let columns: [ArrowColumn] + init(_ schema: ArrowSchema, columns: [ArrowColumn]) { + self.schema = schema + self.columns = columns + self.rowCount = columns[0].length + } + + public class Builder { + let schemaBuilder = ArrowSchema.Builder() + var columns = [ArrowColumn]() + + func addColumn<T>(_ fieldName: String, arrowArray: ArrowArray<T>) throws -> Builder { + return self.addColumn(fieldName, chunked: try ChunkedArray([arrowArray])) + } + + func addColumn<T>(_ fieldName: String, chunked: ChunkedArray<T>) -> Builder { + let field = ArrowField(fieldName, type: chunked.type, isNullable: chunked.nullCount != 0) + self.schemaBuilder.addField(field) + self.columns.append(ArrowColumn(field, chunked: chunked)) + return self + } + + func addColumn<T>(_ field: ArrowField, arrowArray: ArrowArray<T>) throws -> Builder { + self.schemaBuilder.addField(field) + self.columns.append(ArrowColumn(field, chunked: try ChunkedArray([arrowArray]))) + return self + } + + func addColumn<T>(_ field: ArrowField, chunked: ChunkedArray<T>) -> Builder { + self.schemaBuilder.addField(field) + self.columns.append(ArrowColumn(field, chunked: chunked)) + return self + } + + func addColumn(_ column: ArrowColumn) -> Builder { + self.columns.append(column) + return self + } + + func finish() -> ArrowTable { + return ArrowTable(self.schemaBuilder.finish(), columns: self.columns) + } + } +} + +public class RecordBatch { + let schema: ArrowSchema + var columnCount: UInt {get{return UInt(self.columns.count)}} + let columns: [ChunkedArrayHolder] + let length: UInt + init(_ schema: ArrowSchema, columns: [ChunkedArrayHolder]) { + self.schema = schema + self.columns = columns + self.length = columns[0].length + } + + func column(_ index: Int) -> ChunkedArrayHolder { + return self.columns[index] + } + + func column(_ name: String) -> ChunkedArrayHolder? { + if let index = self.schema.fieldIndex(name) { + return self.columns[index] + } + + return nil; + } + + +} diff --git a/Arrow/Sources/Arrow/ArrowType.swift b/Arrow/Sources/Arrow/ArrowType.swift new file mode 100644 index 0000000..e136fad --- /dev/null +++ b/Arrow/Sources/Arrow/ArrowType.swift @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +enum ValidationError: Error { + case unknownType + case outOfBounds(index: UInt) + case arrayHasNoElements +} + +public enum ArrowTypeId { + case Binary + case Boolean + case Date32 + case Date64 + case DateType + case Decimal128 + case Decimal256 + case Dictionary + case Double + case FixedSizeBinary + case FixedWidthType + case Float + //case HalfFloatType + case Int16 + case Int32 + case Int64 + case Int8 + case Integer + case IntervalUnit + case List + case Nested + case Null + case Number + case String + case Struct + case Time32 + case Time64 + case Time + case UInt16 + case UInt32 + case UInt64 + case UInt8 + case Union + case Unknown +} + +public class ArrowType { + public static let ArrowInt8 = Info.PrimitiveInfo(ArrowTypeId.Int8) + public static let ArrowInt16 = Info.PrimitiveInfo(ArrowTypeId.Int16) + public static let ArrowInt32 = Info.PrimitiveInfo(ArrowTypeId.Int32) + public static let ArrowInt64 = Info.PrimitiveInfo(ArrowTypeId.Int64) + public static let ArrowUInt16 = Info.PrimitiveInfo(ArrowTypeId.UInt16) + public static let ArrowUInt32 = Info.PrimitiveInfo(ArrowTypeId.UInt32) + public static let ArrowUInt64 = Info.PrimitiveInfo(ArrowTypeId.UInt64) + public static let ArrowUInt8 = Info.PrimitiveInfo(ArrowTypeId.UInt8) + public static let ArrowFloat = Info.PrimitiveInfo(ArrowTypeId.Float) + public static let ArrowDouble = Info.PrimitiveInfo(ArrowTypeId.Double) + public static let ArrowUnknown = Info.PrimitiveInfo(ArrowTypeId.Unknown) + public static let ArrowString = Info.VariableInfo(ArrowTypeId.String) + public static let ArrowBool = Info.PrimitiveInfo(ArrowTypeId.Boolean) + public static let ArrowDate32 = Info.PrimitiveInfo(ArrowTypeId.Date32) + public static let ArrowDate64 = Info.PrimitiveInfo(ArrowTypeId.Date64) + + public enum Info { + case PrimitiveInfo(ArrowTypeId) + case VariableInfo(ArrowTypeId) + } + + public static func infoForNumericType<T>(_ t: T.Type) -> ArrowType.Info { + if t == Int8.self { + return ArrowType.ArrowInt8 + }else if t == Int16.self { + return ArrowType.ArrowInt16 + }else if t == Int32.self { + return ArrowType.ArrowInt32 + }else if t == Int64.self { + return ArrowType.ArrowInt64 + }else if t == UInt8.self { + return ArrowType.ArrowUInt8 + }else if t == UInt16.self { + return ArrowType.ArrowUInt16 + }else if t == UInt32.self { + return ArrowType.ArrowUInt32 + }else if t == UInt64.self { + return ArrowType.ArrowUInt64 + }else if t == Float.self { + return ArrowType.ArrowFloat + }else if t == Double.self { + return ArrowType.ArrowDouble + }else { + return ArrowType.ArrowUnknown + } + } +} + +extension ArrowType.Info: Equatable { + public static func==(lhs: ArrowType.Info, rhs: ArrowType.Info) -> Bool { + switch(lhs, rhs) { + case (.PrimitiveInfo(let lhsId), .PrimitiveInfo(let rhsId)): + return lhsId == rhsId + case (.VariableInfo(let lhsId), .VariableInfo(let rhsId)): + return lhsId == rhsId + case (.VariableInfo(_), .PrimitiveInfo(_)), (.PrimitiveInfo(_), .VariableInfo(_)): + return false + } + } +} + +func getBytesFor<T>(_ data: T) -> Data? { + let t = T.self + if t == String.self { + let temp = data as! String + return temp.data(using: .utf8); + }else { + return nil + } +} diff --git a/Arrow/Sources/Arrow/BitUtility.swift b/Arrow/Sources/Arrow/BitUtility.swift new file mode 100644 index 0000000..a4abb59 --- /dev/null +++ b/Arrow/Sources/Arrow/BitUtility.swift @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +class BitUtility { + static let twoPow: [UInt8] = [1, 2, 4, 8, 16, 32, 64, 128] + static func isSet(_ bit: UInt, buffer: ArrowBuffer) -> Bool { + let byteIndex = UInt(bit / 8) + let theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self) + let bitToGet = 7 - Int(bit % 8); + return theByte & twoPow[bitToGet] > 0 + } + + static func setBit(_ bit: UInt, buffer: ArrowBuffer) { + let byteIndex = UInt(bit / 8) + var theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self) + theByte |= twoPow[7 - Int(bit % 8)] + buffer.rawPointer.storeBytes(of: theByte, toByteOffset: Int(byteIndex), as: UInt8.self) + } + + static func clearBit(_ bit: UInt, buffer: ArrowBuffer) { + let byteIndex = UInt(bit / 8) + var theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self) + theByte &= ~twoPow[7 - Int(bit % 8)] + buffer.rawPointer.storeBytes(of: theByte, toByteOffset: Int(byteIndex), as: UInt8.self) + } +} + diff --git a/Arrow/Sources/Arrow/ChunkedArray.swift b/Arrow/Sources/Arrow/ChunkedArray.swift new file mode 100644 index 0000000..a6cd29a --- /dev/null +++ b/Arrow/Sources/Arrow/ChunkedArray.swift @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class ChunkedArray<T> { + let arrays: [ArrowArray<T>] + let type: ArrowType.Info + let nullCount: UInt + let length: UInt + var arrayCount: UInt {get{return UInt(self.arrays.count)}} + + init(_ arrays: [ArrowArray<T>]) throws { + if arrays.count == 0 { + throw ValidationError.arrayHasNoElements + } + + self.type = arrays[0].arrowData.type + var len: UInt = 0 + var nullCount: UInt = 0 + for array in arrays { + len += array.length + nullCount += array.nullCount + } + + self.arrays = arrays + self.length = len + self.nullCount = nullCount + } + + subscript(_ index: UInt) -> ArrowArray<T> { + return arrays[Int(index)] + } + +} diff --git a/Arrow/Sources/Arrow/MemoryAllocator.swift b/Arrow/Sources/Arrow/MemoryAllocator.swift new file mode 100644 index 0000000..b4a7bd8 --- /dev/null +++ b/Arrow/Sources/Arrow/MemoryAllocator.swift @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class MemoryAllocator { + let alignment: Int + init(_ alignment: Int) { + self.alignment = alignment + } + + func allocateArray(_ byteCount: Int) -> UnsafeMutableRawPointer { + return UnsafeMutableRawPointer.allocate( + byteCount: byteCount, + alignment: self.alignment); + } +} diff --git a/Arrow/Tests/ArrowTests/ArrowTests.swift b/Arrow/Tests/ArrowTests/ArrowTests.swift new file mode 100644 index 0000000..8aa6a38 --- /dev/null +++ b/Arrow/Tests/ArrowTests/ArrowTests.swift @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import XCTest +@testable import Arrow + +final class ArrowTests: XCTestCase { + func testPrimitiveArray() throws { + // This is an example of a functional test case. + // Use XCTAssert and related functions to verify your tests produce the correct + // results. + let arrayBuilder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder(); + for i in 0..<100 { + arrayBuilder.append(UInt8(i)) + } + + XCTAssertEqual(arrayBuilder.nullCount, 0) + arrayBuilder.append(nil) + XCTAssertEqual(arrayBuilder.length, 101) + XCTAssertEqual(arrayBuilder.capacity, 136) + XCTAssertEqual(arrayBuilder.nullCount, 1) + let array = try arrayBuilder.finish() + XCTAssertEqual(array.length, 101) + XCTAssertEqual(array[1]!, 1) + XCTAssertEqual(array[10]!, 10) + XCTAssertEqual(try array.isNull(100), true) + + let doubleBuilder: NumberArrayBuilder<Double> = try ArrowArrayBuilders.loadNumberArrayBuilder(); + doubleBuilder.append(14) + doubleBuilder.append(40.4) + XCTAssertEqual(doubleBuilder.nullCount, 0) + XCTAssertEqual(doubleBuilder.length, 2) + XCTAssertEqual(doubleBuilder.capacity, 264) + let doubleArray = try doubleBuilder.finish() + XCTAssertEqual(doubleArray.length, 32) + XCTAssertEqual(doubleArray[0]!, 14) + XCTAssertEqual(doubleArray[1]!, 40.4) + } + + func testStringArray() throws { + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder(); + for i in 0..<100 { + if i % 10 == 9 { + stringBuilder.append(nil) + } else { + stringBuilder.append("test" + String(i)) + } + } + XCTAssertEqual(stringBuilder.nullCount, 10) + XCTAssertEqual(stringBuilder.length, 100) + XCTAssertEqual(stringBuilder.capacity, 1032) + let stringArray = try stringBuilder.finish() + XCTAssertEqual(stringArray.length, 100) + for i in 0..<stringArray.length { + if i % 10 == 9 { + XCTAssertEqual(try stringArray.isNull(i), true) + } else { + XCTAssertEqual(stringArray[i]!, "test" + String(i)) + } + } + + XCTAssertEqual(stringArray[1]!, "test1") + XCTAssertEqual(stringArray[0]!, "test0") + } + + func testBoolArray() throws { + let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder() + boolBuilder.append(true) + boolBuilder.append(nil) + boolBuilder.append(false) + boolBuilder.append(false) + XCTAssertEqual(boolBuilder.nullCount, 1) + XCTAssertEqual(boolBuilder.length, 4) + XCTAssertEqual(boolBuilder.capacity, 72) + let boolArray = try boolBuilder.finish() + XCTAssertEqual(boolArray.length, 32) + XCTAssertEqual(boolArray[1], nil) + XCTAssertEqual(boolArray[0]!, true) + XCTAssertEqual(boolArray[2]!, false) + } + + func testSchema() throws { + let schemaBuilder = ArrowSchema.Builder(); + let schema = schemaBuilder.addField("col1", type: ArrowType.ArrowInt8, isNullable: true) + .addField("col2", type: ArrowType.ArrowBool, isNullable: false) + .finish() + XCTAssertEqual(schema.fields.count, 2) + XCTAssertEqual(schema.fields[0].name, "col1") + XCTAssertEqual(schema.fields[0].type, ArrowType.ArrowInt8) + XCTAssertEqual(schema.fields[0].isNullable, true) + XCTAssertEqual(schema.fields[1].name, "col2") + XCTAssertEqual(schema.fields[1].type, ArrowType.ArrowBool) + XCTAssertEqual(schema.fields[1].isNullable, false) + } + + func testTable() throws { + let uint8Builder: NumberArrayBuilder<UInt8> = try ArrowArrayBuilders.loadNumberArrayBuilder(); + uint8Builder.append(10) + uint8Builder.append(22) + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder(); + stringBuilder.append("test10") + stringBuilder.append("test22") + + let table = try ArrowTable.Builder() + .addColumn("col1", arrowArray: uint8Builder.finish()) + .addColumn("col2", arrowArray: stringBuilder.finish()) + .finish(); + + let schema = table.schema + XCTAssertEqual(schema.fields.count, 2) + XCTAssertEqual(schema.fields[0].name, "col1") + XCTAssertEqual(schema.fields[0].type, ArrowType.ArrowUInt8) + XCTAssertEqual(schema.fields[0].isNullable, false) + XCTAssertEqual(schema.fields[1].name, "col2") + XCTAssertEqual(schema.fields[1].type, ArrowType.ArrowString) + XCTAssertEqual(schema.fields[1].isNullable, false) + XCTAssertEqual(table.columns.count, 2) + let col1: ChunkedArray<UInt8> = table.columns[0].data(); + let col2: ChunkedArray<String> = table.columns[1].data(); + XCTAssertEqual(col1.length, 32) + XCTAssertEqual(col2.length, 32) + } + + func testDate() throws { + let date32Builder: Date32ArrayBuilder = try ArrowArrayBuilders.loadDate32ArrayBuilder(); + let date2 = Date(timeIntervalSinceReferenceDate: 86400 * 1) + let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000) + date32Builder.append(date1) + date32Builder.append(date2) + date32Builder.append(nil) + XCTAssertEqual(date32Builder.nullCount, 1) + XCTAssertEqual(date32Builder.length, 3) + XCTAssertEqual(date32Builder.capacity, 136) + let date32Array = try date32Builder.finish() + XCTAssertEqual(date32Array.length, 32) + XCTAssertEqual(date32Array[1], date2) + XCTAssertEqual(date32Array[0]!, date1) + + let date64Builder: Date64ArrayBuilder = try ArrowArrayBuilders.loadDate64ArrayBuilder(); + date64Builder.append(date1) + date64Builder.append(date2) + date64Builder.append(nil) + XCTAssertEqual(date64Builder.nullCount, 1) + XCTAssertEqual(date64Builder.length, 3) + XCTAssertEqual(date64Builder.capacity, 264) + let date64Array = try date64Builder.finish() + XCTAssertEqual(date64Array.length, 32) + XCTAssertEqual(date64Array[1], date2) + XCTAssertEqual(date64Array[0]!, date1) + + } +}
