paddyhoran commented on a change in pull request #7004: URL: https://github.com/apache/arrow/pull/7004#discussion_r419653824
########## File path: rust/arrow/src/array/union.rs ########## @@ -0,0 +1,1174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Contains the `UnionArray` and `UnionBuilder` types. +//! +//! Each slot in a `UnionArray` can have a value chosen from a number of types. Each of the +//! possible types are named like the fields of a [`StructArray`](crate::array::StructArray). +//! A `UnionArray` can have two possible memory layouts, "dense" or "sparse". For more information +//! on please see the [specification](https://arrow.apache.org/docs/format/Columnar.html#union-layout). +//! +//! Builders are provided for `UnionArray`'s involving primitive types. `UnionArray`'s of nested +//! types are also supported but not via `UnionBuilder`, see the tests for examples. +//! +//! # Example: Dense Memory Layout +//! +//! ``` +//! use arrow::array::UnionBuilder; +//! use arrow::datatypes::{Float64Type, Int32Type}; +//! +//! # fn main() -> arrow::error::Result<()> { +//! let mut builder = UnionBuilder::new_dense(3); +//! builder.append::<Int32Type>("a", 1).unwrap(); +//! builder.append::<Float64Type>("b", 3.0).unwrap(); +//! builder.append::<Int32Type>("a", 4).unwrap(); +//! let union = builder.build().unwrap(); +//! +//! assert_eq!(union.type_id(0), 0_i8); +//! assert_eq!(union.type_id(1), 1_i8); +//! assert_eq!(union.type_id(2), 0_i8); +//! +//! assert_eq!(union.value_offset(0), 0_i32); +//! assert_eq!(union.value_offset(1), 0_i32); +//! assert_eq!(union.value_offset(2), 1_i32); +//! +//! # Ok(()) +//! # } +//! ``` +//! +//! # Example: Sparse Memory Layout +//! ``` +//! use arrow::array::UnionBuilder; +//! use arrow::datatypes::{Float64Type, Int32Type}; +//! +//! # fn main() -> arrow::error::Result<()> { +//! let mut builder = UnionBuilder::new_sparse(3); +//! builder.append::<Int32Type>("a", 1).unwrap(); +//! builder.append::<Float64Type>("b", 3.0).unwrap(); +//! builder.append::<Int32Type>("a", 4).unwrap(); +//! let union = builder.build().unwrap(); +//! +//! assert_eq!(union.type_id(0), 0_i8); +//! assert_eq!(union.type_id(1), 1_i8); +//! assert_eq!(union.type_id(2), 0_i8); +//! +//! assert_eq!(union.value_offset(0), 0_i32); +//! assert_eq!(union.value_offset(1), 1_i32); +//! assert_eq!(union.value_offset(2), 2_i32); +//! +//! # Ok(()) +//! # } +//! ``` +use crate::array::{ + builder::{builder_to_mutable_buffer, mutable_buffer_to_builder, BufferBuilderTrait}, + make_array, Array, ArrayData, ArrayDataBuilder, ArrayDataRef, ArrayRef, + BooleanBufferBuilder, BufferBuilder, Int32BufferBuilder, Int8BufferBuilder, +}; +use crate::buffer::{Buffer, MutableBuffer}; +use crate::datatypes::*; +use crate::error::{ArrowError, Result}; + +use crate::util::bit_util; +use core::fmt; +use std::any::Any; +use std::collections::HashMap; +use std::mem::size_of; + +/// An Array that can represent slots of varying types +pub struct UnionArray { + data: ArrayDataRef, + boxed_fields: Vec<ArrayRef>, +} + +impl UnionArray { + /// Creates a new `UnionArray`. + /// + /// Accepts type ids, child arrays and optionally offsets (for dense unions) to create + /// a new `UnionArray`. This method makes no attempt to validate the data provided by the + /// caller and assumes that each of the components are correct and consistent with each other. + /// See `try_new` for an alternative that validates the data provided. + /// + /// # Data Consistency + /// + /// The `type_ids` `Buffer` should contain `i8` values. These values should be greater than + /// zero and must be less than the number of children provided in `child_arrays`. These values + /// are used to index into the `child_arrays`. + /// + /// The `value_offsets` `Buffer` is only provided in the case of a dense union, sparse unions + /// should use `None`. If provided the `value_offsets` `Buffer` should contain `i32` values. + /// These values should be greater than zero and must be less than the length of the overall + /// array. + /// + /// In both cases above we use signed integer types to maintain compatibility with other + /// Arrow implementations. + /// + /// In both of the cases above we are accepting `Buffer`'s which are assumed to be representing + /// `i8` and `i32` values respectively. `Buffer` objects are untyped and no attempt is made + /// to ensure that the data provided is valid. + pub fn new( + type_ids: Buffer, + value_offsets: Option<Buffer>, + child_arrays: Vec<(Field, ArrayRef)>, + bitmap_data: Option<(Buffer, usize)>, + ) -> Self { + let (field_types, field_values): (Vec<_>, Vec<_>) = + child_arrays.into_iter().unzip(); + let len = type_ids.len(); + let mut builder = ArrayData::builder(DataType::Union(field_types)) + .add_buffer(type_ids) + .child_data(field_values.into_iter().map(|a| a.data()).collect()) + .len(len); + if let Some((bitmap, null_count)) = bitmap_data { + builder = builder.null_bit_buffer(bitmap).null_count(null_count); + } + let data = match value_offsets { + Some(b) => builder.add_buffer(b).build(), + None => builder.build(), + }; + Self::from(data) + } + /// Attempts to create a new `UnionArray` and validates the inputs provided. + pub fn try_new( + type_ids: Buffer, + value_offsets: Option<Buffer>, + child_arrays: Vec<(Field, ArrayRef)>, + bitmap: Option<Buffer>, + ) -> Result<Self> { + let bitmap_data = bitmap.map(|b| { + let null_count = type_ids.len() - bit_util::count_set_bits(b.data()); + (b, null_count) + }); + + if let Some(b) = &value_offsets { + let nulls = match bitmap_data { + Some((_, n)) => n, + None => 0, + }; + if ((type_ids.len() - nulls) * 4) != b.len() { + return Err(ArrowError::InvalidArgumentError( + "Type Ids and Offsets represent a different number of array slots." + .to_string(), + )); + } + } + + // Check the type_ids + let type_id_slice: &[i8] = unsafe { type_ids.typed_data() }; + let invalid_type_ids = type_id_slice + .iter() + .filter(|i| *i < &0) + .collect::<Vec<&i8>>(); + if invalid_type_ids.len() > 0 { + return Err(ArrowError::InvalidArgumentError(format!( + "Type Ids must be positive and cannot be greater than the number of \ + child arrays, found:\n{:?}", + invalid_type_ids + ))); + } + + // Check the value offsets if provided + if let Some(offset_buffer) = &value_offsets { + let max_len = type_ids.len() as i32; + let offsets_slice: &[i32] = unsafe { offset_buffer.typed_data() }; + let invalid_offsets = offsets_slice + .iter() + .filter(|i| *i < &0 || *i > &max_len) + .collect::<Vec<&i32>>(); + if invalid_offsets.len() > 0 { + return Err(ArrowError::InvalidArgumentError(format!( + "Offsets must be positive and within the length of the Array, \ + found:\n{:?}", + invalid_offsets + ))); + } + } + + Ok(Self::new( + type_ids, + value_offsets, + child_arrays, + bitmap_data, + )) + } + + /// Accesses the child array for `type_id`. + /// + /// # Panics + /// + /// Panics if the `type_id` provided is less than zero or greater than the number of types + /// in the `Union`. + pub fn child(&self, type_id: i8) -> ArrayRef { + assert!(0 <= type_id); + assert!((type_id as usize) < self.boxed_fields.len()); + self.boxed_fields[type_id as usize].clone() + } + + /// Returns the `type_id` for the array slot at `index`. + /// + /// # Panics + /// + /// Panics if `index` is greater than the length of the array. + pub fn type_id(&self, index: usize) -> i8 { + assert!(index - self.offset() < self.len()); + self.data().buffers()[0].data()[index] as i8 + } + + /// Returns the offset into the underlying values array for the array slot at `index`. + /// + /// # Panics + /// + /// Panics if `index` is greater than the length of the array. + pub fn value_offset(&self, index: usize) -> i32 { + assert!(index - self.offset() < self.len()); + if self.is_dense() { + let valid_slots = match self.data.null_buffer() { + Some(b) => bit_util::count_set_bits_offset(b.data(), 0, index), + None => index, + }; + self.data().buffers()[1].data()[valid_slots * size_of::<i32>()] as i32 + } else { + index as i32 + } + } + + /// Returns the array's value at `index`. + /// + /// # Panics + /// + /// Panics if `index` is greater than the length of the array. + pub fn value(&self, index: usize) -> ArrayRef { + let type_id = self.type_id(self.offset() + index); + let value_offset = self.value_offset(self.offset() + index) as usize; + let child_data = self.boxed_fields[type_id as usize].clone(); + child_data.slice(value_offset, 1) + } + + /// Returns the names of the types in the union. + pub fn type_names(&self) -> Vec<&str> { + match self.data.data_type() { + DataType::Union(fields) => fields + .iter() + .map(|f| f.name().as_str()) + .collect::<Vec<&str>>(), + _ => unreachable!("Union array's data type is not a union!"), + } + } + + /// Returns whether the `UnionArray` is dense (or sparse if `false`). + fn is_dense(&self) -> bool { + self.data().buffers().len() == 2 + } +} + +impl From<ArrayDataRef> for UnionArray { + fn from(data: ArrayDataRef) -> Self { + let mut boxed_fields = vec![]; + for cd in data.child_data() { + boxed_fields.push(make_array(cd.clone())); + } + Self { data, boxed_fields } + } +} + +impl Array for UnionArray { + fn as_any(&self) -> &Any { + self + } + + fn data(&self) -> ArrayDataRef { + self.data.clone() + } + + fn data_ref(&self) -> &ArrayDataRef { + &self.data + } +} + +impl fmt::Debug for UnionArray { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let header = if self.is_dense() { + "UnionArray(Dense)\n[\n" + } else { + "UnionArray(Sparse)\n[\n" + }; + write!(f, "{}", header)?; + + write!(f, "-- type id buffer:\n")?; Review comment: Yep, will update. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
