alamb commented on a change in pull request #1006: URL: https://github.com/apache/arrow-datafusion/pull/1006#discussion_r738699587
########## File path: datafusion/src/physical_plan/expressions/get_indexed_field.rs ########## @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! get field of a struct array Review comment: ```suggestion //! get field of a `ListArray` ``` ########## File path: datafusion/src/lib.rs ########## @@ -231,6 +231,7 @@ pub mod variable; pub use arrow; pub use parquet; +pub mod field_util; Review comment: ```suggestion pub (crate) mod field_util; ``` Unless there is some reason to expose this module, I think it might be nice to leave it internal so that we can move it around as needed ########## File path: datafusion/src/physical_plan/expressions/get_indexed_field.rs ########## @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! get field of a struct array + +use std::{any::Any, sync::Arc}; + +use arrow::{ + datatypes::{DataType, Schema}, + record_batch::RecordBatch, +}; + +use crate::arrow::array::Array; +use crate::arrow::compute::concat; +use crate::scalar::ScalarValue; +use crate::{ + error::DataFusionError, + error::Result, + field_util::get_indexed_field as get_data_type_field, + physical_plan::{ColumnarValue, PhysicalExpr}, +}; +use arrow::array::ListArray; +use std::fmt::Debug; + +/// expression to get a field of a struct array. +#[derive(Debug)] +pub struct GetIndexedFieldExpr { + arg: Arc<dyn PhysicalExpr>, + key: ScalarValue, +} + +impl GetIndexedFieldExpr { + /// Create new get field expression + pub fn new(arg: Arc<dyn PhysicalExpr>, key: ScalarValue) -> Self { + Self { arg, key } + } + + /// Get the input expression + pub fn arg(&self) -> &Arc<dyn PhysicalExpr> { + &self.arg + } +} + +impl std::fmt::Display for GetIndexedFieldExpr { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "({}).[{}]", self.arg, self.key) + } +} + +impl PhysicalExpr for GetIndexedFieldExpr { + fn as_any(&self) -> &dyn Any { + self + } + + fn data_type(&self, input_schema: &Schema) -> Result<DataType> { + let data_type = self.arg.data_type(input_schema)?; + get_data_type_field(&data_type, &self.key).map(|f| f.data_type().clone()) + } + + fn nullable(&self, input_schema: &Schema) -> Result<bool> { + let data_type = self.arg.data_type(input_schema)?; + get_data_type_field(&data_type, &self.key).map(|f| f.is_nullable()) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> { + let arg = self.arg.evaluate(batch)?; + match arg { + ColumnarValue::Array(array) => match (array.data_type(), &self.key) { + (DataType::List(_), ScalarValue::Int64(Some(i))) => { Review comment: if `self.key` is null, I think we could also return null So perhaps we could add a new case like this (untested): ```rust (DataType::List(_), _) if self.key.is_null=> { let scalar_null: ScalarValue = array.data_type().try_into()?; Ok(ColumnarValue::Scalar(scalar_null)) } ``` ########## File path: datafusion/src/field_util.rs ########## @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Utility functions for complex field access + +use arrow::datatypes::{DataType, Field}; + +use crate::error::{DataFusionError, Result}; +use crate::scalar::ScalarValue; + +/// Returns the field access indexed by `key` from a [`DataType::List`] +/// # Error +/// Errors if +/// * the `data_type` is not a Struct or, +/// * there is no field key is not of the required index type +pub fn get_indexed_field(data_type: &DataType, key: &ScalarValue) -> Result<Field> { + match (data_type, key) { + (DataType::List(lt), ScalarValue::Int64(Some(i))) => { + if *i < 0 { + Err(DataFusionError::Plan( + format!("List based indexed access requires a positive int, was {0}", i), + )) + } else { + Ok(Field::new(&i.to_string(), lt.data_type().clone(), false)) + } + } + (DataType::List(_), _) => { + Err(DataFusionError::Plan( + "Only ints are valid as an indexed field in a list" + .to_string(), + )) + } + _ => Err(DataFusionError::Plan( + "The expression to get an indexed field is only valid for `List` or 'Dictionary'" Review comment: ```suggestion "The expression to get an indexed field is only valid for `List` types" ``` ########## File path: datafusion/src/physical_plan/expressions/get_indexed_field.rs ########## @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! get field of a struct array + +use std::{any::Any, sync::Arc}; + +use arrow::{ + datatypes::{DataType, Schema}, + record_batch::RecordBatch, +}; + +use crate::arrow::array::Array; +use crate::arrow::compute::concat; +use crate::scalar::ScalarValue; +use crate::{ + error::DataFusionError, + error::Result, + field_util::get_indexed_field as get_data_type_field, + physical_plan::{ColumnarValue, PhysicalExpr}, +}; +use arrow::array::ListArray; +use std::fmt::Debug; + +/// expression to get a field of a struct array. +#[derive(Debug)] +pub struct GetIndexedFieldExpr { + arg: Arc<dyn PhysicalExpr>, + key: ScalarValue, +} + +impl GetIndexedFieldExpr { + /// Create new get field expression + pub fn new(arg: Arc<dyn PhysicalExpr>, key: ScalarValue) -> Self { + Self { arg, key } + } + + /// Get the input expression + pub fn arg(&self) -> &Arc<dyn PhysicalExpr> { + &self.arg + } +} + +impl std::fmt::Display for GetIndexedFieldExpr { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "({}).[{}]", self.arg, self.key) + } +} + +impl PhysicalExpr for GetIndexedFieldExpr { + fn as_any(&self) -> &dyn Any { + self + } + + fn data_type(&self, input_schema: &Schema) -> Result<DataType> { + let data_type = self.arg.data_type(input_schema)?; + get_data_type_field(&data_type, &self.key).map(|f| f.data_type().clone()) + } + + fn nullable(&self, input_schema: &Schema) -> Result<bool> { + let data_type = self.arg.data_type(input_schema)?; + get_data_type_field(&data_type, &self.key).map(|f| f.is_nullable()) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> { + let arg = self.arg.evaluate(batch)?; + match arg { + ColumnarValue::Array(array) => match (array.data_type(), &self.key) { + (DataType::List(_), ScalarValue::Int64(Some(i))) => { + let as_list_array = + array.as_any().downcast_ref::<ListArray>().unwrap(); + let x: Vec<Arc<dyn Array>> = as_list_array + .iter() + .filter_map(|o| o.map(|list| list.slice(*i as usize, 1))) + .collect(); + let vec = x.iter().map(|a| a.as_ref()).collect::<Vec<&dyn Array>>(); + let iter = concat(vec.as_slice()).unwrap(); + Ok(ColumnarValue::Array(iter)) + } + _ => Err(DataFusionError::NotImplemented( + "get indexed field is only possible on lists".to_string(), Review comment: Untested: ```suggestion format!("get indexed field is only possible on lists with int64 indexes. Tried {} with {} index", array.data_type(), self.key()) ``` ########## File path: datafusion/src/physical_plan/planner.rs ########## @@ -141,6 +143,10 @@ fn create_physical_name(e: &Expr, is_first_expr: bool) -> Result<String> { let expr = create_physical_name(expr, false)?; Ok(format!("{} IS NOT NULL", expr)) } + Expr::GetIndexedField { expr, key } => { Review comment: 👍 ########## File path: datafusion/src/physical_plan/expressions/get_indexed_field.rs ########## @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! get field of a struct array + +use std::{any::Any, sync::Arc}; + +use arrow::{ + datatypes::{DataType, Schema}, + record_batch::RecordBatch, +}; + +use crate::arrow::array::Array; +use crate::arrow::compute::concat; +use crate::scalar::ScalarValue; +use crate::{ + error::DataFusionError, + error::Result, + field_util::get_indexed_field as get_data_type_field, + physical_plan::{ColumnarValue, PhysicalExpr}, +}; +use arrow::array::ListArray; +use std::fmt::Debug; + +/// expression to get a field of a struct array. +#[derive(Debug)] +pub struct GetIndexedFieldExpr { + arg: Arc<dyn PhysicalExpr>, + key: ScalarValue, +} + +impl GetIndexedFieldExpr { + /// Create new get field expression + pub fn new(arg: Arc<dyn PhysicalExpr>, key: ScalarValue) -> Self { + Self { arg, key } + } + + /// Get the input expression + pub fn arg(&self) -> &Arc<dyn PhysicalExpr> { + &self.arg + } +} + +impl std::fmt::Display for GetIndexedFieldExpr { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "({}).[{}]", self.arg, self.key) + } +} + +impl PhysicalExpr for GetIndexedFieldExpr { + fn as_any(&self) -> &dyn Any { + self + } + + fn data_type(&self, input_schema: &Schema) -> Result<DataType> { + let data_type = self.arg.data_type(input_schema)?; + get_data_type_field(&data_type, &self.key).map(|f| f.data_type().clone()) + } + + fn nullable(&self, input_schema: &Schema) -> Result<bool> { + let data_type = self.arg.data_type(input_schema)?; + get_data_type_field(&data_type, &self.key).map(|f| f.is_nullable()) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> { + let arg = self.arg.evaluate(batch)?; + match arg { + ColumnarValue::Array(array) => match (array.data_type(), &self.key) { + (DataType::List(_), ScalarValue::Int64(Some(i))) => { + let as_list_array = + array.as_any().downcast_ref::<ListArray>().unwrap(); + let x: Vec<Arc<dyn Array>> = as_list_array + .iter() + .filter_map(|o| o.map(|list| list.slice(*i as usize, 1))) + .collect(); + let vec = x.iter().map(|a| a.as_ref()).collect::<Vec<&dyn Array>>(); + let iter = concat(vec.as_slice()).unwrap(); + Ok(ColumnarValue::Array(iter)) + } + _ => Err(DataFusionError::NotImplemented( + "get indexed field is only possible on lists".to_string(), + )), + }, + ColumnarValue::Scalar(_) => Err(DataFusionError::NotImplemented( + "field is not yet implemented for scalar values".to_string(), Review comment: ```suggestion "field access is not yet implemented for scalar values".to_string(), ``` ########## File path: datafusion/src/logical_plan/expr.rs ########## @@ -245,6 +246,13 @@ pub enum Expr { IsNull(Box<Expr>), /// arithmetic negation of an expression, the operand must be of a signed numeric data type Negative(Box<Expr>), + /// Returns the field of a [`ListArray`] or ['DictionaryArray'] by name Review comment: ```suggestion /// Returns the field of a [`ListArray`] by name ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
