geoffreyclaude commented on code in PR #22996: URL: https://github.com/apache/datafusion/pull/22996#discussion_r3435738659
########## datafusion/functions-aggregate/src/map_agg.rs: ########## @@ -0,0 +1,710 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `MAP_AGG` aggregate implementation: [`MapAgg`] + +use std::collections::{HashSet, VecDeque}; +use std::mem::{size_of, size_of_val, take}; +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray, MapArray, StructArray}; +use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::compute::SortOptions; +use arrow::datatypes::{DataType, Field, FieldRef, Fields}; + +use datafusion_common::utils::{SingleRowListArrayBuilder, compare_rows, get_row_at_idx}; +use datafusion_common::{Result, ScalarValue, exec_err}; +use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; +use datafusion_expr::utils::format_state_name; +use datafusion_expr::{ + Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility, +}; +use datafusion_functions_aggregate_common::merge_arrays::merge_ordered_arrays; +use datafusion_functions_aggregate_common::order::AggregateOrderSensitivity; +use datafusion_functions_aggregate_common::utils::ordering_fields; +use datafusion_macros::user_doc; +use datafusion_physical_expr_common::sort_expr::LexOrdering; + +use crate::utils::{map_row_to_scalars, struct_to_rows}; + +make_udaf_expr_and_func!( + MapAgg, + map_agg, + "Aggregate key-value pairs into a map", + map_agg_udaf +); + +#[user_doc( + doc_section(label = "General Functions"), + description = "Aggregate key-value pairs from two columns into a single map per group. Pairs with a NULL key are skipped; NULL values are retained. On a duplicate key the first value wins; use ORDER BY to make which value wins deterministic.", + syntax_example = "map_agg(key, value [ORDER BY expression])", + sql_example = r#" +```sql +> SELECT map_agg(name, score) FROM scores GROUP BY department; ++-------------------------------+ +| map_agg(name, score) | ++-------------------------------+ +| {Alice: 95, Bob: 87} | ++-------------------------------+ +``` +"#, + standard_argument(name = "key",), + standard_argument(name = "value",) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct MapAgg { + signature: Signature, +} + +impl Default for MapAgg { + fn default() -> Self { + Self { + signature: Signature::any(2, Volatility::Immutable), + } + } +} + +impl AggregateUDFImpl for MapAgg { + fn name(&self) -> &str { + "map_agg" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> { + Ok(map_type(&arg_types[0], &arg_types[1])) + } + + fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> { + let key_type = args.input_fields[0].data_type(); + let value_type = args.input_fields[1].data_type(); + + let mut fields = vec![Arc::new(Field::new( + format_state_name(args.name, "map_agg"), + map_type(key_type, value_type), + true, + ))]; + + if !args.ordering_fields.is_empty() { + fields.push(Arc::new(Field::new_list( + format_state_name(args.name, "map_agg_orderings"), + Field::new_list_field( + DataType::Struct(Fields::from(args.ordering_fields.to_vec())), + true, + ), + false, + ))); + } + + Ok(fields) + } + + fn order_sensitivity(&self) -> AggregateOrderSensitivity { Review Comment: I’m not sure this should be a hard ordering requirement. The accumulator already stores the ORDER BY values and sorts internally, so the query can still produce a correct answer without the input already being sorted. Marking this as hard can force extra sorts and can also reject queries with multiple ordered aggregates that could otherwise be evaluated. Could this follow the array_agg pattern instead, where ordering is helpful but not required for correctness? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
