neilconway commented on code in PR #26600: URL: https://github.com/apache/datafusion/pull/26600#discussion_r4355271915
########## datafusion-examples/examples/dataframe/main.rs: ########## @@ -21,12 +21,15 @@ //! //! ## Usage //! ```bash -//! cargo run --example dataframe -- [all|dataframe|deserialize_to_struct|cache_factory] +//! cargo run --example dataframe -- [all|aggregate|dataframe|deserialize_to_struct|cache_factory] //! ``` //! //! Each subcommand runs a corresponding example: //! - `all` — run all examples included in this module //! +//! - `aggregate` +//! (file: aggregate.rs, desc: Global and grouped aggregation with select and `aggregate`) Review Comment: Should be \`select\` I think ########## datafusion-examples/examples/dataframe/aggregate.rs: ########## @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! See `main.rs` for how to run it. + +use datafusion::error::Result; +use datafusion::functions_aggregate::expr_fn::{avg, count, max, min}; +use datafusion::functions_aggregate::min_max::max_udaf; +use datafusion::logical_expr::expr::WindowFunction; +use datafusion::prelude::*; +use datafusion_examples::utils::datasets::ExampleDataset; + +/// This example demonstrates aggregation with the DataFrame API. +/// +/// `DataFrame::select` accepts aggregate expressions: a list made only of +/// aggregates (and scalar expressions over them) is planned as a global +/// aggregation, exactly like `aggregate(vec![], ...)`. Grouping still goes +/// through `aggregate(group_by, aggs)`. +/// +/// * [global_aggregate]: `aggregate(vec![], aggs)` vs `select(aggs)` +/// * [expressions_over_aggregates]: compute on top of aggregates in a single `select` +/// * [grouped_aggregate]: GROUP BY uses `aggregate`; `select` never infers a GROUP BY +/// * [aggregate_vs_window]: aggregate (one row) vs window (one row per input row) +pub async fn aggregate_example() -> Result<()> { + let ctx = SessionContext::new(); + let dataset = ExampleDataset::Cars; + ctx.register_csv("cars", dataset.path_str()?, CsvReadOptions::default()) + .await?; + + global_aggregate(&ctx).await?; + expressions_over_aggregates(&ctx).await?; + grouped_aggregate(&ctx).await?; + aggregate_vs_window(&ctx).await?; + Ok(()) +} + +/// SELECT count(car) AS n, +/// avg(speed) AS avg_speed, +/// max(speed) AS max_speed +/// FROM cars +async fn global_aggregate(ctx: &SessionContext) -> Result<()> { + let df = ctx.table("cars").await?; + let aggs = vec![ + count(col("car")).alias("n"), + avg(col("speed")).alias("avg_speed"), + max(col("speed")).alias("max_speed"), + ]; + + // The explicit form: `aggregate` with no group expressions + println!("global aggregation via aggregate(vec![], aggs)"); + df.clone().aggregate(vec![], aggs.clone())?.show().await?; + + // The same query via `select`: a list containing only aggregates is a + // global aggregation and returns exactly one row + println!("global aggregation via select(aggs)"); + df.select(aggs)?.show().await?; + + Ok(()) +} + +/// SELECT max(speed) - min(speed) AS speed_range, +/// avg(speed) * 3.6 AS avg_speed_kmh +/// FROM cars +async fn expressions_over_aggregates(ctx: &SessionContext) -> Result<()> { + let df = ctx.table("cars").await?; + + // Before: aggregate first, then project over the aggregate's output columns Review Comment: "Before" what? Readers in the future won't understand new vs old APIs for this. I'd describe the API shape or how the two forms are different. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
