gabotechs commented on code in PR #113: URL: https://github.com/apache/datafusion-iceberg/pull/113#discussion_r4626725223
########## crates/datafusion/src/read/file_scan_task_provider/lazy.rs: ########## @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +use std::fmt::Formatter; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::stats::Precision; +use datafusion::common::{Result, exec_err}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::Partitioning; +use datafusion::prelude::Expr; +use futures::TryStreamExt; +use iceberg::expr::Predicate; +use iceberg::table::Table; + +use super::eager::EagerFileScanTaskProvider; +use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext}; +use crate::IcebergDataFusionConfig; +use crate::read::convert_filters_to_predicate; +use crate::to_datafusion_error; + +/// Snapshot summary field from the spec's "Optional Snapshot Summary Fields". +const TOTAL_RECORDS: &str = "total-records"; + +/// Discovers files while executing, in a single partition: the files of a +/// snapshot that may match a predicate. +#[derive(Debug, Clone)] +pub struct LazyFileScanTaskProvider { + table: Table, + schema: SchemaRef, + snapshot_id: Option<i64>, + projection: Vec<String>, + predicate: Option<Predicate>, +} + +impl LazyFileScanTaskProvider { + /// Consumes discovery and retains its tasks in round-robin groups, using the + /// context's target partition count. The result no longer retains the table. + /// Empty discovery produces one empty group. Dropping this future drops the + /// discovery stream; this method does not spawn a background collector. + pub async fn into_eager( + self, + context: Arc<TaskContext>, + ) -> Result<EagerFileScanTaskProvider> { + let target = context.session_config().target_partitions(); + if target == 0 { + return exec_err!( + "Iceberg eager planning requires at least one target partition" + ); + } + let estimated_num_rows = self.estimated_num_rows(); + let mut tasks = Arc::new(self).open( + 0, + FileScanTaskProviderContext { + task_context: &context, + }, + )?; + let mut groups = Vec::new(); + let mut partition = 0; + while let Some(task) = tasks.try_next().await.map_err(to_datafusion_error)? { + if partition == groups.len() { + groups.push(Vec::new()); + } + groups[partition].push(task); + partition = (partition + 1) % target; + } + if groups.is_empty() { + groups.push(Vec::new()); + } + let mut provider = EagerFileScanTaskProvider::try_new(groups)?; + provider.estimated_num_rows = estimated_num_rows; + Ok(provider) + } + + /// Table retained for discovery, including its metadata and storage access. + pub fn table(&self) -> &Table { + &self.table + } + + /// Output schema after projection, matching the columns selected by discovery. + pub fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + /// Names of the columns selected during discovery, in output schema order. + pub fn projection(&self) -> &[String] { + &self.projection + } + + /// Snapshot to scan, or the table's current one if `None`. + pub fn snapshot_id(&self) -> Option<i64> { + self.snapshot_id + } + + /// Filter used to skip files, if any. + pub fn predicate(&self) -> Option<&Predicate> { + self.predicate.as_ref() + } +} + +/// Builds a task provider and its projected schema from a table scan request. +/// +/// Configuration is supplied at construction. `iceberg.execution.eager_file_planning` +/// selects eager collection; otherwise discovery remains deferred until execution. +/// The projected schema is returned alongside the provider so callers never need +/// to recover it by downcasting an eager or lazy implementation. Reader settings +/// and fetch belong to the data-source builder. +/// +/// ``` +/// # use std::sync::Arc; +/// # use datafusion::{arrow::datatypes::SchemaRef, common::Result, execution::TaskContext, prelude::Expr}; +/// # use iceberg::table::Table; +/// # use iceberg_datafusion::read::{FileScanTaskProvider, FileScanTaskProviderBuilder}; +/// async fn plan_tasks( +/// context: Arc<TaskContext>, +/// table: Table, +/// schema: SchemaRef, +/// projection: Option<&Vec<usize>>, +/// filters: &[Expr], +/// ) -> Result<(SchemaRef, Arc<dyn FileScanTaskProvider>)> { +/// FileScanTaskProviderBuilder::new(context, table, schema) +/// .with_projection(projection) +/// .with_filters(filters) +/// .build() +/// .await +/// } +/// ``` +#[derive(Debug, Clone)] +pub struct FileScanTaskProviderBuilder { + context: Arc<TaskContext>, + table: Table, + schema: SchemaRef, + projection: Option<Vec<usize>>, + snapshot_id: Option<i64>, + predicate: Option<Predicate>, +} + +impl FileScanTaskProviderBuilder { + /// Starts with the planning context and the schema advertised by the table + /// provider. Projection indices refer to this schema, not the table's current + /// schema. Pass an already-projected schema and omit projection to retain it. + pub fn new(context: Arc<TaskContext>, table: Table, schema: SchemaRef) -> Self { + Self { + context, + table, + schema, + projection: None, + snapshot_id: None, + predicate: None, + } + } + + /// Selects columns from the supplied schema in this order, or all its columns + /// for `None`. Indices are validated when [`Self::build`] is called. + pub fn with_projection(mut self, projection: Option<&Vec<usize>>) -> Self { + self.projection = projection.cloned(); + self + } + + /// Selects a snapshot, or the retained table's current snapshot for `None`. + /// The supplied schema is not replaced with the snapshot's schema. + pub fn with_snapshot_id(mut self, snapshot_id: Option<i64>) -> Self { + self.snapshot_id = snapshot_id; + self + } + + /// Sets the predicate used to prune files and filter their rows. + pub fn with_predicate(mut self, predicate: Option<Predicate>) -> Self { + self.predicate = predicate; + self + } + + /// Pushes down supported parts of DataFusion filters. The caller must still + /// apply the original filters, as unsupported parts are omitted. + pub fn with_filters(self, filters: &[Expr]) -> Self { + self.with_predicate(convert_filters_to_predicate(filters)) + } + + /// Validates projection and builds the configured eager or lazy provider. + /// Lazy mode performs no file discovery; eager mode awaits [`LazyFileScanTaskProvider::into_eager`]. + /// The returned schema matches the columns selected by both implementations. + /// + /// # Errors + /// + /// Returns projection errors, or discovery errors when eager planning is enabled. + pub async fn build(self) -> Result<(SchemaRef, Arc<dyn FileScanTaskProvider>)> { + let schema = match self.projection { + Some(projection) => Arc::new(self.schema.project(&projection)?), + None => self.schema, + }; + let projection = schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect(); + let discovery = LazyFileScanTaskProvider { + table: self.table, + schema: Arc::clone(&schema), + projection, + snapshot_id: self.snapshot_id, + predicate: self.predicate, + }; + let provider: Arc<dyn FileScanTaskProvider> = + if IcebergDataFusionConfig::from_config_or_default( + self.context.session_config(), + ) + .execution + .eager_file_planning + { + Arc::new(discovery.into_eager(self.context).await?) + } else { + Arc::new(discovery) + }; Review Comment: Nice ########## crates/datafusion/src/read/file_scan_task_provider/lazy.rs: ########## @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +use std::fmt::Formatter; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::stats::Precision; +use datafusion::common::{Result, exec_err}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::Partitioning; +use datafusion::prelude::Expr; +use futures::TryStreamExt; +use iceberg::expr::Predicate; +use iceberg::table::Table; + +use super::eager::EagerFileScanTaskProvider; +use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext}; +use crate::IcebergDataFusionConfig; +use crate::read::convert_filters_to_predicate; +use crate::to_datafusion_error; + +/// Snapshot summary field from the spec's "Optional Snapshot Summary Fields". +const TOTAL_RECORDS: &str = "total-records"; + +/// Discovers files while executing, in a single partition: the files of a +/// snapshot that may match a predicate. +#[derive(Debug, Clone)] +pub struct LazyFileScanTaskProvider { + table: Table, + schema: SchemaRef, + snapshot_id: Option<i64>, + projection: Vec<String>, + predicate: Option<Predicate>, +} + +impl LazyFileScanTaskProvider { + /// Consumes discovery and retains its tasks in round-robin groups, using the + /// context's target partition count. The result no longer retains the table. + /// Empty discovery produces one empty group. Dropping this future drops the + /// discovery stream; this method does not spawn a background collector. + pub async fn into_eager( Review Comment: Really like this! it feels like pieces fitting together smoothly ########## crates/datafusion/tests/file_scan_task_provider.rs: ########## @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; +use std::sync::Arc; + +use datafusion::arrow::array::RecordBatch; +use datafusion::common::Result; +use datafusion::common::stats::Precision; +use datafusion::datasource::source::{DataSource, DataSourceExec}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::{ExecutionPlan, displayable}; +use futures::TryStreamExt; +use iceberg_datafusion::IcebergDataFusionConfig; +use iceberg_datafusion::read::{EagerFileScanTaskProvider, IcebergDataSource}; +use iceberg_datafusion::test_utils::IcebergTestHarness; + +#[tokio::test] +async fn lazy_scan_filters_and_projects_taxi_rows() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_config(|config| config.with_target_partitions(2)) + .build() + .await?; + + let (plan, rows) = harness + .query( + "SELECT pickup_date, COUNT(*) AS trips FROM taxi \ + WHERE pickup_date IN (DATE '2024-01-10', DATE '2024-01-11') \ + GROUP BY pickup_date ORDER BY pickup_date", + ) + .await?; + + insta::assert_snapshot!(plan, @" + SortPreservingMergeExec: [pickup_date@0 ASC NULLS LAST] + ProjectionExec: expr=[pickup_date@0 as pickup_date, count(Int64(1))@1 as trips] + SortExec: expr=[pickup_date@0 ASC NULLS LAST], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([pickup_date@0], 2), input_partitions=2 + AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] + FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 + RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=lazy, predicate=(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11) + "); + insta::assert_snapshot!(rows, @" + +-------------+-------+ + | pickup_date | trips | + +-------------+-------+ + | 2024-01-10 | 25000 | + | 2024-01-11 | 25000 | + +-------------+-------+ + "); + Ok(()) +} + +#[tokio::test] +async fn eager_scan_plans_only_the_matching_taxi_files() -> Result<()> { + // Taxi has seven daily files; this query needs only two of them. + let harness = IcebergTestHarness::builder() + .with_config(|config| { + config + .with_target_partitions(2) + .with_option_extension(IcebergDataFusionConfig::default()) + .set_bool("iceberg.execution.eager_file_planning", true) + }) + .build() + .await?; + + let (plan, rows) = harness + .query( + "SELECT pickup_date, COUNT(*) AS trips FROM taxi \ + WHERE pickup_date IN (DATE '2024-01-10', DATE '2024-01-11') \ + GROUP BY pickup_date ORDER BY pickup_date", + ) + .await?; + + insta::assert_snapshot!(plan, @" + SortPreservingMergeExec: [pickup_date@0 ASC NULLS LAST] + ProjectionExec: expr=[pickup_date@0 as pickup_date, count(Int64(1))@1 as trips] + SortExec: expr=[pickup_date@0 ASC NULLS LAST], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([pickup_date@0], 2), input_partitions=2 + AggregateExec: mode=Partial, gby=[pickup_date@0 as pickup_date], aggr=[count(Int64(1))] + FilterExec: pickup_date@0 = 2024-01-10 OR pickup_date@0 = 2024-01-11 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=2, predicates=[(pickup_date = 2024-01-10) OR (pickup_date = 2024-01-11)] + "); + insta::assert_snapshot!(rows, @" + +-------------+-------+ + | pickup_date | trips | + +-------------+-------+ + | 2024-01-10 | 25000 | + | 2024-01-11 | 25000 | + +-------------+-------+ + "); + Ok(()) +} + +#[tokio::test] +async fn eager_scan_with_no_matching_files_is_executable() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_config(|config| { + config + .with_target_partitions(2) + .with_option_extension(IcebergDataFusionConfig::default()) + .set_bool("iceberg.execution.eager_file_planning", true) + }) + .build() + .await?; + + let (plan, rows) = harness + .query("SELECT COUNT(*) AS trips FROM taxi WHERE pickup_date = DATE '2025-01-01'") + .await?; + + insta::assert_snapshot!(plan, @" + ProjectionExec: expr=[count(Int64(1))@0 as trips] + AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + CoalescePartitionsExec + AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + FilterExec: pickup_date@0 = 2025-01-01, projection=[] + RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 + DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=0 + "); + insta::assert_snapshot!(rows, @" + +-------+ + | trips | + +-------+ + | 0 | + +-------+ + "); + Ok(()) +} + +#[tokio::test] +async fn replacing_the_provider_preserves_the_fetch_limit() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_config(|config| { + config + .with_target_partitions(2) + .with_option_extension(IcebergDataFusionConfig::default()) + .set_bool("iceberg.execution.eager_file_planning", true) + }) + .build() + .await?; + let plan = harness.plan("SELECT pickup_date FROM taxi LIMIT 2").await?; + let source = iceberg_source(&plan).unwrap(); + let assigned_file = eager_files(source).tasks().next().unwrap().clone(); + + let assigned = source.with_file_scan_task_provider(Arc::new( + EagerFileScanTaskProvider::try_new(vec![vec![assigned_file]])?, + ))?; + let assigned_plan = DataSourceExec::from_data_source(assigned); + + insta::assert_snapshot!(displayable(assigned_plan.as_ref()).indent(true).to_string(), @"DataSourceExec: format=iceberg, projection=[pickup_date], provider=eager, planned_files=1, fetch=2"); + Ok(()) +} + +#[tokio::test] +async fn eager_snapshot_statistics_apply_fetch_per_partition() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_config(|config| { + config + .with_target_partitions(2) + .with_option_extension(IcebergDataFusionConfig::default()) + .set_bool("iceberg.execution.eager_file_planning", true) + }) + .build() + .await?; + let plan = harness.plan("SELECT pickup_date FROM taxi LIMIT 2").await?; + let source = iceberg_source(&plan).unwrap(); + + // Eager conversion retains the snapshot estimate. Fetch bounds each Review Comment: Just merged that, in case you want to try it out ########## crates/datafusion/src/read/scan.rs: ########## @@ -228,66 +230,81 @@ impl DataSource for IcebergDataSource { partition: usize, context: Arc<TaskContext>, ) -> Result<SendableRecordBatchStream> { - let fut = get_batch_stream( - self.table.clone(), - self.snapshot_id, - Some(self.column_names()), - self.predicate.clone(), - context - .session_config() - .options() - .execution - .parquet - .enable_page_index, - context.session_config().batch_size(), - ); - let stream = futures::stream::once(fut).try_flatten(); - let stream = Box::pin(RecordBatchStreamAdapter::new(self.schema.clone(), stream)); - // With no fetch, the limit passes every row through, still recording - // the output metrics. + if partition >= self.output_partitioning().partition_count() { + return exec_err!("Invalid Iceberg scan partition {partition}"); + } + let tasks = Arc::clone(&self.file_scan_task_provider).open( + partition, + FileScanTaskProviderContext { + task_context: &context, + }, + )?; + let stream = self + .reader + .clone() + .read(tasks) + .map_err(to_datafusion_error)? + .stream() + .map_err(to_datafusion_error); Ok(Box::pin(LimitStream::new( - stream, + Box::pin(RecordBatchStreamAdapter::new(self.schema.clone(), stream)), 0, self.fetch, BaselineMetrics::new(&self.metrics, partition), ))) } - fn fmt_as( - &self, - _t: DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!(f, "format=iceberg",)?; - write!(f, ", projection=[{}]", self.column_names().join(", "))?; - if let Some(predicate) = &self.predicate { - write!(f, ", predicate={predicate}")?; - } + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + let projection: Vec<&str> = self + .schema + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect(); + write!(f, "format=iceberg, projection=[{}]", projection.join(", "))?; + self.file_scan_task_provider.fmt_as(f)?; if let Some(fetch) = self.fetch { write!(f, ", fetch={fetch}")?; } Ok(()) } fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) + self.file_scan_task_provider.partitioning() + } + + fn metrics(&self) -> ExecutionPlanMetricsSet { + self.metrics.clone() Review Comment: At some point, we are going to want to merge these with the ones provided by the `file_scan_task_provider`, which will likely capture its own stats. ########## crates/datafusion/src/read/file_scan_task_provider/provider.rs: ########## @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +use std::any::Any; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +use datafusion::common::Result; +use datafusion::common::stats::Precision; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::Partitioning; + +/// Supplies the file tasks an [`IcebergDataSource`](crate::read::IcebergDataSource) +/// reads, split into output partitions. +/// +/// The data source owns reading, limits and metrics; implementations only +/// decide which files each partition reads and when they are discovered. +/// Implementations are `Any`, so consumers such as codecs can downcast a +/// source's [`file_scan_task_provider`](crate::read::IcebergDataSource::file_scan_task_provider) +/// to a concrete type: +/// +/// ``` +/// # use std::any::Any; +/// # use iceberg_datafusion::read::{IcebergDataSource, EagerFileScanTaskProvider}; +/// fn planned_files(source: &IcebergDataSource) -> Option<&EagerFileScanTaskProvider> { +/// (source.file_scan_task_provider() as &dyn Any).downcast_ref::<EagerFileScanTaskProvider>() +/// } +/// ``` +pub trait FileScanTaskProvider: Any + Debug + Send + Sync { + /// How the data source's output is partitioned, with at least one + /// partition. Any expressions refer to columns of the data source's + /// projected schema. + fn partitioning(&self) -> Partitioning; + + /// Streams the file tasks of `partition`, which is below the + /// [`Self::partitioning`] partition count. + fn open( + self: Arc<Self>, + partition: usize, + context: FileScanTaskProviderContext, + ) -> Result<iceberg::scan::FileScanTaskStream>; + + /// Estimated rows across all partitions before the source's fetch, without + /// discovering or reading files. Use an inexact estimate when predicates or + /// deletes may remove rows. Unknown by default for externally supplied tasks. + fn estimated_num_rows(&self) -> Precision<usize> { + Precision::Absent + } + Review Comment: I wonder how this will evolve in the future. I can imagine how the eager version has something to say not just about row count, but also about byte size or even per-column NDVs... I see in this PR the way this work is: - The FileScanTableProvider is responsible for just estimating the row count - The IcebergDataSource is responsible for building an `Statistics` object inferred from the row count What do you think about slightly changing the shape so that: - The FileScanTableProvider returns a full `Statistics` object, whatever it can provide, it will - The IcebergDataSource receives the `Statistics` from the `FileScanTableProvider, and it back-fills whatever the FileScanTableProvider left as `Precision::Absent` in a best-effort manner That does not sound like it corners us, and I'd expect to provide better public API stability guarantees, otherwise it's very likely that we'll need to change the `estimated_num_rows` method very soon. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
