gabotechs commented on code in PR #67:
URL: https://github.com/apache/datafusion-iceberg/pull/67#discussion_r4597017317


##########
crates/datafusion/src/read/builder.rs:
##########
@@ -0,0 +1,453 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Session-configured construction of table scans and already-planned scans.
+
+use std::fmt::{Debug, Formatter};
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::common::Result;
+use datafusion::datasource::source::DataSource;
+use datafusion::execution::TaskContext;
+use datafusion::prelude::Expr;
+use iceberg::arrow::ArrowReader;
+use iceberg::expr::Predicate;
+use iceberg::io::FileIO;
+use iceberg::scan::FileScanTask;
+use iceberg::table::Table;
+
+use super::eager::EagerIcebergDataSource;
+use super::file_planning::PlannedFileScan;
+use super::predicate::convert_filters_to_predicate;
+use super::scan::{IcebergDataSource, build_table_scan};
+use crate::IcebergDataFusionConfig;
+
+/// Builds a scan using the session settings and memory pool in a task context.
+///
+/// Obtain the context from `SessionContext::task_ctx()` or the `Session` 
passed to
+/// a table provider. Eager planning, target partitions, batch size and 
page-index
+/// settings come from that context, not separate builder options.
+///
+/// Select either a table or already-planned files after supplying the schema.
+/// Only table input exposes discovery projection, snapshot and filter 
settings.
+/// Table scans finish asynchronously; reconstruction finishes synchronously 
and
+/// never discovers files. Both return an `Arc<dyn DataSource>`.
+///
+/// ```
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::common::Result;
+/// # use datafusion::datasource::source::DataSource;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::table::Table;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// # use std::sync::Arc;
+/// async fn scan(table: Table, schema: SchemaRef, context: &TaskContext)
+///     -> Result<Arc<dyn DataSource>>
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_table(table)
+///         .build()
+///         .await
+/// }
+/// ```
+///
+/// Reconstruction uses the same entry point but has no asynchronous discovery:
+/// ```
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::common::Result;
+/// # use datafusion::datasource::source::DataSource;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::arrow::ArrowReader;
+/// # use iceberg::io::FileIO;
+/// # use iceberg::scan::FileScanTask;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// # use std::sync::Arc;
+/// fn restore(context: &TaskContext, schema: SchemaRef,
+///     groups: Vec<Vec<FileScanTask>>, file_io: FileIO, reader: ArrowReader)
+///     -> Result<Arc<dyn DataSource>>
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_planned_files(groups)
+///         .with_file_io(file_io)
+///         .with_reader(reader)
+///         .build()
+/// }
+/// ```
+///
+/// Planned input cannot request snapshot discovery:
+/// ```compile_fail,E0599
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::arrow::ArrowReader;
+/// # use iceberg::io::FileIO;
+/// # use iceberg::scan::FileScanTask;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// fn invalid(context: &TaskContext, schema: SchemaRef,
+///     groups: Vec<Vec<FileScanTask>>, file_io: FileIO, reader: ArrowReader)
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_planned_files(groups)
+///         .with_file_io(file_io)
+///         .with_reader(reader)
+///         .with_snapshot_id(Some(1));
+/// }
+/// ```
+///
+/// Reconstruction cannot finish without a process-local reader:
+/// ```compile_fail,E0599
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::io::FileIO;
+/// # use iceberg::scan::FileScanTask;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// fn invalid(context: &TaskContext, schema: SchemaRef,
+///     groups: Vec<Vec<FileScanTask>>, file_io: FileIO)
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_planned_files(groups)
+///         .with_file_io(file_io)
+///         .build();
+/// }
+/// ```
+#[derive(Clone)]
+#[must_use]
+pub struct IcebergDataSourceBuilder<'a, Input = NeedsSchema> {
+    context: &'a TaskContext,
+    input: Input,
+    fetch: Option<usize>,

Review Comment:
   Pretty fancy! Unfortunately this still has the same drawbacks of the 
positional arguments approach: if we need to require one more input, then 
there'll be a new `Input = NewRequiredState` and the API will still break but 
in a more convoluted way.
   
   Even if I like the type system trickery being done here, I don't think it's 
contributing positively to the ergonomics of the `IcebergDataSourceBuilder`, as 
it does not solve the breaking API problem, and it brings a significant amount 
of complexity.
   
   I think you might have taken my comment in 
https://github.com/apache/datafusion-iceberg/pull/67#discussion_r4585121006 too 
literally, keeping the builder follow a builder pattern is good, but maybe not 
at all costs, and even if this does adhere very strictly to the builder 
pattern, the cost is a decent amount of complexity without really solving the 
breaking APIs issue.



##########
crates/datafusion/tests/common/scan_fixture.rs:
##########
@@ -0,0 +1,240 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::error::Error;
+use std::fs::File;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray};
+use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+use iceberg::io::FileIO;
+use iceberg::metadata_columns::{
+    RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS,
+    RESERVED_FIELD_ID_DELETE_FILE_PATH, RESERVED_FIELD_ID_DELETE_FILE_POS,
+};
+use iceberg::spec::{
+    DataContentType, DataFileBuilder, DataFileFormat, FormatVersion, 
ManifestListWriter,
+    ManifestWriterBuilder, NestedField, Operation, PrimitiveType, Schema, 
Snapshot,
+    SortOrder, Struct, Summary, TableMetadataBuilder, Type, 
UnboundPartitionSpec,
+};
+use iceberg::table::Table;
+use iceberg::{Runtime, TableIdent};
+use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY};
+use tempfile::TempDir;
+
+/// A small unpartitioned table with an `id` column. Owns its temporary files.
+pub struct ScanFixture {

Review Comment:
   This is the main thing I'm trying to avoid with 
https://github.com/apache/datafusion-iceberg/pull/87, I'd really like to avoid 
having huge one-off fixtures that are only useful for one integration test.
   
   I don't want to block this PR just for this, but I would expect this custom 
fixture to go away when #87 lands.



##########
crates/datafusion/src/config.rs:
##########
@@ -0,0 +1,142 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::any::Any;
+
+use datafusion::common::{Result, config_err, extensions_options};
+use datafusion::config::{ConfigEntry, ConfigExtension, ExtensionOptions};
+
+/// Session options for Iceberg tables, under `iceberg.<group>.<option>`.
+///
+/// Register this extension before setting Iceberg options:
+/// ```
+/// use datafusion::prelude::SessionConfig;
+/// use iceberg_datafusion::IcebergDataFusionConfig;
+///
+/// let config = SessionConfig::new()
+///     .with_option_extension(IcebergDataFusionConfig::default())
+///     .set_bool("iceberg.execution.eager_file_planning", true);
+/// ```
+#[derive(Debug, Clone, Default)]
+#[non_exhaustive]
+pub struct IcebergDataFusionConfig {
+    /// Options for executing table scans.
+    pub execution: IcebergExecutionConfig,
+}
+
+// The nested extensions_options! macro drops parent names from entries(). Keep
+// the group here so every listed key can be passed back to set(), including 
when
+// consumers propagate options to another session or process.
+impl ExtensionOptions for IcebergDataFusionConfig {
+    fn as_any(&self) -> &dyn Any {

Review Comment:
   You should not need a manual implementation of this, I think you should have 
some examples upstream about how to nest config structs inside of other configs.
   
   You probably want to look at `config_namespace!` instead.



##########
crates/datafusion/src/read/eager.rs:
##########
@@ -0,0 +1,196 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::common::tree_node::TreeNodeRecursion;
+use datafusion::common::{Statistics, exec_err};
+use datafusion::datasource::source::DataSource;
+use datafusion::error::Result;
+use datafusion::execution::{SendableRecordBatchStream, TaskContext};
+use datafusion::physical_expr::projection::ProjectionExprs;
+use datafusion::physical_expr::{EquivalenceProperties, Partitioning, 
PhysicalExpr};
+use datafusion::physical_plan::DisplayFormatType;
+use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
+use futures::StreamExt;
+use iceberg::arrow::ArrowReader;
+use iceberg::io::FileIO;
+use iceberg::scan::FileScanTask;
+
+use super::file_planning::PlannedFileScan;
+use super::reader::ScanReader;
+
+/// An Iceberg scan whose complete file assignment is available before 
execution.
+/// Reads already-selected file groups without retaining a table or discovery 
request.
+/// Consumers can inspect and regroup these tasks, or reconstruct a source on 
another
+/// process using a locally configured reader.
+#[derive(Clone)]
+pub struct EagerIcebergDataSource {
+    schema: SchemaRef,
+    fetch: Option<usize>,
+    reader: ScanReader,
+    file_io: FileIO,
+    files: Arc<PlannedFileScan>,
+}
+impl std::fmt::Debug for EagerIcebergDataSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

Review Comment:
   A bit oddly formatted this file, there's no spaces anywhere. It passes 
`rustfmt` so I guess it's fine.



##########
crates/datafusion/src/read/builder.rs:
##########
@@ -0,0 +1,453 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Session-configured construction of table scans and already-planned scans.
+
+use std::fmt::{Debug, Formatter};
+use std::sync::Arc;
+
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::common::Result;
+use datafusion::datasource::source::DataSource;
+use datafusion::execution::TaskContext;
+use datafusion::prelude::Expr;
+use iceberg::arrow::ArrowReader;
+use iceberg::expr::Predicate;
+use iceberg::io::FileIO;
+use iceberg::scan::FileScanTask;
+use iceberg::table::Table;
+
+use super::eager::EagerIcebergDataSource;
+use super::file_planning::PlannedFileScan;
+use super::predicate::convert_filters_to_predicate;
+use super::scan::{IcebergDataSource, build_table_scan};
+use crate::IcebergDataFusionConfig;
+
+/// Builds a scan using the session settings and memory pool in a task context.
+///
+/// Obtain the context from `SessionContext::task_ctx()` or the `Session` 
passed to
+/// a table provider. Eager planning, target partitions, batch size and 
page-index
+/// settings come from that context, not separate builder options.
+///
+/// Select either a table or already-planned files after supplying the schema.
+/// Only table input exposes discovery projection, snapshot and filter 
settings.
+/// Table scans finish asynchronously; reconstruction finishes synchronously 
and
+/// never discovers files. Both return an `Arc<dyn DataSource>`.
+///
+/// ```
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::common::Result;
+/// # use datafusion::datasource::source::DataSource;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::table::Table;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// # use std::sync::Arc;
+/// async fn scan(table: Table, schema: SchemaRef, context: &TaskContext)
+///     -> Result<Arc<dyn DataSource>>
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_table(table)
+///         .build()
+///         .await
+/// }
+/// ```
+///
+/// Reconstruction uses the same entry point but has no asynchronous discovery:
+/// ```
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::common::Result;
+/// # use datafusion::datasource::source::DataSource;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::arrow::ArrowReader;
+/// # use iceberg::io::FileIO;
+/// # use iceberg::scan::FileScanTask;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// # use std::sync::Arc;
+/// fn restore(context: &TaskContext, schema: SchemaRef,
+///     groups: Vec<Vec<FileScanTask>>, file_io: FileIO, reader: ArrowReader)
+///     -> Result<Arc<dyn DataSource>>
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_planned_files(groups)
+///         .with_file_io(file_io)
+///         .with_reader(reader)
+///         .build()
+/// }
+/// ```
+///
+/// Planned input cannot request snapshot discovery:
+/// ```compile_fail,E0599
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::arrow::ArrowReader;
+/// # use iceberg::io::FileIO;
+/// # use iceberg::scan::FileScanTask;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// fn invalid(context: &TaskContext, schema: SchemaRef,
+///     groups: Vec<Vec<FileScanTask>>, file_io: FileIO, reader: ArrowReader)
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_planned_files(groups)
+///         .with_file_io(file_io)
+///         .with_reader(reader)
+///         .with_snapshot_id(Some(1));
+/// }
+/// ```
+///
+/// Reconstruction cannot finish without a process-local reader:
+/// ```compile_fail,E0599
+/// # use datafusion::arrow::datatypes::SchemaRef;
+/// # use datafusion::execution::TaskContext;
+/// # use iceberg::io::FileIO;
+/// # use iceberg::scan::FileScanTask;
+/// # use iceberg_datafusion::read::IcebergDataSourceBuilder;
+/// fn invalid(context: &TaskContext, schema: SchemaRef,
+///     groups: Vec<Vec<FileScanTask>>, file_io: FileIO)
+/// {
+///     IcebergDataSourceBuilder::new(context)
+///         .with_schema(schema)
+///         .with_planned_files(groups)
+///         .with_file_io(file_io)
+///         .build();
+/// }
+/// ```
+#[derive(Clone)]
+#[must_use]
+pub struct IcebergDataSourceBuilder<'a, Input = NeedsSchema> {
+    context: &'a TaskContext,
+    input: Input,
+    fetch: Option<usize>,
+}
+
+impl<Input> Debug for IcebergDataSourceBuilder<'_, Input> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IcebergDataSourceBuilder")
+            .field("fetch", &self.fetch)
+            .finish_non_exhaustive()
+    }
+}
+
+/// Initial builder state, before supplying the output schema.
+#[derive(Clone, Debug)]
+pub struct NeedsSchema;
+
+/// Builder state before selecting table discovery or already-planned files.
+#[derive(Clone, Debug)]
+pub struct NeedsInput {
+    schema: SchemaRef,
+}
+
+/// A table discovery request. Its settings cannot be applied to planned files.
+#[derive(Clone)]
+pub struct TableScanInput {
+    schema: SchemaRef,
+    table: Table,
+    projection: Option<Vec<usize>>,
+    snapshot_id: Option<i64>,
+    predicate: Option<Predicate>,
+    reader: Option<ArrowReader>,
+}
+
+/// Planned input whose FileIO and reader are supplied in subsequent steps.
+#[derive(Clone)]
+pub struct PlannedFilesInput<Reader> {
+    schema: SchemaRef,
+    groups: Vec<Vec<FileScanTask>>,
+    reader: Reader,
+}
+
+/// Planned files still require process-local file access.
+#[derive(Clone, Debug)]
+pub struct NeedsFileIo;
+
+/// Planned files have file access and still require a configured reader.
+#[derive(Clone, Debug)]
+pub struct NeedsReader {
+    file_io: FileIO,
+}
+
+/// Complete process-local file access for planned input.
+#[derive(Clone)]
+pub struct PlannedReader {
+    file_io: FileIO,
+    reader: ArrowReader,
+}
+
+impl<'a> IcebergDataSourceBuilder<'a> {
+    /// Starts construction with the session's execution context.
+    pub fn new(context: &'a TaskContext) -> Self {
+        Self {
+            context,
+            input: NeedsSchema,
+            fetch: None,
+        }
+    }
+
+    /// Supplies the provider's schema for discovery, or the already-projected
+    /// output schema when reconstructing planned files.
+    pub fn with_schema(
+        self,
+        schema: SchemaRef,
+    ) -> IcebergDataSourceBuilder<'a, NeedsInput> {
+        self.map_input(|_| NeedsInput { schema })
+    }
+}
+
+impl<'a, Input> IcebergDataSourceBuilder<'a, Input> {
+    fn map_input<Next>(
+        self,
+        f: impl FnOnce(Input) -> Next,
+    ) -> IcebergDataSourceBuilder<'a, Next> {
+        IcebergDataSourceBuilder {
+            context: self.context,
+            input: f(self.input),
+            fetch: self.fetch,
+        }
+    }
+
+    /// Replaces the execution context, including session settings and query 
memory
+    /// pool. Construction has not discovered files or reserved descriptor 
memory yet.
+    pub fn with_context<'b>(
+        self,
+        context: &'b TaskContext,
+    ) -> IcebergDataSourceBuilder<'b, Input> {
+        IcebergDataSourceBuilder {
+            context,
+            input: self.input,
+            fetch: self.fetch,
+        }
+    }
+
+    /// Stops each output partition after `fetch` rows, or reads all rows for 
`None`.
+    pub fn with_fetch(mut self, fetch: Option<usize>) -> Self {
+        self.fetch = fetch;
+        self
+    }
+}
+
+impl<'a> IcebergDataSourceBuilder<'a, NeedsInput> {
+    /// Selects discovery from a table. The session chooses eager or lazy 
planning.
+    pub fn with_table(
+        self,
+        table: Table,
+    ) -> IcebergDataSourceBuilder<'a, TableScanInput> {
+        self.map_input(|input| TableScanInput {
+            schema: input.schema,
+            table,
+            projection: None,
+            snapshot_id: None,
+            predicate: None,
+            reader: None,
+        })
+    }
+
+    /// Selects already-planned groups, each representing one output partition.
+    /// Supply FileIO and a process-local reader next. No table or catalog is 
needed.
+    /// Supplied group order and empty individual groups are preserved.
+    pub fn with_planned_files(

Review Comment:
   :thinking: I imagine this is one of the methods that need to be public so 
that reconstruction works.
   
   What if, rather than allowing folks to re-construct from the outside, we 
just provide the protobuf codecs ourselves in this project. Would that simplify 
the API we want to expose for constructing the datasources?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to