alexanderbianchi commented on code in PR #109:
URL:
https://github.com/apache/datafusion-iceberg/pull/109#discussion_r4608105493
##########
crates/datafusion/src/read/scan.rs:
##########
@@ -15,118 +15,138 @@
// specific language governing permissions and limitations
// under the License.
-use std::pin::Pin;
use std::sync::Arc;
-use datafusion::arrow::array::RecordBatch;
+use super::file_scan_task_provider::{
+ FileScanTaskProvider, FileScanTaskProviderContext,
LazyFileScanTaskProvider,
+};
+use super::predicate::convert_filters_to_predicate;
+use crate::read::EagerFileScanTaskProvider;
+use crate::to_datafusion_error;
+
use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
+use datafusion::catalog::Session;
use datafusion::common::Statistics;
use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::datasource::source::DataSource;
use datafusion::error::Result;
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
use datafusion::physical_expr::projection::ProjectionExprs;
use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr};
+use datafusion::physical_plan::limit::LimitStream;
+use datafusion::physical_plan::metrics::{BaselineMetrics,
ExecutionPlanMetricsSet};
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{DisplayFormatType, Partitioning};
use datafusion::prelude::Expr;
-use futures::{Stream, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
use iceberg::expr::Predicate;
+use iceberg::scan::FileScanTask;
use iceberg::table::Table;
-use super::predicate::convert_filters_to_predicate;
-use crate::to_datafusion_error;
-
-/// Manages the scanning process of an Iceberg [`Table`], encapsulating the
-/// necessary details and computed properties required for execution planning.
+/// Reads an Iceberg [`Table`]: the file tasks supplied by its
+/// [`FileScanTaskProvider`], as batches of its projected schema.
+///
+/// Built with [`IcebergDataSourceBuilder`],
+/// which discovers files lazily during execution. Swap in another provider
+/// with [`Self::with_file_scan_task_provider`].
#[derive(Debug, Clone)]
pub struct IcebergDataSource {
- /// A table in the catalog.
+ /// Table the files belong to, used to build the reader.
table: Table,
- /// Snapshot of the table to scan.
- snapshot_id: Option<i64>,
/// Schema returned by this data source after projection.
schema: ArrowSchemaRef,
- /// Projection column names in the output schema.
- projection: Vec<String>,
- /// Filter to apply to the table scan
- predicate: Option<Predicate>,
/// Optional limit on the number of rows to return
fetch: Option<usize>,
+ /// Files to read, by output partition.
+ file_scan_task_provider: Arc<dyn FileScanTaskProvider>,
+ /// Output rows and compute time recorded while reading.
+ metrics: ExecutionPlanMetricsSet,
}
impl IcebergDataSource {
- /// Starts building a source that scans `table` and returns the columns of
- /// `schema`, all of them unless
[`IcebergDataSourceBuilder::with_projection`]
- /// selects some. Columns are read from Iceberg by name.
- pub fn builder(table: Table, schema: ArrowSchemaRef) ->
IcebergDataSourceBuilder {
- IcebergDataSourceBuilder {
- table,
- schema,
- projection: None,
- snapshot_id: None,
- predicate: None,
- fetch: None,
- }
- }
-
pub fn table(&self) -> &Table {
&self.table
}
- pub fn snapshot_id(&self) -> Option<i64> {
- self.snapshot_id
- }
-
pub fn schema(&self) -> ArrowSchemaRef {
self.schema.clone()
}
- pub fn projection(&self) -> &[String] {
- &self.projection
+ pub fn fetch(&self) -> Option<usize> {
+ self.fetch
}
- pub fn predicate(&self) -> Option<&Predicate> {
- self.predicate.as_ref()
+ /// Supplies the files this source reads. Downcast to inspect a concrete
+ /// provider.
+ pub fn file_scan_task_provider(&self) -> &dyn FileScanTaskProvider {
+ self.file_scan_task_provider.as_ref()
}
- pub fn fetch(&self) -> Option<usize> {
- self.fetch
+ /// Returns a source that reads the files of `file_scan_task_provider`
+ /// instead, with fresh metrics. The
+ /// tasks must read this source's projected columns.
+ pub fn with_file_scan_task_provider(
+ &self,
+ file_scan_task_provider: Arc<dyn FileScanTaskProvider>,
+ ) -> Self {
+ Self {
+ file_scan_task_provider,
+ metrics: Default::default(),
+ ..self.clone()
+ }
}
}
-/// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`].
+/// Builds an [`IcebergDataSource`].
+///
+/// [`Self::new`] takes what every scan needs; everything else is optional and
+/// set with the `with_*` methods, each defaulting to `None`.
///
/// A table provider passes the table's schema and selects columns with
/// [`Self::with_projection`]. To rebuild a source, such as in another process,
/// pass its [`IcebergDataSource::schema`], which is already projected, and no
/// projection.
-#[derive(Debug, Clone)]
-pub struct IcebergDataSourceBuilder {
- table: Table,
+#[derive(Clone)]
+pub struct IcebergDataSourceBuilder<'a> {
+ table: &'a Table,
schema: ArrowSchemaRef,
projection: Option<Vec<usize>>,
snapshot_id: Option<i64>,
predicate: Option<Predicate>,
fetch: Option<usize>,
+ collected_file_scan_tasks: Option<Vec<Vec<FileScanTask>>>,
Review Comment:
This is kinda an abstraction leak from the FileScanTaskProvider. We don't
need the Table after we have the fileScanTasks and the Table itself is needed
only for Lazy.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]