alexanderbianchi commented on code in PR #67:
URL: https://github.com/apache/datafusion-iceberg/pull/67#discussion_r4577171974
##########
crates/datafusion/src/read/scan.rs:
##########
@@ -145,32 +216,128 @@ impl IcebergDataSourceBuilder {
self
}
+ /// Finish configuring the scan and optionally select its files before
execution.
+ ///
+ /// Honors [`IcebergPlanningConfig::eager_file_planning`]. The returned
builder
+ /// only exposes `build`, so filters, projection, and snapshot cannot
change
+ /// after files have been selected.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the projection is invalid, file planning fails, or
+ /// the retained tasks exceed the query memory pool.
+ pub async fn with_planned_files(
+ self,
+ context: Arc<TaskContext>,
+ ) -> Result<PlannedIcebergDataSourceBuilder> {
+ if !context
+ .session_config()
+ .options()
+ .extensions
+ .get::<IcebergPlanningConfig>()
+ .is_some_and(|config| config.eager_file_planning)
+ {
+ return Ok(PlannedIcebergDataSourceBuilder {
+ builder: self,
+ planned: None,
+ });
+ }
+ let target_partitions = context.session_config().target_partitions();
+ let schema = self.projected_schema()?;
+ let projection: Vec<String> = schema
+ .fields()
+ .iter()
+ .map(|field| field.name().clone())
+ .collect();
+ let scan = build_table_scan(
+ &self.table,
+ self.snapshot_id,
+ Some(&projection),
+ self.predicate.as_ref(),
+ )?;
+ // Dropping physical planning drops collection and its memory
reservation.
+ let planned = PlannedFileScan::collect(scan, &context).await?;
+ let partitions =
planned.tasks().len().max(1).min(target_partitions.max(1));
+
+ Ok(PlannedIcebergDataSourceBuilder {
+ builder: self,
+ planned: Some(PlannedScan {
+ files: Arc::new(planned),
+ partitions,
+ }),
+ })
+ }
+
/// Builds the source, which reads the columns of its projected schema by
name.
///
/// # Errors
///
/// Returns an error if the projection is out of bounds for the schema.
pub fn build(self) -> Result<IcebergDataSource> {
- let schema = match &self.projection {
- None => self.schema,
- Some(projection) => Arc::new(self.schema.project(projection)?),
- };
+ self.build_with_planned_files(None)
+ }
+
+ fn projected_schema(&self) -> Result<ArrowSchemaRef> {
+ match &self.projection {
+ None => Ok(self.schema.clone()),
+ Some(projection) => Ok(Arc::new(self.schema.project(projection)?)),
+ }
+ }
+
+ fn build_with_planned_files(
+ self,
+ planned: Option<PlannedScan>,
+ ) -> Result<IcebergDataSource> {
+ let schema = self.projected_schema()?;
let projection = schema
.fields()
.iter()
.map(|field| field.name().clone())
.collect();
+ let reader = self.table.reader_builder();
+ let reader = if planned
+ .as_ref()
+ .is_some_and(|planned| planned.partitions > 1)
+ {
+ reader.with_data_file_concurrency_limit(1)
+ } else {
+ reader
+ };
+ let reader = ScanReader(reader.build());
Ok(IcebergDataSource {
table: self.table,
snapshot_id: self.snapshot_id,
schema,
projection,
predicate: self.predicate,
fetch: self.fetch,
+ planned,
+ reader,
})
}
}
+/// A configured scan ready to build after optional eager file planning.
+///
+/// Created by [`IcebergDataSourceBuilder::with_planned_files`]. Configuration
+/// is fixed so the retained tasks stay consistent with the resulting source.
+#[derive(Debug, Clone)]
+pub struct PlannedIcebergDataSourceBuilder {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]