andygrove commented on code in PR #2784:
URL:
https://github.com/apache/datafusion-ballista/pull/2784#discussion_r4642993326
##########
integrations/iceberg/tests/scan.rs:
##########
@@ -42,36 +44,172 @@ mod standalone {
assert_standalone_scans(memory_catalog(dir.path()).await).await
}
- // A known limitation: a view stores its planned query, and tables are
- // pinned when a query is planned.
#[tokio::test]
- async fn should_keep_a_views_version() -> Result<()> {
+ async fn should_read_a_views_latest_version() -> Result<()> {
let dir = tempfile::tempdir()?;
- let catalog = memory_catalog(dir.path()).await;
- let state = SessionStateBuilder::new()
- .with_config(SessionConfig::new_with_ballista())
- .with_default_features()
- .build();
- let ctx =
SessionContext::standalone_with_state(register_iceberg(state)).await?;
- ctx.register_table("t", Arc::new(catalog_table(&catalog).await))?;
+ let (catalog, ctx) = view_context(dir.path()).await?;
insert(&catalog, "(1, 'a')").await;
ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?;
+ // A view over a view, reading `t` again in a subquery.
+ ctx.sql("CREATE VIEW w AS SELECT * FROM v WHERE id IN (SELECT id FROM
t)")
+ .await?;
+ insert(&catalog, "(2, 'b')").await;
+
+ for view in ["v", "w"] {
+ let sql = format!("SELECT * FROM {view} ORDER BY id");
+ let result = ctx.sql(&sql).await?.collect().await?;
+ assert_batches_eq!(LATEST, &result);
+ }
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn should_read_a_dataframe_views_latest_version() -> Result<()> {
+ let dir = tempfile::tempdir()?;
+ let (catalog, ctx) = view_context(dir.path()).await?;
+
+ insert(&catalog, "(1, 'a')").await;
+ let df = ctx.sql("SELECT * FROM t").await?;
+ ctx.register_table("v", df.clone().into_view())?;
+ ctx.register_table("tv", df.into_temporary_view())?;
insert(&catalog, "(2, 'b')").await;
- let result = ctx.sql("SELECT * FROM v").await?.collect().await?;
+ for (view, table_type) in [("v", TableType::View), ("tv",
TableType::Temporary)] {
+ let sql = format!("SELECT * FROM {view} ORDER BY id");
+ let result = ctx.sql(&sql).await?.collect().await?;
+ assert_batches_eq!(LATEST, &result);
+ assert_eq!(ctx.table_provider(view).await?.table_type(),
table_type);
+ }
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn should_read_a_view_created_while_its_table_was_gone() ->
Result<()> {
+ let dir = tempfile::tempdir()?;
+ let (catalog, ctx) = view_context(dir.path()).await?;
+
+ insert(&catalog, "(1, 'a')").await;
+ ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?;
+ // While `t` is gone, `w` is created over `v`, and reading it fails.
+ rename_table(&catalog, "t", "gone").await;
+ ctx.sql("CREATE VIEW w AS SELECT * FROM v").await?;
+ assert_table_gone(&ctx, "SELECT * FROM w").await;
+ // Once `t` is back, `w` reads its latest version.
+ rename_table(&catalog, "gone", "t").await;
+ insert(&catalog, "(2, 'b')").await;
+ let result = ctx
+ .sql("SELECT * FROM w ORDER BY id")
+ .await?
+ .collect()
+ .await?;
+ assert_batches_eq!(LATEST, &result);
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn should_fail_to_read_a_view_whose_table_is_gone() -> Result<()> {
+ let dir = tempfile::tempdir()?;
+ let (catalog, ctx) = view_context(dir.path()).await?;
+ ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?;
+ rename_table(&catalog, "t", "gone").await;
+
+ // Without a filter, the client fails to encode the view's scan.
+ assert_table_gone(&ctx, "SELECT * FROM v").await;
+ // With one, pushing the filter into the view's scan fails first.
+ assert_eq!(
Review Comment:
Small one. The filter case here still asserts the full string, including
DataFusion's optimizer wrapper and the iceberg error's debug text. Could it use
`assert_table_gone` like the others? Then a DataFusion or iceberg bump won't
break it.
##########
integrations/iceberg/src/catalog.rs:
##########
@@ -109,18 +125,20 @@ impl SchemaProvider for PinningSchema {
let Some(table) = self.0.table(name).await? else {
return Ok(None);
};
- let Some(provider) =
table.downcast_ref::<IcebergCatalogTableProvider>() else {
- return Ok(Some(table));
+ if let Some(provider) =
table.downcast_ref::<IcebergCatalogTableProvider>() {
+ let table = load(provider).await;
Review Comment:
One thing I noticed while tracing the error path. A plain `SELECT * FROM t`
on a table that was dropped used to fail at lookup with a clean `No such
table`. Now it fails when the client encodes the plan, so it comes back as an
`Internal` error with the "likely a bug in DataFusion" text from
`distributed_query.rs`. That's probably the most common way users will hit a
missing table, so it feels like a step back for them. Would you be up for
landing #2885 before or with this one? It would also be nice to have a test
that reads `t` directly while it's gone, alongside the view cases.
##########
integrations/iceberg/src/catalog.rs:
##########
@@ -145,8 +163,132 @@ impl SchemaProvider for PinningSchema {
}
}
+/// A catalog-backed table: its catalog, by address, and its identifier.
+type TableKey = (usize, TableIdent);
+
+fn key(provider: &IcebergCatalogTableProvider) -> TableKey {
+ let catalog = Arc::as_ptr(provider.catalog()).cast::<()>() as usize;
+ (catalog, provider.table_ident().clone())
+}
+
+/// A loaded table, or the error loading it returned, shared by every scan of
it.
+type Loaded = std::result::Result<Table, Arc<DataFusionError>>;
+
+/// The version of `provider`'s table its catalog serves now, or why it
couldn't
+/// be loaded.
+async fn load(provider: &IcebergCatalogTableProvider) -> Loaded {
+ provider
+ .catalog()
+ .load_table(provider.table_ident())
+ .await
+ .map_err(|error| Arc::new(to_datafusion_error(error)))
+}
+
+/// The catalog-backed table a scan's pinned table was pinned from, if any.
+fn origin(scan: &TableScan) -> Option<&IcebergCatalogTableProvider> {
+ let source = scan.source.downcast_ref::<DefaultTableSource>()?;
+ let pinned = source.table_provider.downcast_ref::<PinnedTable>()?;
+ pinned.origin.as_ref()
+}
+
+/// Returns a view's `plan` with each catalog-backed Iceberg table it reads
+/// pinned to the version its catalog serves now, loading each table once, or
+/// `None` if it reads none. A table that can't be loaded is pinned to the
error,
+/// so the plan still knows which table to load next time. DataFusion inlines a
+/// view when it plans a scan of it, so the plan of a view over other views
+/// already holds their tables, and needs no recursion.
+async fn repin(plan: &LogicalPlan) -> Result<Option<LogicalPlan>> {
+ let mut providers = HashMap::new();
+ plan.apply_with_subqueries(|node| {
+ if let LogicalPlan::TableScan(scan) = node
+ && let Some(origin) = origin(scan)
+ {
+ providers
+ .entry(key(origin))
+ .or_insert_with(|| origin.clone());
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ if providers.is_empty() {
+ return Ok(None);
+ }
+ let loads = providers
+ .into_iter()
+ .map(|(key, provider)| async move { (key, load(&provider).await) });
+ let tables: HashMap<_, _> = join_all(loads).await.into_iter().collect();
+ let plan = plan
+ .clone()
+ .transform_up_with_subqueries(|node| {
+ let LogicalPlan::TableScan(mut scan) = node else {
+ return Ok(Transformed::no(node));
+ };
+ let Some(origin) = origin(&scan) else {
+ return Ok(Transformed::no(LogicalPlan::TableScan(scan)));
+ };
+ let table = tables[&key(origin)].clone();
Review Comment:
Tiny robustness thought on `tables[&key(origin)]`. The two traversals should
always agree, but a `.get()` that leaves the scan alone if there's no entry
would avoid a panic if they ever drift apart. What do you think?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]