gabotechs commented on code in PR #27099:
URL: https://github.com/apache/datafusion/pull/27099#discussion_r4611630619
##########
datafusion/substrait/tests/cases/roundtrip_logical_plan.rs:
##########
@@ -2755,6 +2756,140 @@ async fn
roundtrip_lambda_parameter_struct_field_names() -> Result<()> {
Ok(())
}
+/// Round trips `sql` and checks that the output type and rows don't change.
Substrait
+/// literals carry no struct field names, so a struct literal combined with a
struct
+/// column must take the column's names for the two to have the same type.
+///
+/// `sql` can read two tables: `struct_rows`, whose column `st` is a
`Struct{a, b}`, and
+/// `zones`, whose column `z` is a `List<Struct{subnet: Struct{id},
zone_name}>`. Plans
+/// are not compared as text: for a set operation, the consumer puts the
output alias
+/// above the union rather than in each input.
+async fn assert_struct_literal_roundtrips(sql: &str) -> Result<()> {
+ let ctx = higher_order_function_ctx().await?;
+ for table in [
+ "CREATE TABLE struct_rows AS SELECT named_struct('a', 1, 'b', 'x') AS
st \
+ UNION ALL SELECT NULL AS st",
+ "CREATE TABLE zones AS SELECT [named_struct('subnet',
named_struct('id', 'subnet-1'), \
+ 'zone_name', 'zone-a')] AS z UNION ALL SELECT NULL AS z",
+ ] {
+ ctx.sql(table).await?.collect().await?;
+ }
+
+ let plan = ctx.sql(sql).await?.into_optimized_plan()?;
+ let expected = DataFrame::new(ctx.state(), plan.clone()).collect().await?;
+ let plan2 = substrait_roundtrip(&plan, &ctx).await?;
+ assert_eq!(
+ plan.schema().field(0).data_type(),
+ plan2.schema().field(0).data_type()
+ );
+ let actual = DataFrame::new(ctx.state(), plan2).collect().await?;
+ assert_eq!(
+ batches_to_sort_string(&expected),
+ batches_to_sort_string(&actual)
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_coalesce() -> Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT coalesce(st, named_struct('a', 0, 'b', 'z')) FROM struct_rows",
+ )
+ .await
+}
Review Comment:
This tests look really good!
##########
datafusion/substrait/tests/cases/roundtrip_logical_plan.rs:
##########
@@ -2755,6 +2756,140 @@ async fn
roundtrip_lambda_parameter_struct_field_names() -> Result<()> {
Ok(())
}
+/// Round trips `sql` and checks that the output type and rows don't change.
Substrait
+/// literals carry no struct field names, so a struct literal combined with a
struct
+/// column must take the column's names for the two to have the same type.
+///
+/// `sql` can read two tables: `struct_rows`, whose column `st` is a
`Struct{a, b}`, and
+/// `zones`, whose column `z` is a `List<Struct{subnet: Struct{id},
zone_name}>`. Plans
+/// are not compared as text: for a set operation, the consumer puts the
output alias
+/// above the union rather than in each input.
+async fn assert_struct_literal_roundtrips(sql: &str) -> Result<()> {
+ let ctx = higher_order_function_ctx().await?;
+ for table in [
+ "CREATE TABLE struct_rows AS SELECT named_struct('a', 1, 'b', 'x') AS
st \
+ UNION ALL SELECT NULL AS st",
+ "CREATE TABLE zones AS SELECT [named_struct('subnet',
named_struct('id', 'subnet-1'), \
+ 'zone_name', 'zone-a')] AS z UNION ALL SELECT NULL AS z",
+ ] {
+ ctx.sql(table).await?.collect().await?;
+ }
+
+ let plan = ctx.sql(sql).await?.into_optimized_plan()?;
+ let expected = DataFrame::new(ctx.state(), plan.clone()).collect().await?;
+ let plan2 = substrait_roundtrip(&plan, &ctx).await?;
+ assert_eq!(
+ plan.schema().field(0).data_type(),
+ plan2.schema().field(0).data_type()
+ );
+ let actual = DataFrame::new(ctx.state(), plan2).collect().await?;
+ assert_eq!(
+ batches_to_sort_string(&expected),
+ batches_to_sort_string(&actual)
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_coalesce() -> Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT coalesce(st, named_struct('a', 0, 'b', 'z')) FROM struct_rows",
+ )
+ .await
+}
+
+// The lambda parameter is typed from the `coalesce` result, so it only finds
`subnet`
+// and `id` if the empty list took the column's field names first.
+#[tokio::test]
+async fn roundtrip_struct_literal_next_to_nested_struct_column() -> Result<()>
{
+ assert_struct_literal_roundtrips(
+ "SELECT array_transform(coalesce(z, []), p0 -> get_field(get_field(p0,
'subnet'), 'id')) \
+ FROM zones",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_case_else() -> Result<()> {
+ roundtrip(
+ "SELECT (CASE WHEN column1 IS NOT NULL THEN named_struct('f1',
column1) ELSE NULL END) AS res \
+ FROM (VALUES ('a'), (NULL))",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_nested_case_after_column() -> Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT CASE WHEN st IS NOT NULL THEN st \
+ ELSE CASE WHEN random() > 2 THEN named_struct('a', 0, 'b', 'z') \
+ ELSE named_struct('a', 9, 'b', 'w') END END FROM struct_rows",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_nested_case_before_column() -> Result<()>
{
+ assert_struct_literal_roundtrips(
+ "SELECT CASE WHEN st IS NULL \
+ THEN CASE WHEN random() > 2 THEN named_struct('a', 0, 'b', 'z') \
+ ELSE named_struct('a', 9, 'b', 'w') END ELSE st END FROM struct_rows",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_in_list() -> Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT st IN (named_struct('a', 1, 'b', 'x'), named_struct('a', 2,
'b', 'y')) \
+ FROM struct_rows",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_window_function_argument() -> Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT lag(st, 1, named_struct('a', 0, 'b', 'z')) OVER () FROM
struct_rows",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_first_input() -> Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT NULL AS r FROM struct_rows UNION ALL SELECT st AS r FROM
struct_rows",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_null_second_input() -> Result<()>
{
+ assert_struct_literal_roundtrips(
+ "SELECT st AS r FROM struct_rows UNION ALL SELECT NULL AS r FROM
struct_rows",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_limit() -> Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT * FROM (SELECT NULL AS r FROM struct_rows LIMIT 1) \
+ UNION ALL SELECT st AS r FROM struct_rows",
+ )
+ .await
+}
+
+#[tokio::test]
+async fn roundtrip_struct_literal_union_with_nested_union_input() ->
Result<()> {
+ assert_struct_literal_roundtrips(
+ "SELECT * FROM (SELECT named_struct('a', 0, 'b', 'z') AS r FROM
struct_rows \
+ UNION SELECT named_struct('a', 9, 'b', 'w') AS r FROM struct_rows) \
+ UNION ALL SELECT st AS r FROM struct_rows",
+ )
+ .await
+}
+
Review Comment:
Here are some other test cases that should improve coverage:
```rust
#[tokio::test]
async fn roundtrip_struct_literal_in_function_call() -> Result<()> {
// make_array survives optimization, so it reaches the consumer as a
function call
assert_struct_literal_roundtrips(
"SELECT make_array(st, named_struct('a', 0, 'b', 'z')) FROM
struct_rows",
)
.await
}
// `r` has the shape of `st` but is independent of it, so it keeps its own
names
#[tokio::test]
async fn roundtrip_struct_literal_independent_function_argument() ->
Result<()> {
assert_struct_literal_roundtrips(
"SELECT named_struct('l', st, 'r', named_struct('c0', 42, 'c1',
'q')) FROM struct_rows",
)
.await
}
// more than 3 items, so the optimizer doesn't rewrite the IN list into
`=`/`OR`
#[tokio::test]
async fn roundtrip_struct_literal_in_long_in_list() -> Result<()> {
assert_struct_literal_roundtrips(
"SELECT st IN (named_struct('a', 1, 'b', 'x'), named_struct('a', 2,
'b', 'y'), \
named_struct('a', 3, 'b', 'z'), named_struct('a', 4, 'b', 'w'))
FROM struct_rows",
)
.await
}
#[tokio::test]
async fn roundtrip_struct_literal_in_map_values() -> Result<()> {
assert_struct_literal_roundtrips(
"SELECT CASE WHEN st IS NULL THEN map(['k'], [named_struct('a', 0,
'b', 'z')]) \
ELSE map(['k'], [st]) END FROM struct_rows",
)
.await
}
```
##########
datafusion/substrait/src/logical_plan/consumer/expr/literal.rs:
##########
@@ -573,10 +578,244 @@ pub(crate) fn from_substrait_literal(
Ok(scalar_value)
}
+/// Renames the fields of the struct literals in `exprs` after those of the
first
+/// expression in `exprs` that isn't a literal and has a struct type, such as
a column.
+///
+/// For example, in `coalesce(s, [])` with `s: List<Struct{a, b}>`:
+///
+/// - before: `s` is `List<Struct{a, b}>`, `[]` is `List<Struct{c0, c1}>`
+/// - after: both are `List<Struct{a, b}>`
+///
+/// This is needed because a Substrait literal's type has no field names, so
the
+/// consumer names its fields `c0`, `c1`, ..., while a column keeps the names
from its
+/// table. DataFusion matches struct fields by name, so it can't combine
+/// `Struct{a, b}` with `Struct{c0, c1}`. Substrait matches them by position,
so giving
+/// the literal the column's names doesn't change what the plan means.
+///
+/// A literal whose shape differs from the column's (another number of fields,
other
+/// nesting) is left as it is, and so is everything if no expression has a
struct type.
+pub(crate) fn name_struct_literals_like_first_struct_expr<'a>(
+ exprs: impl IntoIterator<Item = &'a mut Expr>,
+ input_schema: &DFSchema,
+) {
+ let mut exprs = exprs.into_iter().collect::<Vec<_>>();
Review Comment:
> [!WARNING]
> Claude generated comment
This calls `get_type` on every non-literal expression before checking
whether any of them is a struct literal. It runs for every operator, `CASE`, IN
list and window call, so every plan pays for it, even one without structs.
`get_type` walks the whole subtree, so a left-deep chain like `a = 0 OR a = 1
OR ...` costs O(n²) to consume.
I measured consuming an unoptimized plan with an `OR` chain of n terms
(release build):
| n terms | this PR | this PR without the operator rename |
|---|---|---|
| 250 | 2.5 ms | 0.37 ms |
| 500 | 8.6 ms | 0.56 ms |
| 1000 | 33 ms | 1.1 ms |
The scalar function path already avoids this with `has_struct_literal_arg`.
The same early return here should fix it:
```suggestion
let mut exprs = exprs.into_iter().collect::<Vec<_>>();
if !exprs.iter().any(|e| {
has_literal_value(e)
&& e.get_type(input_schema).is_ok_and(|t| contains_struct(&t))
}) {
return;
}
```
Could you check that it brings the timing back to linear?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]