This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new f235c9f622d [SPARK-42098][SQL] Fix ResolveInlineTables can not handle with RuntimeReplaceable expression f235c9f622d is described below commit f235c9f622ddeb4c6a5cce8903130709dcd2217c Author: Jia Fan <fanjiaemi...@qq.com> AuthorDate: Fri Jul 28 12:44:02 2023 +0800 [SPARK-42098][SQL] Fix ResolveInlineTables can not handle with RuntimeReplaceable expression ### What changes were proposed in this pull request? Fix `ResolveInlineTables` can not handle with `RuntimeReplaceable` expression eg: ```sql select * from values (try_add(5, 0)) ``` The error will throw: ```java [INVALID_INLINE_TABLE.CANNOT_EVALUATE_EXPRESSION_IN_INLINE_TABLE] Invalid inline table. Cannot evaluate the expression "try_add(5, 0)" in inline table definition.; line 1 pos 22 ``` ### Why are the changes needed? Fix the bug ResolveInlineTables can not handle with RuntimeReplaceable expression ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? add new test. Closes #42110 from Hisoka-X/SPARK-42098_inline_table_replaceable. Authored-by: Jia Fan <fanjiaemi...@qq.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../catalyst/analysis/ResolveInlineTables.scala | 6 ++++-- .../analyzer-results/inline-table.sql.out | 21 +++++++++++++++++++ .../resources/sql-tests/inputs/inline-table.sql | 5 +++++ .../sql-tests/results/inline-table.sql.out | 24 ++++++++++++++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala index cf706171cd9..c203f9fa39d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala @@ -21,6 +21,7 @@ import scala.util.control.NonFatal import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.AliasHelper +import org.apache.spark.sql.catalyst.optimizer.ReplaceExpressions import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.AlwaysProcess @@ -36,8 +37,9 @@ object ResolveInlineTables extends Rule[LogicalPlan] with CastSupport with Alias AlwaysProcess.fn, ruleId) { case table: UnresolvedInlineTable if table.expressionsResolved => validateInputDimension(table) - validateInputEvaluable(table) - convert(table) + val newTable = ReplaceExpressions(table).asInstanceOf[UnresolvedInlineTable] + validateInputEvaluable(newTable) + convert(newTable) } /** diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out index f3e6eb4d8dc..2a17f092a06 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/inline-table.sql.out @@ -220,3 +220,24 @@ select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991- Project [a#x, b#x] +- SubqueryAlias data +- LocalRelation [a#x, b#x] + + +-- !query +select * from values (try_add(5, 0)) +-- !query analysis +Project [col1#x] ++- LocalRelation [col1#x] + + +-- !query +select * from values (try_divide(5, 0)) +-- !query analysis +Project [col1#x] ++- LocalRelation [col1#x] + + +-- !query +select * from values (10 + try_divide(5, 0)) +-- !query analysis +Project [col1#x] ++- LocalRelation [col1#x] diff --git a/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql b/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql index fd8bb2d837d..6867248f576 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/inline-table.sql @@ -55,3 +55,8 @@ select * from values ("one", count(1)), ("two", 2) as data(a, b); -- string to timestamp select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b); + +-- ReplaceExpressions as row +select * from values (try_add(5, 0)); +select * from values (try_divide(5, 0)); +select * from values (10 + try_divide(5, 0)); diff --git a/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out b/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out index e735dd0dd5a..709d7ab73f6 100644 --- a/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out @@ -242,3 +242,27 @@ select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991- struct<a:timestamp,b:array<timestamp>> -- !query output 1991-12-06 00:00:00 [1991-12-06 01:00:00,1991-12-06 12:00:00] + + +-- !query +select * from values (try_add(5, 0)) +-- !query schema +struct<col1:int> +-- !query output +5 + + +-- !query +select * from values (try_divide(5, 0)) +-- !query schema +struct<col1:double> +-- !query output +NULL + + +-- !query +select * from values (10 + try_divide(5, 0)) +-- !query schema +struct<col1:double> +-- !query output +NULL --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org