This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new ed6c1b33a0a [SPARK-45075][SQL][3.4] Fix alter table with invalid
default value will not report error
ed6c1b33a0a is described below
commit ed6c1b33a0a763680182b29baedebb241e0139a4
Author: Jia Fan <fanjiaemi...@qq.com>
AuthorDate: Mon Sep 11 23:28:20 2023 -0700
[SPARK-45075][SQL][3.4] Fix alter table with invalid default value will not
report error
### What changes were proposed in this pull request?
This is a backporting PR to branch-3.4 from
https://github.com/apache/spark/pull/42810
Changed the way of assert the error to adapt to 3.4
### Why are the changes needed?
Fix bug on 3.4
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
add new test.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #42876 from Hisoka-X/SPARK-45075_followup_3.4_alter_column.
Authored-by: Jia Fan <fanjiaemi...@qq.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../spark/sql/connector/catalog/TableChange.java | 3 +--
.../plans/logical/v2AlterTableCommands.scala | 11 +++++++++--
.../spark/sql/connector/AlterTableTests.scala | 23 ++++++++++++++++++++++
3 files changed, 33 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
index 609cfab2d56..ebecb6f507e 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
@@ -696,9 +696,8 @@ public interface TableChange {
/**
* Returns the column default value SQL string (Spark SQL dialect). The
default value literal
* is not provided as updating column default values does not need to
back-fill existing data.
- * Null means dropping the column default value.
+ * Empty string means dropping the column default value.
*/
- @Nullable
public String newDefaultValue() { return newDefaultValue; }
@Override
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
index eb9d45f06ec..b02c4fac12d 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2AlterTableCommands.scala
@@ -17,9 +17,9 @@
package org.apache.spark.sql.catalyst.plans.logical
-import org.apache.spark.sql.catalyst.analysis.{FieldName, FieldPosition}
+import org.apache.spark.sql.catalyst.analysis.{FieldName, FieldPosition,
ResolvedFieldName}
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.util.TypeUtils
+import org.apache.spark.sql.catalyst.util.{ResolveDefaultColumns, TypeUtils}
import org.apache.spark.sql.connector.catalog.{TableCatalog, TableChange}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types.DataType
@@ -228,6 +228,13 @@ case class AlterColumn(
TableChange.updateColumnPosition(colName, newPosition.position)
}
val defaultValueChange = setDefaultExpression.map { newDefaultExpression =>
+ if (newDefaultExpression.nonEmpty) {
+ // SPARK-45075: We call 'ResolveDefaultColumns.analyze' here to make
sure that the default
+ // value parses successfully, and return an error otherwise
+ val newDataType =
dataType.getOrElse(column.asInstanceOf[ResolvedFieldName].field.dataType)
+ ResolveDefaultColumns.analyze(column.name.last, newDataType,
newDefaultExpression,
+ "ALTER TABLE ALTER COLUMN")
+ }
TableChange.updateColumnDefaultValue(colName, newDefaultExpression)
}
typeChange.toSeq ++ nullabilityChange ++ commentChange ++ positionChange
++ defaultValueChange
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
index 2047212a4ea..8f1faa18933 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
@@ -366,6 +366,29 @@ trait AlterTableTests extends SharedSparkSession with
QueryErrorsBase {
}
}
+ test("SPARK-45075: ALTER COLUMN with invalid default value") {
+ withSQLConf(SQLConf.DEFAULT_COLUMN_ALLOWED_PROVIDERS.key -> s"$v2Format,
") {
+ withTable("t") {
+ sql(s"create table t(i boolean) using $v2Format")
+ // The default value fails to analyze.
+ assert(intercept[AnalysisException](
+ sql("alter table t add column s bigint default badvalue")
+ ).getMessage.contains("Failed to execute ALTER TABLE command because
the destination" +
+ " table column s has a DEFAULT value of badvalue which fails
to resolve as a" +
+ " valid expression: [UNRESOLVED_COLUMN.WITHOUT_SUGGESTION] A
column or function" +
+ " parameter with name `badvalue` cannot be resolved. ;"))
+
+ sql("alter table t add column s bigint default 3L")
+ assert(intercept[AnalysisException](
+ sql("alter table t alter column s set default badvalue")
+ ).getMessage.contains("Failed to execute ALTER TABLE ALTER COLUMN
command because the" +
+ " destination table column s has a DEFAULT value of badvalue
which fails to" +
+ " resolve as a valid expression:
[UNRESOLVED_COLUMN.WITHOUT_SUGGESTION] A" +
+ " column or function parameter with name `badvalue` cannot be
resolved. ;"))
+ }
+ }
+ }
+
test("AlterTable: add complex column") {
val t = s"${catalogAndNamespace}table_name"
withTable(t) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
