This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fbf81ebaef49 [SPARK-47263][SQL] Assign names to the legacy conditions
_LEGACY_ERROR_TEMP_13[44-46]
fbf81ebaef49 is described below
commit fbf81ebaef49baa4c19a936fb3884c2e62e6a49b
Author: xuping <13289341...@163.com>
AuthorDate: Wed Sep 18 22:06:00 2024 +0200
[SPARK-47263][SQL] Assign names to the legacy conditions
_LEGACY_ERROR_TEMP_13[44-46]
### What changes were proposed in this pull request?
rename err class _LEGACY_ERROR_TEMP_13[44-46]: 44 removed, 45 to
DEFAULT_UNSUPPORTED, 46 to ADD_DEFAULT_UNSUPPORTED
### Why are the changes needed?
replace legacy err class name with semantically explicits.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Re run the UT class modified in the PR
(org.apache.spark.sql.sources.InsertSuite &
org.apache.spark.sql.types.StructTypeSuite)
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #46320 from PaysonXu/SPARK-47263.
Authored-by: xuping <13289341...@163.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../src/main/resources/error/error-conditions.json | 27 ++++++++++------------
.../catalyst/util/ResolveDefaultColumnsUtil.scala | 9 +++++---
.../spark/sql/errors/QueryCompilationErrors.scala | 16 ++++---------
.../apache/spark/sql/types/StructTypeSuite.scala | 23 ++++++++++++------
.../org/apache/spark/sql/sources/InsertSuite.scala | 6 ++---
5 files changed, 41 insertions(+), 40 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index 25dd676c4aff..6463cc2c12da 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -1,4 +1,10 @@
{
+ "ADD_DEFAULT_UNSUPPORTED" : {
+ "message" : [
+ "Failed to execute <statementType> command because DEFAULT values are
not supported when adding new columns to previously existing target data source
with table provider: \"<dataSource>\"."
+ ],
+ "sqlState" : "42623"
+ },
"AGGREGATE_FUNCTION_WITH_NONDETERMINISTIC_EXPRESSION" : {
"message" : [
"Non-deterministic expression <sqlExpr> should not appear in the
arguments of an aggregate function."
@@ -1096,6 +1102,12 @@
],
"sqlState" : "42608"
},
+ "DEFAULT_UNSUPPORTED" : {
+ "message" : [
+ "Failed to execute <statementType> command because DEFAULT values are
not supported for target data source with table provider: \"<dataSource>\"."
+ ],
+ "sqlState" : "42623"
+ },
"DISTINCT_WINDOW_FUNCTION_UNSUPPORTED" : {
"message" : [
"Distinct window functions are not supported: <windowExpr>."
@@ -6673,21 +6685,6 @@
"Sinks cannot request distribution and ordering in continuous execution
mode."
]
},
- "_LEGACY_ERROR_TEMP_1344" : {
- "message" : [
- "Invalid DEFAULT value for column <fieldName>: <defaultValue> fails to
parse as a valid literal value."
- ]
- },
- "_LEGACY_ERROR_TEMP_1345" : {
- "message" : [
- "Failed to execute <statementType> command because DEFAULT values are
not supported for target data source with table provider: \"<dataSource>\"."
- ]
- },
- "_LEGACY_ERROR_TEMP_1346" : {
- "message" : [
- "Failed to execute <statementType> command because DEFAULT values are
not supported when adding new columns to previously existing target data source
with table provider: \"<dataSource>\"."
- ]
- },
"_LEGACY_ERROR_TEMP_2000" : {
"message" : [
"<message>. If necessary set <ansiConfig> to false to bypass this error."
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
index 8b7392e71249..693ac8d94dbc 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.util
import scala.collection.mutable.ArrayBuffer
-import org.apache.spark.{SparkThrowable, SparkUnsupportedOperationException}
+import org.apache.spark.{SparkException, SparkThrowable,
SparkUnsupportedOperationException}
import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.sql.AnalysisException
@@ -412,8 +412,11 @@ object ResolveDefaultColumns extends QueryErrorsBase
case _: ExprLiteral | _: Cast => expr
}
} catch {
- case _: AnalysisException | _: MatchError =>
- throw
QueryCompilationErrors.failedToParseExistenceDefaultAsLiteral(field.name, text)
+ // AnalysisException thrown from analyze is already formatted, throw
it directly.
+ case ae: AnalysisException => throw ae
+ case _: MatchError =>
+ throw SparkException.internalError(s"parse existence default as
literal err," +
+ s" field name: ${field.name}, value: $text")
}
// The expression should be a literal value by this point, possibly
wrapped in a cast
// function. This is enforced by the execution of commands that assign
default values.
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index f268ef85ef1d..e324d4e9d2ed 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -3516,29 +3516,21 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase with Compilat
"cond" -> toSQLExpr(cond)))
}
- def failedToParseExistenceDefaultAsLiteral(fieldName: String, defaultValue:
String): Throwable = {
- new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1344",
- messageParameters = Map(
- "fieldName" -> fieldName,
- "defaultValue" -> defaultValue))
- }
-
def defaultReferencesNotAllowedInDataSource(
statementType: String, dataSource: String): Throwable = {
new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1345",
+ errorClass = "DEFAULT_UNSUPPORTED",
messageParameters = Map(
- "statementType" -> statementType,
+ "statementType" -> toSQLStmt(statementType),
"dataSource" -> dataSource))
}
def addNewDefaultColumnToExistingTableNotAllowed(
statementType: String, dataSource: String): Throwable = {
new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1346",
+ errorClass = "ADD_DEFAULT_UNSUPPORTED",
messageParameters = Map(
- "statementType" -> statementType,
+ "statementType" -> toSQLStmt(statementType),
"dataSource" -> dataSource))
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
index 5ec1525bf9b6..6a67525dd02d 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
@@ -564,7 +564,6 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
.putString(ResolveDefaultColumns.EXISTS_DEFAULT_COLUMN_METADATA_KEY,
"1 + 1")
.putString(ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY, "1 + 1")
.build())))
- val error = "fails to parse as a valid literal value"
assert(ResolveDefaultColumns.existenceDefaultValues(source2).length == 1)
assert(ResolveDefaultColumns.existenceDefaultValues(source2)(0) == 2)
@@ -576,9 +575,13 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper
{
.putString(ResolveDefaultColumns.EXISTS_DEFAULT_COLUMN_METADATA_KEY,
"invalid")
.putString(ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY, "invalid")
.build())))
- assert(intercept[AnalysisException] {
- ResolveDefaultColumns.existenceDefaultValues(source3)
- }.getMessage.contains(error))
+
+ checkError(
+ exception = intercept[AnalysisException]{
+ ResolveDefaultColumns.existenceDefaultValues(source3)
+ },
+ condition = "INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION",
+ parameters = Map("statement" -> "", "colName" -> "`c1`", "defaultValue"
-> "invalid"))
// Negative test: StructType.defaultValues fails because the existence
default value fails to
// resolve.
@@ -592,9 +595,15 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper
{
ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY,
"(SELECT 'abc' FROM missingtable)")
.build())))
- assert(intercept[AnalysisException] {
- ResolveDefaultColumns.existenceDefaultValues(source4)
- }.getMessage.contains(error))
+
+ checkError(
+ exception = intercept[AnalysisException]{
+ ResolveDefaultColumns.existenceDefaultValues(source4)
+ },
+ condition = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map("statement" -> "",
+ "colName" -> "`c1`",
+ "defaultValue" -> "(SELECT 'abc' FROM missingtable)"))
}
test("SPARK-46629: Test STRUCT DDL with NOT NULL round trip") {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 57655a58a694..41447d8af574 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -1998,7 +1998,7 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
exception = intercept[AnalysisException] {
sql(s"create table t(a string default 'abc') using parquet")
},
- condition = "_LEGACY_ERROR_TEMP_1345",
+ condition = "DEFAULT_UNSUPPORTED",
parameters = Map("statementType" -> "CREATE TABLE", "dataSource" ->
"parquet"))
withTable("t") {
sql(s"create table t(a string, b int) using parquet")
@@ -2006,7 +2006,7 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
exception = intercept[AnalysisException] {
sql("alter table t add column s bigint default 42")
},
- condition = "_LEGACY_ERROR_TEMP_1345",
+ condition = "DEFAULT_UNSUPPORTED",
parameters = Map(
"statementType" -> "ALTER TABLE ADD COLUMNS",
"dataSource" -> "parquet"))
@@ -2314,7 +2314,7 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
// provider is now in the denylist.
sql(s"alter table t1 add column (b string default 'abc')")
},
- condition = "_LEGACY_ERROR_TEMP_1346",
+ condition = "ADD_DEFAULT_UNSUPPORTED",
parameters = Map(
"statementType" -> "ALTER TABLE ADD COLUMNS",
"dataSource" -> provider))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
