This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a3d999292f8e [SPARK-46524][SQL] Improve error messages for invalid
save mode
a3d999292f8e is described below
commit a3d999292f8e99269dfd0289e2f5aca7e5ea4fae
Author: allisonwang-db <allison.w...@databricks.com>
AuthorDate: Wed Jan 3 15:43:53 2024 +0300
[SPARK-46524][SQL] Improve error messages for invalid save mode
### What changes were proposed in this pull request?
This PR improves the error messages when writing a data frame with an
invalid save mode.
### Why are the changes needed?
To improve the error messages.
Before this PR, Spark throws an java.lang.IllegalArgumentException:
`java.lang.IllegalArgumentException: Unknown save mode: foo. Accepted save
modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists', 'default'.`
After this PR, the error will have a proper error class:
`[INVALID_SAVE_MODE] The specified save mode "foo" is invalid. Valid save
modes include "append", "overwrite", "ignore", "error", "errorifexists", and
"default"."
`
### Does this PR introduce _any_ user-facing change?
Yes. The error messages will be changed.
### How was this patch tested?
New unit test
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #44508 from allisonwang-db/spark-46524-invalid-save-mode.
Authored-by: allisonwang-db <allison.w...@databricks.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
R/pkg/tests/fulltests/test_sparkSQL.R | 2 +-
common/utils/src/main/resources/error/error-classes.json | 6 ++++++
docs/sql-error-conditions.md | 6 ++++++
.../org/apache/spark/sql/errors/QueryCompilationErrors.scala | 7 +++++++
.../src/main/scala/org/apache/spark/sql/DataFrameWriter.scala | 3 +--
.../spark/sql/execution/python/PythonDataSourceSuite.scala | 9 +++++++++
6 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R
b/R/pkg/tests/fulltests/test_sparkSQL.R
index 0d96f708a544..c1a5292195af 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -1414,7 +1414,7 @@ test_that("test HiveContext", {
# Invalid mode
expect_error(saveAsTable(df, "parquetest", "parquet", mode = "abc", path =
parquetDataPath),
- "illegal argument - Unknown save mode: abc")
+ "Error in mode : analysis error - \\[INVALID_SAVE_MODE\\].*")
unsetHiveContext()
}
})
diff --git a/common/utils/src/main/resources/error/error-classes.json
b/common/utils/src/main/resources/error/error-classes.json
index 87e43fe0e38c..bcaf8a74c08d 100644
--- a/common/utils/src/main/resources/error/error-classes.json
+++ b/common/utils/src/main/resources/error/error-classes.json
@@ -2239,6 +2239,12 @@
],
"sqlState" : "42613"
},
+ "INVALID_SAVE_MODE" : {
+ "message" : [
+ "The specified save mode <mode> is invalid. Valid save modes include
\"append\", \"overwrite\", \"ignore\", \"error\", \"errorifexists\", and
\"default\"."
+ ],
+ "sqlState" : "42000"
+ },
"INVALID_SCHEMA" : {
"message" : [
"The input schema <inputSchema> is not a valid schema string."
diff --git a/docs/sql-error-conditions.md b/docs/sql-error-conditions.md
index 3f4074af9b78..c6108e97b4c5 100644
--- a/docs/sql-error-conditions.md
+++ b/docs/sql-error-conditions.md
@@ -1271,6 +1271,12 @@ For more details see
[INVALID_PARTITION_OPERATION](sql-error-conditions-invalid-
Parameterized query must either use positional, or named parameters, but not
both.
+### INVALID_SAVE_MODE
+
+[SQLSTATE:
42000](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)
+
+The specified save mode `<mode>` is invalid. Valid save modes include
"append", "overwrite", "ignore", "error", "errorifexists", and "default".
+
### [INVALID_SCHEMA](sql-error-conditions-invalid-schema-error-class.html)
[SQLSTATE:
42K07](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index bc847d1c0069..b844ee2bdc45 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -3184,6 +3184,13 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase with Compilat
"config" -> SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key))
}
+ def invalidSaveModeError(saveMode: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INVALID_SAVE_MODE",
+ messageParameters = Map("mode" -> toDSOption(saveMode))
+ )
+ }
+
def writeWithSaveModeUnsupportedBySourceError(source: String, createMode:
String): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1308",
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index c8727146160b..2d6d5f0e8b2b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -88,8 +88,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
case "append" => mode(SaveMode.Append)
case "ignore" => mode(SaveMode.Ignore)
case "error" | "errorifexists" | "default" =>
mode(SaveMode.ErrorIfExists)
- case _ => throw new IllegalArgumentException(s"Unknown save mode:
$saveMode. Accepted " +
- "save modes are 'overwrite', 'append', 'ignore', 'error',
'errorifexists', 'default'.")
+ case _ => throw QueryCompilationErrors.invalidSaveModeError(saveMode)
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonDataSourceSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonDataSourceSuite.scala
index 3e7cd82db8d7..487951912bb0 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonDataSourceSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonDataSourceSuite.scala
@@ -629,6 +629,15 @@ class PythonDataSourceSuite extends QueryTest with
SharedSparkSession {
assert(error.getMessage.contains("TableProvider implementation
SimpleDataSource " +
"cannot be written with ErrorIfExists mode, please use Append or
Overwrite modes instead."))
}
+
+ withClue("invalid mode") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ spark.range(1).write.format(dataSourceName).mode("foo").save()
+ },
+ errorClass = "INVALID_SAVE_MODE",
+ parameters = Map("mode" -> "\"foo\""))
+ }
}
test("data source write - overwrite mode") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
