This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new e00f14ff521 [SPARK-41314][SQL] Assign a name to the error class `_LEGACY_ERROR_TEMP_1094` e00f14ff521 is described below commit e00f14ff5216e194fe39ef38d2c9414a22ef696a Author: yangjie01 <yangji...@baidu.com> AuthorDate: Thu Dec 1 11:49:42 2022 +0300 [SPARK-41314][SQL] Assign a name to the error class `_LEGACY_ERROR_TEMP_1094` ### What changes were proposed in this pull request? This pr aims to rename error class `_LEGACY_ERROR_TEMP_1094` to `INVALID_SCHEMA.NON_STRUCT_TYPE`. ### Why are the changes needed? Proper names of error classes to improve user experience with Spark SQL. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Add new tests to check `INVALID_SCHEMA.NON_STRUCT_TYPE` Closes #38856 from LuciferYang/SPARK-41314. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- core/src/main/resources/error/error-classes.json | 10 ++++---- .../spark/sql/catalyst/expressions/ExprUtils.scala | 2 +- .../spark/sql/errors/QueryCompilationErrors.scala | 9 ++++--- .../resources/sql-tests/inputs/csv-functions.sql | 1 + .../sql-tests/results/csv-functions.sql.out | 22 ++++++++++++++++ .../org/apache/spark/sql/CsvFunctionsSuite.scala | 29 ++++++++++++++++++++++ 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index 65b6dc68d12..347b9a14862 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -782,6 +782,11 @@ "The input expression must be string literal and not null." ] }, + "NON_STRUCT_TYPE" : { + "message" : [ + "The input expression should be evaluated to struct type, but got <dataType>." + ] + }, "PARSE_ERROR" : { "message" : [ "Cannot parse the schema:", @@ -2211,11 +2216,6 @@ "Cannot read table property '<key>' as it's corrupted.<details>." ] }, - "_LEGACY_ERROR_TEMP_1094" : { - "message" : [ - "Schema should be struct type but got <dataType>." - ] - }, "_LEGACY_ERROR_TEMP_1097" : { "message" : [ "The field for corrupt records must be string type and nullable." diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala index fbe3d5eb458..2fa970bac0c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala @@ -46,7 +46,7 @@ object ExprUtils extends QueryErrorsBase { def evalSchemaExpr(exp: Expression): StructType = { val dataType = evalTypeExpr(exp) if (!dataType.isInstanceOf[StructType]) { - throw QueryCompilationErrors.schemaIsNotStructTypeError(dataType) + throw QueryCompilationErrors.schemaIsNotStructTypeError(exp, dataType) } dataType.asInstanceOf[StructType] } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index fc9a08104b4..2e20d7aec8d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -1010,10 +1010,13 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase { messageParameters = Map("inputSchema" -> toSQLExpr(exp))) } - def schemaIsNotStructTypeError(dataType: DataType): Throwable = { + def schemaIsNotStructTypeError(exp: Expression, dataType: DataType): Throwable = { new AnalysisException( - errorClass = "_LEGACY_ERROR_TEMP_1094", - messageParameters = Map("dataType" -> dataType.toString)) + errorClass = "INVALID_SCHEMA.NON_STRUCT_TYPE", + messageParameters = Map( + "inputSchema" -> toSQLExpr(exp), + "dataType" -> toSQLType(dataType) + )) } def keyValueInMapNotStringError(m: CreateMap): Throwable = { diff --git a/sql/core/src/test/resources/sql-tests/inputs/csv-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/csv-functions.sql index a1a4bc9de3f..01d436534a1 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/csv-functions.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/csv-functions.sql @@ -4,6 +4,7 @@ select from_csv('26/08/2015', 'time Timestamp', map('timestampFormat', 'dd/MM/yy -- Check if errors handled select from_csv('1', 1); select from_csv('1', 'a InvalidType'); +select from_csv('1', 'Array<int>'); select from_csv('1', 'a INT', named_struct('mode', 'PERMISSIVE')); select from_csv('1', 'a INT', map('mode', 1)); select from_csv(); diff --git a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out index e5e70714a1b..58c0a7b9f3c 100644 --- a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out @@ -59,6 +59,28 @@ org.apache.spark.sql.AnalysisException } +-- !query +select from_csv('1', 'Array<int>') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "INVALID_SCHEMA.NON_STRUCT_TYPE", + "messageParameters" : { + "dataType" : "\"ARRAY<INT>\"", + "inputSchema" : "\"Array<int>\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 34, + "fragment" : "from_csv('1', 'Array<int>')" + } ] +} + + -- !query select from_csv('1', 'a INT', named_struct('mode', 'PERMISSIVE')) -- !query schema diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala index ab4c148da04..fcdc40404e7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala @@ -43,6 +43,35 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession { Row(Row(1)) :: Nil) } + test("from_csv with non struct schema") { + checkError( + exception = intercept[AnalysisException] { + Seq("1").toDS().select(from_csv($"value", lit("ARRAY<int>"), Map[String, String]().asJava)) + }, + errorClass = "INVALID_SCHEMA.NON_STRUCT_TYPE", + parameters = Map( + "inputSchema" -> "\"ARRAY<int>\"", + "dataType" -> "\"ARRAY<INT>\"" + ) + ) + + checkError( + exception = intercept[AnalysisException] { + Seq("1").toDF("csv").selectExpr(s"from_csv(csv, 'ARRAY<int>')") + }, + errorClass = "INVALID_SCHEMA.NON_STRUCT_TYPE", + parameters = Map( + "inputSchema" -> "\"ARRAY<int>\"", + "dataType" -> "\"ARRAY<INT>\"" + ), + context = ExpectedContext( + fragment = "from_csv(csv, 'ARRAY<int>')", + start = 0, + stop = 26 + ) + ) + } + test("from_csv with option (timestampFormat)") { val df = Seq("26/08/2015 18:00").toDS() val schema = new StructType().add("time", TimestampType) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org