This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 23c4cce61b7 [SPARK-45614][SQL] Assign names to error _LEGACY_ERROR_TEMP_215[6,7,8] 23c4cce61b7 is described below commit 23c4cce61b78c1f25e34850c9d0e242dd1fcffe8 Author: dengziming <dengziming1...@gmail.com> AuthorDate: Fri Oct 27 10:44:30 2023 +0300 [SPARK-45614][SQL] Assign names to error _LEGACY_ERROR_TEMP_215[6,7,8] ### What changes were proposed in this pull request? Replace the legacy error class `_LEGACY_ERROR_TEMP_2156` with an internal error. Assign the name `INVALID_PARAMETER_VALUE.START` to the legacy error class `_LEGACY_ERROR_TEMP_2157`. Assign the name `INVALID_PARAMETER_VALUE.LENGTH` to the legacy error class `_LEGACY_ERROR_TEMP_2158`. ### Why are the changes needed? To assign proper name as a part of activity in SPARK-37935. ### Does this PR introduce _any_ user-facing change? Yes, the error message will include the error class name. ### How was this patch tested? `_LEGACY_ERROR_TEMP_2156` can't be produced from user code since it's an internal error, so I added a unit test in `CollectionExpressionsSuite.scala`. `INVALID_PARAMETER_VALUE.START` and `INVALID_PARAMETER_VALUE.LENGTH` can be produced from user code so I add 2 tests in `QueryExecutionErrorsSuite.scala` to validate them. ### Was this patch authored or co-authored using generative AI tooling? No Closes #43481 from dengziming/SPARK-45614. Authored-by: dengziming <dengziming1...@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../src/main/resources/error/error-classes.json | 25 ++++++--------- ...nditions-invalid-parameter-value-error-class.md | 8 +++++ .../expressions/collectionOperations.scala | 5 +-- .../spark/sql/errors/QueryExecutionErrors.scala | 22 +++++++------ .../expressions/CollectionExpressionsSuite.scala | 36 +++++++++++++++++++--- .../sql/errors/QueryExecutionErrorsSuite.scala | 31 ++++++++++++++++++- 6 files changed, 94 insertions(+), 33 deletions(-) diff --git a/common/utils/src/main/resources/error/error-classes.json b/common/utils/src/main/resources/error/error-classes.json index 5b756ad1077..f9cc0a86521 100644 --- a/common/utils/src/main/resources/error/error-classes.json +++ b/common/utils/src/main/resources/error/error-classes.json @@ -1974,6 +1974,11 @@ "expects one of the units without quotes YEAR, QUARTER, MONTH, WEEK, DAY, DAYOFYEAR, HOUR, MINUTE, SECOND, MILLISECOND, MICROSECOND, but got the string literal <invalidValue>." ] }, + "LENGTH" : { + "message" : [ + "Expects `length` greater than or equal to 0, but got <length>." + ] + }, "NULL" : { "message" : [ "expects a non-NULL value." @@ -1989,6 +1994,11 @@ "Expects group index between 0 and <groupCount>, but got <groupIndex>." ] }, + "START" : { + "message" : [ + "Expects a positive or a negative value for `start`, but got 0." + ] + }, "ZERO_INDEX" : { "message" : [ "expects %1$, %2$ and so on, but got %0$." @@ -5749,21 +5759,6 @@ "<userClass> is not annotated with SQLUserDefinedType nor registered with UDTRegistration.}" ] }, - "_LEGACY_ERROR_TEMP_2156" : { - "message" : [ - "The size function doesn't support the operand type <dataType>." - ] - }, - "_LEGACY_ERROR_TEMP_2157" : { - "message" : [ - "Unexpected value for start in function <prettyName>: SQL array indices start at 1." - ] - }, - "_LEGACY_ERROR_TEMP_2158" : { - "message" : [ - "Unexpected value for length in function <prettyName>: length must be greater than or equal to 0." - ] - }, "_LEGACY_ERROR_TEMP_2159" : { "message" : [ "Unsuccessful try to concat arrays with <numberOfElements> elements due to exceeding the array size limit <maxRoundedArrayLength>." diff --git a/docs/sql-error-conditions-invalid-parameter-value-error-class.md b/docs/sql-error-conditions-invalid-parameter-value-error-class.md index 8186a56314d..d58d4fc2f59 100644 --- a/docs/sql-error-conditions-invalid-parameter-value-error-class.md +++ b/docs/sql-error-conditions-invalid-parameter-value-error-class.md @@ -49,6 +49,10 @@ expects an integer value in [0, `<upper>`), but got `<invalidValue>`. expects one of the units without quotes YEAR, QUARTER, MONTH, WEEK, DAY, DAYOFYEAR, HOUR, MINUTE, SECOND, MILLISECOND, MICROSECOND, but got the string literal `<invalidValue>`. +## LENGTH + +Expects `length` greater than or equal to 0, but got `<length>`. + ## NULL expects a non-NULL value. @@ -61,6 +65,10 @@ expects a non-NULL value. Expects group index between 0 and `<groupCount>`, but got `<groupIndex>`. +## START + +Expects a positive or a negative value for `start`, but got 0. + ## ZERO_INDEX expects `%1$`, `%2$` and so on, but got `%0$`. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 759000bc5f5..0a080423b10 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -1768,7 +1768,7 @@ case class Slice(x: Expression, start: Expression, length: Expression) startInt - 1 } if (lengthInt < 0) { - throw QueryExecutionErrors.unexpectedValueForLengthInFunctionError(prettyName) + throw QueryExecutionErrors.unexpectedValueForLengthInFunctionError(prettyName, lengthInt) } // startIndex can be negative if start is negative and its absolute value is greater than the // number of elements in the array @@ -1796,7 +1796,8 @@ case class Slice(x: Expression, start: Expression, length: Expression) | $startIdx = $start - 1; |} |if ($length < 0) { - | throw QueryExecutionErrors.unexpectedValueForLengthInFunctionError("$prettyName"); + | throw QueryExecutionErrors.unexpectedValueForLengthInFunctionError( + | "$prettyName", $length); |} else if ($length > $x.numElements() - $startIdx) { | $resLength = $x.numElements() - $startIdx; |} else { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index a11b929919d..afc244509c4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -1368,25 +1368,27 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } def unsupportedOperandTypeForSizeFunctionError( - dataType: DataType): SparkUnsupportedOperationException = { - new SparkUnsupportedOperationException( - errorClass = "_LEGACY_ERROR_TEMP_2156", - messageParameters = Map( - "dataType" -> dataType.getClass.getCanonicalName)) + dataType: DataType): Throwable = { + SparkException.internalError( + s"The size function doesn't support the operand type ${toSQLType(dataType)}") } def unexpectedValueForStartInFunctionError(prettyName: String): SparkRuntimeException = { new SparkRuntimeException( - errorClass = "_LEGACY_ERROR_TEMP_2157", + errorClass = "INVALID_PARAMETER_VALUE.START", messageParameters = Map( - "prettyName" -> prettyName)) + "parameter" -> toSQLId("start"), + "functionName" -> toSQLId(prettyName))) } - def unexpectedValueForLengthInFunctionError(prettyName: String): SparkRuntimeException = { + def unexpectedValueForLengthInFunctionError( + prettyName: String, length: Int): SparkRuntimeException = { new SparkRuntimeException( - errorClass = "_LEGACY_ERROR_TEMP_2158", + errorClass = "INVALID_PARAMETER_VALUE.LENGTH", messageParameters = Map( - "prettyName" -> prettyName)) + "parameter" -> toSQLId("length"), + "length" -> length.toString, + "functionName" -> toSQLId(prettyName))) } def invalidIndexOfZeroError(context: SQLQueryContext): RuntimeException = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index ff393857c31..b3a21bd9565 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -32,12 +32,14 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils} import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{outstandingZoneIds, LA, UTC} import org.apache.spark.sql.catalyst.util.IntervalUtils._ +import org.apache.spark.sql.errors.DataTypeErrorsBase import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.unsafe.array.ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH import org.apache.spark.unsafe.types.UTF8String -class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { +class CollectionExpressionsSuite + extends SparkFunSuite with ExpressionEvalHelper with DataTypeErrorsBase { implicit def stringToUTF8Str(str: String): UTF8String = UTF8String.fromString(str) @@ -86,6 +88,19 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper } } + test("Unsupported data type for size()") { + val exception = intercept[org.apache.spark.SparkException] { + Size(Literal.create("str", StringType)).eval(EmptyRow) + } + checkError( + exception = exception, + errorClass = "INTERNAL_ERROR", + parameters = Map( + "message" -> ("The size function doesn't support the operand type " + + toSQLType(StringType)) + )) + } + test("MapKeys/MapValues") { val m0 = Literal.create(Map("a" -> "1", "b" -> "2"), MapType(StringType, StringType)) val m1 = Literal.create(Map[String, String](), MapType(StringType, StringType)) @@ -600,10 +615,21 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Slice(a0, Literal(-3), Literal(2)), Seq(4, 5)) checkEvaluation(Slice(a0, Literal(4), Literal(10)), Seq(4, 5, 6)) checkEvaluation(Slice(a0, Literal(-1), Literal(2)), Seq(6)) - checkExceptionInExpression[RuntimeException](Slice(a0, Literal(1), Literal(-1)), - "Unexpected value for length") - checkExceptionInExpression[RuntimeException](Slice(a0, Literal(0), Literal(1)), - "Unexpected value for start") + checkErrorInExpression[SparkRuntimeException]( + expression = Slice(a0, Literal(1), Literal(-1)), + errorClass = "INVALID_PARAMETER_VALUE.LENGTH", + parameters = Map( + "parameter" -> toSQLId("length"), + "length" -> (-1).toString, + "functionName" -> toSQLId("slice") + )) + checkErrorInExpression[SparkRuntimeException]( + expression = Slice(a0, Literal(0), Literal(1)), + errorClass = "INVALID_PARAMETER_VALUE.START", + parameters = Map( + "parameter" -> toSQLId("start"), + "functionName" -> toSQLId("slice") + )) checkEvaluation(Slice(a0, Literal(-20), Literal(1)), Seq.empty[Int]) checkEvaluation(Slice(a1, Literal(-20), Literal(1)), Seq.empty[String]) checkEvaluation(Slice(a0, Literal.create(null, IntegerType), Literal(2)), null) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala index fb1d05f2a9a..945dd782da0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala @@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext import org.apache.spark.sql.catalyst.expressions.objects.InitializeJavaBean import org.apache.spark.sql.catalyst.rules.RuleIdCollection import org.apache.spark.sql.catalyst.util.BadRecordException +import org.apache.spark.sql.errors.DataTypeErrorsBase import org.apache.spark.sql.execution.datasources.jdbc.{DriverRegistry, JDBCOptions} import org.apache.spark.sql.execution.datasources.jdbc.connection.ConnectionProvider import org.apache.spark.sql.execution.datasources.orc.OrcTest @@ -57,7 +58,8 @@ class QueryExecutionErrorsSuite extends QueryTest with ParquetTest with OrcTest - with SharedSparkSession { + with SharedSparkSession + with DataTypeErrorsBase { import testImplicits._ @@ -1064,6 +1066,33 @@ class QueryExecutionErrorsSuite ) } } + + test("Unexpected `start` for slice()") { + checkError( + exception = intercept[SparkRuntimeException] { + sql("select slice(array(1,2,3), 0, 1)").collect() + }, + errorClass = "INVALID_PARAMETER_VALUE.START", + parameters = Map( + "parameter" -> toSQLId("start"), + "functionName" -> toSQLId("slice") + ) + ) + } + + test("Unexpected `length` for slice()") { + checkError( + exception = intercept[SparkRuntimeException] { + sql("select slice(array(1,2,3), 1, -1)").collect() + }, + errorClass = "INVALID_PARAMETER_VALUE.LENGTH", + parameters = Map( + "parameter" -> toSQLId("length"), + "length" -> (-1).toString, + "functionName" -> toSQLId("slice") + ) + ) + } } class FakeFileSystemSetPermission extends LocalFileSystem { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org