This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 44151c7908b [SPARK-40663][SQL] Migrate execution errors onto error classes: _LEGACY_ERROR_TEMP_2026-2050 44151c7908b is described below commit 44151c7908b055737f416c81ac19bb0a06a11049 Author: itholic <haejoon....@databricks.com> AuthorDate: Sun Oct 9 11:19:25 2022 +0300 [SPARK-40663][SQL] Migrate execution errors onto error classes: _LEGACY_ERROR_TEMP_2026-2050 ### What changes were proposed in this pull request? This PR proposes to migrate 25 execution errors onto temporary error classes with the prefix `_LEGACY_ERROR_TEMP_2026` to `_LEGACY_ERROR_TEMP_2050`. The error classes are prefixed with `_LEGACY_ERROR_TEMP_` indicates the dev-facing error messages, and won't be exposed to end users. ### Why are the changes needed? To speed-up the error class migration. The migration on temporary error classes allow us to analyze the errors, so we can detect the most popular error classes. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? ``` $ build/sbt "sql/testOnly org.apache.spark.sql.SQLQueryTestSuite" $ build/sbt "test:testOnly *SQLQuerySuite" ``` Closes #38108 from itholic/SPARK-40540-2026-2050. Lead-authored-by: itholic <haejoon....@databricks.com> Co-authored-by: Haejoon Lee <44108233+itho...@users.noreply.github.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- R/pkg/tests/fulltests/test_sparkSQL.R | 3 +- core/src/main/resources/error/error-classes.json | 130 ++++++++++++++- .../spark/sql/errors/QueryExecutionErrors.scala | 183 ++++++++++++++------- 3 files changed, 253 insertions(+), 63 deletions(-) diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R index 2d55597b55f..90538c3e776 100644 --- a/R/pkg/tests/fulltests/test_sparkSQL.R +++ b/R/pkg/tests/fulltests/test_sparkSQL.R @@ -3961,7 +3961,8 @@ test_that("Call DataFrameWriter.save() API in Java without path and check argume # It makes sure that we can omit path argument in write.df API and then it calls # DataFrameWriter.save() without path. expect_error(write.df(df, source = "csv"), - "Error in save : illegal argument - Expected exactly one path to be specified") + paste("Error in save : org.apache.spark.SparkIllegalArgumentException:", + "Expected exactly one path to be specified")) expect_error(write.json(df, jsonPath), "Error in json : analysis error - Path file:.*already exists") expect_error(write.text(df, jsonPath), diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index 5d9787f10ab..146c99d5ca5 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -3020,11 +3020,6 @@ "<message>. If necessary set <ansiConfig> to false to bypass this error." ] }, - "_LEGACY_ERROR_TEMP_2001" : { - "message" : [ - "<message> If necessary set <ansiConfig> to false to bypass this error" - ] - }, "_LEGACY_ERROR_TEMP_2002" : { "message" : [ "<message>" @@ -3144,5 +3139,130 @@ "message" : [ "<className> must override either <m1> or <m2>" ] + }, + "_LEGACY_ERROR_TEMP_2026" : { + "message" : [ + "Failed to convert value <value> (class of <cls>) with the type of <dataType> to JSON." + ] + }, + "_LEGACY_ERROR_TEMP_2027" : { + "message" : [ + "Unexpected operator <op> in correlated subquery<pos>" + ] + }, + "_LEGACY_ERROR_TEMP_2028" : { + "message" : [ + "This line should be unreachable<err>" + ] + }, + "_LEGACY_ERROR_TEMP_2029" : { + "message" : [ + "Not supported rounding mode: <roundMode>" + ] + }, + "_LEGACY_ERROR_TEMP_2030" : { + "message" : [ + "Can not handle nested schema yet... plan <plan>" + ] + }, + "_LEGACY_ERROR_TEMP_2031" : { + "message" : [ + "The input external row cannot be null." + ] + }, + "_LEGACY_ERROR_TEMP_2032" : { + "message" : [ + "<fieldCannotBeNullMsg>" + ] + }, + "_LEGACY_ERROR_TEMP_2033" : { + "message" : [ + "Unable to create database <name> as failed to create its directory <locationUri>" + ] + }, + "_LEGACY_ERROR_TEMP_2034" : { + "message" : [ + "Unable to drop database <name> as failed to delete its directory <locationUri>" + ] + }, + "_LEGACY_ERROR_TEMP_2035" : { + "message" : [ + "Unable to create table <table> as failed to create its directory <defaultTableLocation>" + ] + }, + "_LEGACY_ERROR_TEMP_2036" : { + "message" : [ + "Unable to delete partition path <partitionPath>" + ] + }, + "_LEGACY_ERROR_TEMP_2037" : { + "message" : [ + "Unable to drop table <table> as failed to delete its directory <dir>" + ] + }, + "_LEGACY_ERROR_TEMP_2038" : { + "message" : [ + "Unable to rename table <oldName> to <newName> as failed to rename its directory <oldDir>" + ] + }, + "_LEGACY_ERROR_TEMP_2039" : { + "message" : [ + "Unable to create partition path <partitionPath>" + ] + }, + "_LEGACY_ERROR_TEMP_2040" : { + "message" : [ + "Unable to rename partition path <oldPartPath>" + ] + }, + "_LEGACY_ERROR_TEMP_2041" : { + "message" : [ + "<methodName> is not implemented" + ] + }, + "_LEGACY_ERROR_TEMP_2042" : { + "message" : [ + "<message>. If necessary set <ansiConfig> to false to bypass this error." + ] + }, + "_LEGACY_ERROR_TEMP_2043" : { + "message" : [ + "- <sqlValue> caused overflow" + ] + }, + "_LEGACY_ERROR_TEMP_2044" : { + "message" : [ + "<sqlValue1> <symbol> <sqlValue2> caused overflow" + ] + }, + "_LEGACY_ERROR_TEMP_2045" : { + "message" : [ + "Unsupported table change: <message>" + ] + }, + "_LEGACY_ERROR_TEMP_2046" : { + "message" : [ + "[BUG] Not a DataSourceRDDPartition: <split>" + ] + }, + "_LEGACY_ERROR_TEMP_2047" : { + "message" : [ + "'path' is not specified" + ] + }, + "_LEGACY_ERROR_TEMP_2048" : { + "message" : [ + "Schema must be specified when creating a streaming source DataFrame. If some files already exist in the directory, then depending on the file format you may be able to create a static DataFrame on that directory with 'spark.read.load(directory)' and infer schema from it." + ] + }, + "_LEGACY_ERROR_TEMP_2049" : { + "message" : [ + "Data source <className> does not support streamed <operator>" + ] + }, + "_LEGACY_ERROR_TEMP_2050" : { + "message" : [ + "Expected exactly one path to be specified, but got: <paths>" + ] } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index b15078d9b91..341c3e72de8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -296,7 +296,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase { def ansiIllegalArgumentError(message: String): SparkIllegalArgumentException = { new SparkIllegalArgumentException( - errorClass = "_LEGACY_ERROR_TEMP_2001", + errorClass = "_LEGACY_ERROR_TEMP_2000", messageParameters = Map( "message" -> message, "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key))) @@ -495,88 +495,141 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase { messageParameters = Map("className" -> className, "m1" -> m1, "m2" -> m2)) } - def failToConvertValueToJsonError(value: AnyRef, cls: Class[_], dataType: DataType): Throwable = { - new RuntimeException(s"Failed to convert value $value (class of $cls) " + - s"with the type of $dataType to JSON.") + def failToConvertValueToJsonError( + value: AnyRef, cls: Class[_], dataType: DataType): SparkRuntimeException = { + new SparkRuntimeException( + errorClass = "_LEGACY_ERROR_TEMP_2026", + messageParameters = Map( + "value" -> value.toString(), + "cls" -> cls.toString(), + "dataType" -> dataType.toString())) } - def unexpectedOperatorInCorrelatedSubquery(op: LogicalPlan, pos: String = ""): Throwable = { - new RuntimeException(s"Unexpected operator $op in correlated subquery" + pos) + def unexpectedOperatorInCorrelatedSubquery( + op: LogicalPlan, pos: String = ""): SparkRuntimeException = { + new SparkRuntimeException( + errorClass = "_LEGACY_ERROR_TEMP_2027", + messageParameters = Map("op" -> op.toString(), "pos" -> pos)) } - def unreachableError(err: String = ""): Throwable = { - new RuntimeException("This line should be unreachable" + err) + def unreachableError(err: String = ""): SparkRuntimeException = { + new SparkRuntimeException( + errorClass = "_LEGACY_ERROR_TEMP_2028", + messageParameters = Map("err" -> err)) } - def unsupportedRoundingMode(roundMode: BigDecimal.RoundingMode.Value): Throwable = { - new RuntimeException(s"Not supported rounding mode: $roundMode") + def unsupportedRoundingMode(roundMode: BigDecimal.RoundingMode.Value): SparkRuntimeException = { + new SparkRuntimeException( + errorClass = "_LEGACY_ERROR_TEMP_2029", + messageParameters = Map("roundMode" -> roundMode.toString())) } - def resolveCannotHandleNestedSchema(plan: LogicalPlan): Throwable = { - new RuntimeException(s"Can not handle nested schema yet... plan $plan") + def resolveCannotHandleNestedSchema(plan: LogicalPlan): SparkRuntimeException = { + new SparkRuntimeException( + errorClass = "_LEGACY_ERROR_TEMP_2030", + messageParameters = Map("plan" -> plan.toString())) } - def inputExternalRowCannotBeNullError(): RuntimeException = { - new RuntimeException("The input external row cannot be null.") + def inputExternalRowCannotBeNullError(): SparkRuntimeException = { + new SparkRuntimeException( + errorClass = "_LEGACY_ERROR_TEMP_2031", + messageParameters = Map.empty) } def fieldCannotBeNullMsg(index: Int, fieldName: String): String = { s"The ${index}th field '$fieldName' of input row cannot be null." } - def fieldCannotBeNullError(index: Int, fieldName: String): RuntimeException = { - new RuntimeException(fieldCannotBeNullMsg(index, fieldName)) + def fieldCannotBeNullError(index: Int, fieldName: String): SparkRuntimeException = { + new SparkRuntimeException( + errorClass = "_LEGACY_ERROR_TEMP_2032", + messageParameters = Map("fieldCannotBeNullMsg" -> fieldCannotBeNullMsg(index, fieldName))) } def unableToCreateDatabaseAsFailedToCreateDirectoryError( dbDefinition: CatalogDatabase, e: IOException): Throwable = { - new SparkException(s"Unable to create database ${dbDefinition.name} as failed " + - s"to create its directory ${dbDefinition.locationUri}", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2033", + messageParameters = Map( + "name" -> dbDefinition.name, + "locationUri" -> dbDefinition.locationUri.toString()), + cause = e) } def unableToDropDatabaseAsFailedToDeleteDirectoryError( dbDefinition: CatalogDatabase, e: IOException): Throwable = { - new SparkException(s"Unable to drop database ${dbDefinition.name} as failed " + - s"to delete its directory ${dbDefinition.locationUri}", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2034", + messageParameters = Map( + "name" -> dbDefinition.name, + "locationUri" -> dbDefinition.locationUri.toString()), + cause = e) } def unableToCreateTableAsFailedToCreateDirectoryError( table: String, defaultTableLocation: Path, e: IOException): Throwable = { - new SparkException(s"Unable to create table $table as failed " + - s"to create its directory $defaultTableLocation", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2035", + messageParameters = Map( + "table" -> table, + "defaultTableLocation" -> defaultTableLocation.toString()), + cause = e) } def unableToDeletePartitionPathError(partitionPath: Path, e: IOException): Throwable = { - new SparkException(s"Unable to delete partition path $partitionPath", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2036", + messageParameters = Map("partitionPath" -> partitionPath.toString()), + cause = e) } def unableToDropTableAsFailedToDeleteDirectoryError( table: String, dir: Path, e: IOException): Throwable = { - new SparkException(s"Unable to drop table $table as failed " + - s"to delete its directory $dir", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2037", + messageParameters = Map("table" -> table, "dir" -> dir.toString()), + cause = e) } def unableToRenameTableAsFailedToRenameDirectoryError( oldName: String, newName: String, oldDir: Path, e: IOException): Throwable = { - new SparkException(s"Unable to rename table $oldName to $newName as failed " + - s"to rename its directory $oldDir", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2038", + messageParameters = Map( + "oldName" -> oldName, + "newName" -> newName, + "oldDir" -> oldDir.toString()), + cause = e) } def unableToCreatePartitionPathError(partitionPath: Path, e: IOException): Throwable = { - new SparkException(s"Unable to create partition path $partitionPath", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2039", + messageParameters = Map("partitionPath" -> partitionPath.toString()), + cause = e) } def unableToRenamePartitionPathError(oldPartPath: Path, e: IOException): Throwable = { - new SparkException(s"Unable to rename partition path $oldPartPath", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2040", + messageParameters = Map("oldPartPath" -> oldPartPath.toString()), + cause = e) } - def methodNotImplementedError(methodName: String): Throwable = { - new UnsupportedOperationException(s"$methodName is not implemented") + def methodNotImplementedError(methodName: String): SparkUnsupportedOperationException = { + new SparkUnsupportedOperationException( + errorClass = "_LEGACY_ERROR_TEMP_2041", + messageParameters = Map("methodName" -> methodName)) } - def arithmeticOverflowError(e: ArithmeticException): ArithmeticException = { - new ArithmeticException(s"${e.getMessage}. If necessary set ${SQLConf.ANSI_ENABLED.key} " + - s"to false to bypass this error.") + def arithmeticOverflowError(e: ArithmeticException): SparkArithmeticException = { + new SparkArithmeticException( + errorClass = "_LEGACY_ERROR_TEMP_2042", + messageParameters = Map( + "message" -> e.getMessage, + "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)), + context = Array.empty, + summary = "") } def arithmeticOverflowError( @@ -596,14 +649,24 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase { summary = getSummary(context)) } - def unaryMinusCauseOverflowError(originValue: Int): ArithmeticException = { - arithmeticOverflowError(s"- ${toSQLValue(originValue, IntegerType)} caused overflow") + def unaryMinusCauseOverflowError(originValue: Int): SparkArithmeticException = { + new SparkArithmeticException( + errorClass = "_LEGACY_ERROR_TEMP_2043", + messageParameters = Map("sqlValue" -> toSQLValue(originValue, IntegerType)), + context = Array.empty, + summary = "") } def binaryArithmeticCauseOverflowError( - eval1: Short, symbol: String, eval2: Short): ArithmeticException = { - arithmeticOverflowError( - s"${toSQLValue(eval1, ShortType)} $symbol ${toSQLValue(eval2, ShortType)} caused overflow") + eval1: Short, symbol: String, eval2: Short): SparkArithmeticException = { + new SparkArithmeticException( + errorClass = "_LEGACY_ERROR_TEMP_2044", + messageParameters = Map( + "sqlValue1" -> toSQLValue(eval1, ShortType), + "symbol" -> symbol, + "sqlValue2" -> toSQLValue(eval2, ShortType)), + context = Array.empty, + summary = "") } def intervalArithmeticOverflowError( @@ -635,25 +698,29 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase { } def unsupportedTableChangeError(e: IllegalArgumentException): Throwable = { - new SparkException(s"Unsupported table change: ${e.getMessage}", e) + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2045", + messageParameters = Map("message" -> e.getMessage), + cause = e) } def notADatasourceRDDPartitionError(split: Partition): Throwable = { - new SparkException(s"[BUG] Not a DataSourceRDDPartition: $split") + new SparkException( + errorClass = "_LEGACY_ERROR_TEMP_2046", + messageParameters = Map("split" -> split.toString()), + cause = null) } - def dataPathNotSpecifiedError(): Throwable = { - new IllegalArgumentException("'path' is not specified") + def dataPathNotSpecifiedError(): SparkIllegalArgumentException = { + new SparkIllegalArgumentException( + errorClass = "_LEGACY_ERROR_TEMP_2047", + messageParameters = Map.empty) } - def createStreamingSourceNotSpecifySchemaError(): Throwable = { - new IllegalArgumentException( - s""" - |Schema must be specified when creating a streaming source DataFrame. If some - |files already exist in the directory, then depending on the file format you - |may be able to create a static DataFrame on that directory with - |'spark.read.load(directory)' and infer schema from it. - """.stripMargin) + def createStreamingSourceNotSpecifySchemaError(): SparkIllegalArgumentException = { + new SparkIllegalArgumentException( + errorClass = "_LEGACY_ERROR_TEMP_2048", + messageParameters = Map.empty) } def inferDateWithLegacyTimeParserError(): Throwable = { @@ -663,14 +730,16 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase { } def streamedOperatorUnsupportedByDataSourceError( - className: String, operator: String): Throwable = { - new UnsupportedOperationException( - s"Data source $className does not support streamed $operator") + className: String, operator: String): SparkUnsupportedOperationException = { + new SparkUnsupportedOperationException( + errorClass = "_LEGACY_ERROR_TEMP_2049", + messageParameters = Map("className" -> className, "operator" -> operator)) } - def multiplePathsSpecifiedError(allPaths: Seq[String]): Throwable = { - new IllegalArgumentException("Expected exactly one path to be specified, but " + - s"got: ${allPaths.mkString(", ")}") + def multiplePathsSpecifiedError(allPaths: Seq[String]): SparkIllegalArgumentException = { + new SparkIllegalArgumentException( + errorClass = "_LEGACY_ERROR_TEMP_2050", + messageParameters = Map("paths" -> allPaths.mkString(", "))) } def failedToFindDataSourceError(provider: String, error: Throwable): Throwable = { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org