This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new af016d9eb87 [SPARK-39050][SQL] Error class: UNSUPPORTED_OPERATION to
UNSUPPORTED_FEATURE
af016d9eb87 is described below
commit af016d9eb87dcdd2423dc6eae691a52a5a23ae63
Author: Serge Rielau <serge.rie...@databricks.com>
AuthorDate: Fri Apr 29 08:57:36 2022 +0300
[SPARK-39050][SQL] Error class: UNSUPPORTED_OPERATION to UNSUPPORTED_FEATURE
### What changes were proposed in this pull request?
UNSUPPORTED_OPERATION will be removed and replaced with the existing
UNSUPPORTED_FEATURE.
This effects three errors:
ARROW TIMESTAMP, ORC TMESTAMP TO TIMESTAMP_NTZ and ORC TMESTAMP_NTZ TO
TIMESTAMP
### Why are the changes needed?
Clean up ERROR CLASSES before publishing them.
### Does this PR introduce _any_ user-facing change?
No, this is still internal, unreleased code
### How was this patch tested?
Run existing QueryExecutionErrorsSuite
Closes #36385 from
srielau/SPARK-39050-UNSUPPORTED_OPERATION-to-UNSUPPORTED_FEATURE.
Authored-by: Serge Rielau <serge.rie...@databricks.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
core/src/main/resources/error/error-classes.json | 6 ++---
.../spark/sql/errors/QueryExecutionErrors.scala | 30 ++++++++++------------
.../org/apache/spark/sql/util/ArrowUtils.scala | 2 +-
.../apache/spark/sql/util/ArrowUtilsSuite.scala | 4 +--
.../sql/errors/QueryExecutionErrorsSuite.scala | 23 +++++++++--------
5 files changed, 31 insertions(+), 34 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index 4738599685b..4908a9b6c2e 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -213,6 +213,9 @@
"NATURAL_CROSS_JOIN" : {
"message" : [ "NATURAL CROSS JOIN." ]
},
+ "ORC_TYPE_CAST" : {
+ "message" : [ "Unable to convert <orcType> of Orc to data type
<toType>." ]
+ },
"PANDAS_UDAF_IN_PIVOT" : {
"message" : [ "Pandas user defined aggregate function in the PIVOT
clause." ]
},
@@ -243,9 +246,6 @@
"UNSUPPORTED_GROUPING_EXPRESSION" : {
"message" : [ "grouping()/grouping_id() can only be used with
GroupingSets/Cube/Rollup" ]
},
- "UNSUPPORTED_OPERATION" : {
- "message" : [ "The operation is not supported: <operation>" ]
- },
"UNTYPED_SCALA_UDF" : {
"message" : [ "You're using untyped Scala UDF, which does not have the
input type information. Spark may blindly pass null to the Scala closure with
primitive-type argument, and the closure will see the default value of the Java
type for the null argument, e.g. `udf((x: Int) => x, IntegerType)`, the result
is 0 for null input. To get rid of this error, you could:\n1. use typed Scala
UDF APIs(without return type parameter), e.g. `udf((x: Int) => x)`\n2. use Java
UDF APIs, e.g. `udf(ne [...]
},
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index d5e42a1dde7..225315d3f02 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -68,6 +68,11 @@ import org.apache.spark.util.CircularBuffer
*/
object QueryExecutionErrors extends QueryErrorsBase {
+ def internalMissingTimezoneIdError(): Throwable = {
+ new SparkIllegalStateException(errorClass = "INTERNAL_ERROR",
+ messageParameters = Array("Missing timezoneId where it is mandatory."))
+ }
+
def logicalHintOperatorNotRemovedDuringAnalysisError(): Throwable = {
new SparkIllegalStateException(errorClass = "INTERNAL_ERROR",
messageParameters = Array(
@@ -1614,15 +1619,6 @@ object QueryExecutionErrors extends QueryErrorsBase {
new SparkException(s"Can not load in UserDefinedType ${name} for user
class ${userClass}.")
}
- def timeZoneIdNotSpecifiedForTimestampTypeError(): Throwable = {
- new SparkUnsupportedOperationException(
- errorClass = "UNSUPPORTED_OPERATION",
- messageParameters = Array(
- s"${toSQLType(TimestampType)} must supply timeZoneId parameter " +
- s"while converting to the arrow timestamp type.")
- )
- }
-
def notPublicClassError(name: String): Throwable = {
new UnsupportedOperationException(
s"$name is not a public class. Only public classes are supported.")
@@ -1936,18 +1932,18 @@ object QueryExecutionErrors extends QueryErrorsBase {
def cannotConvertOrcTimestampToTimestampNTZError(): Throwable = {
new SparkUnsupportedOperationException(
- errorClass = "UNSUPPORTED_OPERATION",
- messageParameters = Array(
- s"Unable to convert ${toSQLType(TimestampType)} of Orc to " +
- s"data type ${toSQLType(TimestampNTZType)}."))
+ errorClass = "UNSUPPORTED_FEATURE",
+ messageParameters = Array("ORC_TYPE_CAST",
+ toSQLType(TimestampType),
+ toSQLType(TimestampNTZType)))
}
def cannotConvertOrcTimestampNTZToTimestampLTZError(): Throwable = {
new SparkUnsupportedOperationException(
- errorClass = "UNSUPPORTED_OPERATION",
- messageParameters = Array(
- s"Unable to convert ${toSQLType(TimestampNTZType)} of Orc to " +
- s"data type ${toSQLType(TimestampType)}."))
+ errorClass = "UNSUPPORTED_FEATURE",
+ messageParameters = Array("ORC_TYPE_CAST",
+ toSQLType(TimestampNTZType),
+ toSQLType(TimestampType)))
}
def writePartitionExceedConfigSizeWhenDynamicPartitionError(
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
index 4254c045ca6..8fa1879548c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
@@ -49,7 +49,7 @@ private[sql] object ArrowUtils {
case DateType => new ArrowType.Date(DateUnit.DAY)
case TimestampType =>
if (timeZoneId == null) {
- throw
QueryExecutionErrors.timeZoneIdNotSpecifiedForTimestampTypeError()
+ throw QueryExecutionErrors.internalMissingTimezoneIdError()
} else {
new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
index 642b387b88e..03178f2b761 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
@@ -50,10 +50,10 @@ class ArrowUtilsSuite extends SparkFunSuite {
roundtrip(DateType)
roundtrip(YearMonthIntervalType())
roundtrip(DayTimeIntervalType())
- val tsExMsg = intercept[UnsupportedOperationException] {
+ val tsExMsg = intercept[org.apache.spark.SparkIllegalStateException] {
roundtrip(TimestampType)
}
- assert(tsExMsg.getMessage.contains("timeZoneId"))
+ assert(tsExMsg.getMessage.contains("timezoneId"))
}
test("timestamp") {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
index 1105503e138..338da79674e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala
@@ -261,17 +261,16 @@ class QueryExecutionErrorsSuite
}
}
- test("UNSUPPORTED_OPERATION: timeZoneId not specified while converting
TimestampType to Arrow") {
+ test("INTERNAL_ERROR: timeZoneId not specified while converting
TimestampType to Arrow") {
checkErrorClass(
- exception = intercept[SparkUnsupportedOperationException] {
+ exception = intercept[SparkIllegalStateException] {
ArrowUtils.toArrowSchema(new StructType().add("value", TimestampType),
null)
},
- errorClass = "UNSUPPORTED_OPERATION",
- msg = "The operation is not supported: \"TIMESTAMP\" must supply
timeZoneId " +
- "parameter while converting to the arrow timestamp type.")
+ errorClass = "INTERNAL_ERROR",
+ msg = "Missing timezoneId where it is mandatory.")
}
- test("UNSUPPORTED_OPERATION - SPARK-36346: can't read Timestamp as
TimestampNTZ") {
+ test("UNSUPPORTED_FEATURE - SPARK-36346: can't read Timestamp as
TimestampNTZ") {
withTempPath { file =>
sql("select timestamp_ltz'2019-03-21
00:02:03'").write.orc(file.getCanonicalPath)
withAllNativeOrcReaders {
@@ -279,14 +278,15 @@ class QueryExecutionErrorsSuite
exception = intercept[SparkException] {
spark.read.schema("time
timestamp_ntz").orc(file.getCanonicalPath).collect()
}.getCause.asInstanceOf[SparkUnsupportedOperationException],
- errorClass = "UNSUPPORTED_OPERATION",
- msg = "The operation is not supported: " +
+ errorClass = "UNSUPPORTED_FEATURE",
+ errorSubClass = Some("ORC_TYPE_CAST"),
+ msg = "The feature is not supported: " +
"Unable to convert \"TIMESTAMP\" of Orc to data type
\"TIMESTAMP_NTZ\".")
}
}
}
- test("UNSUPPORTED_OPERATION - SPARK-38504: can't read TimestampNTZ as
TimestampLTZ") {
+ test("UNSUPPORTED_FEATURE - SPARK-38504: can't read TimestampNTZ as
TimestampLTZ") {
withTempPath { file =>
sql("select timestamp_ntz'2019-03-21
00:02:03'").write.orc(file.getCanonicalPath)
withAllNativeOrcReaders {
@@ -294,8 +294,9 @@ class QueryExecutionErrorsSuite
exception = intercept[SparkException] {
spark.read.schema("time
timestamp_ltz").orc(file.getCanonicalPath).collect()
}.getCause.asInstanceOf[SparkUnsupportedOperationException],
- errorClass = "UNSUPPORTED_OPERATION",
- msg = "The operation is not supported: " +
+ errorClass = "UNSUPPORTED_FEATURE",
+ errorSubClass = Some("ORC_TYPE_CAST"),
+ msg = "The feature is not supported: " +
"Unable to convert \"TIMESTAMP_NTZ\" of Orc to data type
\"TIMESTAMP\".")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
