This is an automated email from the ASF dual-hosted git repository. gengliang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new b70fa24 [SPARK-38710][SQL] Use SparkArithmeticException for arithmetic overflow runtime errors b70fa24 is described below commit b70fa24304e524255df8b19c31666699e2642583 Author: Gengliang Wang <gengli...@apache.org> AuthorDate: Thu Mar 31 23:22:05 2022 +0800 [SPARK-38710][SQL] Use SparkArithmeticException for arithmetic overflow runtime errors ### What changes were proposed in this pull request? On arithmetic overflow runtime errors, Spark should throw SparkArithmeticException instead of `java.lang.ArithmeticException` ### Why are the changes needed? Use a better error exception type. ### Does this PR introduce _any_ user-facing change? Yes, trivial change on the exception type: on arithmetic overflow runtime errors, Spark will throw SparkArithmeticException instead of `java.lang.ArithmeticException` ### How was this patch tested? UT Closes #36022 from gengliangwang/ArithmeticException. Authored-by: Gengliang Wang <gengli...@apache.org> Signed-off-by: Gengliang Wang <gengli...@apache.org> --- core/src/main/resources/error/error-classes.json | 4 ++++ .../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 5 ++--- .../test/resources/sql-tests/results/ansi/interval.sql.out | 14 +++++++------- .../src/test/resources/sql-tests/results/interval.sql.out | 14 +++++++------- .../resources/sql-tests/results/postgreSQL/int4.sql.out | 12 ++++++------ .../resources/sql-tests/results/postgreSQL/int8.sql.out | 8 ++++---- .../sql-tests/results/postgreSQL/window_part2.sql.out | 4 ++-- .../org/apache/spark/sql/DataFrameAggregateSuite.scala | 8 ++++---- 8 files changed, 36 insertions(+), 33 deletions(-) diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index d9e2e74..d7d7702 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -3,6 +3,10 @@ "message" : [ "Field name %s is ambiguous and has %s matching fields in the struct." ], "sqlState" : "42000" }, + "ARITHMETIC_OVERFLOW" : { + "message" : [ "%s.%s If necessary set %s to false (except for ANSI interval type) to bypass this error.%s" ], + "sqlState" : "22003" + }, "CANNOT_CAST_DATATYPE" : { "message" : [ "Cannot cast %s to %s." ], "sqlState" : "22005" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 316cdc9..c86cc89 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -430,9 +430,8 @@ object QueryExecutionErrors { hint: String = "", errorContext: String = ""): ArithmeticException = { val alternative = if (hint.nonEmpty) s" To return NULL instead, use '$hint'." else "" - new ArithmeticException(s"$message.$alternative If necessary set " + - s"${SQLConf.ANSI_ENABLED.key} to false (except for ANSI interval type) to bypass this " + - "error." + errorContext) + new SparkArithmeticException("ARITHMETIC_OVERFLOW", + Array(message, alternative, SQLConf.ANSI_ENABLED.key, errorContext)) } def unaryMinusCauseOverflowError(originValue: AnyVal): ArithmeticException = { diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index e468612..f28e530 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -1755,7 +1755,7 @@ select -(a) from values (interval '-2147483648 months', interval '2147483647 mon -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. @@ -1764,7 +1764,7 @@ select a - b from values (interval '-2147483648 months', interval '2147483647 mo -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. @@ -1773,7 +1773,7 @@ select b + interval '1 month' from values (interval '-2147483648 months', interv -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. @@ -2002,7 +2002,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1 -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1 @@ -2014,7 +2014,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L @@ -2060,7 +2060,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1 -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1 @@ -2072,7 +2072,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index df1db77..58c86c9 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1744,7 +1744,7 @@ select -(a) from values (interval '-2147483648 months', interval '2147483647 mon -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. @@ -1753,7 +1753,7 @@ select a - b from values (interval '-2147483648 months', interval '2147483647 mo -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. @@ -1762,7 +1762,7 @@ select b + interval '1 month' from values (interval '-2147483648 months', interv -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. @@ -1991,7 +1991,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1 -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1 @@ -2003,7 +2003,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L @@ -2049,7 +2049,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1 -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1 @@ -2061,7 +2061,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out index 993fe44..144a015 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out @@ -199,7 +199,7 @@ SELECT '' AS five, i.f1, i.f1 * smallint('2') AS x FROM INT4_TBL i -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 25) == SELECT '' AS five, i.f1, i.f1 * smallint('2') AS x FROM INT4_TBL i @@ -222,7 +222,7 @@ SELECT '' AS five, i.f1, i.f1 * int('2') AS x FROM INT4_TBL i -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 25) == SELECT '' AS five, i.f1, i.f1 * int('2') AS x FROM INT4_TBL i @@ -245,7 +245,7 @@ SELECT '' AS five, i.f1, i.f1 + smallint('2') AS x FROM INT4_TBL i -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 25) == SELECT '' AS five, i.f1, i.f1 + smallint('2') AS x FROM INT4_TBL i @@ -269,7 +269,7 @@ SELECT '' AS five, i.f1, i.f1 + int('2') AS x FROM INT4_TBL i -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 25) == SELECT '' AS five, i.f1, i.f1 + int('2') AS x FROM INT4_TBL i @@ -293,7 +293,7 @@ SELECT '' AS five, i.f1, i.f1 - smallint('2') AS x FROM INT4_TBL i -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 25) == SELECT '' AS five, i.f1, i.f1 - smallint('2') AS x FROM INT4_TBL i @@ -317,7 +317,7 @@ SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT4_TBL i -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 25) == SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT4_TBL i diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out index bb4a770..cc524b5 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out @@ -391,7 +391,7 @@ SELECT '' AS three, q1, q2, q1 * q2 AS multiply FROM INT8_TBL -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 28) == SELECT '' AS three, q1, q2, q1 * q2 AS multiply FROM INT8_TBL @@ -753,7 +753,7 @@ SELECT bigint((-9223372036854775808)) * bigint((-1)) -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT bigint((-9223372036854775808)) * bigint((-1)) @@ -781,7 +781,7 @@ SELECT bigint((-9223372036854775808)) * int((-1)) -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT bigint((-9223372036854775808)) * int((-1)) @@ -809,7 +809,7 @@ SELECT bigint((-9223372036854775808)) * smallint((-1)) -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. == SQL(line 1, position 7) == SELECT bigint((-9223372036854775808)) * smallint((-1)) diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out index ab6170eb..75c40ce 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out @@ -224,7 +224,7 @@ from range(9223372036854775804, 9223372036854775807) x -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. @@ -234,7 +234,7 @@ from range(-9223372036854775806, -9223372036854775805) x -- !query schema struct<> -- !query output -java.lang.ArithmeticException +org.apache.spark.SparkArithmeticException long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index 157736f..425be96 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -1278,12 +1278,12 @@ class DataFrameAggregateSuite extends QueryTest val error = intercept[SparkException] { checkAnswer(df2.select(sum($"year-month")), Nil) } - assert(error.toString contains "java.lang.ArithmeticException: integer overflow") + assert(error.toString contains "SparkArithmeticException: integer overflow") val error2 = intercept[SparkException] { checkAnswer(df2.select(sum($"day")), Nil) } - assert(error2.toString contains "java.lang.ArithmeticException: long overflow") + assert(error2.toString contains "SparkArithmeticException: long overflow") } test("SPARK-34837: Support ANSI SQL intervals by the aggregate function `avg`") { @@ -1412,12 +1412,12 @@ class DataFrameAggregateSuite extends QueryTest val error = intercept[SparkException] { checkAnswer(df2.select(avg($"year-month")), Nil) } - assert(error.toString contains "java.lang.ArithmeticException: integer overflow") + assert(error.toString contains "SparkArithmeticException: integer overflow") val error2 = intercept[SparkException] { checkAnswer(df2.select(avg($"day")), Nil) } - assert(error2.toString contains "java.lang.ArithmeticException: long overflow") + assert(error2.toString contains "SparkArithmeticException: long overflow") val df3 = intervalData.filter($"class" > 4) val avgDF3 = df3.select(avg($"year-month"), avg($"day")) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org