This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 028992399b2 [SPARK-40209][SQL] Don't change the interval value of
Decimal in `changePrecision()` on errors
028992399b2 is described below
commit 028992399b2be19b0e97225ef3e4def4e10b7570
Author: Max Gekk <max.g...@gmail.com>
AuthorDate: Thu Aug 25 10:11:12 2022 +0300
[SPARK-40209][SQL] Don't change the interval value of Decimal in
`changePrecision()` on errors
### What changes were proposed in this pull request?
In the PR, I propose to modify the internal value of Decimal `longVal` in
the method `changePrecision` only in the success cases, and don't touch it on
any errors to keep the original value.
### Why are the changes needed?
To don't confuse users by error messages. The fix might improve user
experience with Spark SQL.
### Does this PR introduce _any_ user-facing change?
Yes. The PR changes user-facing errors.
Before:
```sql
spark-sql> select cast(interval '10.123' second as decimal(1, 0));
[NUMERIC_VALUE_OUT_OF_RANGE] 0.000010 cannot be represented ...
```
After:
```sql
spark-sql> select cast(interval '10.123' second as decimal(1, 0));
[NUMERIC_VALUE_OUT_OF_RANGE] 10.123000 cannot be represented ...
```
### How was this patch tested?
By running the affected test suites:
```
$ build/sbt "sql/testOnly org.apache.spark.sql.SQLQueryTestSuite -- -z
cast.sql"
```
Closes #37649 from MaxGekk/longVal-changePrecision.
Authored-by: Max Gekk <max.g...@gmail.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../scala/org/apache/spark/sql/types/Decimal.scala | 44 +++++++++++-----------
.../resources/sql-tests/results/ansi/cast.sql.out | 2 +-
.../test/resources/sql-tests/results/cast.sql.out | 2 +-
3 files changed, 25 insertions(+), 23 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index aa683a06a8e..69eff6de4b9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -395,69 +395,71 @@ final class Decimal extends Ordered[Decimal] with
Serializable {
return true
}
DecimalType.checkNegativeScale(scale)
- // First, update our longVal if we can, or transfer over to using a
BigDecimal
- if (decimalVal.eq(null)) {
+ var lv = longVal
+ var dv = decimalVal
+ // First, update our lv if we can, or transfer over to using a BigDecimal
+ if (dv.eq(null)) {
if (scale < _scale) {
// Easier case: we just need to divide our scale down
val diff = _scale - scale
val pow10diff = POW_10(diff)
// % and / always round to 0
- val droppedDigits = longVal % pow10diff
- longVal /= pow10diff
+ val droppedDigits = lv % pow10diff
+ lv /= pow10diff
roundMode match {
case ROUND_FLOOR =>
if (droppedDigits < 0) {
- longVal += -1L
+ lv += -1L
}
case ROUND_CEILING =>
if (droppedDigits > 0) {
- longVal += 1L
+ lv += 1L
}
case ROUND_HALF_UP =>
if (math.abs(droppedDigits) * 2 >= pow10diff) {
- longVal += (if (droppedDigits < 0) -1L else 1L)
+ lv += (if (droppedDigits < 0) -1L else 1L)
}
case ROUND_HALF_EVEN =>
val doubled = math.abs(droppedDigits) * 2
- if (doubled > pow10diff || doubled == pow10diff && longVal % 2 !=
0) {
- longVal += (if (droppedDigits < 0) -1L else 1L)
+ if (doubled > pow10diff || doubled == pow10diff && lv % 2 != 0) {
+ lv += (if (droppedDigits < 0) -1L else 1L)
}
case _ =>
throw QueryExecutionErrors.unsupportedRoundingMode(roundMode)
}
} else if (scale > _scale) {
- // We might be able to multiply longVal by a power of 10 and not
overflow, but if not,
+ // We might be able to multiply lv by a power of 10 and not overflow,
but if not,
// switch to using a BigDecimal
val diff = scale - _scale
val p = POW_10(math.max(MAX_LONG_DIGITS - diff, 0))
- if (diff <= MAX_LONG_DIGITS && longVal > -p && longVal < p) {
- // Multiplying longVal by POW_10(diff) will still keep it below
MAX_LONG_DIGITS
- longVal *= POW_10(diff)
+ if (diff <= MAX_LONG_DIGITS && lv > -p && lv < p) {
+ // Multiplying lv by POW_10(diff) will still keep it below
MAX_LONG_DIGITS
+ lv *= POW_10(diff)
} else {
// Give up on using Longs; switch to BigDecimal, which we'll modify
below
- decimalVal = BigDecimal(longVal, _scale)
+ dv = BigDecimal(lv, _scale)
}
}
// In both cases, we will check whether our precision is okay below
}
- if (decimalVal.ne(null)) {
+ if (dv.ne(null)) {
// We get here if either we started with a BigDecimal, or we switched to
one because we would
- // have overflowed our Long; in either case we must rescale decimalVal
to the new scale.
- val newVal = decimalVal.setScale(scale, roundMode)
- if (newVal.precision > precision) {
+ // have overflowed our Long; in either case we must rescale dv to the
new scale.
+ dv = dv.setScale(scale, roundMode)
+ if (dv.precision > precision) {
return false
}
- decimalVal = newVal
} else {
// We're still using Longs, but we should check whether we match the new
precision
val p = POW_10(math.min(precision, MAX_LONG_DIGITS))
- if (longVal <= -p || longVal >= p) {
+ if (lv <= -p || lv >= p) {
// Note that we shouldn't have been able to fix this by switching to
BigDecimal
return false
}
}
-
+ decimalVal = dv
+ longVal = lv
_precision = precision
_scale = scale
true
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/cast.sql.out
b/sql/core/src/test/resources/sql-tests/results/ansi/cast.sql.out
index 8f53e557b59..378f817f7d9 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/cast.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/cast.sql.out
@@ -1495,7 +1495,7 @@ org.apache.spark.SparkArithmeticException
"errorClass" : "NUMERIC_VALUE_OUT_OF_RANGE",
"sqlState" : "22005",
"messageParameters" : {
- "value" : "0.000010",
+ "value" : "10.123000",
"precision" : "1",
"scale" : "0",
"config" : "\"spark.sql.ansi.enabled\""
diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out
b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
index 087dd831195..9271fd6350e 100644
--- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
@@ -869,7 +869,7 @@ org.apache.spark.SparkArithmeticException
"errorClass" : "NUMERIC_VALUE_OUT_OF_RANGE",
"sqlState" : "22005",
"messageParameters" : {
- "value" : "0.000010",
+ "value" : "10.123000",
"precision" : "1",
"scale" : "0",
"config" : "\"spark.sql.ansi.enabled\""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
