This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6a0e671b7672 [SPARK-57937][SQL] Improve decimal division performance
by dividing at the result scale
6a0e671b7672 is described below
commit 6a0e671b7672982dd4138c368ea71660ca70e604
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Jul 5 12:19:19 2026 -0700
[SPARK-57937][SQL] Improve decimal division performance by dividing at the
result scale
### What changes were proposed in this pull request?
This PR changes decimal `Divide` to perform the division directly at the
result scale, instead of dividing at `DecimalType.MAX_SCALE + 1` and then
rounding down to the result scale.
- `Decimal` gets a new method `div(that: Decimal, scale: Int)` that divides
with `RoundingMode.HALF_UP` at the given scale.
- `DivModLike` codegen gains an overridable `decimalOperation` hook
(default: existing `decimalMethod` behavior), and `Divide` overrides it — both
the codegen and the interpreted eval now call `div(b, resultScale)` followed by
the existing overflow check. `IntegralDivide`, `Remainder`, and `Pmod` are
unchanged.
Previously, `Decimal./` always divided at scale `39` with `ROUND_DOWN` and
the subsequent `changePrecision` performed a second division to reach the
result scale. This is equivalent to a single `HALF_UP` division at the result
scale: truncating at 39 fractional digits cannot change a `HALF_UP` rounding
decision at any scale `<= 38`, because every half-way point at scale `s <= 38`
is exactly representable within 39 fractional digits.
### Why are the changes needed?
The fixed scale-39 intermediate makes decimal division needlessly
expensive, especially for small decimal types. For example, `decimal(7,2) /
decimal(7,2)` (result `decimal(17,10)`) produces a scale-39 intermediate whose
unscaled value exceeds the `Long` range, forcing two `BigInteger` (Knuth)
divisions per row — one inside `BigDecimal.divide` and another in
`CheckOverflow`'s rescale, visible in profiles as:
```
java.math.MutableBigInteger.divideMagnitude
java.math.BigDecimal.setScale
org.apache.spark.sql.types.Decimal.changePrecision
org.apache.spark.sql.types.Decimal.toPrecision
```
Dividing once at the result scale keeps the computation in the compact
`long` representation whenever the quotient fits.
### Does this PR introduce _any_ user-facing change?
- Query results are unchanged.
- The only difference is that the `NUMERIC_VALUE_OUT_OF_RANGE` error
message on division overflow now renders the quotient rounded at the result
scale instead of the internal scale-39 intermediate, e.g.
`1000000073899961059796.725866331521039185` instead of
`1000000073899961059796.725866331521039184725213147467478092333`. The error
class, parameters, and whether an error occurs are all unchanged.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #57006 from dongjoon-hyun/SPARK-57937.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/sql/types/Decimal.scala | 9 ++++++++
.../sql/catalyst/expressions/arithmetic.scala | 23 ++++++++++++------
.../org/apache/spark/sql/types/DecimalSuite.scala | 27 ++++++++++++++++++++++
.../DecimalDivideBenchmark-jdk21-results.txt | 16 ++++++-------
.../DecimalDivideBenchmark-jdk25-results.txt | 20 ++++++++--------
.../benchmarks/DecimalDivideBenchmark-results.txt | 16 ++++++-------
.../results/decimalArithmeticOperations.sql.out | 12 +++++-----
7 files changed, 84 insertions(+), 39 deletions(-)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala
b/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index bd94c386ab53..a41847f02c2a 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -530,6 +530,15 @@ final class Decimal extends Ordered[Decimal] with
Serializable {
.divide(that.toJavaBigDecimal, DecimalType.MAX_SCALE + 1,
MATH_CONTEXT.getRoundingMode))
}
+ /**
+ * Divides `this` by `that` rounding the result to `scale` fractional digits
with HALF_UP.
+ * Returns the same result as `this / that` followed by rounding to `scale`,
but with a single
+ * division, which stays in the compact representation when the quotient
fits in a Long.
+ */
+ def div(that: Decimal, scale: Int): Decimal =
+ if (that.isZero) null
+ else Decimal(toJavaBigDecimal.divide(that.toJavaBigDecimal, scale,
RoundingMode.HALF_UP))
+
def %(that: Decimal): Decimal =
if (that.isZero) null
else Decimal(toJavaBigDecimal.remainder(that.toJavaBigDecimal,
MATH_CONTEXT))
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 88b409c4e867..9c72f822143f 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -692,6 +692,10 @@ trait DivModLike extends BinaryArithmetic {
def evalOperation(left: Any, right: Any): Any
+ /** Java code computing the decimal result that is fed into the overflow
check. */
+ protected def decimalOperation(eval1: String, eval2: String, scale: Int):
String =
+ s"$eval1.$decimalMethod($eval2)"
+
/**
* Special case handling due to division/remainder by 0 => null or
ArithmeticException.
*/
@@ -724,14 +728,14 @@ trait DivModLike extends BinaryArithmetic {
if (failOnError) {
s"""
|Decimal $decimalValue = $castUtils.changePrecisionExact(
- | ${eval1.value}.$decimalMethod(${eval2.value}), $precision,
$scale,
+ | ${decimalOperation(eval1.value, eval2.value, scale)},
$precision, $scale,
| $errorContextCode);
|${ev.value} = ${decimalToDataTypeCodeGen(s"$decimalValue")};
|""".stripMargin
} else {
s"""
|Decimal $decimalValue = $castUtils.changePrecisionOrNull(
- | ${eval1.value}.$decimalMethod(${eval2.value}), $precision,
$scale);
+ | ${decimalOperation(eval1.value, eval2.value, scale)},
$precision, $scale);
|if ($decimalValue != null) {
| ${ev.value} = ${decimalToDataTypeCodeGen(s"$decimalValue")};
|} else {
@@ -860,13 +864,18 @@ case class Divide(
}
}
+ // Dividing directly at the result scale returns the same result as dividing
at
+ // `DecimalType.MAX_SCALE + 1` and then rounding to the result scale, but
with a single
+ // division that stays in the compact representation when the quotient fits
in a Long.
+ override protected def decimalOperation(eval1: String, eval2: String, scale:
Int): String =
+ s"$eval1.div($eval2, $scale)"
+
private lazy val div: (Any, Any) => Any = dataType match {
- case d @ DecimalType.Fixed(precision, scale) =>
- val fractional = PhysicalDecimalType(precision, scale).fractional
+ case DecimalType.Fixed(precision, scale) =>
(l, r) => {
- val value = fractional.asInstanceOf[Fractional[Any]].div(l, r)
- checkDecimalOverflow(value.asInstanceOf[Decimal], precision, scale)
- }
+ val value = l.asInstanceOf[Decimal].div(r.asInstanceOf[Decimal], scale)
+ checkDecimalOverflow(value, precision, scale)
+ }
case ft: FractionalType => PhysicalFractionalType.fractional(ft).div
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 4af2c8367819..080c6693280a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -258,6 +258,33 @@ class DecimalSuite extends SparkFunSuite with
PrivateMethodTester with SQLHelper
assert(Decimal(100).quot(Decimal(-33)) === Decimal(BigDecimal("-3")))
}
+ test("SPARK-57937: div with target scale") {
+ // `div(that, scale)` must return the same result as `/` (which divides at
+ // `DecimalType.MAX_SCALE + 1` with ROUND_DOWN) followed by rounding to
the same scale.
+ def checkDiv(a: Decimal, b: Decimal, scale: Int): Unit = {
+ val expected = (a / b).toPrecision(DecimalType.MAX_PRECISION, scale)
+ val actual = a.div(b, scale)
+ assert(actual.toJavaBigDecimal.compareTo(expected.toJavaBigDecimal) ===
0,
+ s"$a.div($b, $scale)")
+ }
+ assert(Decimal(100).div(Decimal(0), 10) === null)
+ // Half-way cases, both signs.
+ checkDiv(Decimal("1.00"), Decimal("8.00"), 2)
+ checkDiv(Decimal("-1.00"), Decimal("8.00"), 2)
+ checkDiv(Decimal("1.05"), Decimal("-2.00"), 3)
+ // Non-terminating expansions.
+ checkDiv(Decimal("1.00"), Decimal("3.00"), 10)
+ checkDiv(Decimal("-2.00"), Decimal("7.00"), 38)
+ val rnd = new scala.util.Random(42)
+ for (_ <- 1 to 1000) {
+ // decimal(7, 2) operands, rounded to the scale of the decimal(17, 10)
result type.
+ val a = Decimal(rnd.nextInt(19999999) - 9999999, 7, 2)
+ val b = Decimal(rnd.nextInt(9999998) + 1, 7, 2)
+ checkDiv(a, b, 10)
+ checkDiv(a, -b, 10)
+ }
+ }
+
test("negate & abs") {
assert(-Decimal(100) === Decimal(BigDecimal("-100")))
assert(-Decimal(-100) === Decimal(BigDecimal("100")))
diff --git a/sql/core/benchmarks/DecimalDivideBenchmark-jdk21-results.txt
b/sql/core/benchmarks/DecimalDivideBenchmark-jdk21-results.txt
index b449110256c4..c58728c42f81 100644
--- a/sql/core/benchmarks/DecimalDivideBenchmark-jdk21-results.txt
+++ b/sql/core/benchmarks/DecimalDivideBenchmark-jdk21-results.txt
@@ -6,18 +6,18 @@ OpenJDK 64-Bit Server VM 21.0.11+10-LTS on Linux
6.17.0-1018-azure
AMD EPYC 7763 64-Core Processor
decimal(7,2) / decimal(7,2): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-a / b (ansi=false) 7797 7858
37 1.3 779.7 1.0X
-a / b (ansi=true) 7886 7904
18 1.3 788.6 1.0X
-a / 2 (ansi=false) 5524 5553
18 1.8 552.4 1.4X
-a / 2 (ansi=true) 5476 5500
24 1.8 547.6 1.4X
+a / b (ansi=false) 4776 4837
90 2.1 477.6 1.0X
+a / b (ansi=true) 4813 4824
9 2.1 481.3 1.0X
+a / 2 (ansi=false) 2799 2828
21 3.6 279.9 1.7X
+a / 2 (ansi=true) 2739 2764
17 3.7 273.9 1.7X
OpenJDK 64-Bit Server VM 21.0.11+10-LTS on Linux 6.17.0-1018-azure
AMD EPYC 7763 64-Core Processor
decimal(38,18) / decimal(38,18): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-a / b (ansi=false) 14130 14148
11 0.7 1413.0 1.0X
-a / b (ansi=true) 14138 14146
6 0.7 1413.8 1.0X
-a / 2 (ansi=false) 8607 8628
27 1.2 860.7 1.6X
-a / 2 (ansi=true) 8579 8623
26 1.2 857.9 1.6X
+a / b (ansi=false) 11056 11066
7 0.9 1105.6 1.0X
+a / b (ansi=true) 11067 11100
22 0.9 1106.7 1.0X
+a / 2 (ansi=false) 5969 5986
15 1.7 596.9 1.9X
+a / 2 (ansi=true) 5975 5984
13 1.7 597.5 1.9X
diff --git a/sql/core/benchmarks/DecimalDivideBenchmark-jdk25-results.txt
b/sql/core/benchmarks/DecimalDivideBenchmark-jdk25-results.txt
index 50913d8981d7..3281042b958a 100644
--- a/sql/core/benchmarks/DecimalDivideBenchmark-jdk25-results.txt
+++ b/sql/core/benchmarks/DecimalDivideBenchmark-jdk25-results.txt
@@ -3,21 +3,21 @@ Decimal division
================================================================================================
OpenJDK 64-Bit Server VM 25.0.3+9-LTS on Linux 6.17.0-1018-azure
-AMD EPYC 9V45 96-Core Processor
+AMD EPYC 7763 64-Core Processor
decimal(7,2) / decimal(7,2): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-a / b (ansi=false) 3110 3121
14 3.2 311.0 1.0X
-a / b (ansi=true) 3023 3070
36 3.3 302.3 1.0X
-a / 2 (ansi=false) 2088 2111
26 4.8 208.8 1.5X
-a / 2 (ansi=true) 2039 2065
30 4.9 203.9 1.5X
+a / b (ansi=false) 2792 2826
23 3.6 279.2 1.0X
+a / b (ansi=true) 2767 2781
15 3.6 276.7 1.0X
+a / 2 (ansi=false) 1773 1788
10 5.6 177.3 1.6X
+a / 2 (ansi=true) 1770 1778
7 5.7 177.0 1.6X
OpenJDK 64-Bit Server VM 25.0.3+9-LTS on Linux 6.17.0-1018-azure
-AMD EPYC 9V45 96-Core Processor
+AMD EPYC 7763 64-Core Processor
decimal(38,18) / decimal(38,18): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-a / b (ansi=false) 5557 5611
43 1.8 555.7 1.0X
-a / b (ansi=true) 5383 5437
43 1.9 538.3 1.0X
-a / 2 (ansi=false) 3275 3313
32 3.1 327.5 1.7X
-a / 2 (ansi=true) 3025 3082
35 3.3 302.5 1.8X
+a / b (ansi=false) 6595 6605
7 1.5 659.5 1.0X
+a / b (ansi=true) 6611 6629
11 1.5 661.1 1.0X
+a / 2 (ansi=false) 3373 3438
36 3.0 337.3 2.0X
+a / 2 (ansi=true) 3468 3499
19 2.9 346.8 1.9X
diff --git a/sql/core/benchmarks/DecimalDivideBenchmark-results.txt
b/sql/core/benchmarks/DecimalDivideBenchmark-results.txt
index ea91d089f6c4..e256bd58163f 100644
--- a/sql/core/benchmarks/DecimalDivideBenchmark-results.txt
+++ b/sql/core/benchmarks/DecimalDivideBenchmark-results.txt
@@ -6,18 +6,18 @@ OpenJDK 64-Bit Server VM 17.0.19+10-LTS on Linux
6.17.0-1018-azure
AMD EPYC 7763 64-Core Processor
decimal(7,2) / decimal(7,2): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-a / b (ansi=false) 9214 9273
35 1.1 921.4 1.0X
-a / b (ansi=true) 9287 9306
19 1.1 928.7 1.0X
-a / 2 (ansi=false) 6273 6296
21 1.6 627.3 1.5X
-a / 2 (ansi=true) 6257 6262
4 1.6 625.7 1.5X
+a / b (ansi=false) 6172 6204
32 1.6 617.2 1.0X
+a / b (ansi=true) 6088 6120
46 1.6 608.8 1.0X
+a / 2 (ansi=false) 3525 3541
17 2.8 352.5 1.8X
+a / 2 (ansi=true) 3574 3592
24 2.8 357.4 1.7X
OpenJDK 64-Bit Server VM 17.0.19+10-LTS on Linux 6.17.0-1018-azure
AMD EPYC 7763 64-Core Processor
decimal(38,18) / decimal(38,18): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-a / b (ansi=false) 15889 15898
9 0.6 1588.9 1.0X
-a / b (ansi=true) 15872 15896
18 0.6 1587.2 1.0X
-a / 2 (ansi=false) 9324 9341
13 1.1 932.4 1.7X
-a / 2 (ansi=true) 9307 9345
23 1.1 930.7 1.7X
+a / b (ansi=false) 12420 12435
17 0.8 1242.0 1.0X
+a / b (ansi=true) 12499 12578
90 0.8 1249.9 1.0X
+a / 2 (ansi=false) 6676 6691
9 1.5 667.6 1.9X
+a / 2 (ansi=true) 6470 6510
23 1.5 647.0 1.9X
diff --git
a/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out
b/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out
index 5110af2189cd..ddc7111dc6f6 100644
---
a/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out
+++
b/sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out
@@ -248,7 +248,7 @@ org.apache.spark.SparkArithmeticException
"config" : "\"spark.sql.ansi.enabled\"",
"precision" : "38",
"scale" : "6",
- "value" :
"1000000000000000000000000000000000000.000000000000000000000000000000000000000"
+ "value" : "1000000000000000000000000000000000000.000000"
},
"queryContext" : [ {
"objectType" : "",
@@ -583,7 +583,7 @@ org.apache.spark.SparkArithmeticException
"config" : "\"spark.sql.ansi.enabled\"",
"precision" : "38",
"scale" : "3",
- "value" :
"1000000000000000000000000000000000000.000000000000000000000000000000000000000"
+ "value" : "1000000000000000000000000000000000000.000"
},
"queryContext" : [ {
"objectType" : "",
@@ -683,7 +683,7 @@ org.apache.spark.SparkArithmeticException
"config" : "\"spark.sql.ansi.enabled\"",
"precision" : "38",
"scale" : "18",
- "value" : "1000000073899961059796.725866331521039184725213147467478092333"
+ "value" : "1000000073899961059796.725866331521039185"
},
"queryContext" : [ {
"objectType" : "",
@@ -708,7 +708,7 @@ org.apache.spark.SparkArithmeticException
"config" : "\"spark.sql.ansi.enabled\"",
"precision" : "38",
"scale" : "2",
- "value" :
"10123456789012345678901234567890123456.000000000000000000000000000000000000000"
+ "value" : "10123456789012345678901234567890123456.00"
},
"queryContext" : [ {
"objectType" : "",
@@ -733,7 +733,7 @@ org.apache.spark.SparkArithmeticException
"config" : "\"spark.sql.ansi.enabled\"",
"precision" : "38",
"scale" : "3",
- "value" :
"101234567890123456789012345678901234.560000000000000000000000000000000000000"
+ "value" : "101234567890123456789012345678901234.560"
},
"queryContext" : [ {
"objectType" : "",
@@ -790,7 +790,7 @@ org.apache.spark.SparkArithmeticException
"config" : "\"spark.sql.ansi.enabled\"",
"precision" : "38",
"scale" : "6",
- "value" :
"101234567890123456789012345678901.234560000000000000000000000000000000000"
+ "value" : "101234567890123456789012345678901.234560"
},
"queryContext" : [ {
"objectType" : "",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
