subject:"spark git commit\: \[SPARK\-20211\]\[SQL\]\[BACKPORT\-2.2\] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between \-1.0 and 1.0"

spark git commit: [SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0

2017-06-14 Thread wenchen

Repository: spark
Updated Branches:
  refs/heads/branch-2.0 0f3598820 -> 0239b1673


[SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values 
when the Input is BigDecimal between -1.0 and 1.0

### What changes were proposed in this pull request?

This PR is to backport https://github.com/apache/spark/pull/18244 to 2.2

---

The precision and scale of decimal values are wrong when the input is 
BigDecimal between -1.0 and 1.0.

The BigDecimal's precision is the digit count starts from the leftmost nonzero 
digit based on the [JAVA's BigDecimal 
definition](https://docs.oracle.com/javase/7/docs/api/java/math/BigDecimal.html).
 However, our Decimal decision follows the database decimal standard, which is 
the total number of digits, including both to the left and the right of the 
decimal point. Thus, this PR is to fix the issue by doing the conversion.

Before this PR, the following queries failed:
```SQL
select 1 > 0.0001
select floor(0.0001)
select ceil(0.0001)
```

### How was this patch tested?
Added test cases.

Author: gatorsmile 

Closes #18297 from gatorsmile/backport18244.

(cherry picked from commit 626511953b87747e933e4f64b9fcd4c4776a5c4e)
Signed-off-by: Wenchen Fan 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0239b167
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0239b167
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0239b167

Branch: refs/heads/branch-2.0
Commit: 0239b167359ce94deb13eaa447a692f2ea590f52
Parents: 0f35988
Author: gatorsmile 
Authored: Wed Jun 14 19:18:28 2017 +0800
Committer: Wenchen Fan 
Committed: Wed Jun 14 19:18:58 2017 +0800

--
 .../org/apache/spark/sql/types/Decimal.scala|  10 +-
 .../apache/spark/sql/types/DecimalSuite.scala   |  10 ++
 .../resources/sql-tests/inputs/arithmetic.sql   |  24 
 .../sql-tests/results/arithmetic.sql.out| 134 ++-
 4 files changed, 176 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/0239b167/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 7085905..e262bd5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -124,7 +124,15 @@ final class Decimal extends Ordered[Decimal] with 
Serializable {
   def set(decimal: BigDecimal): Decimal = {
 this.decimalVal = decimal
 this.longVal = 0L
-this._precision = decimal.precision
+if (decimal.precision <= decimal.scale) {
+  // For Decimal, we expect the precision is equal to or large than the 
scale, however,
+  // in BigDecimal, the digit count starts from the leftmost nonzero digit 
of the exact
+  // result. For example, the precision of 0.01 equals to 1 based on the 
definition, but
+  // the scale is 2. The expected precision should be 3.
+  this._precision = decimal.scale + 1
+} else {
+  this._precision = decimal.precision
+}
 this._scale = decimal.scale
 this
   }

http://git-wip-us.apache.org/repos/asf/spark/blob/0239b167/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 4cf329d..9b401cf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -34,6 +34,16 @@ class DecimalSuite extends SparkFunSuite with 
PrivateMethodTester {
 
   test("creating decimals") {
 checkDecimal(new Decimal(), "0", 1, 0)
+checkDecimal(Decimal(BigDecimal("0.09")), "0.09", 3, 2)
+checkDecimal(Decimal(BigDecimal("0.9")), "0.9", 2, 1)
+checkDecimal(Decimal(BigDecimal("0.90")), "0.90", 3, 2)
+checkDecimal(Decimal(BigDecimal("0.0")), "0.0", 2, 1)
+checkDecimal(Decimal(BigDecimal("0")), "0", 1, 0)
+checkDecimal(Decimal(BigDecimal("1.0")), "1.0", 2, 1)
+checkDecimal(Decimal(BigDecimal("-0.09")), "-0.09", 3, 2)
+checkDecimal(Decimal(BigDecimal("-0.9")), "-0.9", 2, 1)
+checkDecimal(Decimal(BigDecimal("-0.90")), "-0.90", 3, 2)
+checkDecimal(Decimal(BigDecimal("-1.0")), "-1.0", 2, 1)
 checkDecimal(Decimal(BigDecimal("10.030")), "10.030", 5, 3)
 checkDecimal(Decimal(BigDecimal("10.030"), 4, 1),

spark git commit: [SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0

2017-06-14 Thread wenchen

Repository: spark
Updated Branches:
  refs/heads/branch-2.1 ee0e74e65 -> a890466bc


[SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values 
when the Input is BigDecimal between -1.0 and 1.0

### What changes were proposed in this pull request?

This PR is to backport https://github.com/apache/spark/pull/18244 to 2.2

---

The precision and scale of decimal values are wrong when the input is 
BigDecimal between -1.0 and 1.0.

The BigDecimal's precision is the digit count starts from the leftmost nonzero 
digit based on the [JAVA's BigDecimal 
definition](https://docs.oracle.com/javase/7/docs/api/java/math/BigDecimal.html).
 However, our Decimal decision follows the database decimal standard, which is 
the total number of digits, including both to the left and the right of the 
decimal point. Thus, this PR is to fix the issue by doing the conversion.

Before this PR, the following queries failed:
```SQL
select 1 > 0.0001
select floor(0.0001)
select ceil(0.0001)
```

### How was this patch tested?
Added test cases.

Author: gatorsmile 

Closes #18297 from gatorsmile/backport18244.

(cherry picked from commit 626511953b87747e933e4f64b9fcd4c4776a5c4e)
Signed-off-by: Wenchen Fan 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a890466b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a890466b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a890466b

Branch: refs/heads/branch-2.1
Commit: a890466bcc600941927c7040aee409d82b7587d6
Parents: ee0e74e
Author: gatorsmile 
Authored: Wed Jun 14 19:18:28 2017 +0800
Committer: Wenchen Fan 
Committed: Wed Jun 14 19:18:43 2017 +0800

--
 .../org/apache/spark/sql/types/Decimal.scala|  10 +-
 .../apache/spark/sql/types/DecimalSuite.scala   |  10 ++
 .../resources/sql-tests/inputs/arithmetic.sql   |  24 
 .../sql-tests/results/arithmetic.sql.out| 134 ++-
 4 files changed, 176 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/a890466b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 465fb83..1807fd6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -125,7 +125,15 @@ final class Decimal extends Ordered[Decimal] with 
Serializable {
   def set(decimal: BigDecimal): Decimal = {
 this.decimalVal = decimal
 this.longVal = 0L
-this._precision = decimal.precision
+if (decimal.precision <= decimal.scale) {
+  // For Decimal, we expect the precision is equal to or large than the 
scale, however,
+  // in BigDecimal, the digit count starts from the leftmost nonzero digit 
of the exact
+  // result. For example, the precision of 0.01 equals to 1 based on the 
definition, but
+  // the scale is 2. The expected precision should be 3.
+  this._precision = decimal.scale + 1
+} else {
+  this._precision = decimal.precision
+}
 this._scale = decimal.scale
 this
   }

http://git-wip-us.apache.org/repos/asf/spark/blob/a890466b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 52d0692..6a71aca 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -32,6 +32,16 @@ class DecimalSuite extends SparkFunSuite with 
PrivateMethodTester {
 
   test("creating decimals") {
 checkDecimal(new Decimal(), "0", 1, 0)
+checkDecimal(Decimal(BigDecimal("0.09")), "0.09", 3, 2)
+checkDecimal(Decimal(BigDecimal("0.9")), "0.9", 2, 1)
+checkDecimal(Decimal(BigDecimal("0.90")), "0.90", 3, 2)
+checkDecimal(Decimal(BigDecimal("0.0")), "0.0", 2, 1)
+checkDecimal(Decimal(BigDecimal("0")), "0", 1, 0)
+checkDecimal(Decimal(BigDecimal("1.0")), "1.0", 2, 1)
+checkDecimal(Decimal(BigDecimal("-0.09")), "-0.09", 3, 2)
+checkDecimal(Decimal(BigDecimal("-0.9")), "-0.9", 2, 1)
+checkDecimal(Decimal(BigDecimal("-0.90")), "-0.90", 3, 2)
+checkDecimal(Decimal(BigDecimal("-1.0")), "-1.0", 2, 1)
 checkDecimal(Decimal(BigDecimal("10.030")), "10.030", 5, 3)
 checkDecimal(Decimal(BigDecimal("10.030"), 4, 1),

spark git commit: [SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0

2017-06-14 Thread wenchen

Repository: spark
Updated Branches:
  refs/heads/branch-2.2 9bdc83590 -> 626511953


[SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values 
when the Input is BigDecimal between -1.0 and 1.0

### What changes were proposed in this pull request?

This PR is to backport https://github.com/apache/spark/pull/18244 to 2.2

---

The precision and scale of decimal values are wrong when the input is 
BigDecimal between -1.0 and 1.0.

The BigDecimal's precision is the digit count starts from the leftmost nonzero 
digit based on the [JAVA's BigDecimal 
definition](https://docs.oracle.com/javase/7/docs/api/java/math/BigDecimal.html).
 However, our Decimal decision follows the database decimal standard, which is 
the total number of digits, including both to the left and the right of the 
decimal point. Thus, this PR is to fix the issue by doing the conversion.

Before this PR, the following queries failed:
```SQL
select 1 > 0.0001
select floor(0.0001)
select ceil(0.0001)
```

### How was this patch tested?
Added test cases.

Author: gatorsmile 

Closes #18297 from gatorsmile/backport18244.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/62651195
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/62651195
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/62651195

Branch: refs/heads/branch-2.2
Commit: 626511953b87747e933e4f64b9fcd4c4776a5c4e
Parents: 9bdc835
Author: gatorsmile 
Authored: Wed Jun 14 19:18:28 2017 +0800
Committer: Wenchen Fan 
Committed: Wed Jun 14 19:18:28 2017 +0800

--
 .../org/apache/spark/sql/types/Decimal.scala|  10 +-
 .../apache/spark/sql/types/DecimalSuite.scala   |  10 ++
 .../resources/sql-tests/inputs/arithmetic.sql   |  24 
 .../sql-tests/results/arithmetic.sql.out| 134 ++-
 4 files changed, 176 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/62651195/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 80916ee..1f1fb51 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -126,7 +126,15 @@ final class Decimal extends Ordered[Decimal] with 
Serializable {
   def set(decimal: BigDecimal): Decimal = {
 this.decimalVal = decimal
 this.longVal = 0L
-this._precision = decimal.precision
+if (decimal.precision <= decimal.scale) {
+  // For Decimal, we expect the precision is equal to or large than the 
scale, however,
+  // in BigDecimal, the digit count starts from the leftmost nonzero digit 
of the exact
+  // result. For example, the precision of 0.01 equals to 1 based on the 
definition, but
+  // the scale is 2. The expected precision should be 3.
+  this._precision = decimal.scale + 1
+} else {
+  this._precision = decimal.precision
+}
 this._scale = decimal.scale
 this
   }

http://git-wip-us.apache.org/repos/asf/spark/blob/62651195/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 93c231e..144f3d6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -32,6 +32,16 @@ class DecimalSuite extends SparkFunSuite with 
PrivateMethodTester {
 
   test("creating decimals") {
 checkDecimal(new Decimal(), "0", 1, 0)
+checkDecimal(Decimal(BigDecimal("0.09")), "0.09", 3, 2)
+checkDecimal(Decimal(BigDecimal("0.9")), "0.9", 2, 1)
+checkDecimal(Decimal(BigDecimal("0.90")), "0.90", 3, 2)
+checkDecimal(Decimal(BigDecimal("0.0")), "0.0", 2, 1)
+checkDecimal(Decimal(BigDecimal("0")), "0", 1, 0)
+checkDecimal(Decimal(BigDecimal("1.0")), "1.0", 2, 1)
+checkDecimal(Decimal(BigDecimal("-0.09")), "-0.09", 3, 2)
+checkDecimal(Decimal(BigDecimal("-0.9")), "-0.9", 2, 1)
+checkDecimal(Decimal(BigDecimal("-0.90")), "-0.90", 3, 2)
+checkDecimal(Decimal(BigDecimal("-1.0")), "-1.0", 2, 1)
 checkDecimal(Decimal(BigDecimal("10.030")), "10.030", 5, 3)
 checkDecimal(Decimal(BigDecimal("10.030"), 4, 1), "10.0", 4, 1)
 checkDecimal(Decimal(BigDecimal("-9.95"), 4, 1), "-10.0", 4, 1)

spark git commit: [SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0

spark git commit: [SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0

spark git commit: [SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0

3 matches

Site Navigation

Mail list logo

Footer information