Repository: spark
Updated Branches:
refs/heads/branch-1.5 6f05b7aeb -> 8925896b1
[SPARK-10245] [SQL] Fix decimal literals with precision < scale
In BigDecimal or java.math.BigDecimal, the precision could be smaller than
scale, for example, BigDecimal("0.001") has precision = 1 and scale = 3. But
DecimalType require that the precision should be larger than scale, so we
should use the maximum of precision and scale when inferring the schema from
decimal literal.
Author: Davies Liu <dav...@databricks.com>
Closes #8428 from davies/smaller_decimal.
(cherry picked from commit ec89bd840a6862751999d612f586a962cae63f6d)
Signed-off-by: Yin Huai <yh...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8925896b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8925896b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8925896b
Branch: refs/heads/branch-1.5
Commit: 8925896b1eb0a13d723d38fb263d3bec0a01ec10
Parents: 6f05b7a
Author: Davies Liu <dav...@databricks.com>
Authored: Tue Aug 25 14:55:34 2015 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Aug 25 14:55:45 2015 -0700
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/expressions/literals.scala | 7 ++++---
.../sql/catalyst/expressions/LiteralExpressionSuite.scala | 8 +++++---
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 10 ++++++++++
3 files changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8925896b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 34bad23..8c0c5d5 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -36,9 +36,10 @@ object Literal {
case s: Short => Literal(s, ShortType)
case s: String => Literal(UTF8String.fromString(s), StringType)
case b: Boolean => Literal(b, BooleanType)
- case d: BigDecimal => Literal(Decimal(d), DecimalType(d.precision,
d.scale))
- case d: java.math.BigDecimal => Literal(Decimal(d),
DecimalType(d.precision(), d.scale()))
- case d: Decimal => Literal(d, DecimalType(d.precision, d.scale))
+ case d: BigDecimal => Literal(Decimal(d),
DecimalType(Math.max(d.precision, d.scale), d.scale))
+ case d: java.math.BigDecimal =>
+ Literal(Decimal(d), DecimalType(Math.max(d.precision, d.scale),
d.scale()))
+ case d: Decimal => Literal(d, DecimalType(Math.max(d.precision, d.scale),
d.scale))
case t: Timestamp => Literal(DateTimeUtils.fromJavaTimestamp(t),
TimestampType)
case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType)
case a: Array[Byte] => Literal(a, BinaryType)
http://git-wip-us.apache.org/repos/asf/spark/blob/8925896b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index f6404d2..015eb18 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -83,12 +83,14 @@ class LiteralExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
test("decimal") {
- List(0.0, 1.2, 1.1111, 5).foreach { d =>
+ List(-0.0001, 0.0, 0.001, 1.2, 1.1111, 5).foreach { d =>
checkEvaluation(Literal(Decimal(d)), Decimal(d))
checkEvaluation(Literal(Decimal(d.toInt)), Decimal(d.toInt))
checkEvaluation(Literal(Decimal(d.toLong)), Decimal(d.toLong))
- checkEvaluation(Literal(Decimal((d * 1000L).toLong, 10, 1)),
- Decimal((d * 1000L).toLong, 10, 1))
+ checkEvaluation(Literal(Decimal((d * 1000L).toLong, 10, 3)),
+ Decimal((d * 1000L).toLong, 10, 3))
+ checkEvaluation(Literal(BigDecimal(d.toString)), Decimal(d))
+ checkEvaluation(Literal(new java.math.BigDecimal(d.toString)),
Decimal(d))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/8925896b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index dcb4e83..aa07665 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -1627,6 +1627,16 @@ class SQLQuerySuite extends QueryTest with
SharedSQLContext {
Row(null))
}
+ test("precision smaller than scale") {
+ checkAnswer(sql("select 10.00"), Row(BigDecimal("10.00")))
+ checkAnswer(sql("select 1.00"), Row(BigDecimal("1.00")))
+ checkAnswer(sql("select 0.10"), Row(BigDecimal("0.10")))
+ checkAnswer(sql("select 0.01"), Row(BigDecimal("0.01")))
+ checkAnswer(sql("select 0.001"), Row(BigDecimal("0.001")))
+ checkAnswer(sql("select -0.01"), Row(BigDecimal("-0.01")))
+ checkAnswer(sql("select -0.001"), Row(BigDecimal("-0.001")))
+ }
+
test("external sorting updates peak execution memory") {
withSQLConf((SQLConf.EXTERNAL_SORT.key, "true")) {
val sc = sqlContext.sparkContext
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
