Repository: spark
Updated Branches:
refs/heads/master bff021dfa -> ea3b1e352
[SPARK-20763][SQL] The function of `month` and `day` return the value which is
not we expected.
## What changes were proposed in this pull request?
spark-sql>select month("1582-09-28");
spark-sql>10
For this case, the expected result is 9, but it is 10.
spark-sql>select day("1582-04-18");
spark-sql>28
For this case, the expected result is 18, but it is 28.
when the date before "1582-10-04", the function of `month` and `day` return
the value which is not we expected.
## How was this patch tested?
unit tests
Author: liuxian <liu.xi...@zte.com.cn>
Closes #17997 from 10110346/wip_lx_0516.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ea3b1e35
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ea3b1e35
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ea3b1e35
Branch: refs/heads/master
Commit: ea3b1e352a605cd35cdee987d0e5eb8528ef1b45
Parents: bff021d
Author: liuxian <liu.xi...@zte.com.cn>
Authored: Fri May 19 10:25:21 2017 -0700
Committer: Xiao Li <gatorsm...@gmail.com>
Committed: Fri May 19 10:25:21 2017 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/util/DateTimeUtils.scala | 9 ++++++++-
.../expressions/DateExpressionsSuite.scala | 18 ++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ea3b1e35/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index b9a3d98..efb4229 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -603,7 +603,14 @@ object DateTimeUtils {
*/
private[this] def getYearAndDayInYear(daysSince1970: SQLDate): (Int, Int) = {
// add the difference (in days) between 1.1.1970 and the artificial year 0
(-17999)
- val daysNormalized = daysSince1970 + toYearZero
+ var daysSince1970Tmp = daysSince1970
+ // Since Julian calendar was replaced with the Gregorian calendar,
+ // the 10 days after Oct. 4 were skipped.
+ // (1582-10-04) -141428 days since 1970-01-01
+ if (daysSince1970 <= -141428) {
+ daysSince1970Tmp -= 10
+ }
+ val daysNormalized = daysSince1970Tmp + toYearZero
val numOfQuarterCenturies = daysNormalized / daysIn400Years
val daysInThis400 = daysNormalized % daysIn400Years + 1
val (years, dayInYear) = numYears(daysInThis400)
http://git-wip-us.apache.org/repos/asf/spark/blob/ea3b1e35/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index d3bac0a..4ce6853 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -76,6 +76,9 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
checkEvaluation(DayOfYear(Literal.create(null, DateType)), null)
+
+ checkEvaluation(DayOfYear(Literal(new Date(sdf.parse("1582-10-15
13:10:15").getTime))), 288)
+ checkEvaluation(DayOfYear(Literal(new Date(sdf.parse("1582-10-04
13:10:15").getTime))), 277)
checkConsistencyBetweenInterpretedAndCodegen(DayOfYear, DateType)
}
@@ -96,6 +99,8 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
}
+ checkEvaluation(Year(Literal(new Date(sdf.parse("1582-01-01
13:10:15").getTime))), 1582)
+ checkEvaluation(Year(Literal(new Date(sdf.parse("1581-12-31
13:10:15").getTime))), 1581)
checkConsistencyBetweenInterpretedAndCodegen(Year, DateType)
}
@@ -116,6 +121,9 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
}
+
+ checkEvaluation(Quarter(Literal(new Date(sdf.parse("1582-10-01
13:10:15").getTime))), 4)
+ checkEvaluation(Quarter(Literal(new Date(sdf.parse("1582-09-30
13:10:15").getTime))), 3)
checkConsistencyBetweenInterpretedAndCodegen(Quarter, DateType)
}
@@ -125,6 +133,10 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(Month(Cast(Literal(sdfDate.format(d)), DateType, gmtId)),
4)
checkEvaluation(Month(Cast(Literal(ts), DateType, gmtId)), 11)
+ checkEvaluation(Month(Literal(new Date(sdf.parse("1582-04-28
13:10:15").getTime))), 4)
+ checkEvaluation(Month(Literal(new Date(sdf.parse("1582-10-04
13:10:15").getTime))), 10)
+ checkEvaluation(Month(Literal(new Date(sdf.parse("1582-10-15
13:10:15").getTime))), 10)
+
val c = Calendar.getInstance()
(2003 to 2004).foreach { y =>
(0 to 3).foreach { m =>
@@ -146,6 +158,10 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(DayOfMonth(Cast(Literal(sdfDate.format(d)), DateType,
gmtId)), 8)
checkEvaluation(DayOfMonth(Cast(Literal(ts), DateType, gmtId)), 8)
+ checkEvaluation(DayOfMonth(Literal(new Date(sdf.parse("1582-04-28
13:10:15").getTime))), 28)
+ checkEvaluation(DayOfMonth(Literal(new Date(sdf.parse("1582-10-15
13:10:15").getTime))), 15)
+ checkEvaluation(DayOfMonth(Literal(new Date(sdf.parse("1582-10-04
13:10:15").getTime))), 4)
+
val c = Calendar.getInstance()
(1999 to 2000).foreach { y =>
c.set(y, 0, 1, 0, 0, 0)
@@ -186,6 +202,8 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
checkEvaluation(WeekOfYear(Cast(Literal(sdfDate.format(d)), DateType,
gmtId)), 15)
checkEvaluation(WeekOfYear(Cast(Literal(ts), DateType, gmtId)), 45)
checkEvaluation(WeekOfYear(Cast(Literal("2011-05-06"), DateType, gmtId)),
18)
+ checkEvaluation(WeekOfYear(Literal(new Date(sdf.parse("1582-10-15
13:10:15").getTime))), 40)
+ checkEvaluation(WeekOfYear(Literal(new Date(sdf.parse("1582-10-04
13:10:15").getTime))), 40)
checkConsistencyBetweenInterpretedAndCodegen(WeekOfYear, DateType)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
