This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 50a2967 [SPARK-30958][SQL] do not set default era for
DateTimeFormatter
50a2967 is described below
commit 50a29672e061ca17598e657593db45f55aebc34b
Author: Wenchen Fan <wenc...@databricks.com>
AuthorDate: Mon Mar 16 16:48:31 2020 +0900
[SPARK-30958][SQL] do not set default era for DateTimeFormatter
### What changes were proposed in this pull request?
It's not needed at all as now we replace "y" with "u" if there is no "G".
So the era is either explicitly specified (e.g. "yyyy G") or can be inferred
from the year (e.g. "uuuu").
### Why are the changes needed?
By default we use "uuuu" as the year pattern, which indicates the era
already. If we set a default era, it can get conflicted and fail the parsing.
### Does this PR introduce any user-facing change?
yea, now spark can parse date/timestamp with negative year via the "yyyy"
pattern, which will be converted to "uuuu" under the hood.
### How was this patch tested?
new tests
Closes #27707 from cloud-fan/bug.
Authored-by: Wenchen Fan <wenc...@databricks.com>
Signed-off-by: HyukjinKwon <gurwls...@apache.org>
---
.../catalyst/util/DateTimeFormatterHelper.scala | 1 -
.../apache/spark/sql/util/DateFormatterSuite.scala | 26 ++++++++++++++++--
.../spark/sql/util/TimestampFormatterSuite.scala | 32 ++++++++++++++++++++--
3 files changed, 53 insertions(+), 6 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index fab5041..a59f49b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -95,7 +95,6 @@ private object DateTimeFormatterHelper {
def toFormatter(builder: DateTimeFormatterBuilder, locale: Locale):
DateTimeFormatter = {
builder
- .parseDefaulting(ChronoField.ERA, 1)
.parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
.parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
.parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
index d617b1c..a40dbcc 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -17,9 +17,9 @@
package org.apache.spark.sql.util
-import java.time.{LocalDate, ZoneOffset}
+import java.time.{DateTimeException, LocalDate, ZoneOffset}
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId,
localDateToDays}
@@ -114,4 +114,26 @@ class DateFormatterSuite extends SparkFunSuite with
SQLHelper {
assert(formatter.parse("tomorrow UTC") === today + 1)
}
}
+
+ test("SPARK-30958: parse date with negative year") {
+ val formatter1 = DateFormatter("yyyy-MM-dd", ZoneOffset.UTC)
+ assert(formatter1.parse("-1234-02-22") ===
localDateToDays(LocalDate.of(-1234, 2, 22)))
+
+ def assertParsingError(f: => Unit): Unit = {
+ intercept[Exception](f) match {
+ case e: SparkUpgradeException =>
+ assert(e.getCause.isInstanceOf[DateTimeException])
+ case e =>
+ assert(e.isInstanceOf[DateTimeException])
+ }
+ }
+
+ // "yyyy" with "G" can't parse negative year or year 0000.
+ val formatter2 = DateFormatter("G yyyy-MM-dd", ZoneOffset.UTC)
+ assertParsingError(formatter2.parse("BC -1234-02-22"))
+ assertParsingError(formatter2.parse("AD 0000-02-22"))
+
+ assert(formatter2.parse("BC 1234-02-22") ===
localDateToDays(LocalDate.of(-1233, 2, 22)))
+ assert(formatter2.parse("AD 1234-02-22") ===
localDateToDays(LocalDate.of(1234, 2, 22)))
+ }
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index 082849c..959ef8e 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -17,12 +17,12 @@
package org.apache.spark.sql.util
-import java.time.{Instant, LocalDateTime, LocalTime, ZoneOffset}
+import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime,
ZoneOffset}
import java.util.concurrent.TimeUnit
import org.scalatest.Matchers
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils,
TimestampFormatter}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
@@ -112,7 +112,7 @@ class TimestampFormatterSuite extends SparkFunSuite with
SQLHelper with Matchers
}
}
- test(" case insensitive parsing of am and pm") {
+ test("case insensitive parsing of am and pm") {
val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a",
ZoneOffset.UTC)
val micros = formatter.parse("2009 Mar 20 11:30:01 am")
assert(micros === TimeUnit.SECONDS.toMicros(
@@ -233,4 +233,30 @@ class TimestampFormatterSuite extends SparkFunSuite with
SQLHelper with Matchers
"2019-10-14 09:39:07")
}
}
+
+ test("SPARK-30958: parse timestamp with negative year") {
+ val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
+ assert(formatter1.parse("-1234-02-22 02:22:22") === instantToMicros(
+ LocalDateTime.of(-1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+
+ def assertParsingError(f: => Unit): Unit = {
+ intercept[Exception](f) match {
+ case e: SparkUpgradeException =>
+ assert(e.getCause.isInstanceOf[DateTimeException])
+ case e =>
+ assert(e.isInstanceOf[DateTimeException])
+ }
+ }
+
+ // "yyyy" with "G" can't parse negative year or year 0000.
+ val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss",
ZoneOffset.UTC)
+ assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
+ assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
+
+ assert(formatter2.parse("BC 1234-02-22 02:22:22") === instantToMicros(
+ LocalDateTime.of(-1233, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+ assert(formatter2.parse("AD 1234-02-22 02:22:22") === instantToMicros(
+ LocalDateTime.of(1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
