This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new df55945 [SPARK-35772][SQL][TESTS] Check all year-month interval types
in `HiveInspectors` tests
df55945 is described below
commit df55945804918f4d147dcef7a9b5f18bff4cabc9
Author: Angerszhuuuu <angers....@gmail.com>
AuthorDate: Wed Jun 23 08:54:07 2021 +0300
[SPARK-35772][SQL][TESTS] Check all year-month interval types in
`HiveInspectors` tests
### What changes were proposed in this pull request?
Check all year-month interval types in HiveInspectors tests.
### Why are the changes needed?
To improve test coverage.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added UT.
Closes #32970 from AngersZhuuuu/SPARK-35772.
Authored-by: Angerszhuuuu <angers....@gmail.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../execution/HiveScriptTransformationSuite.scala | 46 ++++++++++++++++------
1 file changed, 35 insertions(+), 11 deletions(-)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
index 8cea781..d84a766 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.execution._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.types._
+import org.apache.spark.sql.types.YearMonthIntervalType._
import org.apache.spark.unsafe.types.CalendarInterval
class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with
TestHiveSingleton {
@@ -521,22 +522,20 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
}
- test("SPARK-34879: HiveInspectors supports DayTimeIntervalType and
YearMonthIntervalType") {
+ test("SPARK-34879: HiveInspectors supports DayTimeIntervalType") {
assume(TestUtils.testCommandAvailable("/bin/bash"))
withTempView("v") {
val df = Seq(
(Duration.ofDays(1),
Duration.ofSeconds(100).plusNanos(123456),
- Duration.of(Long.MaxValue, ChronoUnit.MICROS),
- Period.ofMonths(10)),
+ Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
(Duration.ofDays(1),
Duration.ofSeconds(100).plusNanos(1123456789),
- Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND),
- Period.ofMonths(10))
- ).toDF("a", "b", "c", "d")
+ Duration.ofSeconds(Long.MaxValue /
DateTimeConstants.MICROS_PER_SECOND))
+ ).toDF("a", "b", "c")
df.createTempView("v")
- // Hive serde supports DayTimeIntervalType/YearMonthIntervalType as
input and output data type
+ // Hive serde supports DayTimeIntervalType as input and output data type
checkAnswer(
df,
(child: SparkPlan) => createScriptTransformationExec(
@@ -545,12 +544,37 @@ class HiveScriptTransformationSuite extends
BaseScriptTransformationSuite with T
// TODO(SPARK-35733): Check all day-time interval types in
HiveInspectors tests
AttributeReference("a", DayTimeIntervalType())(),
AttributeReference("b", DayTimeIntervalType())(),
- AttributeReference("c", DayTimeIntervalType())(),
- // TODO(SPARK-35772): Check all year-month interval types in
HiveInspectors tests
- AttributeReference("d", YearMonthIntervalType())()),
+ AttributeReference("c", DayTimeIntervalType())()),
+ child = child,
+ ioschema = hiveIOSchema),
+ df.select($"a", $"b", $"c").collect())
+ }
+ }
+
+ test("SPARK-35722: HiveInspectors supports all type of
YearMonthIntervalType") {
+ assume(TestUtils.testCommandAvailable("/bin/bash"))
+ withTempView("v") {
+ val schema = StructType(Seq(
+ StructField("a", YearMonthIntervalType(YEAR)),
+ StructField("b", YearMonthIntervalType(YEAR, MONTH)),
+ StructField("c", YearMonthIntervalType(MONTH))
+ ))
+ val df = spark.createDataFrame(sparkContext.parallelize(Seq(
+ Row(Period.ofMonths(13), Period.ofMonths(13), Period.ofMonths(13))
+ )), schema)
+
+ // Hive serde supports YearMonthIntervalType as input and output data
type
+ checkAnswer(
+ df,
+ (child: SparkPlan) => createScriptTransformationExec(
+ script = "cat",
+ output = Seq(
+ AttributeReference("a", YearMonthIntervalType(YEAR))(),
+ AttributeReference("b", YearMonthIntervalType(YEAR, MONTH))(),
+ AttributeReference("c", YearMonthIntervalType(MONTH))()),
child = child,
ioschema = hiveIOSchema),
- df.select($"a", $"b", $"c", $"d").collect())
+ df.select($"a", $"b", $"c").collect())
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
