This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 3ace01b [SPARK-36120][SQL] Support TimestampNTZ type in cache table 3ace01b is described below commit 3ace01b25bf1c69901a07c11acbae6fa996fc06c Author: Gengliang Wang <gengli...@apache.org> AuthorDate: Tue Jul 13 17:23:48 2021 +0300 [SPARK-36120][SQL] Support TimestampNTZ type in cache table ### What changes were proposed in this pull request? Support TimestampNTZ type column in SQL command Cache table ### Why are the changes needed? Cache table should support the new timestamp type. ### Does this PR introduce _any_ user-facing change? Yes, the TimemstampNTZ type column can used in `CACHE TABLE` ### How was this patch tested? Unit test Closes #33322 from gengliangwang/cacheTable. Authored-by: Gengliang Wang <gengli...@apache.org> Signed-off-by: Max Gekk <max.g...@gmail.com> (cherry picked from commit 067432705fbec970bd713adf37d2aa17c6bcf5a0) Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../spark/sql/execution/columnar/ColumnBuilder.scala | 3 ++- .../apache/spark/sql/execution/columnar/ColumnType.scala | 2 +- .../sql/execution/columnar/GenerateColumnAccessor.scala | 2 +- .../scala/org/apache/spark/sql/CachedTableSuite.scala | 15 ++++++++++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala index e9251e8..9ddc665 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala @@ -175,7 +175,8 @@ private[columnar] object ColumnBuilder { case ByteType => new ByteColumnBuilder case ShortType => new ShortColumnBuilder case IntegerType | DateType | _: YearMonthIntervalType => new IntColumnBuilder - case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnBuilder + case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType => + new LongColumnBuilder case FloatType => new FloatColumnBuilder case DoubleType => new DoubleColumnBuilder case StringType => new StringColumnBuilder diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala index 8e99368..419dcc6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala @@ -818,7 +818,7 @@ private[columnar] object ColumnType { case ByteType => BYTE case ShortType => SHORT case IntegerType | DateType | _: YearMonthIntervalType => INT - case LongType | TimestampType | _: DayTimeIntervalType => LONG + case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType => LONG case FloatType => FLOAT case DoubleType => DOUBLE case StringType => STRING diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala index 190c2c3..6e666d4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala @@ -81,7 +81,7 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera case ByteType => classOf[ByteColumnAccessor].getName case ShortType => classOf[ShortColumnAccessor].getName case IntegerType | DateType | _: YearMonthIntervalType => classOf[IntColumnAccessor].getName - case LongType | TimestampType | _: DayTimeIntervalType => + case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType => classOf[LongColumnAccessor].getName case FloatType => classOf[FloatColumnAccessor].getName case DoubleType => classOf[DoubleColumnAccessor].getName diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala index 1915044..db717d2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql import java.io.{File, FilenameFilter} import java.nio.file.{Files, Paths} -import java.time.{Duration, Period} +import java.time.{Duration, LocalDateTime, Period} import scala.collection.mutable.HashSet import scala.concurrent.duration._ @@ -1538,6 +1538,19 @@ class CachedTableSuite extends QueryTest with SQLTestUtils } } + test("SPARK-36120: Support cache/uncache table with TimestampNTZ type") { + val tableName = "ntzCache" + withTable(tableName) { + sql(s"CACHE TABLE $tableName AS SELECT TIMESTAMP_NTZ'2021-01-01 00:00:00'") + checkAnswer(spark.table(tableName), Row(LocalDateTime.parse("2021-01-01T00:00:00"))) + spark.table(tableName).queryExecution.withCachedData.collect { + case cached: InMemoryRelation => + assert(cached.stats.sizeInBytes === 8) + } + sql(s"UNCACHE TABLE $tableName") + } + } + private def testCreateTemporaryViewUsingWithCache(ident: TableIdentifier): Unit = { withTempDir { dir => val path1 = new File(dir, "t1").getCanonicalPath --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org