This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 9df907a44e [VL] Support casting integral type to timestamp type (#8593)
9df907a44e is described below
commit 9df907a44e89073d2bbf3cbdd9e947c265c21386
Author: PHILO-HE <[email protected]>
AuthorDate: Fri Mar 14 16:51:14 2025 +0800
[VL] Support casting integral type to timestamp type (#8593)
---
.../substrait/SubstraitToVeloxPlanValidator.cc | 11 ++++++++++-
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../parquet/GlutenParquetColumnIndexSuite.scala | 22 +++++++++++++++++++++-
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../parquet/GlutenParquetColumnIndexSuite.scala | 22 +++++++++++++++++++++-
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../parquet/GlutenParquetColumnIndexSuite.scala | 22 +++++++++++++++++++++-
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../parquet/GlutenParquetColumnIndexSuite.scala | 22 +++++++++++++++++++++-
9 files changed, 102 insertions(+), 5 deletions(-)
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index 1c737ac7db..6328c786e4 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -262,7 +262,16 @@ bool SubstraitToVeloxPlanValidator::isAllowedCast(const
TypePtr& fromType, const
}
// Limited support for X to Timestamp.
- if (toType->isTimestamp() && !(fromType->isDate() || fromType->isVarchar()))
{
+ if (toType->isTimestamp()) {
+ if (fromType->isDate()) {
+ return true;
+ }
+ if (fromType->isVarchar()) {
+ return true;
+ }
+ if (fromType->isTinyint() || fromType->isSmallint() ||
fromType->isInteger() || fromType->isBigint()) {
+ return true;
+ }
LOG_VALIDATION_MSG("Casting from " + fromType->toString() + " to TIMESTAMP
is not supported.");
return false;
}
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 8dfd3a2392..ef0a6de568 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -875,6 +875,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenParquetColumnIndexSuite]
// Rewrite by just removing test timestamp.
.exclude("test reading unaligned pages - test all types")
+ // Rewrite by converting smaller integral value to timestamp.
+ .exclude("test reading unaligned pages - test all types (dict encode)")
enableSuite[GlutenParquetCompressionCodecPrecedenceSuite]
enableSuite[GlutenParquetEncodingSuite]
enableSuite[GlutenParquetFileFormatV1Suite]
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
index 1e0597565d..f910ad8871 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
@@ -38,7 +38,27 @@ class GlutenParquetColumnIndexSuite extends
ParquetColumnIndexSuite with GlutenS
"cast(id as float) as _5",
"cast(id as double) as _6",
"cast(id as decimal(20,0)) as _7",
- "cast(cast(1618161925000 + id * 1000 * 60 * 60 * 24 as timestamp) as
date) as _9"
+ // We changed 1618161925000 to 1618161925 to avoid reaching the
limitation of Velox:
+ // Timepoint is outside of supported year range.
+ "cast(cast(1618161925 + id * 60 * 60 * 24 as timestamp) as date) as _9"
+ )
+ checkUnalignedPages(df)(actions: _*)
+ }
+
+ testGluten("test reading unaligned pages - test all types (dict encode)") {
+ val df = spark
+ .range(0, 2000)
+ .selectExpr(
+ "id as _1",
+ "cast(id % 10 as byte) as _2",
+ "cast(id % 10 as short) as _3",
+ "cast(id % 10 as int) as _4",
+ "cast(id % 10 as float) as _5",
+ "cast(id % 10 as double) as _6",
+ "cast(id % 10 as decimal(20,0)) as _7",
+ "cast(id % 2 as boolean) as _8",
+ "cast(cast(1618161925 + (id % 10) * 60 * 60 * 24 as timestamp) as
date) as _9",
+ "cast(1618161925 + (id % 10) as timestamp) as _10"
)
checkUnalignedPages(df)(actions: _*)
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 253930d937..e5c84129d1 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -669,6 +669,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenParquetColumnIndexSuite]
// Rewrite by just removing test timestamp.
.exclude("test reading unaligned pages - test all types")
+ // Rewrite by converting smaller integral value to timestamp.
+ .exclude("test reading unaligned pages - test all types (dict encode)")
enableSuite[GlutenParquetCompressionCodecPrecedenceSuite]
enableSuite[GlutenParquetDeltaByteArrayEncodingSuite]
enableSuite[GlutenParquetDeltaEncodingInteger]
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
index 4ff298e6cc..60e1ca04a2 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
@@ -38,7 +38,27 @@ class GlutenParquetColumnIndexSuite extends
ParquetColumnIndexSuite with GlutenS
"cast(id as float) as _5",
"cast(id as double) as _6",
"cast(id as decimal(20,0)) as _7",
- "cast(cast(1618161925000 + id * 1000 * 60 * 60 * 24 as timestamp) as
date) as _9"
+ // We changed 1618161925000 to 1618161925 to avoid reaching the
limitation of Velox:
+ // Timepoint is outside of supported year range.
+ "cast(cast(1618161925 + id * 60 * 60 * 24 as timestamp) as date) as _9"
+ )
+ checkUnalignedPages(df)(actions: _*)
+ }
+
+ testGluten("test reading unaligned pages - test all types (dict encode)") {
+ val df = spark
+ .range(0, 2000)
+ .selectExpr(
+ "id as _1",
+ "cast(id % 10 as byte) as _2",
+ "cast(id % 10 as short) as _3",
+ "cast(id % 10 as int) as _4",
+ "cast(id % 10 as float) as _5",
+ "cast(id % 10 as double) as _6",
+ "cast(id % 10 as decimal(20,0)) as _7",
+ "cast(id % 2 as boolean) as _8",
+ "cast(cast(1618161925 + (id % 10) * 60 * 60 * 24 as timestamp) as
date) as _9",
+ "cast(1618161925 + (id % 10) as timestamp) as _10"
)
checkUnalignedPages(df)(actions: _*)
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 5d28e39405..1b8d4b9737 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -664,6 +664,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenParquetColumnIndexSuite]
// Rewrite by just removing test timestamp.
.exclude("test reading unaligned pages - test all types")
+ // Rewrite by converting smaller integral value to timestamp.
+ .exclude("test reading unaligned pages - test all types (dict encode)")
enableSuite[GlutenParquetCompressionCodecPrecedenceSuite]
enableSuite[GlutenParquetDeltaByteArrayEncodingSuite]
enableSuite[GlutenParquetDeltaEncodingInteger]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
index 4ff298e6cc..60e1ca04a2 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
@@ -38,7 +38,27 @@ class GlutenParquetColumnIndexSuite extends
ParquetColumnIndexSuite with GlutenS
"cast(id as float) as _5",
"cast(id as double) as _6",
"cast(id as decimal(20,0)) as _7",
- "cast(cast(1618161925000 + id * 1000 * 60 * 60 * 24 as timestamp) as
date) as _9"
+ // We changed 1618161925000 to 1618161925 to avoid reaching the
limitation of Velox:
+ // Timepoint is outside of supported year range.
+ "cast(cast(1618161925 + id * 60 * 60 * 24 as timestamp) as date) as _9"
+ )
+ checkUnalignedPages(df)(actions: _*)
+ }
+
+ testGluten("test reading unaligned pages - test all types (dict encode)") {
+ val df = spark
+ .range(0, 2000)
+ .selectExpr(
+ "id as _1",
+ "cast(id % 10 as byte) as _2",
+ "cast(id % 10 as short) as _3",
+ "cast(id % 10 as int) as _4",
+ "cast(id % 10 as float) as _5",
+ "cast(id % 10 as double) as _6",
+ "cast(id % 10 as decimal(20,0)) as _7",
+ "cast(id % 2 as boolean) as _8",
+ "cast(cast(1618161925 + (id % 10) * 60 * 60 * 24 as timestamp) as
date) as _9",
+ "cast(1618161925 + (id % 10) as timestamp) as _10"
)
checkUnalignedPages(df)(actions: _*)
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index b0a9e99e98..7bae3d5ff6 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -675,6 +675,8 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenParquetColumnIndexSuite]
// Rewrite by just removing test timestamp.
.exclude("test reading unaligned pages - test all types")
+ // Rewrite by converting smaller integral value to timestamp.
+ .exclude("test reading unaligned pages - test all types (dict encode)")
enableSuite[GlutenParquetCompressionCodecPrecedenceSuite]
enableSuite[GlutenParquetDeltaByteArrayEncodingSuite]
enableSuite[GlutenParquetDeltaEncodingInteger]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
index 4ff298e6cc..60e1ca04a2 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala
@@ -38,7 +38,27 @@ class GlutenParquetColumnIndexSuite extends
ParquetColumnIndexSuite with GlutenS
"cast(id as float) as _5",
"cast(id as double) as _6",
"cast(id as decimal(20,0)) as _7",
- "cast(cast(1618161925000 + id * 1000 * 60 * 60 * 24 as timestamp) as
date) as _9"
+ // We changed 1618161925000 to 1618161925 to avoid reaching the
limitation of Velox:
+ // Timepoint is outside of supported year range.
+ "cast(cast(1618161925 + id * 60 * 60 * 24 as timestamp) as date) as _9"
+ )
+ checkUnalignedPages(df)(actions: _*)
+ }
+
+ testGluten("test reading unaligned pages - test all types (dict encode)") {
+ val df = spark
+ .range(0, 2000)
+ .selectExpr(
+ "id as _1",
+ "cast(id % 10 as byte) as _2",
+ "cast(id % 10 as short) as _3",
+ "cast(id % 10 as int) as _4",
+ "cast(id % 10 as float) as _5",
+ "cast(id % 10 as double) as _6",
+ "cast(id % 10 as decimal(20,0)) as _7",
+ "cast(id % 2 as boolean) as _8",
+ "cast(cast(1618161925 + (id % 10) * 60 * 60 * 24 as timestamp) as
date) as _9",
+ "cast(1618161925 + (id % 10) as timestamp) as _10"
)
checkUnalignedPages(df)(actions: _*)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
