This is an automated email from the ASF dual-hosted git repository.
taiyangli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 61460d078 [GLUTEN-6809][CH] Support function
unix_seconds/unix_date/unix_micros/unix_millis (#7094)
61460d078 is described below
commit 61460d0787f954ba9104de720deebd0eb8443ac9
Author: 李扬 <[email protected]>
AuthorDate: Wed Sep 4 09:50:11 2024 +0800
[GLUTEN-6809][CH] Support function
unix_seconds/unix_date/unix_micros/unix_millis (#7094)
* support function unix_seconds
* support function unix_date
* support function unix_millis and unix_micros
* add uts
* add uts
---
.../org/apache/gluten/utils/CHExpressionUtil.scala | 4 ----
.../execution/GlutenFunctionValidateSuite.scala | 21 +++++++++++++++++++--
.../CommonScalarFunctionParser.cpp | 7 ++++++-
.../utils/clickhouse/ClickHouseTestSettings.scala | 3 ---
.../utils/clickhouse/ClickHouseTestSettings.scala | 3 ---
.../utils/clickhouse/ClickHouseTestSettings.scala | 3 ---
.../utils/clickhouse/ClickHouseTestSettings.scala | 3 ---
7 files changed, 25 insertions(+), 19 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index bb4710ef2..1d0f13055 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -199,8 +199,6 @@ object CHExpressionUtil {
ARRAY_REMOVE -> DefaultValidator(),
ARRAYS_ZIP -> DefaultValidator(),
DATE_FROM_UNIX_DATE -> DefaultValidator(),
- UNIX_DATE -> DefaultValidator(),
- UNIX_SECONDS -> DefaultValidator(),
MONOTONICALLY_INCREASING_ID -> DefaultValidator(),
SPARK_PARTITION_ID -> DefaultValidator(),
URL_DECODE -> DefaultValidator(),
@@ -218,8 +216,6 @@ object CHExpressionUtil {
REGR_SXY -> DefaultValidator(),
TO_UTC_TIMESTAMP -> UtcTimestampValidator(),
FROM_UTC_TIMESTAMP -> UtcTimestampValidator(),
- UNIX_MILLIS -> DefaultValidator(),
- UNIX_MICROS -> DefaultValidator(),
TIMESTAMP_MILLIS -> DefaultValidator(),
TIMESTAMP_MICROS -> DefaultValidator(),
STACK -> DefaultValidator(),
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
index 1278264b4..0b91522ae 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
@@ -757,9 +757,26 @@ class GlutenFunctionValidateSuite extends
GlutenClickHouseWholeStageTransformerS
}
test("test function array_except") {
+ val sql =
+ """
+ |SELECT array_except(array(id, id+1, id+2), array(id+2, id+3))
+ |FROM RANGE(10)
+ |""".stripMargin
+ runQueryAndCompare(sql)(checkGlutenOperatorMatch[ProjectExecTransformer])
+ }
+
+ test("test functions unix_seconds/unix_date/unix_millis/unix_micros") {
val sql = """
- |SELECT array_except(array(id, id+1, id+2), array(id+2, id+3))
- |FROM RANGE(10)
+ |SELECT
+ | id,
+ | unix_seconds(cast(concat('2024-09-03 17:23:1',
+ | cast(id as string)) as timestamp)),
+ | unix_date(cast(concat('2024-09-1', cast(id as string)) as
date)),
+ | unix_millis(cast(concat('2024-09-03 17:23:10.11',
+ | cast(id as string)) as timestamp)),
+ | unix_micros(cast(concat('2024-09-03 17:23:10.12345',
+ | cast(id as string)) as timestamp))
+ |FROM range(10)
|""".stripMargin
runQueryAndCompare(sql)(checkGlutenOperatorMatch[ProjectExecTransformer])
}
diff --git
a/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
index b65595ef8..ae654bd29 100644
---
a/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
+++
b/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
@@ -53,7 +53,7 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(LT, lt, less);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(And, and, and);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Or, or, or);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Equal, equal, equals);
-REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Not, not, not);
+REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Not, not, not );
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Xor, xor, xor);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Cast, cast, CAST);
@@ -143,6 +143,7 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(In, in, in);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Coalesce, coalesce, coalesce);
+// date time functions
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(FromUnixtime, from_unixtime,
fromUnixTimestampInJodaSyntax);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateAdd, date_add, addDays);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateSub, date_sub, subtractDays);
@@ -153,6 +154,10 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateTrunc,
date_trunc, dateTrunc);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(FloorDatetime, floor_datetime,
dateTrunc);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Floor, floor, sparkFloor);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(MothsBetween, months_between,
sparkMonthsBetween);
+REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixSeconds, unix_seconds,
toUnixTimestamp);
+REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixDate, unix_date, toInt32);
+REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMillis, unix_millis,
toUnixTimestamp64Milli);
+REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMicros, unix_micros,
toUnixTimestamp64Micro);
// array functions
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Array, array, array);
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 90d1dbae8..fb9ce5afb 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -739,10 +739,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is
missing")
.exclude("DATE_FROM_UNIX_DATE")
- .exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
- .exclude("UNIX_MILLIS")
- .exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 1115af316..705f5beaf 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -768,10 +768,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is
missing")
.exclude("DATE_FROM_UNIX_DATE")
- .exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
- .exclude("UNIX_MILLIS")
- .exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 9ededd6c5..5f30dea84 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -656,10 +656,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is
missing")
.exclude("DATE_FROM_UNIX_DATE")
- .exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
- .exclude("UNIX_MILLIS")
- .exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index fd1326cb1..6a2241f7e 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -656,10 +656,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is
missing")
.exclude("DATE_FROM_UNIX_DATE")
- .exclude("UNIX_DATE")
.exclude("UNIX_SECONDS")
- .exclude("UNIX_MILLIS")
- .exclude("UNIX_MICROS")
.exclude("TIMESTAMP_SECONDS")
.exclude("TIMESTAMP_MILLIS")
.exclude("TIMESTAMP_MICROS")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
