This is an automated email from the ASF dual-hosted git repository.
taiyang-li pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 31adaf8756 [GLUTEN-4730][CH] feat: Support date_from_unix_date
function: Add ClickHouse backend implementation (#12410)
31adaf8756 is described below
commit 31adaf8756f0017cceb15e608d40e3b53152b6b1
Author: Navaneeth Sujith <[email protected]>
AuthorDate: Wed Jul 8 23:11:14 2026 -0700
[GLUTEN-4730][CH] feat: Support date_from_unix_date function: Add
ClickHouse backend implementation (#12410)
* Addressed comments.
* removed trailing spaces
* added a curly brace
* test to see if this is the fix
---
.../Functions/SparkFunctionDateFromUnixDate.cpp | 28 ++++++++
.../Functions/SparkFunctionDateFromUnixDate.h | 82 ++++++++++++++++++++++
.../CommonScalarFunctionParser.cpp | 1 +
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 -
.../spark/sql/GlutenDateFunctionsSuite.scala | 17 +++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 -
6 files changed, 128 insertions(+), 2 deletions(-)
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.cpp
b/cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.cpp
new file mode 100644
index 0000000000..fd67b789a4
--- /dev/null
+++ b/cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.cpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Functions/SparkFunctionDateFromUnixDate.h>
+
+namespace local_engine
+{
+
+REGISTER_FUNCTION(SparkFunctionDateFromUnixDate)
+{
+ factory.registerFunction<local_engine::SparkFunctionDateFromUnixDate>();
+}
+
+}
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.h
b/cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.h
new file mode 100644
index 0000000000..1e1b95be47
--- /dev/null
+++ b/cpp-ch/local-engine/Functions/SparkFunctionDateFromUnixDate.h
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include <Columns/ColumnVector.h>
+#include <DataTypes/DataTypeDate32.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/IDataType.h>
+#include <Functions/FunctionFactory.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+}
+
+namespace local_engine
+{
+class SparkFunctionDateFromUnixDate : public DB::IFunction
+{
+public:
+ static constexpr auto name = "sparkDateFromUnixDate";
+ static DB::FunctionPtr create(DB::ContextPtr) { return
std::make_shared<SparkFunctionDateFromUnixDate>(); }
+
+ SparkFunctionDateFromUnixDate() = default;
+ ~SparkFunctionDateFromUnixDate() override = default;
+
+ String getName() const override { return name; }
+ bool isSuitableForShortCircuitArgumentsExecution(const
DB::DataTypesWithConstInfo &) const override { return true; }
+ size_t getNumberOfArguments() const override { return 1; }
+ bool isVariadic() const override { return false; }
+ bool useDefaultImplementationForConstants() const override { return true; }
+ DB::DataTypePtr getReturnTypeImpl(const DB::ColumnsWithTypeAndName &)
const override { return std::make_shared<DB::DataTypeDate32>(); }
+
+ DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments,
const DB::DataTypePtr & result_type, size_t input_rows) const override
+ {
+ if (arguments.size() != 1)
+ throw
DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}
requires exactly 1 argument", name);
+
+ const DB::ColumnWithTypeAndName & first_arg = arguments[0];
+ DB::WhichDataType which(first_arg.type);
+ if (which.isInt32())
+ return executeInternal<Int32>(first_arg.column, input_rows);
+ else
+ throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Function {} requires Int32 argument", name);
+ }
+
+ template<typename T>
+ DB::ColumnPtr executeInternal(const DB::ColumnPtr & col, size_t
input_rows) const
+ {
+ const DB::ColumnVector<T> * col_src =
checkAndGetColumn<DB::ColumnVector<T>>(col.get());
+ DB::MutableColumnPtr res =
DB::ColumnVector<Int32>::create(col->size());
+ DB::PaddedPODArray<Int32> & data = assert_cast<DB::ColumnVector<Int32>
*>(res.get())->getData();
+
+ for (size_t i = 0; i < input_rows; ++i)
+ {
+ // Spark's date_from_unix_date returns a Date that is the given
number of days since 1970-01-01.
+ // CH Date32 is also stored as days since 1970-01-01, so the value
passes through unchanged.
+ const T unix_date = col_src->getElement(i);
+ data[i] = static_cast<Int32>(unix_date);
+ }
+ return res;
+ }
+};
+}
\ No newline at end of file
diff --git
a/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
index ae760543bb..4e82457d1c 100644
---
a/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
+++
b/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
@@ -154,6 +154,7 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Floor, floor,
sparkFloor);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(MothsBetween, months_between,
sparkMonthsBetween);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixSeconds, unix_seconds,
toUnixTimestamp);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixDate, unix_date, toInt32);
+REGISTER_COMMON_SCALAR_FUNCTION_PARSER(DateFromUnixDate, date_from_unix_date,
sparkDateFromUnixDate);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMillis, unix_millis,
toUnixTimestamp64Milli);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(UnixMicros, unix_micros,
toUnixTimestamp64Micro);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(TimestampMillis, timestamp_millis,
fromUnixTimestamp64Milli);
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index dc887eb3b1..7a77f7c4f3 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -748,7 +748,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp_ntz")
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is
missing")
- .exclude("DATE_FROM_UNIX_DATE")
.exclude("UNIX_SECONDS")
.exclude("TIMESTAMP_SECONDS") // refer to
https://github.com/ClickHouse/ClickHouse/issues/69280
.exclude("TIMESTAMP_MICROS") // refer to
https://github.com/apache/gluten/issues/7127
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index f9c5995caf..f40bc9e20a 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -281,4 +281,21 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite
with GlutenSQLTestsTra
val df1 = Seq(x1, x2).toDF("x")
checkAnswer(df1.select(to_date(col("x"))), Row(Date.valueOf("2016-02-29"))
:: Row(null) :: Nil)
}
+ testGluten("date_from_unix_date") {
+ // -100000 and 200000 are outside ClickHouse's native Date32 range
+ // [1900-01-01, 2299-12-31]. They guard against implementations that clamp
to that
+ // range (e.g. mapping to CH toDate32), which would silently diverge from
Spark.
+ val df = Seq(Some(0), Some(1000), Some(-100000), Some(200000),
None).toDF("unix_date")
+ val expected = Seq(
+ Row(Date.valueOf("1970-01-01")),
+ Row(Date.valueOf("1972-09-27")),
+ Row(Date.valueOf("1696-03-17")),
+ Row(Date.valueOf("2517-08-01")),
+ Row(null))
+
+ // date_from_unix_date is only available in the Scala functions API since
Spark 3.5,
+ // so go through expr() here.
+ checkAnswer(df.select(expr("date_from_unix_date(unix_date)")), expected)
+ checkAnswer(df.selectExpr("date_from_unix_date(unix_date)"), expected)
+ }
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 279e99dc0f..53078da7a3 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -646,7 +646,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("to_timestamp_ntz")
.exclude("to_timestamp exception mode")
.exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is
missing")
- .exclude("DATE_FROM_UNIX_DATE")
.exclude("UNIX_SECONDS")
.exclude("TIMESTAMP_SECONDS") // refer to
https://github.com/ClickHouse/ClickHouse/issues/69280
.exclude("TIMESTAMP_MICROS") // refer to
https://github.com/apache/gluten/issues/7127
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
