This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0c7dcc6333 [VL] Support format_number function (#12754)
0c7dcc6333 is described below
commit 0c7dcc6333ed3f9ad2d5e7bf2184717e7ce48120
Author: Kaifei Yi <[email protected]>
AuthorDate: Sat Aug 15 00:41:05 2026 +0800
[VL] Support format_number function (#12754)
---
.../org/apache/gluten/utils/CHExpressionUtil.scala | 3 ++-
.../gluten/expression/ExpressionRestrictions.scala | 18 ++++++++++++-
.../functions/ScalarFunctionsValidateSuite.scala | 30 ++++++++++++++++++++++
docs/velox-backend-scalar-function-support.md | 4 +--
.../gluten/expression/ExpressionMappings.scala | 1 +
.../apache/gluten/expression/ExpressionNames.scala | 1 +
6 files changed, 53 insertions(+), 4 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index 76f8d6c1f1..1a7dc3aa13 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -214,6 +214,7 @@ object CHExpressionUtil {
DIV -> DefaultValidator(),
REGEXP_INSTR -> DefaultValidator(),
DAY_NAME -> DefaultValidator(),
- MONTH_NAME -> DefaultValidator()
+ MONTH_NAME -> DefaultValidator(),
+ FORMAT_NUMBER -> DefaultValidator()
)
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
index 04f776fa42..b6bfd4119b 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionRestrictions.scala
@@ -101,6 +101,21 @@ object RaiseErrorRestrictions extends
ExpressionRestrictions {
override val restrictionMessages: Array[String] =
Array(ONLY_SUPPORT_ERROR_MESSAGE)
}
+object FormatNumberRestrictions extends ExpressionRestrictions {
+ val NOT_SUPPORT_DECIMAL_INPUT: String =
+ s"${ExpressionNames.FORMAT_NUMBER} only supports tinyint, smallint,
integer, bigint, " +
+ s"float and double input; DecimalType input is not supported in Velox"
+
+ val NOT_SUPPORT_STRING_FORMAT: String =
+ s"${ExpressionNames.FORMAT_NUMBER} with a string format argument (e.g.
'#,###.##') is not " +
+ s"supported in Velox; only an integer number of decimal places is
supported"
+
+ override val functionName: String = ExpressionNames.FORMAT_NUMBER
+
+ override val restrictionMessages: Array[String] =
+ Array(NOT_SUPPORT_DECIMAL_INPUT, NOT_SUPPORT_STRING_FORMAT)
+}
+
object ExpressionRestrictions {
// Called by gen-function-support-docs.py to get all restrictions.
def listAllRestrictions(): Array[ExpressionRestrictions] = {
@@ -109,7 +124,8 @@ object ExpressionRestrictions {
FromJsonRestrictions,
ToJsonRestrictions,
Unbase64Restrictions,
- Base64Restrictions
+ Base64Restrictions,
+ FormatNumberRestrictions
)
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 418cae1b9b..557b86a17d 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -749,6 +749,36 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("format_number") {
+ // Integer / bigint input with different decimal places.
+ runQueryAndCompare("SELECT format_number(l_partkey, 0) FROM lineitem limit
50") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ runQueryAndCompare("SELECT format_number(l_orderkey, 2) FROM lineitem
limit 50") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ // Floating-point input, exercising HALF_EVEN rounding and thousands
separators.
+ runQueryAndCompare(
+ "SELECT format_number(cast(l_quantity as double), 1) FROM lineitem limit
50") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ runQueryAndCompare(
+ "SELECT format_number(cast(l_discount as double), 3) FROM lineitem limit
50") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ // Velox format_number only supports
tinyint/smallint/integer/bigint/float/double.
+ // Decimal input has no matching signature, so it must fall back to
vanilla Spark.
+ runQueryAndCompare("SELECT format_number(l_quantity, 1) FROM lineitem
limit 50") {
+ checkSparkPlan[ProjectExec]
+ }
+ // Velox only implements the integer decimal-places form. The
string-format form
+ // (e.g. '#,###.##') has no matching signature, so it must fall back to
vanilla Spark.
+ runQueryAndCompare(
+ "SELECT format_number(cast(l_quantity as double), '#,###.##') FROM
lineitem limit 50") {
+ checkSparkPlan[ProjectExec]
+ }
+ }
+
testWithMinSparkVersion("mask", "3.4") {
runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") {
checkGlutenPlan[ProjectExecTransformer]
diff --git a/docs/velox-backend-scalar-function-support.md
b/docs/velox-backend-scalar-function-support.md
index 8732d4ed4d..6924767eea 100644
--- a/docs/velox-backend-scalar-function-support.md
+++ b/docs/velox-backend-scalar-function-support.md
@@ -1,6 +1,6 @@
# Scalar Functions Support Status
-**Out of 357 scalar functions in Spark 3.5, Gluten currently fully supports
245 functions and partially supports 27 functions.**
+**Out of 357 scalar functions in Spark 3.5, Gluten currently fully supports
245 functions and partially supports 28 functions.**
## Array Functions
@@ -370,7 +370,7 @@
| encode | Encode | |
|
| endswith | EndsWithExpressionBuilder | PS | BinaryType
unsupported |
| find_in_set | FindInSet | S |
|
-| format_number | FormatNumber | |
|
+| format_number | FormatNumber | PS | format_number
only supports tinyint, smallint, integer, bigint, float and double input;
DecimalType input is not supported in Velox<br>format_number with a string
format argument (e.g. '#,###.##') is not supported in Velox; only an integer
number of decimal places is supported |
| format_string | FormatString | |
|
| initcap | InitCap | S |
|
| instr | StringInstr | S |
|
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index d0ed9347b6..5193f31d66 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -107,6 +107,7 @@ object ExpressionMappings {
Sig[UnBase64](UNBASE64),
Sig[Base64](BASE64),
Sig[FormatString](FORMAT_STRING),
+ Sig[FormatNumber](FORMAT_NUMBER),
// URL functions
Sig[ParseUrl](PARSE_URL),
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index def43746c1..a6b1818ff9 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -143,6 +143,7 @@ object ExpressionNames {
final val BASE64 = "base64"
final val MASK = "mask"
final val FORMAT_STRING = "format_string"
+ final val FORMAT_NUMBER = "format_number"
final val LUHN_CHECK = "luhn_check"
final val TO_PRETTY_STRING = "to_pretty_string"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
