This is an automated email from the ASF dual-hosted git repository. ruifengz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 15c98e0e5e07 [SPARK-48954] try_mod() replaces try_remainder() 15c98e0e5e07 is described below commit 15c98e0e5e070d61e32a8eec935488efd9605480 Author: Serge Rielau <se...@rielau.com> AuthorDate: Sun Jul 21 16:47:46 2024 +0800 [SPARK-48954] try_mod() replaces try_remainder() ### What changes were proposed in this pull request? for consistency try_remainder() gets renamed to try_mod(). this is Spark 4.0.0 only, so no need for config. ### Why are the changes needed? To keep consistent naming. ### Does this PR introduce _any_ user-facing change? Yes, replaces try_remainder() with try_mod() ### How was this patch tested? Existing try_remainder() tests ### Was this patch authored or co-authored using generative AI tooling? No Closes #47427 from srielau/SPARK-48954-try-mod. Authored-by: Serge Rielau <se...@rielau.com> Signed-off-by: Ruifeng Zheng <ruife...@apache.org> --- .../scala/org/apache/spark/sql/functions.scala | 2 +- docs/sql-ref-ansi-compliance.md | 2 +- .../source/reference/pyspark.sql/functions.rst | 2 +- python/pyspark/sql/connect/functions/builtin.py | 6 ++-- python/pyspark/sql/functions/builtin.py | 32 +++++++++++----------- .../sql/tests/connect/test_connect_column.py | 8 ++---- .../sql/catalyst/analysis/FunctionRegistry.scala | 2 +- .../spark/sql/catalyst/expressions/TryEval.scala | 4 +-- .../sql/catalyst/expressions/arithmetic.scala | 2 +- .../sql/catalyst/expressions/TryEvalSuite.scala | 2 +- .../scala/org/apache/spark/sql/functions.scala | 2 +- .../sql-functions/sql-expression-schema.md | 2 +- .../org/apache/spark/sql/MathFunctionsSuite.scala | 6 ++-- 13 files changed, 34 insertions(+), 38 deletions(-) diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala index 02b25dd6cbb5..c0bf9c9d013c 100644 --- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala +++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala @@ -1947,7 +1947,7 @@ object functions { * @group math_funcs * @since 4.0.0 */ - def try_remainder(left: Column, right: Column): Column = Column.fn("try_remainder", left, right) + def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) /** * Returns `left``*``right` and the result is null on overflow. The acceptable input types are diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index 54f9fd439548..443bc8409efc 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -374,7 +374,7 @@ When ANSI mode is on, it throws exceptions for invalid operations. You can use t - `try_subtract`: identical to the add operator `-`, except that it returns `NULL` result instead of throwing an exception on integral value overflow. - `try_multiply`: identical to the add operator `*`, except that it returns `NULL` result instead of throwing an exception on integral value overflow. - `try_divide`: identical to the division operator `/`, except that it returns `NULL` result instead of throwing an exception on dividing 0. - - `try_remainder`: identical to the remainder operator `%`, except that it returns `NULL` result instead of throwing an exception on dividing 0. + - `try_mod`: identical to the remainder operator `%`, except that it returns `NULL` result instead of throwing an exception on dividing 0. - `try_sum`: identical to the function `sum`, except that it returns `NULL` result instead of throwing an exception on integral/decimal/interval value overflow. - `try_avg`: identical to the function `avg`, except that it returns `NULL` result instead of throwing an exception on decimal/interval value overflow. - `try_element_at`: identical to the function `element_at`, except that it returns `NULL` result instead of throwing an exception on array's index out of bound. diff --git a/python/docs/source/reference/pyspark.sql/functions.rst b/python/docs/source/reference/pyspark.sql/functions.rst index c7ae525429ca..7585448204f6 100644 --- a/python/docs/source/reference/pyspark.sql/functions.rst +++ b/python/docs/source/reference/pyspark.sql/functions.rst @@ -142,8 +142,8 @@ Mathematical Functions tanh try_add try_divide + try_mod try_multiply - try_remainder try_subtract unhex width_bucket diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py index 7f7ea3c6f45d..37398ca9ccf3 100644 --- a/python/pyspark/sql/connect/functions/builtin.py +++ b/python/pyspark/sql/connect/functions/builtin.py @@ -934,11 +934,11 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: try_divide.__doc__ = pysparkfuncs.try_divide.__doc__ -def try_remainder(left: "ColumnOrName", right: "ColumnOrName") -> Column: - return _invoke_function_over_columns("try_remainder", left, right) +def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: + return _invoke_function_over_columns("try_mod", left, right) -try_remainder.__doc__ = pysparkfuncs.try_remainder.__doc__ +try_mod.__doc__ = pysparkfuncs.try_mod.__doc__ def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 87b4fbc49518..6ee311780520 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -658,7 +658,7 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: @_try_remote_functions -def try_remainder(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ Returns the remainder after `dividend`/`divisor`. Its result is always null if `divisor` is 0. @@ -679,14 +679,14 @@ def try_remainder(left: "ColumnOrName", right: "ColumnOrName") -> Column: >>> import pyspark.sql.functions as sf >>> spark.createDataFrame( ... [(6000, 15), (3, 2), (1234, 0)], ["a", "b"] - ... ).select(sf.try_remainder("a", "b")).show() - +-------------------+ - |try_remainder(a, b)| - +-------------------+ - | 0| - | 1| - | NULL| - +-------------------+ + ... ).select(sf.try_mod("a", "b")).show() + +-------------+ + |try_mod(a, b)| + +-------------+ + | 0| + | 1| + | NULL| + +-------------+ Example 2: Exception during division, resulting in NULL when ANSI mode is on @@ -695,16 +695,16 @@ def try_remainder(left: "ColumnOrName", right: "ColumnOrName") -> Column: >>> spark.conf.set("spark.sql.ansi.enabled", "true") >>> try: ... df = spark.range(1) - ... df.select(sf.try_remainder(df.id, sf.lit(0))).show() + ... df.select(sf.try_mod(df.id, sf.lit(0))).show() ... finally: ... spark.conf.set("spark.sql.ansi.enabled", origin) - +--------------------+ - |try_remainder(id, 0)| - +--------------------+ - | NULL| - +--------------------+ + +--------------+ + |try_mod(id, 0)| + +--------------+ + | NULL| + +--------------+ """ - return _invoke_function_over_columns("try_remainder", left, right) + return _invoke_function_over_columns("try_mod", left, right) @_try_remote_functions diff --git a/python/pyspark/sql/tests/connect/test_connect_column.py b/python/pyspark/sql/tests/connect/test_connect_column.py index c797087aef0a..39b46153d4b1 100644 --- a/python/pyspark/sql/tests/connect/test_connect_column.py +++ b/python/pyspark/sql/tests/connect/test_connect_column.py @@ -824,12 +824,8 @@ class SparkConnectColumnTests(SparkConnectSQLTestCase): ) self.assert_eq( - cdf.select( - cdf.a % cdf["b"], cdf["a"] % 2, CF.try_remainder(CF.lit(12), cdf.c) - ).toPandas(), - sdf.select( - sdf.a % sdf["b"], sdf["a"] % 2, SF.try_remainder(SF.lit(12), sdf.c) - ).toPandas(), + cdf.select(cdf.a % cdf["b"], cdf["a"] % 2, CF.try_mod(CF.lit(12), cdf.c)).toPandas(), + sdf.select(sdf.a % sdf["b"], sdf["a"] % 2, SF.try_mod(SF.lit(12), sdf.c)).toPandas(), ) self.assert_eq( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index d47a34c3626c..739fac1f33fd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -451,7 +451,7 @@ object FunctionRegistry { // "try_*" function which always return Null instead of runtime error. expression[TryAdd]("try_add"), expression[TryDivide]("try_divide"), - expression[TryRemainder]("try_remainder"), + expression[TryMod]("try_mod"), expression[TrySubtract]("try_subtract"), expression[TryMultiply]("try_multiply"), expression[TryElementAt]("try_element_at"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala index 05eafe01906a..b7d0ffdb75fb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala @@ -150,7 +150,7 @@ case class TryDivide(left: Expression, right: Expression, replacement: Expressio since = "4.0.0", group = "math_funcs") // scalastyle:on line.size.limit -case class TryRemainder(left: Expression, right: Expression, replacement: Expression) +case class TryMod(left: Expression, right: Expression, replacement: Expression) extends RuntimeReplaceable with InheritAnalysisRules { def this(left: Expression, right: Expression) = this(left, right, (left.dataType, right.dataType) match { @@ -160,7 +160,7 @@ case class TryRemainder(left: Expression, right: Expression, replacement: Expres } ) - override def prettyName: String = "try_remainder" + override def prettyName: String = "try_mod" override def parameters: Seq[Expression] = Seq(left, right) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index f1b192a3e21f..f889c3ebc4d9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -904,7 +904,7 @@ case class Remainder( override def inputType: AbstractDataType = NumericType - // `try_remainder` has exactly the same behavior as the legacy divide, so here it only executes + // `try_mod` has exactly the same behavior as the legacy divide, so here it only executes // the error code path when `evalMode` is `ANSI`. protected override def failOnError: Boolean = evalMode == EvalMode.ANSI diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryEvalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryEvalSuite.scala index e082f2e3accc..cb5d02edfed9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryEvalSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryEvalSuite.scala @@ -46,7 +46,7 @@ class TryEvalSuite extends SparkFunSuite with ExpressionEvalHelper { } } - test("try_remainder") { + test("try_mod") { Seq( (3.0, 2.0, 1.0), (1.0, 0.0, null), diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index dd9b8cd26ad2..88303b1979a7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -1950,7 +1950,7 @@ object functions { * @group math_funcs * @since 4.0.0 */ - def try_remainder(left: Column, right: Column): Column = Column.fn("try_remainder", left, right) + def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) /** * Returns `left``*``right` and the result is null on overflow. The acceptable input types are diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 228c8f5df692..7f0e20ba345e 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -352,9 +352,9 @@ | org.apache.spark.sql.catalyst.expressions.TryAesDecrypt | try_aes_decrypt | SELECT try_aes_decrypt(unhex('6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM') | struct<try_aes_decrypt(unhex(6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210), 0000111122223333, GCM, DEFAULT, ):binary> | | org.apache.spark.sql.catalyst.expressions.TryDivide | try_divide | SELECT try_divide(3, 2) | struct<try_divide(3, 2):double> | | org.apache.spark.sql.catalyst.expressions.TryElementAt | try_element_at | SELECT try_element_at(array(1, 2, 3), 2) | struct<try_element_at(array(1, 2, 3), 2):int> | +| org.apache.spark.sql.catalyst.expressions.TryMod | try_mod | SELECT try_mod(3, 2) | struct<try_mod(3, 2):int> | | org.apache.spark.sql.catalyst.expressions.TryMultiply | try_multiply | SELECT try_multiply(2, 3) | struct<try_multiply(2, 3):int> | | org.apache.spark.sql.catalyst.expressions.TryReflect | try_reflect | SELECT try_reflect('java.util.UUID', 'randomUUID') | struct<try_reflect(java.util.UUID, randomUUID):string> | -| org.apache.spark.sql.catalyst.expressions.TryRemainder | try_remainder | SELECT try_remainder(3, 2) | struct<try_remainder(3, 2):int> | | org.apache.spark.sql.catalyst.expressions.TrySubtract | try_subtract | SELECT try_subtract(2, 1) | struct<try_subtract(2, 1):int> | | org.apache.spark.sql.catalyst.expressions.TryToBinary | try_to_binary | SELECT try_to_binary('abc', 'utf-8') | struct<try_to_binary(abc, utf-8):binary> | | org.apache.spark.sql.catalyst.expressions.TryToNumber | try_to_number | SELECT try_to_number('454', '999') | struct<try_to_number(454, 999):decimal(3,0)> | diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala index ac14b345a762..f1d0815c181b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala @@ -707,15 +707,15 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession { df1.select(try_divide(make_interval(col("year"), col("month")), lit(0)))) } - test("try_remainder") { + test("try_mod") { val df = Seq((10, 3), (5, 5), (5, 0)).toDF("birth", "age") - checkAnswer(df.selectExpr("try_remainder(birth, age)"), Seq(Row(1), Row(0), Row(null))) + checkAnswer(df.selectExpr("try_mod(birth, age)"), Seq(Row(1), Row(0), Row(null))) val dfDecimal = Seq( (BigDecimal(10), BigDecimal(3)), (BigDecimal(5), BigDecimal(5)), (BigDecimal(5), BigDecimal(0))).toDF("birth", "age") - checkAnswer(dfDecimal.selectExpr("try_remainder(birth, age)"), Seq(Row(1), Row(0), Row(null))) + checkAnswer(dfDecimal.selectExpr("try_mod(birth, age)"), Seq(Row(1), Row(0), Row(null))) } test("try_element_at") { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org