This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push: new c7671a2a066 [DOCS] Update concat and concat_ws documentation to point out unexpected behavior c7671a2a066 is described below commit c7671a2a0667d0d8f8bc191e5be93cd21b0bb4db Author: Christopher Watford <132389385+watford...@users.noreply.github.com> AuthorDate: Mon Jul 31 10:11:28 2023 +0800 [DOCS] Update concat and concat_ws documentation to point out unexpected behavior Add documentation covering unexpected behavior of concat and concat_ws with respect to null values. ### What changes were proposed in this pull request? Adds additional documentation to `concat` and `concat_ws`. ### Why are the changes needed? The behavior of `concat` and `concat_ws` were unexpected w.r.t. null values and the documentation did not help make their behavior clear. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Closes #42153 from watfordkcf/patch-1. Lead-authored-by: Christopher Watford <132389385+watford...@users.noreply.github.com> Co-authored-by: Christopher Watford <christopher.watf...@kcftech.com> Signed-off-by: Kent Yao <y...@apache.org> (cherry picked from commit ff022e54c2caedb164ce943dc628fcf607eddcdd) Signed-off-by: Kent Yao <y...@apache.org> --- .../apache/spark/sql/catalyst/expressions/stringExpressions.scala | 6 +++++- sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 1e58384c81d..507e4200378 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -53,13 +53,17 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String} */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`.", + usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`, skipping null values.", examples = """ Examples: > SELECT _FUNC_(' ', 'Spark', 'SQL'); Spark SQL > SELECT _FUNC_('s'); + > SELECT _FUNC_('/', 'foo', null, 'bar'); + foo/bar + > SELECT _FUNC_(null, 'Spark', 'SQL'); + NULL """, since = "1.5.0", group = "string_funcs") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index cb5c1ad5c49..960ce731ac6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -2643,6 +2643,8 @@ object functions { * Concatenates multiple input string columns together into a single string column, * using the given separator. * + * @note Input strings which are null are skipped. + * * @group string_funcs * @since 1.5.0 */ @@ -3959,6 +3961,8 @@ object functions { * Concatenates multiple input columns together into a single column. * The function works with strings, binary and compatible array columns. * + * @note Returns null if any of the input columns are null. + * * @group collection_funcs * @since 1.5.0 */ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org