This is an automated email from the ASF dual-hosted git repository. gengliang pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 4648412 [SPARK-36585][SQL][DOCS] Support setting "since" version in FunctionRegistry 4648412 is described below commit 464841224ccf14b8fd8a1358e4e6b219f36fd1be Author: Gengliang Wang <gengli...@apache.org> AuthorDate: Wed Aug 25 22:32:20 2021 +0800 [SPARK-36585][SQL][DOCS] Support setting "since" version in FunctionRegistry ### What changes were proposed in this pull request? Spark 3.2.0 includes two new functions `regexp` and `regexp_like`, which are identical to `rlike`. However, in the generated documentation. the since versions of both functions are `1.0.0` since they are based on the expression `RLike`: - https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp - https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp_like This PR is to: * Support setting `since` version in FunctionRegistry * Correct the `since` version of `regexp` and `regexp_like` ### Why are the changes needed? Correct the SQL doc ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Run ``` sh sql/create-docs.sh ``` and check the SQL doc manually Closes #33834 from gengliangwang/allowSQLFunVersion. Authored-by: Gengliang Wang <gengli...@apache.org> Signed-off-by: Gengliang Wang <gengli...@apache.org> (cherry picked from commit 18143fb42604a3ad7ed4bfd89084f7f0a6bce4d6) Signed-off-by: Gengliang Wang <gengli...@apache.org> --- .../sql/catalyst/analysis/FunctionRegistry.scala | 33 ++++++++++++++-------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 63b1525..00ece4d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -107,7 +107,9 @@ object FunctionRegistryBase { * Return an expression info and a function builder for the function as defined by * T using the given name. */ - def build[T : ClassTag](name: String): (ExpressionInfo, Seq[Expression] => T) = { + def build[T : ClassTag]( + name: String, + since: Option[String]): (ExpressionInfo, Seq[Expression] => T) = { val runtimeClass = scala.reflect.classTag[T].runtimeClass // For `RuntimeReplaceable`, skip the constructor with most arguments, which is the main // constructor and contains non-parameter `child` and should not be used as function builder. @@ -150,13 +152,13 @@ object FunctionRegistryBase { } } - (expressionInfo(name), builder) + (expressionInfo(name, since), builder) } /** * Creates an [[ExpressionInfo]] for the function as defined by T using the given name. */ - def expressionInfo[T : ClassTag](name: String): ExpressionInfo = { + def expressionInfo[T : ClassTag](name: String, since: Option[String]): ExpressionInfo = { val clazz = scala.reflect.classTag[T].runtimeClass val df = clazz.getAnnotation(classOf[ExpressionDescription]) if (df != null) { @@ -170,7 +172,7 @@ object FunctionRegistryBase { df.examples(), df.note(), df.group(), - df.since(), + since.getOrElse(df.since()), df.deprecated(), df.source()) } else { @@ -495,12 +497,12 @@ object FunctionRegistry { expression[RegExpExtract]("regexp_extract"), expression[RegExpExtractAll]("regexp_extract_all"), expression[RegExpReplace]("regexp_replace"), - expression[RLike]("regexp_like", true), - expression[RLike]("regexp", true), expression[StringRepeat]("repeat"), expression[StringReplace]("replace"), expression[Overlay]("overlay"), expression[RLike]("rlike"), + expression[RLike]("regexp_like", true, Some("3.2.0")), + expression[RLike]("regexp", true, Some("3.2.0")), expression[StringRPad]("rpad"), expression[StringTrimRight]("rtrim"), expression[Sentences]("sentences"), @@ -735,10 +737,19 @@ object FunctionRegistry { val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet - /** See usage above. */ - private def expression[T <: Expression : ClassTag](name: String, setAlias: Boolean = false) - : (String, (ExpressionInfo, FunctionBuilder)) = { - val (expressionInfo, builder) = FunctionRegistryBase.build[T](name) + /** + * Create a SQL function builder and corresponding `ExpressionInfo`. + * @param name The function name. + * @param setAlias The alias name used in SQL representation string. + * @param since The Spark version since the function is added. + * @tparam T The actual expression class. + * @return (function name, (expression information, function builder)) + */ + private def expression[T <: Expression : ClassTag]( + name: String, + setAlias: Boolean = false, + since: Option[String] = None): (String, (ExpressionInfo, FunctionBuilder)) = { + val (expressionInfo, builder) = FunctionRegistryBase.build[T](name, since) val newBuilder = (expressions: Seq[Expression]) => { val expr = builder(expressions) if (setAlias) expr.setTagValue(FUNC_ALIAS, name) @@ -813,7 +824,7 @@ object TableFunctionRegistry { private def logicalPlan[T <: LogicalPlan : ClassTag](name: String) : (String, (ExpressionInfo, TableFunctionBuilder)) = { - val (info, builder) = FunctionRegistryBase.build[T](name) + val (info, builder) = FunctionRegistryBase.build[T](name, since = None) val newBuilder = (expressions: Seq[Expression]) => { try { builder(expressions) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org