This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new e4b5977c9b7 [SPARK-46437][DOCS] Remove cruft from the built-in SQL functions documentation e4b5977c9b7 is described below commit e4b5977c9b7b808d32a6370dccc33eaeb235085e Author: Nicholas Chammas <nicholas.cham...@gmail.com> AuthorDate: Fri Dec 22 09:54:13 2023 +0900 [SPARK-46437][DOCS] Remove cruft from the built-in SQL functions documentation ### What changes were proposed in this pull request? - Remove a bunch of Liquid directives that are not necessary. - Add a table of contents to the built-in SQL functions page. - Move the generated HTML for built-in SQL functions to a subdirectory. ### Why are the changes needed? To reduce confusion for maintainers. ### Does this PR introduce _any_ user-facing change? Yes. It adds a table of contents and change the heading style of the examples. Otherwise, the generated docs are identical. ### How was this patch tested? I built Spark, ran `./sql/create-docs.sh`, and reviewed the generated docs in my browser. The page is too long to screenshot completely, but here are a couple of screenshots. <img width="500" alt="Screenshot 2023-12-20 at 7 06 36 PM" src="https://github.com/apache/spark/assets/1039369/b285d8a2-6eab-488d-9e28-2fdc9cc833a9"> <img width="500" alt="Screenshot 2023-12-20 at 7 06 48 PM" src="https://github.com/apache/spark/assets/1039369/2f9670bc-773a-48a8-a0d0-54206b8a4887"> ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44393 from nchammas/sql-builtin-funcs-cruft. Authored-by: Nicholas Chammas <nicholas.cham...@gmail.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- docs/.gitignore | 1 + docs/sql-ref-functions-builtin.md | 281 ++++++++++++++------------------------ docs/sql-ref-functions.md | 2 +- sql/gen-sql-functions-docs.py | 15 +- 4 files changed, 113 insertions(+), 186 deletions(-) diff --git a/docs/.gitignore b/docs/.gitignore index 9df83f37815..bcfdcbf5dcc 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1 +1,2 @@ generated-*.html +_generated_function_html/ diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md index 0ff1432fabf..88ed309a883 100644 --- a/docs/sql-ref-functions-builtin.md +++ b/docs/sql-ref-functions-builtin.md @@ -17,202 +17,125 @@ license: | limitations under the License. --- -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-agg-funcs-table.html' %} +* Table of contents +{:toc} + ### Aggregate Functions -{% include_relative generated-agg-funcs-table.html %} -#### Examples -{% include_relative generated-agg-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-window-funcs-table.html' %} +{% include_relative _generated_function_html/agg-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/agg-funcs-examples.html %} + ### Window Functions -{% include_relative generated-window-funcs-table.html %} -#### Examples -{% include_relative generated-window-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-array-funcs-table.html' %} +{% include_relative _generated_function_html/window-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/window-funcs-examples.html %} + ### Array Functions -{% include_relative generated-array-funcs-table.html %} -#### Examples -{% include_relative generated-array-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-collection-funcs-table.html' %} +{% include_relative _generated_function_html/array-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/array-funcs-examples.html %} + ### Collection Functions -{% include_relative generated-collection-funcs-table.html %} -#### Examples -{% include_relative generated-collection-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} - -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-struct-funcs-table.html' %} +{% include_relative _generated_function_html/collection-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/collection-funcs-examples.html %} + ### STRUCT Functions -{% include_relative generated-struct-funcs-table.html %} -#### Examples -{% include_relative generated-struct-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-map-funcs-table.html' %} +{% include_relative _generated_function_html/struct-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/struct-funcs-examples.html %} + ### Map Functions -{% include_relative generated-map-funcs-table.html %} -#### Examples -{% include_relative generated-map-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-datetime-funcs-table.html' %} +{% include_relative _generated_function_html/map-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/map-funcs-examples.html %} + ### Date and Timestamp Functions -{% include_relative generated-datetime-funcs-table.html %} -#### Examples -{% include_relative generated-datetime-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-math-funcs-table.html' %} +{% include_relative _generated_function_html/datetime-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/datetime-funcs-examples.html %} + ### Mathematical Functions -{% include_relative generated-math-funcs-table.html %} -#### Examples -{% include_relative generated-math-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-string-funcs-table.html' %} +{% include_relative _generated_function_html/math-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/math-funcs-examples.html %} + ### String Functions -{% include_relative generated-string-funcs-table.html %} -#### Examples -{% include_relative generated-string-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-conditional-funcs-table.html' %} +{% include_relative _generated_function_html/string-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/string-funcs-examples.html %} + ### Conditional Functions -{% include_relative generated-conditional-funcs-table.html %} -#### Examples -{% include_relative generated-conditional-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-hash-funcs-table.html' %} +{% include_relative _generated_function_html/conditional-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/conditional-funcs-examples.html %} + ### Hash Functions -{% include_relative generated-hash-funcs-table.html %} -#### Examples -{% include_relative generated-hash-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} - -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-csv-funcs-table.html' %} +{% include_relative _generated_function_html/hash-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/hash-funcs-examples.html %} + ### CSV Functions -{% include_relative generated-csv-funcs-table.html %} -#### Examples -{% include_relative generated-csv-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} - -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-json-funcs-table.html' %} +{% include_relative _generated_function_html/csv-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/csv-funcs-examples.html %} + ### JSON Functions -{% include_relative generated-json-funcs-table.html %} -#### Examples -{% include_relative generated-json-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} - -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-xml-funcs-table.html' %} +{% include_relative _generated_function_html/json-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/json-funcs-examples.html %} + ### XML Functions -{% include_relative generated-xml-funcs-table.html %} -#### Examples -{% include_relative generated-xml-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} - -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-url-funcs-table.html' %} +{% include_relative _generated_function_html/xml-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/xml-funcs-examples.html %} + ### URL Functions -{% include_relative generated-url-funcs-table.html %} -#### Examples -{% include_relative generated-url-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-bitwise-funcs-table.html' %} +{% include_relative _generated_function_html/url-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/url-funcs-examples.html %} + ### Bitwise Functions -{% include_relative generated-bitwise-funcs-table.html %} -#### Examples -{% include_relative generated-bitwise-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-conversion-funcs-table.html' %} +{% include_relative _generated_function_html/bitwise-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/bitwise-funcs-examples.html %} + ### Conversion Functions -{% include_relative generated-conversion-funcs-table.html %} -#### Examples -{% include_relative generated-conversion-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-predicate-funcs-table.html' %} +{% include_relative _generated_function_html/conversion-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/conversion-funcs-examples.html %} + ### Predicate Functions -{% include_relative generated-predicate-funcs-table.html %} -#### Examples -{% include_relative generated-predicate-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-misc-funcs-table.html' %} +{% include_relative _generated_function_html/predicate-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/predicate-funcs-examples.html %} + ### Misc Functions -{% include_relative generated-misc-funcs-table.html %} -#### Examples -{% include_relative generated-misc-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} - -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-generator-funcs-table.html' %} +{% include_relative _generated_function_html/misc-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/misc-funcs-examples.html %} + ### Generator Functions -{% include_relative generated-generator-funcs-table.html %} -#### Examples -{% include_relative generated-generator-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_relative _generated_function_html/generator-funcs-table.html %} + +**Examples** +{% include_relative _generated_function_html/generator-funcs-examples.html %} diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md index cc9edd61f41..b4891fe72eb 100644 --- a/docs/sql-ref-functions.md +++ b/docs/sql-ref-functions.md @@ -20,7 +20,7 @@ license: | --- Spark SQL provides two function features to meet a wide range of user needs: built-in functions and user-defined functions (UDFs). -Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task. +Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/index.html) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task. ### Built-in Functions diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py index 053e11d1029..82522478140 100644 --- a/sql/gen-sql-functions-docs.py +++ b/sql/gen-sql-functions-docs.py @@ -16,9 +16,10 @@ # import itertools -import os import re +import shutil from collections import namedtuple +from pathlib import Path # To avoid adding a new direct dependency, we import markdown from within mkdocs. from mkdocs.structure.pages import markdown @@ -26,6 +27,8 @@ from mkdocs.structure.pages import markdown from pyspark.java_gateway import launch_gateway +SPARK_PROJECT_ROOT = Path(__file__).parents[1] + ExpressionInfo = namedtuple("ExpressionInfo", "name usage examples group") groups = { @@ -210,7 +213,7 @@ def generate_functions_table_html(jvm, html_output_dir): for key, infos in _list_grouped_function_infos(jvm): function_table = _make_pretty_usage(infos) key = key.replace("_", "-") - with open("%s/generated-%s-table.html" % (html_output_dir, key), 'w') as table_html: + with open(html_output_dir / f"{key}-table.html", 'w') as table_html: table_html.write(function_table) @@ -233,8 +236,7 @@ def generate_functions_examples_html(jvm, jspark, html_output_dir): examples = _make_pretty_examples(jspark, infos) key = key.replace("_", "-") if examples is not None: - with open("%s/generated-%s-examples.html" % ( - html_output_dir, key), 'w') as examples_html: + with open(html_output_dir / f"{key}-examples.html", 'w') as examples_html: examples_html.write(examples) @@ -242,7 +244,8 @@ if __name__ == "__main__": jvm = launch_gateway().jvm jspark = jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate() jspark.sparkContext().setLogLevel("ERROR") # Make it less noisy. - spark_root_dir = os.path.dirname(os.path.dirname(__file__)) - html_output_dir = os.path.join(spark_root_dir, "docs") + html_output_dir = SPARK_PROJECT_ROOT / "docs" / "_generated_function_html" + shutil.rmtree(html_output_dir, ignore_errors=True) + html_output_dir.mkdir() generate_functions_table_html(jvm, html_output_dir) generate_functions_examples_html(jvm, jspark, html_output_dir) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org