This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e4b5977c9b7 [SPARK-46437][DOCS] Remove cruft from the built-in SQL
functions documentation
e4b5977c9b7 is described below
commit e4b5977c9b7b808d32a6370dccc33eaeb235085e
Author: Nicholas Chammas <nicholas.cham...@gmail.com>
AuthorDate: Fri Dec 22 09:54:13 2023 +0900
[SPARK-46437][DOCS] Remove cruft from the built-in SQL functions
documentation
### What changes were proposed in this pull request?
- Remove a bunch of Liquid directives that are not necessary.
- Add a table of contents to the built-in SQL functions page.
- Move the generated HTML for built-in SQL functions to a subdirectory.
### Why are the changes needed?
To reduce confusion for maintainers.
### Does this PR introduce _any_ user-facing change?
Yes. It adds a table of contents and change the heading style of the
examples.
Otherwise, the generated docs are identical.
### How was this patch tested?
I built Spark, ran `./sql/create-docs.sh`, and reviewed the generated docs
in my browser.
The page is too long to screenshot completely, but here are a couple of
screenshots.
<img width="500" alt="Screenshot 2023-12-20 at 7 06 36 PM"
src="https://github.com/apache/spark/assets/1039369/b285d8a2-6eab-488d-9e28-2fdc9cc833a9";>
<img width="500" alt="Screenshot 2023-12-20 at 7 06 48 PM"
src="https://github.com/apache/spark/assets/1039369/2f9670bc-773a-48a8-a0d0-54206b8a4887";>
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44393 from nchammas/sql-builtin-funcs-cruft.
Authored-by: Nicholas Chammas <nicholas.cham...@gmail.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
docs/.gitignore | 1 +
docs/sql-ref-functions-builtin.md | 281 ++++++++++++++------------------------
docs/sql-ref-functions.md | 2 +-
sql/gen-sql-functions-docs.py | 15 +-
4 files changed, 113 insertions(+), 186 deletions(-)
diff --git a/docs/.gitignore b/docs/.gitignore
index 9df83f37815..bcfdcbf5dcc 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -1 +1,2 @@
generated-*.html
+_generated_function_html/
diff --git a/docs/sql-ref-functions-builtin.md
b/docs/sql-ref-functions-builtin.md
index 0ff1432fabf..88ed309a883 100644
--- a/docs/sql-ref-functions-builtin.md
+++ b/docs/sql-ref-functions-builtin.md
@@ -17,202 +17,125 @@ license: |
limitations under the License.
---
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-agg-funcs-table.html' %}
+* Table of contents
+{:toc}
+
### Aggregate Functions
-{% include_relative generated-agg-funcs-table.html %}
-#### Examples
-{% include_relative generated-agg-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-window-funcs-table.html' %}
+{% include_relative _generated_function_html/agg-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/agg-funcs-examples.html %}
+
### Window Functions
-{% include_relative generated-window-funcs-table.html %}
-#### Examples
-{% include_relative generated-window-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-array-funcs-table.html' %}
+{% include_relative _generated_function_html/window-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/window-funcs-examples.html %}
+
### Array Functions
-{% include_relative generated-array-funcs-table.html %}
-#### Examples
-{% include_relative generated-array-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
-{% if static_file.name == 'generated-collection-funcs-table.html' %}
+{% include_relative _generated_function_html/array-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/array-funcs-examples.html %}
+
### Collection Functions
-{% include_relative generated-collection-funcs-table.html %}
-#### Examples
-{% include_relative generated-collection-funcs-examples.html %}
-{% break %}
-{% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
-{% if static_file.name == 'generated-struct-funcs-table.html' %}
+{% include_relative _generated_function_html/collection-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/collection-funcs-examples.html %}
+
### STRUCT Functions
-{% include_relative generated-struct-funcs-table.html %}
-#### Examples
-{% include_relative generated-struct-funcs-examples.html %}
-{% break %}
-{% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-map-funcs-table.html' %}
+{% include_relative _generated_function_html/struct-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/struct-funcs-examples.html %}
+
### Map Functions
-{% include_relative generated-map-funcs-table.html %}
-#### Examples
-{% include_relative generated-map-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-datetime-funcs-table.html' %}
+{% include_relative _generated_function_html/map-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/map-funcs-examples.html %}
+
### Date and Timestamp Functions
-{% include_relative generated-datetime-funcs-table.html %}
-#### Examples
-{% include_relative generated-datetime-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-math-funcs-table.html' %}
+{% include_relative _generated_function_html/datetime-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/datetime-funcs-examples.html %}
+
### Mathematical Functions
-{% include_relative generated-math-funcs-table.html %}
-#### Examples
-{% include_relative generated-math-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-string-funcs-table.html' %}
+{% include_relative _generated_function_html/math-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/math-funcs-examples.html %}
+
### String Functions
-{% include_relative generated-string-funcs-table.html %}
-#### Examples
-{% include_relative generated-string-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-conditional-funcs-table.html' %}
+{% include_relative _generated_function_html/string-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/string-funcs-examples.html %}
+
### Conditional Functions
-{% include_relative generated-conditional-funcs-table.html %}
-#### Examples
-{% include_relative generated-conditional-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
-{% if static_file.name == 'generated-hash-funcs-table.html' %}
+{% include_relative _generated_function_html/conditional-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/conditional-funcs-examples.html %}
+
### Hash Functions
-{% include_relative generated-hash-funcs-table.html %}
-#### Examples
-{% include_relative generated-hash-funcs-examples.html %}
-{% break %}
-{% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
-{% if static_file.name == 'generated-csv-funcs-table.html' %}
+{% include_relative _generated_function_html/hash-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/hash-funcs-examples.html %}
+
### CSV Functions
-{% include_relative generated-csv-funcs-table.html %}
-#### Examples
-{% include_relative generated-csv-funcs-examples.html %}
-{% break %}
-{% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
-{% if static_file.name == 'generated-json-funcs-table.html' %}
+{% include_relative _generated_function_html/csv-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/csv-funcs-examples.html %}
+
### JSON Functions
-{% include_relative generated-json-funcs-table.html %}
-#### Examples
-{% include_relative generated-json-funcs-examples.html %}
-{% break %}
-{% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
-{% if static_file.name == 'generated-xml-funcs-table.html' %}
+{% include_relative _generated_function_html/json-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/json-funcs-examples.html %}
+
### XML Functions
-{% include_relative generated-xml-funcs-table.html %}
-#### Examples
-{% include_relative generated-xml-funcs-examples.html %}
-{% break %}
-{% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
-{% if static_file.name == 'generated-url-funcs-table.html' %}
+{% include_relative _generated_function_html/xml-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/xml-funcs-examples.html %}
+
### URL Functions
-{% include_relative generated-url-funcs-table.html %}
-#### Examples
-{% include_relative generated-url-funcs-examples.html %}
-{% break %}
-{% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-bitwise-funcs-table.html' %}
+{% include_relative _generated_function_html/url-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/url-funcs-examples.html %}
+
### Bitwise Functions
-{% include_relative generated-bitwise-funcs-table.html %}
-#### Examples
-{% include_relative generated-bitwise-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-conversion-funcs-table.html' %}
+{% include_relative _generated_function_html/bitwise-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/bitwise-funcs-examples.html %}
+
### Conversion Functions
-{% include_relative generated-conversion-funcs-table.html %}
-#### Examples
-{% include_relative generated-conversion-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-predicate-funcs-table.html' %}
+{% include_relative _generated_function_html/conversion-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/conversion-funcs-examples.html %}
+
### Predicate Functions
-{% include_relative generated-predicate-funcs-table.html %}
-#### Examples
-{% include_relative generated-predicate-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-misc-funcs-table.html' %}
+{% include_relative _generated_function_html/predicate-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/predicate-funcs-examples.html %}
+
### Misc Functions
-{% include_relative generated-misc-funcs-table.html %}
-#### Examples
-{% include_relative generated-misc-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
-
-{% for static_file in site.static_files %}
- {% if static_file.name == 'generated-generator-funcs-table.html' %}
+{% include_relative _generated_function_html/misc-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/misc-funcs-examples.html %}
+
### Generator Functions
-{% include_relative generated-generator-funcs-table.html %}
-#### Examples
-{% include_relative generated-generator-funcs-examples.html %}
- {% break %}
- {% endif %}
-{% endfor %}
+{% include_relative _generated_function_html/generator-funcs-table.html %}
+
+**Examples**
+{% include_relative _generated_function_html/generator-funcs-examples.html %}
diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
index cc9edd61f41..b4891fe72eb 100644
--- a/docs/sql-ref-functions.md
+++ b/docs/sql-ref-functions.md
@@ -20,7 +20,7 @@ license: |
---
Spark SQL provides two function features to meet a wide range of user needs:
built-in functions and user-defined functions (UDFs).
-Built-in functions are commonly used routines that Spark SQL predefines and a
complete list of the functions can be found in the [Built-in
Functions](api/sql/) API document. UDFs allow users to define their own
functions when the system’s built-in functions are not enough to perform the
desired task.
+Built-in functions are commonly used routines that Spark SQL predefines and a
complete list of the functions can be found in the [Built-in
Functions](api/sql/index.html) API document. UDFs allow users to define their
own functions when the system’s built-in functions are not enough to perform
the desired task.
### Built-in Functions
diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py
index 053e11d1029..82522478140 100644
--- a/sql/gen-sql-functions-docs.py
+++ b/sql/gen-sql-functions-docs.py
@@ -16,9 +16,10 @@
#
import itertools
-import os
import re
+import shutil
from collections import namedtuple
+from pathlib import Path
# To avoid adding a new direct dependency, we import markdown from within
mkdocs.
from mkdocs.structure.pages import markdown
@@ -26,6 +27,8 @@ from mkdocs.structure.pages import markdown
from pyspark.java_gateway import launch_gateway
+SPARK_PROJECT_ROOT = Path(__file__).parents[1]
+
ExpressionInfo = namedtuple("ExpressionInfo", "name usage examples group")
groups = {
@@ -210,7 +213,7 @@ def generate_functions_table_html(jvm, html_output_dir):
for key, infos in _list_grouped_function_infos(jvm):
function_table = _make_pretty_usage(infos)
key = key.replace("_", "-")
- with open("%s/generated-%s-table.html" % (html_output_dir, key), 'w')
as table_html:
+ with open(html_output_dir / f"{key}-table.html", 'w') as table_html:
table_html.write(function_table)
@@ -233,8 +236,7 @@ def generate_functions_examples_html(jvm, jspark,
html_output_dir):
examples = _make_pretty_examples(jspark, infos)
key = key.replace("_", "-")
if examples is not None:
- with open("%s/generated-%s-examples.html" % (
- html_output_dir, key), 'w') as examples_html:
+ with open(html_output_dir / f"{key}-examples.html", 'w') as
examples_html:
examples_html.write(examples)
@@ -242,7 +244,8 @@ if __name__ == "__main__":
jvm = launch_gateway().jvm
jspark = jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate()
jspark.sparkContext().setLogLevel("ERROR") # Make it less noisy.
- spark_root_dir = os.path.dirname(os.path.dirname(__file__))
- html_output_dir = os.path.join(spark_root_dir, "docs")
+ html_output_dir = SPARK_PROJECT_ROOT / "docs" / "_generated_function_html"
+ shutil.rmtree(html_output_dir, ignore_errors=True)
+ html_output_dir.mkdir()
generate_functions_table_html(jvm, html_output_dir)
generate_functions_examples_html(jvm, jspark, html_output_dir)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
