This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 1787a5261e87 [SPARK-46887][DOCS] Document a few missed `spark.ui.*` configs to `Configuration` page 1787a5261e87 is described below commit 1787a5261e87e0214a3f803f6534c5e52a0138e6 Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Sat Jan 27 16:48:26 2024 -0800 [SPARK-46887][DOCS] Document a few missed `spark.ui.*` configs to `Configuration` page ### What changes were proposed in this pull request? This PR aims to document a few missed `spark.ui.*` configurations for Apache Spark 4. This PR focuses only public configurations and excludes `internal` configuration like `spark.ui.jettyStopTimeout`. ### Why are the changes needed? To improve documentations. After this PR, I verified the following configurations are documented at least once in `Configuration` or `Security` page. ``` $ git grep 'ConfigBuilder("spark.ui.' core/src/main/scala/org/apache/spark/internal/config/Status.scala: val LIVE_ENTITY_UPDATE_PERIOD = ConfigBuilder("spark.ui.liveUpdate.period") core/src/main/scala/org/apache/spark/internal/config/Status.scala: val LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD = ConfigBuilder("spark.ui.liveUpdate.minFlushPeriod") core/src/main/scala/org/apache/spark/internal/config/Status.scala: val MAX_RETAINED_JOBS = ConfigBuilder("spark.ui.retainedJobs") core/src/main/scala/org/apache/spark/internal/config/Status.scala: val MAX_RETAINED_STAGES = ConfigBuilder("spark.ui.retainedStages") core/src/main/scala/org/apache/spark/internal/config/Status.scala: val MAX_RETAINED_TASKS_PER_STAGE = ConfigBuilder("spark.ui.retainedTasks") core/src/main/scala/org/apache/spark/internal/config/Status.scala: val MAX_RETAINED_DEAD_EXECUTORS = ConfigBuilder("spark.ui.retainedDeadExecutors") core/src/main/scala/org/apache/spark/internal/config/Status.scala: val MAX_RETAINED_ROOT_NODES = ConfigBuilder("spark.ui.dagGraph.retainedRootRDDs") core/src/main/scala/org/apache/spark/internal/config/Status.scala: val LIVE_UI_LOCAL_STORE_DIR = ConfigBuilder("spark.ui.store.path") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_SHOW_CONSOLE_PROGRESS = ConfigBuilder("spark.ui.showConsoleProgress") core/src/main/scala/org/apache/spark/internal/config/UI.scala: ConfigBuilder("spark.ui.consoleProgress.update.interval") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_ENABLED = ConfigBuilder("spark.ui.enabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_PORT = ConfigBuilder("spark.ui.port") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_FILTERS = ConfigBuilder("spark.ui.filters") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_ALLOW_FRAMING_FROM = ConfigBuilder("spark.ui.allowFramingFrom") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_REVERSE_PROXY = ConfigBuilder("spark.ui.reverseProxy") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_REVERSE_PROXY_URL = ConfigBuilder("spark.ui.reverseProxyUrl") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_KILL_ENABLED = ConfigBuilder("spark.ui.killEnabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_THREAD_DUMPS_ENABLED = ConfigBuilder("spark.ui.threadDumpsEnabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_FLAMEGRAPH_ENABLED = ConfigBuilder("spark.ui.threadDump.flamegraphEnabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_HEAP_HISTOGRAM_ENABLED = ConfigBuilder("spark.ui.heapHistogramEnabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_PROMETHEUS_ENABLED = ConfigBuilder("spark.ui.prometheus.enabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_X_XSS_PROTECTION = ConfigBuilder("spark.ui.xXssProtection") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_X_CONTENT_TYPE_OPTIONS = ConfigBuilder("spark.ui.xContentTypeOptions.enabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_STRICT_TRANSPORT_SECURITY = ConfigBuilder("spark.ui.strictTransportSecurity") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_REQUEST_HEADER_SIZE = ConfigBuilder("spark.ui.requestHeaderSize") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_TIMELINE_ENABLED = ConfigBuilder("spark.ui.timelineEnabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_TIMELINE_TASKS_MAXIMUM = ConfigBuilder("spark.ui.timeline.tasks.maximum") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_TIMELINE_JOBS_MAXIMUM = ConfigBuilder("spark.ui.timeline.jobs.maximum") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_TIMELINE_STAGES_MAXIMUM = ConfigBuilder("spark.ui.timeline.stages.maximum") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_TIMELINE_EXECUTORS_MAXIMUM = ConfigBuilder("spark.ui.timeline.executors.maximum") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_VIEW_ACLS = ConfigBuilder("spark.ui.view.acls") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_VIEW_ACLS_GROUPS = ConfigBuilder("spark.ui.view.acls.groups") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val PROXY_REDIRECT_URI = ConfigBuilder("spark.ui.proxyRedirectUri") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val CUSTOM_EXECUTOR_LOG_URL = ConfigBuilder("spark.ui.custom.executor.log.url") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_SQL_GROUP_SUB_EXECUTION_ENABLED = ConfigBuilder("spark.ui.groupSQLSubExecutionEnabled") core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_JETTY_STOP_TIMEOUT = ConfigBuilder("spark.ui.jettyStopTimeout") ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual review. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44913 from dongjoon-hyun/SPARK-46887. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../org/apache/spark/internal/config/UI.scala | 2 + docs/configuration.md | 48 ++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/internal/config/UI.scala b/core/src/main/scala/org/apache/spark/internal/config/UI.scala index 086c83552732..0af38a2eb90b 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/UI.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/UI.scala @@ -93,6 +93,7 @@ private[spark] object UI { .createWithDefault(true) val UI_THREAD_DUMPS_ENABLED = ConfigBuilder("spark.ui.threadDumpsEnabled") + .doc("Whether to show a link for executor thread dumps in Stages and Executor pages.") .version("1.2.0") .booleanConf .createWithDefault(true) @@ -104,6 +105,7 @@ private[spark] object UI { .createWithDefault(true) val UI_HEAP_HISTOGRAM_ENABLED = ConfigBuilder("spark.ui.heapHistogramEnabled") + .doc("Whether to show a link for executor heap histogram in Executor page.") .version("3.5.0") .booleanConf .createWithDefault(true) diff --git a/docs/configuration.md b/docs/configuration.md index beb52c62d6c2..e771c323d369 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1425,6 +1425,14 @@ Apart from these, the following properties are also available, and may be useful </td> <td>2.1.0</td> </tr> +<tr> + <td><code>spark.ui.groupSQLSubExecutionEnabled</code></td> + <td>true</td> + <td> + Whether to group sub executions together in SQL UI when they belong to the same root execution + </td> + <td>3.4.0</td> +</tr> <tr> <td><code>spark.ui.enabled</code></td> <td>true</td> @@ -1450,6 +1458,30 @@ Apart from these, the following properties are also available, and may be useful </td> <td>1.0.0</td> </tr> +<tr> + <td><code>spark.ui.threadDumpsEnabled</code></td> + <td>true</td> + <td> + Whether to show a link for executor thread dumps in Stages and Executor pages. + </td> + <td>1.2.0</td> +</tr> +<tr> + <td><code>spark.ui.threadDump.flamegraphEnabled</code></td> + <td>true</td> + <td> + Whether to render the Flamegraph for executor thread dumps. + </td> + <td>4.0.0</td> +</tr> +<tr> + <td><code>spark.ui.heapHistogramEnabled</code></td> + <td>true</td> + <td> + Whether to show a link for executor heap histogram in Executor page. + </td> + <td>3.5.0</td> +</tr> <tr> <td><code>spark.ui.liveUpdate.period</code></td> <td>100ms</td> @@ -1568,6 +1600,14 @@ Apart from these, the following properties are also available, and may be useful </td> <td>1.2.1</td> </tr> +<tr> + <td><code>spark.ui.consoleProgress.update.interval</code></td> + <td>200</td> + <td> + An interval in milliseconds to update the progress bar in the console. + </td> + <td>2.1.0</td> +</tr> <tr> <td><code>spark.ui.custom.executor.log.url</code></td> <td>(none)</td> @@ -1584,6 +1624,14 @@ Apart from these, the following properties are also available, and may be useful </td> <td>3.0.0</td> </tr> +<tr> + <td><code>spark.ui.prometheus.enabled</code></td> + <td>true</td> + <td> + Expose executor metrics at /metrics/executors/prometheus at driver web page. + </td> + <td>3.0.0</td> +</tr> <tr> <td><code>spark.worker.ui.retainedExecutors</code></td> <td>1000</td> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org