This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 24b4581fa818 [SPARK-47894][CORE][WEBUI] Add `Environment` page to
Master UI
24b4581fa818 is described below
commit 24b4581fa818da89a5aff57437addcece707e678
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Wed Apr 17 20:29:54 2024 -0700
[SPARK-47894][CORE][WEBUI] Add `Environment` page to Master UI
### What changes were proposed in this pull request?
This PR aims to add `Environment` page to `Spark Master UI`.
### Why are the changes needed?
To improve `Spark Standalone` cluster UX by providing `Spark Master` JVM's
information
- `Runtime Information`
- `Spark Properties`
- `Hadoop Properties`
- `System Properties`
- `Metrics Properties`
- `Classpath Entries`
<img width="371" alt="Screenshot 2024-04-17 at 17 08 09"
src="https://github.com/apache/spark/assets/9700541/2b02abbd-e08f-4b0f-834a-160ea6fd00c7";>
<img width="612" alt="Screenshot 2024-04-17 at 17 09 02"
src="https://github.com/apache/spark/assets/9700541/664d113a-b677-41a7-9e8c-841e087aae1d";>
### Does this PR introduce _any_ user-facing change?
Yes, but this is a new UI.
### How was this patch tested?
Pass the CIs with the newly added test case.
Or manual check the UI after running `Master`.
```
$ SPARK_MASTER_OPTS="-Dspark.master.rest.enabled=true
-Dspark.deploy.maxDrivers=2" sbin/start-master.sh
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #46111 from dongjoon-hyun/SPARK-47894.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../spark/deploy/master/ui/EnvironmentPage.scala | 141 +++++++++++++++++++++
.../apache/spark/deploy/master/ui/MasterPage.scala | 5 +-
.../spark/deploy/master/ui/MasterWebUI.scala | 5 +
.../master/ui/ReadOnlyMasterWebUISuite.scala | 14 +-
4 files changed, 162 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/EnvironmentPage.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/EnvironmentPage.scala
new file mode 100644
index 000000000000..190e821524ba
--- /dev/null
+++
b/core/src/main/scala/org/apache/spark/deploy/master/ui/EnvironmentPage.scala
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.master.ui
+
+import scala.xml.Node
+
+import jakarta.servlet.http.HttpServletRequest
+
+import org.apache.spark.{SparkConf, SparkEnv}
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.ui._
+import org.apache.spark.util.Utils
+
+private[ui] class EnvironmentPage(
+ parent: MasterWebUI,
+ conf: SparkConf) extends WebUIPage("Environment") {
+
+ def render(request: HttpServletRequest): Seq[Node] = {
+ val details = SparkEnv.environmentDetails(conf,
SparkHadoopUtil.get.newConfiguration(conf),
+ "", Seq.empty, Seq.empty, Seq.empty, Map.empty)
+ val jvmInformation = details("JVM Information").sorted
+ val sparkProperties = Utils.redact(conf, details("Spark
Properties")).sorted
+ val hadoopProperties = Utils.redact(conf, details("Hadoop
Properties")).sorted
+ val systemProperties = Utils.redact(conf, details("System
Properties")).sorted
+ val metricsProperties = Utils.redact(conf, details("Metrics
Properties")).sorted
+ val classpathEntries = details("Classpath Entries").sorted
+
+ val runtimeInformationTable = UIUtils.listingTable(propertyHeader,
propertyRow,
+ jvmInformation, fixedWidth = true, headerClasses = headerClasses)
+ val sparkPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
+ sparkProperties, fixedWidth = true, headerClasses = headerClasses)
+ val hadoopPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
+ hadoopProperties, fixedWidth = true, headerClasses = headerClasses)
+ val systemPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
+ systemProperties, fixedWidth = true, headerClasses = headerClasses)
+ val metricsPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
+ metricsProperties, fixedWidth = true, headerClasses = headerClasses)
+ val classpathEntriesTable = UIUtils.listingTable(classPathHeader,
classPathRow,
+ classpathEntries, fixedWidth = true, headerClasses = headerClasses)
+
+ val content =
+ <div>
+ <p><a href="/">Back to Master</a></p>
+ </div>
+ <span>
+ <span class="collapse-aggregated-runtimeInformation collapse-table"
+ onClick="collapseTable('collapse-aggregated-runtimeInformation',
+ 'aggregated-runtimeInformation')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a>Runtime Information</a>
+ </h4>
+ </span>
+ <div class="aggregated-runtimeInformation collapsible-table">
+ {runtimeInformationTable}
+ </div>
+ <span class="collapse-aggregated-sparkProperties collapse-table"
+ onClick="collapseTable('collapse-aggregated-sparkProperties',
+ 'aggregated-sparkProperties')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a>Spark Properties</a>
+ </h4>
+ </span>
+ <div class="aggregated-sparkProperties collapsible-table">
+ {sparkPropertiesTable}
+ </div>
+ <span class="collapse-aggregated-hadoopProperties collapse-table"
+ onClick="collapseTable('collapse-aggregated-hadoopProperties',
+ 'aggregated-hadoopProperties')">
+ <h4>
+ <span class="collapse-table-arrow arrow-closed"></span>
+ <a>Hadoop Properties</a>
+ </h4>
+ </span>
+ <div class="aggregated-hadoopProperties collapsible-table collapsed">
+ {hadoopPropertiesTable}
+ </div>
+ <span class="collapse-aggregated-systemProperties collapse-table"
+ onClick="collapseTable('collapse-aggregated-systemProperties',
+ 'aggregated-systemProperties')">
+ <h4>
+ <span class="collapse-table-arrow arrow-closed"></span>
+ <a>System Properties</a>
+ </h4>
+ </span>
+ <div class="aggregated-systemProperties collapsible-table collapsed">
+ {systemPropertiesTable}
+ </div>
+ <span class="collapse-aggregated-metricsProperties collapse-table"
+ onClick="collapseTable('collapse-aggregated-metricsProperties',
+ 'aggregated-metricsProperties')">
+ <h4>
+ <span class="collapse-table-arrow arrow-closed"></span>
+ <a>Metrics Properties</a>
+ </h4>
+ </span>
+ <div class="aggregated-metricsProperties collapsible-table collapsed">
+ {metricsPropertiesTable}
+ </div>
+ <span class="collapse-aggregated-classpathEntries collapse-table"
+ onClick="collapseTable('collapse-aggregated-classpathEntries',
+ 'aggregated-classpathEntries')">
+ <h4>
+ <span class="collapse-table-arrow arrow-closed"></span>
+ <a>Classpath Entries</a>
+ </h4>
+ </span>
+ <div class="aggregated-classpathEntries collapsible-table collapsed">
+ {classpathEntriesTable}
+ </div>
+ </span>
+ UIUtils.basicSparkPage(request, content, "Environment")
+ }
+
+ private def propertyHeader = Seq("Name", "Value")
+ private def classPathHeader = Seq("Resource", "Source")
+ private def headerClasses = Seq("sorttable_alpha", "sorttable_alpha")
+ private def headerClassesNoSortValues = Seq("sorttable_numeric",
"sorttable_nosort")
+
+ private def jvmRowDataPre(kv: (String, String)) =
+ <tr><td>{kv._1}</td><td><pre>{kv._2}</pre></td></tr>
+ private def propertyRow(kv: (String, String)) =
<tr><td>{kv._1}</td><td>{kv._2}</td></tr>
+ private def classPathRow(data: (String, String)) =
<tr><td>{data._1}</td><td>{data._2}</td></tr>
+}
+
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index d07f299d52ba..1248b1c368e7 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -169,8 +169,9 @@ private[ui] class MasterPage(parent: MasterWebUI) extends
WebUIPage("") {
{state.completedDrivers.count(_.state == DriverState.ERROR)}
Error,
{state.completedDrivers.count(_.state ==
DriverState.RELAUNCHING)} Relaunching)
</li>
- <li><strong>Status:</strong>
- <a href={"/logPage/?self&logType=out"}>{state.status}</a>
+ <li><strong>Status:</strong> {state.status}
+ (<a href={"/environment"}>Environment</a>,
+ <a href={"/logPage/?self&logType=out"}>Log</a>)
</li>
</ul>
</div>
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
index da3c91956689..d7e650a794dd 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
@@ -54,6 +54,11 @@ class MasterWebUI(
val masterPage = new MasterPage(this)
attachPage(new ApplicationPage(this))
attachPage(new LogPage(this))
+ val envPage = new EnvironmentPage(this, master.conf)
+ attachPage(envPage)
+ this.attachHandler(createServletHandler("/environment",
+ (request: HttpServletRequest) => envPage.render(request),
+ master.conf))
attachPage(masterPage)
addStaticHandler(MasterWebUI.STATIC_RESOURCE_DIR)
addRenderLogHandler(this, master.conf)
diff --git
a/core/src/test/scala/org/apache/spark/deploy/master/ui/ReadOnlyMasterWebUISuite.scala
b/core/src/test/scala/org/apache/spark/deploy/master/ui/ReadOnlyMasterWebUISuite.scala
index d7f05754a7cd..20ff932eb01a 100644
---
a/core/src/test/scala/org/apache/spark/deploy/master/ui/ReadOnlyMasterWebUISuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/master/ui/ReadOnlyMasterWebUISuite.scala
@@ -17,7 +17,9 @@
package org.apache.spark.deploy.master.ui
-import jakarta.servlet.http.HttpServletResponse.SC_METHOD_NOT_ALLOWED
+import scala.io.Source
+
+import jakarta.servlet.http.HttpServletResponse.{SC_METHOD_NOT_ALLOWED, SC_OK}
import org.mockito.Mockito.{mock, when}
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
@@ -74,4 +76,14 @@ class ReadOnlyMasterWebUISuite extends SparkFunSuite {
val body = convPostDataToString(hostnames.map(("host", _)))
assert(sendHttpRequest(url, "POST", body).getResponseCode ===
SC_METHOD_NOT_ALLOWED)
}
+
+ test("SPARK-47894: /environment") {
+ val url =
s"http://${Utils.localHostNameForURI()}:${masterWebUI.boundPort}/environment"
+ val conn = sendHttpRequest(url, "GET", "")
+ assert(conn.getResponseCode === SC_OK)
+ val result = Source.fromInputStream(conn.getInputStream).mkString
+ assert(result.contains("Runtime Information"))
+ assert(result.contains("Spark Properties"))
+ assert(result.contains("Hadoop Properties"))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
