This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e9339e60dd41 [SPARK-45774][CORE][UI] Support
`spark.master.ui.historyServerUrl` in `ApplicationPage`
e9339e60dd41 is described below
commit e9339e60dd418559619e4beae3de0a58ca3b2a31
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Fri Nov 3 23:17:06 2023 +0800
[SPARK-45774][CORE][UI] Support `spark.master.ui.historyServerUrl` in
`ApplicationPage`
### What changes were proposed in this pull request?
This PR aims to support a new configuration
`spark.master.ui.historyServerUrl` to show a new link, **Application History
UI**, from `Application Page` to `Spark History Server` in `Spark Standalone
Cluster`s.
This is useful when `spark.eventLog.enabled=true` and `Spark History
Server` exists. A user can launch `Spark Master` with
`spark.master.ui.historyServerUrl` configuration to link to it.
Please note that `Spark History Server` is an orthogonal service from not
only `Spark Master` but also `Spark Applications`. For example, `Spark
Application`s know only `spark.eventLog.dir`, not SHS info. Moreover, you can
launch multiple SHS based on the same event location with load balancer.
Lastly, SHS can be behind proxy layer too. So, only the users know the exposed
SHS URL.
### Why are the changes needed?
**Application Detail UI**
Apache Spark `Master` currently shows `Application Detail UI` link for only
live Spark jobs.
<img width="436" alt="Screenshot 2023-11-02 at 8 30 20 PM"
src="https://github.com/apache/spark/assets/9700541/28e6398c-3bb5-4f8c-9713-3a95c451759e";>
**Application History UI**
This PR adds `Application History UI` link for completed jobs as the best
effort when `spark.ui.historyServerUrl` is given.
<img width="513" alt="Screenshot 2023-11-02 at 8 38 37 PM"
src="https://github.com/apache/spark/assets/9700541/ca8b5436-f726-4b79-97c3-416cc59ea01b";>
### Does this PR introduce _any_ user-facing change?
No. This is disabled by default.
### How was this patch tested?
Pass the CIs with the newly added test suite.
Also, manual procedure
```
$ SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=$HOME/data/history"
sbin/start-history-server.sh
$
SPARK_MASTER_OPTS=-Dspark.master.ui.historyServerUrl=http://localhost:18080
sbin/start-master.sh
$ sbin/start-worker.sh spark://localhost:7077
$ bin/spark-shell --master spark://127.0.0.1:7077 -c
spark.eventLog.enabled=true -c spark.eventLog.dir=/Users/dongjoon/data/history
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43643 from dongjoon-hyun/SPARK-45774.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Kent Yao <y...@apache.org>
---
.../org/apache/spark/deploy/master/Master.scala | 1 +
.../spark/deploy/master/ui/ApplicationPage.scala | 5 ++
.../org/apache/spark/internal/config/package.scala | 8 +++
.../deploy/master/ui/ApplicationPageSuite.scala | 73 ++++++++++++++++++++++
4 files changed, 87 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 058b944c591a..d5de1366ac05 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -121,6 +121,7 @@ private[deploy] class Master(
if (defaultCores < 1) {
throw new SparkException(s"${DEFAULT_CORES.key} must be positive")
}
+ val historyServerUrl = conf.get(MASTER_UI_HISTORY_SERVER_URL)
// Alternative application submission gateway that is stable across Spark
versions
private val restServerEnabled = conf.get(MASTER_REST_SERVER_ENABLED)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
index 202926233d97..3087d5e8c966 100644
---
a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala
@@ -98,6 +98,11 @@ private[ui] class ApplicationPage(parent: MasterWebUI)
extends WebUIPage("app")
<a href={UIUtils.makeHref(parent.master.reverseProxy,
app.id, app.desc.appUiUrl)}>Application Detail UI</a>
</strong></li>
+ } else if (parent.master.historyServerUrl.nonEmpty) {
+ <li><strong>
+ <a
href={s"${parent.master.historyServerUrl.get}/history/${app.id}"}>
+ Application History UI</a>
+ </strong></li>
}
}
</ul>
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 143dd0c44ce8..93a42eec8326 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1837,6 +1837,14 @@ package object config {
.intConf
.createWithDefault(8080)
+ private[spark] val MASTER_UI_HISTORY_SERVER_URL =
+ ConfigBuilder("spark.master.ui.historyServerUrl")
+ .doc("The URL where Spark history server is running. Please note that
this assumes " +
+ "that all Spark jobs share the same event log location where the
history server accesses.")
+ .version("4.0.0")
+ .stringConf
+ .createOptional
+
private[spark] val IO_COMPRESSION_SNAPPY_BLOCKSIZE =
ConfigBuilder("spark.io.compression.snappy.blockSize")
.doc("Block size in bytes used in Snappy compression, in the case when "
+
diff --git
a/core/src/test/scala/org/apache/spark/deploy/master/ui/ApplicationPageSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/master/ui/ApplicationPageSuite.scala
new file mode 100644
index 000000000000..9890ac24e168
--- /dev/null
+++
b/core/src/test/scala/org/apache/spark/deploy/master/ui/ApplicationPageSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.master.ui
+
+import java.util.Date
+import javax.servlet.http.HttpServletRequest
+
+import org.mockito.Mockito.{mock, when}
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.deploy.ApplicationDescription
+import org.apache.spark.deploy.DeployMessages.{MasterStateResponse,
RequestMasterState}
+import org.apache.spark.deploy.master.{ApplicationInfo, ApplicationState,
Master}
+import org.apache.spark.resource.ResourceProfile
+import org.apache.spark.rpc.RpcEndpointRef
+
+class ApplicationPageSuite extends SparkFunSuite {
+
+ private val master = mock(classOf[Master])
+
when(master.historyServerUrl).thenReturn(Some("http://my-history-server:18080";))
+
+ private val rp = new ResourceProfile(Map.empty, Map.empty)
+ private val desc = ApplicationDescription("name", Some(4), null, "appUiUrl",
rp)
+ private val appFinished = new ApplicationInfo(0, "app-finished", desc, new
Date, null, 1)
+ appFinished.markFinished(ApplicationState.FINISHED)
+ private val appLive = new ApplicationInfo(0, "app-live", desc, new Date,
null, 1)
+
+ private val state = mock(classOf[MasterStateResponse])
+ when(state.completedApps).thenReturn(Array(appFinished))
+ when(state.activeApps).thenReturn(Array(appLive))
+
+ private val rpc = mock(classOf[RpcEndpointRef])
+ when(rpc.askSync[MasterStateResponse](RequestMasterState)).thenReturn(state)
+
+ private val masterWebUI = mock(classOf[MasterWebUI])
+ when(masterWebUI.master).thenReturn(master)
+ when(masterWebUI.masterEndpointRef).thenReturn(rpc)
+
+ test("SPARK-45774: Application Detail UI") {
+ val request = mock(classOf[HttpServletRequest])
+ when(request.getParameter("appId")).thenReturn("app-live")
+
+ val result = new ApplicationPage(masterWebUI).render(request).toString()
+ assert(result.contains("Application Detail UI"))
+ assert(!result.contains("Application History UI"))
+ assert(!result.contains(master.historyServerUrl.get))
+ }
+
+ test("SPARK-45774: Application History UI") {
+ val request = mock(classOf[HttpServletRequest])
+ when(request.getParameter("appId")).thenReturn("app-finished")
+
+ val result = new ApplicationPage(masterWebUI).render(request).toString()
+ assert(!result.contains("Application Detail UI"))
+ assert(result.contains("Application History UI"))
+ assert(result.contains(master.historyServerUrl.get))
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
