Repository: spark Updated Branches: refs/heads/master 5f7dbdba6 -> a4a0addcc
[SPARK-13492][MESOS] Configurable Mesos framework webui URL. ## What changes were proposed in this pull request? Previously the Mesos framework webui URL was being derived only from the Spark UI address leaving no possibility to configure it. This commit makes it configurable. If unset it falls back to the previous behavior. Motivation: This change is necessary in order to be able to install Spark on DCOS and to be able to give it a custom service link. The configured `webui_url` is configured to point to a reverse proxy in the DCOS environment. ## How was this patch tested? Locally, using unit tests and on DCOS testing and stable revision. Author: Sergiusz Urbaniak <s...@mesosphere.io> Closes #11369 from s-urbaniak/sur-webui-url. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a4a0addc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a4a0addc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a4a0addc Branch: refs/heads/master Commit: a4a0addccffb7cd0ece7947d55ce2538afa54c97 Parents: 5f7dbdb Author: Sergiusz Urbaniak <s...@mesosphere.io> Authored: Wed Mar 9 18:10:01 2016 -0800 Committer: Andrew Or <and...@databricks.com> Committed: Wed Mar 9 18:10:01 2016 -0800 ---------------------------------------------------------------------- .../spark/ui/static/historypage-template.html | 2 +- .../org/apache/spark/ui/static/historypage.js | 8 ++--- .../deploy/mesos/MesosClusterDispatcher.scala | 2 +- .../mesos/MesosClusterDispatcherArguments.scala | 2 +- .../mesos/CoarseMesosSchedulerBackend.scala | 3 +- .../cluster/mesos/MesosSchedulerBackend.scala | 3 +- .../CoarseMesosSchedulerBackendSuite.scala | 30 ++++++++++++++++ .../mesos/MesosSchedulerBackendSuite.scala | 37 +++++++++++++++++++- docs/running-on-mesos.md | 16 +++++++++ 9 files changed, 93 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index e5ed5b3..5a7a252 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -67,7 +67,7 @@ <td class="rowGroupColumn"><span title="{{id}}"><a href="{{url}}">{{id}}</a></span></td> <td class="rowGroupColumn">{{name}}</td> {{#attempts}} - <td class="attemptIDSpan"><a href="/history/{{id}}/{{attemptId}}/">{{attemptId}}</a></td> + <td class="attemptIDSpan"><a href="history/{{id}}/{{attemptId}}/">{{attemptId}}</a></td> <td>{{startTime}}</td> <td>{{endTime}}</td> <td><span title="{{duration}}" class="durationClass">{{duration}}</span></td> http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/main/resources/org/apache/spark/ui/static/historypage.js ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 2976099..6096513 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -110,7 +110,7 @@ $(document).ready(function() { requestedIncomplete = getParameterByName("showIncomplete", searchString); requestedIncomplete = (requestedIncomplete == "true" ? true : false); - $.getJSON("/api/v1/applications", function(response,status,jqXHR) { + $.getJSON("api/v1/applications", function(response,status,jqXHR) { var array = []; var hasMultipleAttempts = false; for (i in response) { @@ -139,9 +139,9 @@ $(document).ready(function() { var url = null if (maxAttemptId == null) { - url = "/history/" + id + "/" + url = "history/" + id + "/" } else { - url = "/history/" + id + "/" + maxAttemptId + "/" + url = "history/" + id + "/" + maxAttemptId + "/" } var app_clone = {"id" : id, "name" : name, "url" : url, "attempts" : [attempt]}; @@ -150,7 +150,7 @@ $(document).ready(function() { } var data = {"applications": array} - $.get("/static/historypage-template.html", function(template) { + $.get("static/historypage-template.html", function(template) { historySummary.append(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; var conf = { http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala index 9b31497..7091513 100644 --- a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala +++ b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala @@ -73,7 +73,7 @@ private[mesos] class MesosClusterDispatcher( def start(): Unit = { webUi.bind() - scheduler.frameworkUrl = webUi.activeWebUiUrl + scheduler.frameworkUrl = conf.get("spark.mesos.dispatcher.webui.url", webUi.activeWebUiUrl) scheduler.start() server.start() } http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcherArguments.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcherArguments.scala b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcherArguments.scala index 38935e3..b97805a 100644 --- a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcherArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcherArguments.scala @@ -47,7 +47,7 @@ private[mesos] class MesosClusterDispatcherArguments(args: Array[String], conf: port = value parse(tail) - case ("--webui-port" | "-p") :: IntParam(value) :: tail => + case ("--webui-port") :: IntParam(value) :: tail => webUiPort = value parse(tail) http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala index 622f361..e118098 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala @@ -149,7 +149,8 @@ private[spark] class CoarseMesosSchedulerBackend( sc.sparkUser, sc.appName, sc.conf, - sc.ui.map(_.appUIAddress)) + sc.conf.getOption("spark.mesos.driver.webui.url").orElse(sc.ui.map(_.appUIAddress)) + ) startScheduler(driver) } http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index 8929d8a..1a94aee 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -78,7 +78,8 @@ private[spark] class MesosSchedulerBackend( sc.sparkUser, sc.appName, sc.conf, - sc.ui.map(_.appUIAddress)) + sc.conf.getOption("spark.mesos.driver.webui.url").orElse(sc.ui.map(_.appUIAddress)) + ) startScheduler(driver) } http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala index 2df0540..dd76644 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackendSuite.scala @@ -208,6 +208,35 @@ class CoarseMesosSchedulerBackendSuite extends SparkFunSuite verify(driver, times(1)).killTask(createTaskId("0")) } + test("weburi is set in created scheduler driver") { + setBackend() + val taskScheduler = mock[TaskSchedulerImpl] + when(taskScheduler.sc).thenReturn(sc) + val driver = mock[SchedulerDriver] + when(driver.start()).thenReturn(Protos.Status.DRIVER_RUNNING) + val securityManager = mock[SecurityManager] + + val backend = new CoarseMesosSchedulerBackend(taskScheduler, sc, "master", securityManager) { + override protected def createSchedulerDriver( + masterUrl: String, + scheduler: Scheduler, + sparkUser: String, + appName: String, + conf: SparkConf, + webuiUrl: Option[String] = None, + checkpoint: Option[Boolean] = None, + failoverTimeout: Option[Double] = None, + frameworkId: Option[String] = None): SchedulerDriver = { + markRegistered() + assert(webuiUrl.isDefined) + assert(webuiUrl.get.equals("http://webui";)) + driver + } + } + + backend.start() + } + private def verifyDeclinedOffer(driver: SchedulerDriver, offerId: OfferID, filter: Boolean = false): Unit = { @@ -316,6 +345,7 @@ class CoarseMesosSchedulerBackendSuite extends SparkFunSuite .setMaster("local[*]") .setAppName("test-mesos-dynamic-alloc") .setSparkHome("/path") + .set("spark.mesos.driver.webui.url", "http://webui";) if (sparkConfVars != null) { for (attr <- sparkConfVars) { http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala index 3fb3279..7d6b7bd 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala @@ -26,9 +26,9 @@ import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer +import org.apache.mesos.{Protos, Scheduler, SchedulerDriver} import org.apache.mesos.Protos._ import org.apache.mesos.Protos.Value.Scalar -import org.apache.mesos.SchedulerDriver import org.mockito.{ArgumentCaptor, Matchers} import org.mockito.Matchers._ import org.mockito.Mockito._ @@ -42,6 +42,41 @@ import org.apache.spark.scheduler.cluster.ExecutorInfo class MesosSchedulerBackendSuite extends SparkFunSuite with LocalSparkContext with MockitoSugar { + test("weburi is set in created scheduler driver") { + val conf = new SparkConf + conf.set("spark.mesos.driver.webui.url", "http://webui";) + conf.set("spark.app.name", "name1") + + val sc = mock[SparkContext] + when(sc.conf).thenReturn(conf) + when(sc.sparkUser).thenReturn("sparkUser1") + when(sc.appName).thenReturn("appName1") + + val taskScheduler = mock[TaskSchedulerImpl] + val driver = mock[SchedulerDriver] + when(driver.start()).thenReturn(Protos.Status.DRIVER_RUNNING) + + val backend = new MesosSchedulerBackend(taskScheduler, sc, "master") { + override protected def createSchedulerDriver( + masterUrl: String, + scheduler: Scheduler, + sparkUser: String, + appName: String, + conf: SparkConf, + webuiUrl: Option[String] = None, + checkpoint: Option[Boolean] = None, + failoverTimeout: Option[Double] = None, + frameworkId: Option[String] = None): SchedulerDriver = { + markRegistered() + assert(webuiUrl.isDefined) + assert(webuiUrl.get.equals("http://webui";)) + driver + } + } + + backend.start() + } + test("Use configured mesosExecutor.cores for ExecutorInfo") { val mesosExecutorCores = 3 val conf = new SparkConf http://git-wip-us.apache.org/repos/asf/spark/blob/a4a0addc/docs/running-on-mesos.md ---------------------------------------------------------------------- diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 912a010..3a832de 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -390,6 +390,22 @@ See the [configuration page](configuration.html) for information on Spark config </ul> </td> </tr> +<tr> + <td><code>spark.mesos.driver.webui.url</code></td> + <td><code>(none)</code></td> + <td> + Set the Spark Mesos driver webui_url for interacting with the framework. + If unset it will point to Spark's internal web UI. + </td> +</tr> +<tr> + <td><code>spark.mesos.dispatcher.webui.url</code></td> + <td><code>(none)</code></td> + <td> + Set the Spark Mesos dispatcher webui_url for interacting with the framework. + If unset it will point to Spark's internal web UI. + </td> +</tr> </table> # Troubleshooting and Debugging --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
