This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 29d9021 [SPARK-24621][WEBUI] Show secure URLs on web pages 29d9021 is described below commit 29d902124581b2163e49baa6d379025a6cf1fa1a Author: Gabor Somogyi <gabor.g.somo...@gmail.com> AuthorDate: Sun Mar 10 19:28:35 2019 -0500 [SPARK-24621][WEBUI] Show secure URLs on web pages ## What changes were proposed in this pull request? Web UI URLs are pointing to `http://` targets even if SSL is enabled. In this PR I've changed the code to point to `https://` URLs. ## How was this patch tested? Existing unit tests + manually by starting standalone master/worker/spark-shell. Please see jira. Closes #23991 from gaborgsomogyi/SPARK-24621. Authored-by: Gabor Somogyi <gabor.g.somo...@gmail.com> Signed-off-by: Sean Owen <sean.o...@databricks.com> --- .../src/main/scala/org/apache/spark/deploy/master/Master.scala | 2 +- .../scala/org/apache/spark/deploy/worker/ExecutorRunner.scala | 3 ++- .../src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 3 ++- core/src/main/scala/org/apache/spark/ui/WebUI.scala | 10 ++++++++-- .../test/scala/org/apache/spark/deploy/DeployTestUtils.scala | 1 + .../org/apache/spark/deploy/worker/ExecutorRunnerTest.scala | 4 ++-- 6 files changed, 16 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index 356e52c..50e282f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -143,7 +143,7 @@ private[deploy] class Master( logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}") webUi = new MasterWebUI(this, webUiPort) webUi.bind() - masterWebUiUrl = "http://" + masterPublicAddress + ":" + webUi.boundPort + masterWebUiUrl = s"${webUi.scheme}$masterPublicAddress:${webUi.boundPort}" if (reverseProxy) { masterWebUiUrl = conf.get(UI_REVERSE_PROXY_URL).orElse(Some(masterWebUiUrl)).get webUi.addProxy() diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala index ead28f1..9d2301c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala @@ -45,6 +45,7 @@ private[deploy] class ExecutorRunner( val memory: Int, val worker: RpcEndpointRef, val workerId: String, + val webUiScheme: String, val host: String, val webUiPort: Int, val publicAddress: String, @@ -165,7 +166,7 @@ private[deploy] class ExecutorRunner( if (conf.get(UI_REVERSE_PROXY)) { s"/proxy/$workerId/logPage/?appId=$appId&executorId=$execId&logType=" } else { - s"http://$publicAddress:$webUiPort/logPage/?appId=$appId&executorId=$execId&logType=" + s"$webUiScheme$publicAddress:$webUiPort/logPage/?appId=$appId&executorId=$execId&logType=" } builder.environment.put("SPARK_LOG_URL_STDERR", s"${baseUrl}stderr") builder.environment.put("SPARK_LOG_URL_STDOUT", s"${baseUrl}stdout") diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index 2fc2be5..4bd7aaa 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -211,7 +211,7 @@ private[deploy] class Worker( webUi = new WorkerWebUI(this, workDir, webUiPort) webUi.bind() - workerWebUiUrl = s"http://$publicAddress:${webUi.boundPort}" + workerWebUiUrl = s"${webUi.scheme}$publicAddress:${webUi.boundPort}" registerWithMaster() metricsSystem.registerSource(workerSource) @@ -528,6 +528,7 @@ private[deploy] class Worker( memory_, self, workerId, + webUi.scheme, host, webUi.boundPort, publicAddress, diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala index ebf8655..8165942 100644 --- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala @@ -139,11 +139,17 @@ private[spark] abstract class WebUI( } } + /** @return Whether SSL enabled. Only valid after [[bind]]. */ + def isSecure: Boolean = serverInfo.map(_.securePort.isDefined).getOrElse(false) + + /** @return The scheme of web interface. Only valid after [[bind]]. */ + def scheme: String = if (isSecure) "https://" else "http://" + /** @return The url of web interface. Only valid after [[bind]]. */ - def webUrl: String = s"http://$publicHostName:$boundPort" + def webUrl: String = s"${scheme}$publicHostName:${boundPort}" /** @return The actual port to which this server is bound. Only valid after [[bind]]. */ - def boundPort: Int = serverInfo.map(_.boundPort).getOrElse(-1) + def boundPort: Int = serverInfo.map(si => si.securePort.getOrElse(si.boundPort)).getOrElse(-1) /** Stops the server behind this web interface. Only valid after [[bind]]. */ def stop(): Unit = { diff --git a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala index 55a541d..784981e 100644 --- a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala +++ b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala @@ -64,6 +64,7 @@ private[deploy] object DeployTestUtils { 1234, null, "workerId", + "http://", "host", 123, "publicAddress", diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala index 0240bf8..988c65f 100644 --- a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala +++ b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala @@ -29,8 +29,8 @@ class ExecutorRunnerTest extends SparkFunSuite { val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")) val appDesc = new ApplicationDescription("app name", Some(8), 500, Command("foo", Seq(appId), Map(), Seq(), Seq(), Seq()), "appUiUrl") - val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "worker321", 123, - "publicAddr", new File(sparkHome), new File("ooga"), "blah", conf, Seq("localDir"), + val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "http://", "worker321", + 123, "publicAddr", new File(sparkHome), new File("ooga"), "blah", conf, Seq("localDir"), ExecutorState.RUNNING) val builder = CommandUtils.buildProcessBuilder( appDesc.command, new SecurityManager(conf), 512, sparkHome, er.substituteVariables) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org