This is an automated email from the ASF dual-hosted git repository. vanzin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new d4542a8 [SPARK-27061][K8S] Expose Driver UI port on driver service to access … d4542a8 is described below commit d4542a8ba80c833f8ca9a1d2a7fd25e5a3792c74 Author: chandulal.kavar <ccka...@gmail.com> AuthorDate: Mon Mar 11 10:41:31 2019 -0700 [SPARK-27061][K8S] Expose Driver UI port on driver service to access … ## What changes were proposed in this pull request? Expose Spark UI port on driver service to access logs from service. ## How was this patch tested? The patch was tested using unit tests being contributed as a part of the PR Closes #23990 from chandulal/SPARK-27061. Authored-by: chandulal.kavar <ccka...@gmail.com> Signed-off-by: Marcelo Vanzin <van...@cloudera.com> --- .../spark/deploy/k8s/features/DriverServiceFeatureStep.scala | 6 ++++++ .../deploy/k8s/features/DriverServiceFeatureStepSuite.scala | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala index cec8769..925bcdf 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala @@ -54,6 +54,7 @@ private[spark] class DriverServiceFeatureStep( config.DRIVER_PORT.key, DEFAULT_DRIVER_PORT) private val driverBlockManagerPort = kubernetesConf.sparkConf.getInt( config.DRIVER_BLOCK_MANAGER_PORT.key, DEFAULT_BLOCKMANAGER_PORT) + private val driverUIPort = kubernetesConf.get(config.UI.UI_PORT) override def configurePod(pod: SparkPod): SparkPod = pod @@ -82,6 +83,11 @@ private[spark] class DriverServiceFeatureStep( .withPort(driverBlockManagerPort) .withNewTargetPort(driverBlockManagerPort) .endPort() + .addNewPort() + .withName(UI_PORT_NAME) + .withPort(driverUIPort) + .withNewTargetPort(driverUIPort) + .endPort() .endSpec() .build() Seq(driverService) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala index fbd99b7..9068289 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.deploy.k8s.submit.JavaMainAppResource import org.apache.spark.internal.config._ +import org.apache.spark.internal.config.UI._ import org.apache.spark.util.ManualClock class DriverServiceFeatureStepSuite extends SparkFunSuite { @@ -38,10 +39,11 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite { "label1key" -> "label1value", "label2key" -> "label2value") - test("Headless service has a port for the driver RPC and the block manager.") { + test("Headless service has a port for the driver RPC, the block manager and driver ui.") { val sparkConf = new SparkConf(false) .set(DRIVER_PORT, 9000) .set(DRIVER_BLOCK_MANAGER_PORT, 8080) + .set(UI_PORT, 4080) val kconf = KubernetesTestConf.createDriverConf( sparkConf = sparkConf, labels = DRIVER_LABELS) @@ -56,6 +58,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite { verifyService( 9000, 8080, + 4080, s"${kconf.resourceNamePrefix}${DriverServiceFeatureStep.DRIVER_SVC_POSTFIX}", driverService) } @@ -85,6 +88,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite { verifyService( DEFAULT_DRIVER_PORT, DEFAULT_BLOCKMANAGER_PORT, + UI_PORT.defaultValue.get, s"${kconf.resourceNamePrefix}${DriverServiceFeatureStep.DRIVER_SVC_POSTFIX}", resolvedService) val additionalProps = configurationStep.getAdditionalPodSystemProperties() @@ -152,6 +156,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite { private def verifyService( driverPort: Int, blockManagerPort: Int, + drierUIPort: Int, expectedServiceName: String, service: Service): Unit = { assert(service.getMetadata.getName === expectedServiceName) @@ -159,7 +164,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite { DRIVER_LABELS.foreach { case (k, v) => assert(service.getSpec.getSelector.get(k) === v) } - assert(service.getSpec.getPorts.size() === 2) + assert(service.getSpec.getPorts.size() === 3) val driverServicePorts = service.getSpec.getPorts.asScala assert(driverServicePorts.head.getName === DRIVER_PORT_NAME) assert(driverServicePorts.head.getPort.intValue() === driverPort) @@ -167,5 +172,8 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite { assert(driverServicePorts(1).getName === BLOCK_MANAGER_PORT_NAME) assert(driverServicePorts(1).getPort.intValue() === blockManagerPort) assert(driverServicePorts(1).getTargetPort.getIntVal === blockManagerPort) + assert(driverServicePorts(2).getName === UI_PORT_NAME) + assert(driverServicePorts(2).getPort.intValue() === drierUIPort) + assert(driverServicePorts(2).getTargetPort.getIntVal === drierUIPort) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org