Github user ArtRand commented on a diff in the pull request:
https://github.com/apache/spark/pull/20167#discussion_r164825718
--- Diff:
resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
---
@@ -71,40 +74,64 @@ trait MesosSchedulerUtils extends Logging {
failoverTimeout: Option[Double] = None,
frameworkId: Option[String] = None): SchedulerDriver = {
val fwInfoBuilder =
FrameworkInfo.newBuilder().setUser(sparkUser).setName(appName)
- val credBuilder = Credential.newBuilder()
+
fwInfoBuilder.setHostname(Option(conf.getenv("SPARK_PUBLIC_DNS")).getOrElse(
+ conf.get(DRIVER_HOST_ADDRESS)))
webuiUrl.foreach { url => fwInfoBuilder.setWebuiUrl(url) }
checkpoint.foreach { checkpoint =>
fwInfoBuilder.setCheckpoint(checkpoint) }
failoverTimeout.foreach { timeout =>
fwInfoBuilder.setFailoverTimeout(timeout) }
frameworkId.foreach { id =>
fwInfoBuilder.setId(FrameworkID.newBuilder().setValue(id).build())
}
-
fwInfoBuilder.setHostname(Option(conf.getenv("SPARK_PUBLIC_DNS")).getOrElse(
- conf.get(DRIVER_HOST_ADDRESS)))
- conf.getOption("spark.mesos.principal").foreach { principal =>
- fwInfoBuilder.setPrincipal(principal)
- credBuilder.setPrincipal(principal)
- }
- conf.getOption("spark.mesos.secret").foreach { secret =>
- credBuilder.setSecret(secret)
- }
- if (credBuilder.hasSecret && !fwInfoBuilder.hasPrincipal) {
- throw new SparkException(
- "spark.mesos.principal must be configured when spark.mesos.secret
is set")
- }
+
conf.getOption("spark.mesos.role").foreach { role =>
fwInfoBuilder.setRole(role)
}
val maxGpus = conf.getInt("spark.mesos.gpus.max", 0)
if (maxGpus > 0) {
fwInfoBuilder.addCapabilities(Capability.newBuilder().setType(Capability.Type.GPU_RESOURCES))
}
+ val credBuilder = buildCredentials(conf, fwInfoBuilder)
if (credBuilder.hasPrincipal) {
new MesosSchedulerDriver(
scheduler, fwInfoBuilder.build(), masterUrl, credBuilder.build())
} else {
new MesosSchedulerDriver(scheduler, fwInfoBuilder.build(), masterUrl)
}
}
+
+ def buildCredentials(
+ conf: SparkConf,
+ fwInfoBuilder: Protos.FrameworkInfo.Builder):
Protos.Credential.Builder = {
+ val credBuilder = Credential.newBuilder()
+ conf.getOption("spark.mesos.principal")
+ .orElse(Option(conf.getenv("SPARK_MESOS_PRINCIPAL")))
--- End diff --
Sorry for the delay. I have a use case where I start the Dispatcher in the
Mesos cluster and then execute `spark-submit` cluster calls from within the
container. Unfortunately this requires me to unset a few environment variables
(`MESOS_EXECUTOR_ID MESOS_FRAMEWORK_ID MESOS_SLAVE_ID MESOS_SLAVE_PID
MESOS_TASK_ID`) because they interfere with `spark-submit` due to this function
in the rest client.
If the Dispatcher is started in a mode where it needs these Mesos
authentication credentials, can we assume that we'll want to always forward
them this same way? I realize I might be getting into the weeds here and this
might me a _me_ problem. But I thought I'd bring it up.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
