Github user mccheah commented on a diff in the pull request:

    https://github.com/apache/spark/pull/21669#discussion_r221755757
  
    --- Diff: 
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/HadoopGlobalFeatureDriverStep.scala
 ---
    @@ -0,0 +1,151 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *    http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.spark.deploy.k8s.features
    +
    +import java.io.File
    +
    +import scala.collection.JavaConverters._
    +
    +import com.google.common.base.Charsets
    +import com.google.common.io.Files
    +import io.fabric8.kubernetes.api.model.{ConfigMapBuilder, HasMetadata}
    +
    +import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesUtils, 
SparkPod}
    +import org.apache.spark.deploy.k8s.Config._
    +import org.apache.spark.deploy.k8s.Constants._
    +import org.apache.spark.deploy.k8s.KubernetesDriverSpecificConf
    +import org.apache.spark.deploy.k8s.features.hadoopsteps._
    +import org.apache.spark.internal.Logging
    +
    + /**
    +  * Runs the necessary Hadoop-based logic based on Kerberos configs and 
the presence of the
    +  * HADOOP_CONF_DIR. This runs various bootstrap methods defined in 
HadoopBootstrapUtil.
    +  */
    +private[spark] class HadoopGlobalFeatureDriverStep(
    +    kubernetesConf: KubernetesConf[KubernetesDriverSpecificConf])
    +    extends KubernetesFeatureConfigStep with Logging {
    +
    +    private val conf = kubernetesConf.sparkConf
    +    private val maybePrincipal = 
conf.get(org.apache.spark.internal.config.PRINCIPAL)
    +    private val maybeKeytab = 
conf.get(org.apache.spark.internal.config.KEYTAB)
    +    private val maybeExistingSecretName = 
conf.get(KUBERNETES_KERBEROS_DT_SECRET_NAME)
    +    private val maybeExistingSecretItemKey =
    +      conf.get(KUBERNETES_KERBEROS_DT_SECRET_ITEM_KEY)
    +    private val kubeTokenManager = kubernetesConf.tokenManager
    +    private val isKerberosEnabled = kubeTokenManager.isSecurityEnabled
    +
    +    require(maybeKeytab.forall( _ => isKerberosEnabled ),
    +      "You must enable Kerberos support if you are specifying a Kerberos 
Keytab")
    +
    +    require(maybeExistingSecretName.forall( _ => isKerberosEnabled ),
    +      "You must enable Kerberos support if you are specifying a Kerberos 
Secret")
    +
    +    KubernetesUtils.requireBothOrNeitherDefined(
    +      maybeKeytab,
    +      maybePrincipal,
    +      "If a Kerberos principal is specified you must also specify a 
Kerberos keytab",
    +      "If a Kerberos keytab is specified you must also specify a Kerberos 
principal")
    +
    +    KubernetesUtils.requireBothOrNeitherDefined(
    +      maybeExistingSecretName,
    +      maybeExistingSecretItemKey,
    +      "If a secret data item-key where the data of the Kerberos Delegation 
Token is specified" +
    +        " you must also specify the name of the secret",
    +      "If a secret storing a Kerberos Delegation Token is specified you 
must also" +
    +        " specify the item-key where the data is stored")
    +
    +    require(kubernetesConf.hadoopConfDir.isDefined, "Ensure that 
HADOOP_CONF_DIR is defined")
    +    private val hadoopConfDir = kubernetesConf.hadoopConfDir.get
    +    private val hadoopConfigurationFiles = 
kubeTokenManager.getHadoopConfFiles(hadoopConfDir)
    +
    +    // Either use pre-existing secret or login to create new Secret with 
DT stored within
    +    private val hadoopSpec: Option[KerberosConfigSpec] = (for {
    +      secretName <- maybeExistingSecretName
    +      secretItemKey <- maybeExistingSecretItemKey
    +    } yield {
    +      KerberosConfigSpec(
    +         dtSecret = None,
    +         dtSecretName = secretName,
    +         dtSecretItemKey = secretItemKey,
    +         jobUserName = kubeTokenManager.getCurrentUser.getShortUserName)
    +    }).orElse(
    +      if (isKerberosEnabled) {
    +         Some(HadoopKerberosLogin.buildSpec(
    +             conf,
    +             kubernetesConf.appResourceNamePrefix,
    +             kubeTokenManager))
    +       } else None )
    +
    +    override def configurePod(pod: SparkPod): SparkPod = {
    +      val hadoopBasedSparkPod = HadoopBootstrapUtil.bootstrapHadoopConfDir(
    +        hadoopConfDir,
    +        kubernetesConf.hadoopConfigMapName,
    +        kubeTokenManager,
    +        pod)
    +      (for {
    +        hSpec <- hadoopSpec
    --- End diff --
    
    Can we avoid for... yield - again can use a tuple since both objects have 
to be present for any of this stuff to work.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to