Github user vanzin commented on a diff in the pull request: https://github.com/apache/spark/pull/22760#discussion_r230223525 --- Diff: resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/hadooputils/HadoopBootstrapUtilSuite.scala --- @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.features.hadooputils + +import java.io.File +import java.util.UUID + +import scala.collection.JavaConverters._ + +import com.google.common.io.Files +import io.fabric8.kubernetes.api.model._ +import org.scalatest.BeforeAndAfter + +import org.apache.spark.SparkFunSuite +import org.apache.spark.deploy.k8s.Constants._ +import org.apache.spark.deploy.k8s.SparkPod +import org.apache.spark.deploy.k8s.features.KubernetesFeaturesTestUtils._ +import org.apache.spark.util.Utils + +class HadoopBootstrapUtilSuite extends SparkFunSuite with BeforeAndAfter{ + private val sparkPod = SparkPod.initialPod() + private val hadoopBootstrapUtil = new HadoopBootstrapUtil + private var tmpDir: File = _ + private var tmpFile: File = _ + + before { + tmpDir = Utils.createTempDir() + tmpFile = createTempFile(tmpDir, "contents") + } + + after { + tmpFile.delete() + tmpDir.delete() + } + + test("Testing bootstrapKerberosPod with file location of krb5") { + val dtSecretName = "EXAMPLE_SECRET_NAME" + val dtSecretItemKey = "EXAMPLE_ITEM_KEY" + val userName = "SPARK_USER_NAME" + val fileLocation = Some(tmpFile.getAbsolutePath) + val stringPath = tmpFile.toPath.getFileName.toString + val newKrb5ConfName = Some("/etc/krb5.conf") + val resultingPod = hadoopBootstrapUtil.bootstrapKerberosPod( + dtSecretName, + dtSecretItemKey, + userName, + fileLocation, + newKrb5ConfName, + None, + sparkPod) + val expectedVolumes = Seq( + new VolumeBuilder() + .withName(KRB_FILE_VOLUME) + .withNewConfigMap() + .withName(newKrb5ConfName.get) + .withItems(new KeyToPathBuilder() + .withKey(stringPath) + .withPath(stringPath) + .build()) + .endConfigMap() + .build(), + new VolumeBuilder() + .withName(SPARK_APP_HADOOP_SECRET_VOLUME_NAME) + .withNewSecret() + .withSecretName(dtSecretName) + .endSecret() + .build() + ) + assert(podHasVolumes(resultingPod.pod, expectedVolumes)) + assert(containerHasEnvVars(resultingPod.container, Map( + ENV_HADOOP_TOKEN_FILE_LOCATION -> s"$SPARK_APP_HADOOP_CREDENTIALS_BASE_DIR/$dtSecretItemKey", + ENV_SPARK_USER -> userName) + )) + assert(containerHasVolumeMounts(resultingPod.container, Map( + SPARK_APP_HADOOP_SECRET_VOLUME_NAME -> SPARK_APP_HADOOP_CREDENTIALS_BASE_DIR, + KRB_FILE_VOLUME -> (KRB_FILE_DIR_PATH + "/krb5.conf")) + )) + } + + test("Testing bootstrapKerberosPod with configMap of krb5") { + val dtSecretName = "EXAMPLE_SECRET_NAME" + val dtSecretItemKey = "EXAMPLE_ITEM_KEY" + val userName = "SPARK_USER_NAME" + val existingKrb5ConfName = Some("krb5CMap") + val resultingPod = hadoopBootstrapUtil.bootstrapKerberosPod( + dtSecretName, + dtSecretItemKey, + userName, + None, + None, + existingKrb5ConfName, + sparkPod) + val expectedVolumes = Seq( + new VolumeBuilder() + .withName(KRB_FILE_VOLUME) + .withNewConfigMap() + .withName(existingKrb5ConfName.get) + .endConfigMap() + .build(), + new VolumeBuilder() + .withName(SPARK_APP_HADOOP_SECRET_VOLUME_NAME) + .withNewSecret() + .withSecretName(dtSecretName) + .endSecret() + .build() + ) + assert(podHasVolumes(resultingPod.pod, expectedVolumes)) + assert(containerHasEnvVars(resultingPod.container, Map( + ENV_HADOOP_TOKEN_FILE_LOCATION -> s"$SPARK_APP_HADOOP_CREDENTIALS_BASE_DIR/$dtSecretItemKey", + ENV_SPARK_USER -> userName) + )) + assert(containerHasVolumeMounts(resultingPod.container, Map( + SPARK_APP_HADOOP_SECRET_VOLUME_NAME -> SPARK_APP_HADOOP_CREDENTIALS_BASE_DIR) + )) + } + + test("Testing bootstrapSparkUserPod") { + val userName = "SPARK_USER_NAME" + val resultingPod = hadoopBootstrapUtil.bootstrapSparkUserPod(userName, sparkPod) + assert(containerHasEnvVars(resultingPod.container, Map(ENV_SPARK_USER -> userName))) + } + + test("Testing bootstrapHadoopConfDir with fileLocation HADOOP_CONF") { + val hadoopConfDir = Some(tmpDir.getAbsolutePath) + val stringPath = tmpFile.toPath.getFileName.toString + val newHadoopConfigMapName = Some("hconfMapName") + val resultingPod = hadoopBootstrapUtil.bootstrapHadoopConfDir( + hadoopConfDir, + newHadoopConfigMapName, + None, + sparkPod + ) + assert(containerHasVolumeMounts(resultingPod.container, Map( + HADOOP_FILE_VOLUME -> HADOOP_CONF_DIR_PATH) + )) + val expectedVolumes = Seq( + new VolumeBuilder() + .withName(HADOOP_FILE_VOLUME) + .withNewConfigMap() + .withName(newHadoopConfigMapName.get) + .withItems(new KeyToPathBuilder() + .withKey(stringPath) + .withPath(stringPath) + .build()) + .endConfigMap() + .build() + ) + + assert(podHasVolumes(resultingPod.pod, expectedVolumes)) + assert(containerHasVolumeMounts(resultingPod.container, Map( + HADOOP_FILE_VOLUME -> HADOOP_CONF_DIR_PATH) + )) + assert(containerHasEnvVars(resultingPod.container, Map( + ENV_HADOOP_CONF_DIR -> HADOOP_CONF_DIR_PATH) + )) + } + + test("Testing bootstrapHadoopConfDir with configMap HADOOP_CONF") { + val existingHadoopConfigMapName = Some("hconfMapName") + val resultingPod = hadoopBootstrapUtil.bootstrapHadoopConfDir( + None, + None, + existingHadoopConfigMapName, + sparkPod + ) + assert(containerHasVolumeMounts(resultingPod.container, Map( + HADOOP_FILE_VOLUME -> HADOOP_CONF_DIR_PATH) + )) + val expectedVolumes = Seq( + new VolumeBuilder() + .withName(HADOOP_FILE_VOLUME) + .withNewConfigMap() + .withName(existingHadoopConfigMapName.get) + .endConfigMap() + .build()) + assert(podHasVolumes(resultingPod.pod, expectedVolumes)) + } + + test("Testing buildKrb5ConfigMap") { + val configMapName = "hconfMapName" + val resultingCMap = hadoopBootstrapUtil.buildkrb5ConfigMap( + configMapName, + tmpFile.getAbsolutePath + ) + assert(resultingCMap === new ConfigMapBuilder() + .withNewMetadata() + .withName(configMapName) + .endMetadata() + .addToData(Map(tmpFile.toPath.getFileName.toString -> "contents").asJava) + .build()) + } + + test("Testing buildHadoopConfigMap") { + val configMapName = "hconfMapName" + val resultingCMap = hadoopBootstrapUtil.buildHadoopConfigMap( + configMapName, + Seq(tmpFile) + ) + assert(resultingCMap === new ConfigMapBuilder() + .withNewMetadata() + .withName(configMapName) + .endMetadata() + .addToData(Map(tmpFile.toPath.getFileName.toString -> "contents").asJava) --- End diff -- Same.
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org