[spark] branch branch-2.4 updated: [SPARK-27872][K8S][2.4] Fix executor service account inconsistency
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 5a50e30 [SPARK-27872][K8S][2.4] Fix executor service account
inconsistency
5a50e30 is described below
commit 5a50e30881d11971efe91b95ca6a20bb33868d2a
Author: nssalian
AuthorDate: Sun Oct 11 20:37:41 2020 -0700
[SPARK-27872][K8S][2.4] Fix executor service account inconsistency
### What changes were proposed in this pull request?
Similar patch to #24748 but applied to the branch-2.4.
Backporting the fix to releases 2.4.x.
Closes #29877 from nssalian/SPARK-27872.
Authored-by: nssalian
Signed-off-by: Dongjoon Hyun
---
.../scala/org/apache/spark/deploy/k8s/Config.scala | 7 ++--
.../apache/spark/deploy/k8s/KubernetesUtils.scala | 13 +++
.../DriverKubernetesCredentialsFeatureStep.scala | 11 +-
.../ExecutorKubernetesCredentialsFeatureStep.scala | 43 ++
.../cluster/k8s/KubernetesExecutorBuilder.scala| 9 -
.../k8s/KubernetesExecutorBuilderSuite.scala | 9 -
.../k8s/integrationtest/BasicTestsSuite.scala | 7
.../k8s/integrationtest/KubernetesSuite.scala | 4 ++
8 files changed, 87 insertions(+), 16 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index c7338a7..02471af 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -61,10 +61,9 @@ private[spark] object Config extends Logging {
.stringConf
.createOptional
- val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver"
- val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver.mounted"
+ val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
"spark.kubernetes.authenticate.driver"
+ val KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX =
"spark.kubernetes.authenticate.executor"
+ val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
"spark.kubernetes.authenticate.driver.mounted"
val KUBERNETES_AUTH_CLIENT_MODE_PREFIX = "spark.kubernetes.authenticate"
val OAUTH_TOKEN_CONF_SUFFIX = "oauthToken"
val OAUTH_TOKEN_FILE_CONF_SUFFIX = "oauthTokenFile"
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index 588cd9d..9e32d6e 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.deploy.k8s
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder,
ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting,
ContainerStatus, Pod, PodBuilder}
+
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils
@@ -60,4 +62,15 @@ private[spark] object KubernetesUtils {
}
def parseMasterUrl(url: String): String = url.substring("k8s://".length)
+
+ def buildPodWithServiceAccount(serviceAccount: Option[String], pod:
SparkPod): Option[Pod] = {
+serviceAccount.map { account =>
+ new PodBuilder(pod.pod)
+.editOrNewSpec()
+ .withServiceAccount(account)
+ .withServiceAccountName(account)
+.endSpec()
+.build()
+}
+ }
}
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
index ff5ad66..6079381 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
@@ -27,6 +27,7 @@ import io.fabric8.kubernetes.api.model.{ContainerBuilder,
HasMetadata, PodBuilde
import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.KubernetesUtils.buildPodWithServiceAccount
private[spark] class DriverKubernetesCredentialsFeatureStep(kubernetesConf:
KubernetesConf[_])
extends KubernetesFeatureConfigStep {
@@ -70,15 +71,7 @@ private[spark] class
DriverKubernetesCredentialsFeatureStep(kubernetesConf: Kube
[spark] branch branch-2.4 updated: [SPARK-27872][K8S][2.4] Fix executor service account inconsistency
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 5a50e30 [SPARK-27872][K8S][2.4] Fix executor service account
inconsistency
5a50e30 is described below
commit 5a50e30881d11971efe91b95ca6a20bb33868d2a
Author: nssalian
AuthorDate: Sun Oct 11 20:37:41 2020 -0700
[SPARK-27872][K8S][2.4] Fix executor service account inconsistency
### What changes were proposed in this pull request?
Similar patch to #24748 but applied to the branch-2.4.
Backporting the fix to releases 2.4.x.
Closes #29877 from nssalian/SPARK-27872.
Authored-by: nssalian
Signed-off-by: Dongjoon Hyun
---
.../scala/org/apache/spark/deploy/k8s/Config.scala | 7 ++--
.../apache/spark/deploy/k8s/KubernetesUtils.scala | 13 +++
.../DriverKubernetesCredentialsFeatureStep.scala | 11 +-
.../ExecutorKubernetesCredentialsFeatureStep.scala | 43 ++
.../cluster/k8s/KubernetesExecutorBuilder.scala| 9 -
.../k8s/KubernetesExecutorBuilderSuite.scala | 9 -
.../k8s/integrationtest/BasicTestsSuite.scala | 7
.../k8s/integrationtest/KubernetesSuite.scala | 4 ++
8 files changed, 87 insertions(+), 16 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index c7338a7..02471af 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -61,10 +61,9 @@ private[spark] object Config extends Logging {
.stringConf
.createOptional
- val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver"
- val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver.mounted"
+ val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
"spark.kubernetes.authenticate.driver"
+ val KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX =
"spark.kubernetes.authenticate.executor"
+ val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
"spark.kubernetes.authenticate.driver.mounted"
val KUBERNETES_AUTH_CLIENT_MODE_PREFIX = "spark.kubernetes.authenticate"
val OAUTH_TOKEN_CONF_SUFFIX = "oauthToken"
val OAUTH_TOKEN_FILE_CONF_SUFFIX = "oauthTokenFile"
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index 588cd9d..9e32d6e 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.deploy.k8s
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder,
ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting,
ContainerStatus, Pod, PodBuilder}
+
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils
@@ -60,4 +62,15 @@ private[spark] object KubernetesUtils {
}
def parseMasterUrl(url: String): String = url.substring("k8s://".length)
+
+ def buildPodWithServiceAccount(serviceAccount: Option[String], pod:
SparkPod): Option[Pod] = {
+serviceAccount.map { account =>
+ new PodBuilder(pod.pod)
+.editOrNewSpec()
+ .withServiceAccount(account)
+ .withServiceAccountName(account)
+.endSpec()
+.build()
+}
+ }
}
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
index ff5ad66..6079381 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
@@ -27,6 +27,7 @@ import io.fabric8.kubernetes.api.model.{ContainerBuilder,
HasMetadata, PodBuilde
import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.KubernetesUtils.buildPodWithServiceAccount
private[spark] class DriverKubernetesCredentialsFeatureStep(kubernetesConf:
KubernetesConf[_])
extends KubernetesFeatureConfigStep {
@@ -70,15 +71,7 @@ private[spark] class
DriverKubernetesCredentialsFeatureStep(kubernetesConf: Kube
[spark] branch branch-2.4 updated: [SPARK-27872][K8S][2.4] Fix executor service account inconsistency
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 5a50e30 [SPARK-27872][K8S][2.4] Fix executor service account
inconsistency
5a50e30 is described below
commit 5a50e30881d11971efe91b95ca6a20bb33868d2a
Author: nssalian
AuthorDate: Sun Oct 11 20:37:41 2020 -0700
[SPARK-27872][K8S][2.4] Fix executor service account inconsistency
### What changes were proposed in this pull request?
Similar patch to #24748 but applied to the branch-2.4.
Backporting the fix to releases 2.4.x.
Closes #29877 from nssalian/SPARK-27872.
Authored-by: nssalian
Signed-off-by: Dongjoon Hyun
---
.../scala/org/apache/spark/deploy/k8s/Config.scala | 7 ++--
.../apache/spark/deploy/k8s/KubernetesUtils.scala | 13 +++
.../DriverKubernetesCredentialsFeatureStep.scala | 11 +-
.../ExecutorKubernetesCredentialsFeatureStep.scala | 43 ++
.../cluster/k8s/KubernetesExecutorBuilder.scala| 9 -
.../k8s/KubernetesExecutorBuilderSuite.scala | 9 -
.../k8s/integrationtest/BasicTestsSuite.scala | 7
.../k8s/integrationtest/KubernetesSuite.scala | 4 ++
8 files changed, 87 insertions(+), 16 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index c7338a7..02471af 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -61,10 +61,9 @@ private[spark] object Config extends Logging {
.stringConf
.createOptional
- val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver"
- val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver.mounted"
+ val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
"spark.kubernetes.authenticate.driver"
+ val KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX =
"spark.kubernetes.authenticate.executor"
+ val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
"spark.kubernetes.authenticate.driver.mounted"
val KUBERNETES_AUTH_CLIENT_MODE_PREFIX = "spark.kubernetes.authenticate"
val OAUTH_TOKEN_CONF_SUFFIX = "oauthToken"
val OAUTH_TOKEN_FILE_CONF_SUFFIX = "oauthTokenFile"
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index 588cd9d..9e32d6e 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.deploy.k8s
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder,
ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting,
ContainerStatus, Pod, PodBuilder}
+
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils
@@ -60,4 +62,15 @@ private[spark] object KubernetesUtils {
}
def parseMasterUrl(url: String): String = url.substring("k8s://".length)
+
+ def buildPodWithServiceAccount(serviceAccount: Option[String], pod:
SparkPod): Option[Pod] = {
+serviceAccount.map { account =>
+ new PodBuilder(pod.pod)
+.editOrNewSpec()
+ .withServiceAccount(account)
+ .withServiceAccountName(account)
+.endSpec()
+.build()
+}
+ }
}
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
index ff5ad66..6079381 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
@@ -27,6 +27,7 @@ import io.fabric8.kubernetes.api.model.{ContainerBuilder,
HasMetadata, PodBuilde
import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.KubernetesUtils.buildPodWithServiceAccount
private[spark] class DriverKubernetesCredentialsFeatureStep(kubernetesConf:
KubernetesConf[_])
extends KubernetesFeatureConfigStep {
@@ -70,15 +71,7 @@ private[spark] class
DriverKubernetesCredentialsFeatureStep(kubernetesConf: Kube
[spark] branch branch-2.4 updated: [SPARK-27872][K8S][2.4] Fix executor service account inconsistency
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 5a50e30 [SPARK-27872][K8S][2.4] Fix executor service account
inconsistency
5a50e30 is described below
commit 5a50e30881d11971efe91b95ca6a20bb33868d2a
Author: nssalian
AuthorDate: Sun Oct 11 20:37:41 2020 -0700
[SPARK-27872][K8S][2.4] Fix executor service account inconsistency
### What changes were proposed in this pull request?
Similar patch to #24748 but applied to the branch-2.4.
Backporting the fix to releases 2.4.x.
Closes #29877 from nssalian/SPARK-27872.
Authored-by: nssalian
Signed-off-by: Dongjoon Hyun
---
.../scala/org/apache/spark/deploy/k8s/Config.scala | 7 ++--
.../apache/spark/deploy/k8s/KubernetesUtils.scala | 13 +++
.../DriverKubernetesCredentialsFeatureStep.scala | 11 +-
.../ExecutorKubernetesCredentialsFeatureStep.scala | 43 ++
.../cluster/k8s/KubernetesExecutorBuilder.scala| 9 -
.../k8s/KubernetesExecutorBuilderSuite.scala | 9 -
.../k8s/integrationtest/BasicTestsSuite.scala | 7
.../k8s/integrationtest/KubernetesSuite.scala | 4 ++
8 files changed, 87 insertions(+), 16 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index c7338a7..02471af 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -61,10 +61,9 @@ private[spark] object Config extends Logging {
.stringConf
.createOptional
- val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver"
- val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver.mounted"
+ val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
"spark.kubernetes.authenticate.driver"
+ val KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX =
"spark.kubernetes.authenticate.executor"
+ val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
"spark.kubernetes.authenticate.driver.mounted"
val KUBERNETES_AUTH_CLIENT_MODE_PREFIX = "spark.kubernetes.authenticate"
val OAUTH_TOKEN_CONF_SUFFIX = "oauthToken"
val OAUTH_TOKEN_FILE_CONF_SUFFIX = "oauthTokenFile"
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index 588cd9d..9e32d6e 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.deploy.k8s
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder,
ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting,
ContainerStatus, Pod, PodBuilder}
+
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils
@@ -60,4 +62,15 @@ private[spark] object KubernetesUtils {
}
def parseMasterUrl(url: String): String = url.substring("k8s://".length)
+
+ def buildPodWithServiceAccount(serviceAccount: Option[String], pod:
SparkPod): Option[Pod] = {
+serviceAccount.map { account =>
+ new PodBuilder(pod.pod)
+.editOrNewSpec()
+ .withServiceAccount(account)
+ .withServiceAccountName(account)
+.endSpec()
+.build()
+}
+ }
}
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
index ff5ad66..6079381 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
@@ -27,6 +27,7 @@ import io.fabric8.kubernetes.api.model.{ContainerBuilder,
HasMetadata, PodBuilde
import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.KubernetesUtils.buildPodWithServiceAccount
private[spark] class DriverKubernetesCredentialsFeatureStep(kubernetesConf:
KubernetesConf[_])
extends KubernetesFeatureConfigStep {
@@ -70,15 +71,7 @@ private[spark] class
DriverKubernetesCredentialsFeatureStep(kubernetesConf: Kube
[spark] branch branch-2.4 updated: [SPARK-27872][K8S][2.4] Fix executor service account inconsistency
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 5a50e30 [SPARK-27872][K8S][2.4] Fix executor service account
inconsistency
5a50e30 is described below
commit 5a50e30881d11971efe91b95ca6a20bb33868d2a
Author: nssalian
AuthorDate: Sun Oct 11 20:37:41 2020 -0700
[SPARK-27872][K8S][2.4] Fix executor service account inconsistency
### What changes were proposed in this pull request?
Similar patch to #24748 but applied to the branch-2.4.
Backporting the fix to releases 2.4.x.
Closes #29877 from nssalian/SPARK-27872.
Authored-by: nssalian
Signed-off-by: Dongjoon Hyun
---
.../scala/org/apache/spark/deploy/k8s/Config.scala | 7 ++--
.../apache/spark/deploy/k8s/KubernetesUtils.scala | 13 +++
.../DriverKubernetesCredentialsFeatureStep.scala | 11 +-
.../ExecutorKubernetesCredentialsFeatureStep.scala | 43 ++
.../cluster/k8s/KubernetesExecutorBuilder.scala| 9 -
.../k8s/KubernetesExecutorBuilderSuite.scala | 9 -
.../k8s/integrationtest/BasicTestsSuite.scala | 7
.../k8s/integrationtest/KubernetesSuite.scala | 4 ++
8 files changed, 87 insertions(+), 16 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index c7338a7..02471af 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -61,10 +61,9 @@ private[spark] object Config extends Logging {
.stringConf
.createOptional
- val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver"
- val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver.mounted"
+ val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
"spark.kubernetes.authenticate.driver"
+ val KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX =
"spark.kubernetes.authenticate.executor"
+ val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
"spark.kubernetes.authenticate.driver.mounted"
val KUBERNETES_AUTH_CLIENT_MODE_PREFIX = "spark.kubernetes.authenticate"
val OAUTH_TOKEN_CONF_SUFFIX = "oauthToken"
val OAUTH_TOKEN_FILE_CONF_SUFFIX = "oauthTokenFile"
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index 588cd9d..9e32d6e 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.deploy.k8s
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder,
ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting,
ContainerStatus, Pod, PodBuilder}
+
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils
@@ -60,4 +62,15 @@ private[spark] object KubernetesUtils {
}
def parseMasterUrl(url: String): String = url.substring("k8s://".length)
+
+ def buildPodWithServiceAccount(serviceAccount: Option[String], pod:
SparkPod): Option[Pod] = {
+serviceAccount.map { account =>
+ new PodBuilder(pod.pod)
+.editOrNewSpec()
+ .withServiceAccount(account)
+ .withServiceAccountName(account)
+.endSpec()
+.build()
+}
+ }
}
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
index ff5ad66..6079381 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverKubernetesCredentialsFeatureStep.scala
@@ -27,6 +27,7 @@ import io.fabric8.kubernetes.api.model.{ContainerBuilder,
HasMetadata, PodBuilde
import org.apache.spark.deploy.k8s.{KubernetesConf, SparkPod}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.deploy.k8s.KubernetesUtils.buildPodWithServiceAccount
private[spark] class DriverKubernetesCredentialsFeatureStep(kubernetesConf:
KubernetesConf[_])
extends KubernetesFeatureConfigStep {
@@ -70,15 +71,7 @@ private[spark] class
DriverKubernetesCredentialsFeatureStep(kubernetesConf: Kube
[spark] branch branch-2.4 updated: [SPARK-27872][K8S][2.4] Fix executor service account inconsistency
This is an automated email from the ASF dual-hosted git repository.
eje pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new bf32ac8 [SPARK-27872][K8S][2.4] Fix executor service account
inconsistency
bf32ac8 is described below
commit bf32ac8efa9818be551fe720a71eaba50d5d41ad
Author: nssalian
AuthorDate: Fri Sep 25 12:05:40 2020 -0700
[SPARK-27872][K8S][2.4] Fix executor service account inconsistency
### What changes were proposed in this pull request?
Similar patch to https://github.com/apache/spark/pull/24748 but applied to
the branch-2.4.
Backporting the fix to releases 2.4.x.
Please let me know if I missed some step; I haven't contributed to spark in
a long time.
Closes #29844 from nssalian/patch-SPARK-27872.
Authored-by: nssalian
Signed-off-by: Erik Erlandson
---
.../scala/org/apache/spark/deploy/k8s/Config.scala | 16 +---
.../apache/spark/deploy/k8s/KubernetesUtils.scala | 15
.../DriverKubernetesCredentialsFeatureStep.scala | 13 ++-
.../ExecutorKubernetesCredentialsFeatureStep.scala | 45 ++
.../cluster/k8s/KubernetesExecutorBuilder.scala| 11 +-
.../k8s/KubernetesExecutorBuilderSuite.scala | 9 -
.../k8s/integrationtest/BasicTestsSuite.scala | 7
.../k8s/integrationtest/KubernetesSuite.scala | 4 ++
8 files changed, 102 insertions(+), 18 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index c7338a7..cfff6b9 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -61,10 +61,9 @@ private[spark] object Config extends Logging {
.stringConf
.createOptional
- val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver"
- val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
- "spark.kubernetes.authenticate.driver.mounted"
+ val KUBERNETES_AUTH_DRIVER_CONF_PREFIX =
"spark.kubernetes.authenticate.driver"
+ val KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX =
"spark.kubernetes.authenticate.executor"
+ val KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX =
"spark.kubernetes.authenticate.driver.mounted"
val KUBERNETES_AUTH_CLIENT_MODE_PREFIX = "spark.kubernetes.authenticate"
val OAUTH_TOKEN_CONF_SUFFIX = "oauthToken"
val OAUTH_TOKEN_FILE_CONF_SUFFIX = "oauthTokenFile"
@@ -72,7 +71,7 @@ private[spark] object Config extends Logging {
val CLIENT_CERT_FILE_CONF_SUFFIX = "clientCertFile"
val CA_CERT_FILE_CONF_SUFFIX = "caCertFile"
- val KUBERNETES_SERVICE_ACCOUNT_NAME =
+ val KUBERNETES_DRIVER_SERVICE_ACCOUNT_NAME =
ConfigBuilder(s"$KUBERNETES_AUTH_DRIVER_CONF_PREFIX.serviceAccountName")
.doc("Service account that is used when running the driver pod. The
driver pod uses " +
"this service account when requesting executor pods from the API
server. If specific " +
@@ -81,6 +80,13 @@ private[spark] object Config extends Logging {
.stringConf
.createOptional
+ val KUBERNETES_EXECUTOR_SERVICE_ACCOUNT_NAME =
+ConfigBuilder(s"$KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX.serviceAccountName")
+ .doc("Service account that is used when running the executor pod." +
+"If this parameter is not setup, the service account defaults to
none.")
+ .stringConf
+ .createWithDefault("none")
+
val KUBERNETES_DRIVER_LIMIT_CORES =
ConfigBuilder("spark.kubernetes.driver.limit.cores")
.doc("Specify the hard cpu limit for the driver pod")
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index 588cd9d..aa3cf83 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -16,6 +16,10 @@
*/
package org.apache.spark.deploy.k8s
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder,
+ ContainerStateRunning, ContainerStateTerminated,
+ ContainerStateWaiting, ContainerStatus, Pod, PodBuilder}
+
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils
@@ -60,4 +64,15 @@ private[spark] object KubernetesUtils {
}
def parseMasterUrl(url: String): String = url.substring("k8s://".length)
+
+ def buildPodWithServiceAccount(serviceAccount: Option[String], pod:
SparkPod): Option[Pod] = {
+serviceAccount.map { account =>
+ new PodBuilder(pod.pod)
+.editOrNewSpec()
+
