This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b65c8a106e2 [SPARK-38503][K8S] Warn if
`VolcanoFeatureStep.getAdditionalPreKubernetesResources` is used in executor
side
b65c8a106e2 is described below
commit b65c8a106e289410397deea1cd8837c277f68598
Author: Yikun Jiang <yikunk...@gmail.com>
AuthorDate: Wed Aug 10 10:40:37 2022 -0700
[SPARK-38503][K8S] Warn if
`VolcanoFeatureStep.getAdditionalPreKubernetesResources` is used in executor
side
### What changes were proposed in this pull request?
- Add warn for getAdditionalPreKubernetesResources in executor side
### Why are the changes needed?
Only `spark.kubernetes.scheduler.volcano.podGroupTemplateFile` is going to
be supported in v3.3, and it only be called when building driver side feature
steps.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- K8S IT passed
- CI passed
Closes #35786 from Yikun/SPARK-38455-FOLLOWUP.
Authored-by: Yikun Jiang <yikunk...@gmail.com>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
.../org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala | 8 +++++++-
.../spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala | 7 +++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
index 091923a78ef..ab84393dd5d 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
@@ -21,9 +21,10 @@ import io.fabric8.volcano.client.DefaultVolcanoClient
import io.fabric8.volcano.scheduling.v1beta1.{PodGroup, PodGroupSpec}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverConf,
KubernetesExecutorConf, SparkPod}
+import org.apache.spark.internal.Logging
private[spark] class VolcanoFeatureStep extends
KubernetesDriverCustomFeatureConfigStep
- with KubernetesExecutorCustomFeatureConfigStep {
+ with KubernetesExecutorCustomFeatureConfigStep with Logging {
import VolcanoFeatureStep._
private var kubernetesConf: KubernetesConf = _
@@ -40,6 +41,11 @@ private[spark] class VolcanoFeatureStep extends
KubernetesDriverCustomFeatureCon
}
override def getAdditionalPreKubernetesResources(): Seq[HasMetadata] = {
+ if (kubernetesConf.isInstanceOf[KubernetesExecutorConf]) {
+ logWarning("VolcanoFeatureStep#getAdditionalPreKubernetesResources() is
not supported " +
+ "for executor.")
+ return Seq.empty
+ }
val client = new DefaultVolcanoClient
val template = kubernetesConf.getOption(POD_GROUP_TEMPLATE_FILE_KEY)
val pg = template.map(client.podGroups.load(_).get).getOrElse(new
PodGroup())
diff --git
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
index d0d1f5ee5e1..dab414e0e19 100644
---
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
+++
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
@@ -66,4 +66,11 @@ class VolcanoFeatureStepSuite extends SparkFunSuite {
assert(podGroup.getSpec.getPriorityClassName == "driver-priority")
assert(podGroup.getSpec.getQueue == "driver-queue")
}
+
+ test("SPARK-38503: return empty for executor pre resource") {
+ val kubernetesConf = KubernetesTestConf.createExecutorConf(new SparkConf())
+ val step = new VolcanoFeatureStep()
+ step.init(kubernetesConf)
+ assert(step.getAdditionalPreKubernetesResources() === Seq.empty)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
