This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new b560584 [SPARK-34407][K8S] KubernetesClusterSchedulerBackend.stop should clean up K8s resources b560584 is described below commit b56058456fa10695ee08a17f6cf7080a5edc96e7 Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Mon Feb 8 21:47:23 2021 -0800 [SPARK-34407][K8S] KubernetesClusterSchedulerBackend.stop should clean up K8s resources This PR aims to fix `KubernetesClusterSchedulerBackend.stop` to wrap `super.stop` with `Utils.tryLogNonFatalError`. [CoarseGrainedSchedulerBackend.stop](https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala#L559) may throw `SparkException` and this causes K8s resource (pod and configmap) leakage. No. This is a bug fix. Pass the CI with the newly added test case. Closes #31533 from dongjoon-hyun/SPARK-34407. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> (cherry picked from commit ea339c38b43c59931257386efdd490507f7de64d) Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../cluster/k8s/KubernetesClusterSchedulerBackend.scala | 6 +++++- .../cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala index 105841a..f801802 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala @@ -83,7 +83,11 @@ private[spark] class KubernetesClusterSchedulerBackend( } override def stop(): Unit = { - super.stop() + // When `CoarseGrainedSchedulerBackend.stop` throws `SparkException`, + // K8s cluster scheduler should log and proceed in order to delete the K8s cluster resources. + Utils.tryLogNonFatalError { + super.stop() + } Utils.tryLogNonFatalError { snapshotsStore.stop() diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala index 7e1e39c..260be03 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala @@ -33,7 +33,7 @@ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.deploy.k8s.Fabric8Aliases._ import org.apache.spark.rpc.{RpcEndpoint, RpcEndpointRef, RpcEnv} import org.apache.spark.scheduler.{ExecutorKilled, TaskSchedulerImpl} -import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.RemoveExecutor +import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.{RemoveExecutor, StopDriver} import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils.TEST_SPARK_APP_ID @@ -170,4 +170,14 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn TimeUnit.MILLISECONDS) verify(labeledPods).delete() } + + test("SPARK-34407: CoarseGrainedSchedulerBackend.stop may throw SparkException") { + schedulerBackendUnderTest.start() + + when(driverEndpointRef.askSync[Boolean](StopDriver)).thenThrow(new RuntimeException) + schedulerBackendUnderTest.stop() + + // Verify the last operation of `schedulerBackendUnderTest.stop`. + verify(kubernetesClient).close() + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org