Github user vanzin commented on a diff in the pull request:

    https://github.com/apache/spark/pull/19041#discussion_r138757437
  
    --- Diff: core/src/main/scala/org/apache/spark/RecoverCacheShutdown.scala 
---
    @@ -0,0 +1,234 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *    http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.spark
    +
    +import java.util.concurrent.{ScheduledFuture, TimeUnit}
    +
    +import scala.collection.mutable
    +import scala.concurrent.{ExecutionContext, Future}
    +import scala.util.Failure
    +
    +import org.apache.spark.internal.Logging
    +import org.apache.spark.rpc.RpcEndpointRef
    +import org.apache.spark.storage.{BlockId, BlockManagerId, RDDBlockId}
    +import org.apache.spark.storage.BlockManagerMessages.{GetCachedBlocks, 
GetMemoryStatus, GetSizeOfBlocks, ReplicateOneBlock}
    +import org.apache.spark.util.ThreadUtils
    +
    +/**
    + * Responsible for asynchronously replicating all of an executors cached 
blocks, and then shutting
    + * it down.
    + */
    +final private class RecoverCacheShutdown(
    +    state: RecoverCacheShutdownState,
    +    conf: SparkConf)
    +  extends Logging {
    +
    +  private val threadPool = 
ThreadUtils.newDaemonCachedThreadPool("recover-cache-shutdown-pool")
    +  private implicit val asyncExecutionContext = 
ExecutionContext.fromExecutorService(threadPool)
    +
    +  /**
    +   * Start the recover cache shutdown process for these executors
    +   *
    +   * @param execIds the executors to start shutting down
    +   */
    +  def startExecutorKill(execIds: Seq[String]): Unit = {
    +    logDebug(s"Recover cached data before shutting down executors 
${execIds.mkString(", ")}.")
    +    checkForReplicableBlocks(execIds)
    +  }
    +
    +  /**
    +   * Stops all thread pools
    +   *
    +   * @return
    +   */
    +  def stop(): java.util.List[Runnable] = {
    +    threadPool.shutdownNow()
    +    state.stop()
    +  }
    +
    +  /**
    +   * Get list of cached blocks from BlockManagerMaster. If there are 
cached blocks, replicate them,
    +   * otherwise kill the executors
    +   *
    +   * @param execIds the executors to check
    +   */
    +  private def checkForReplicableBlocks(execIds: Seq[String]) = 
state.getBlocks(execIds).foreach {
    +    case (executorId, NoMoreBlocks) => state.killExecutor(executorId)
    --- End diff --
    
    not 100% scalac will accept it, but maybe:
    
        case (executorId, NoMoreBlocks | NotEnoughMemory) =>
    
    Or you could invert things.
    
        case (executorId, HasCachedBlocks) => replicateBlocks(executorId)
        case (executorId, _) = kill()



---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to