This is an automated email from the ASF dual-hosted git repository. vanzin pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git.
from 1a68fc3 [SPARK-27816][SQL] make TreeNode tag type safe add e9f3f62 [SPARK-27677][CORE] Serve local disk persisted blocks by the external service after releasing executor by dynamic allocation No new revisions were added by this update. Summary of changes: .../network/server/ChunkFetchRequestHandler.java | 3 + .../network/server/OneForOneStreamManager.java | 5 +- .../network/ChunkFetchRequestHandlerSuite.java | 42 ++-- .../network/TransportRequestHandlerSuite.java | 47 ++-- .../server/OneForOneStreamManagerSuite.java | 56 ++++- .../apache/spark/network/shuffle/Constants.java | 14 +- .../shuffle/ExternalShuffleBlockHandler.java | 77 +++++-- .../shuffle/ExternalShuffleBlockResolver.java | 79 +++++-- .../network/shuffle/ExternalShuffleClient.java | 42 +++- .../spark/network/shuffle/ShuffleClient.java | 6 - .../shuffle/protocol/BlockTransferMessage.java | 5 +- .../{StreamHandle.java => BlocksRemoved.java} | 41 ++-- .../{OpenBlocks.java => RemoveBlocks.java} | 21 +- .../CleanupNonShuffleServiceServedFilesSuite.java | 256 +++++++++++++++++++++ .../shuffle/ExternalShuffleBlockHandlerSuite.java | 63 +++-- .../shuffle/ExternalShuffleIntegrationSuite.java | 96 +++++++- .../shuffle/NonShuffleFilesCleanupSuite.java | 221 ------------------ .../network/shuffle/TestShuffleDataContext.java | 34 ++- .../apache/spark/ExecutorAllocationManager.scala | 12 +- .../src/main/scala/org/apache/spark/SparkEnv.scala | 37 ++- .../org/apache/spark/deploy/worker/Worker.scala | 7 +- .../org/apache/spark/internal/config/package.scala | 13 +- .../org/apache/spark/storage/BlockManager.scala | 65 +++--- .../apache/spark/storage/BlockManagerMaster.scala | 9 +- .../spark/storage/BlockManagerMasterEndpoint.scala | 159 ++++++++++--- .../spark/storage/BlockManagerMessages.scala | 3 +- .../org/apache/spark/storage/StorageUtils.scala | 19 +- .../apache/spark/ExternalShuffleServiceSuite.scala | 43 +++- .../spark/storage/BlockManagerInfoSuite.scala | 160 +++++++++++++ .../storage/BlockManagerReplicationSuite.scala | 6 +- .../apache/spark/storage/BlockManagerSuite.scala | 73 +++++- .../streaming/ReceivedBlockHandlerSuite.scala | 4 +- 32 files changed, 1237 insertions(+), 481 deletions(-) copy mllib-local/src/main/scala/org/apache/spark/ml/impl/Utils.scala => common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/Constants.java (81%) copy common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/{StreamHandle.java => BlocksRemoved.java} (57%) copy common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/{OpenBlocks.java => RemoveBlocks.java} (84%) create mode 100644 common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/CleanupNonShuffleServiceServedFilesSuite.java delete mode 100644 common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/NonShuffleFilesCleanupSuite.java create mode 100644 core/src/test/scala/org/apache/spark/storage/BlockManagerInfoSuite.scala --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org