This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 28a99ded8 [CELEBORN-687] Fix shuffleResourceExists, reduce unexpected 
slot release request
28a99ded8 is described below

commit 28a99ded8dce6a1a20f258d8f5d00e0bb37c8bb5
Author: Shuang <[email protected]>
AuthorDate: Fri Jun 16 18:37:48 2023 +0800

    [CELEBORN-687] Fix shuffleResourceExists, reduce unexpected slot release 
request
    
    ### What changes were proposed in this pull request?
    Check ShufflePartitionLocationInfo whether empty or not for every worker
    
    ### Why are the changes needed?
    Actually shuffleResources would only remove related partitionLocations 
after stageEnd , then workers with empty partitionLocations will left(for 
speculative task), so shuffleResourceExists need check 
ShufflePartitionLocationInfo for every worker otherwise it would print wrong 
message and send release slot requests twice.
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Manual test
    
    ### Before this pr
    
    <img width="1252" alt="image" 
src="https://github.com/apache/incubator-celeborn/assets/28799061/06b71162-e78b-4163-8f52-24b50bc6c540";>
    
    
![image](https://github.com/apache/incubator-celeborn/assets/28799061/fec263e0-9641-4d17-a837-ab03c36c5e6d)
    
    ### After this pr
    
    
![image](https://github.com/apache/incubator-celeborn/assets/28799061/8f2f9653-ff58-4a6e-ae06-1023922ca5bf)
    
    Closes #1599 from RexXiong/CELEBORN-687.
    
    Authored-by: Shuang <[email protected]>
    Signed-off-by: zky.zhoukeyong <[email protected]>
---
 .../src/main/scala/org/apache/celeborn/client/LifecycleManager.scala  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git 
a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala 
b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
index 6100c75b0..77dcfc559 100644
--- a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
@@ -1133,8 +1133,8 @@ class LifecycleManager(appId: String, val conf: 
CelebornConf) extends RpcEndpoin
   }
 
   private def shuffleResourceExists(shuffleId: Int): Boolean = {
-    val workers = workerSnapshots(shuffleId)
-    workers != null && !workers.isEmpty
+    val workerPartitionInfos = workerSnapshots(shuffleId)
+    workerPartitionInfos != null && 
workerPartitionInfos.values().asScala.exists(!_.isEmpty())
   }
 
   // Once a partition is released, it will be never needed anymore

Reply via email to