(incubator-uniffle) branch master updated: [MINOR] fix(spark-client): Fix fallback to sort fail issue (#2329)

[maobaolong]

This is an automated email from the ASF dual-hosted git repository.

maobaolong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new c14bff7ea [MINOR] fix(spark-client): Fix fallback to sort fail issue 
(#2329)
c14bff7ea is described below

commit c14bff7ea003e0324fbcbe115f0cdd84cb359d9d
Author: maobaolong <baoloong...@tencent.com>
AuthorDate: Tue Jan 7 14:41:50 2025 +0800

    [MINOR] fix(spark-client): Fix fallback to sort fail issue (#2329)
    
    ### What changes were proposed in this pull request?
    
    Fix fallback to sort fail issue
    
    ### Why are the changes needed?
    
    Support fallback to sort shuffle service.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Existing UTs.
---
 .../java/org/apache/spark/shuffle/DelegationRssShuffleManager.java    | 4 ++++
 .../java/org/apache/spark/shuffle/DelegationRssShuffleManager.java    | 4 ++++
 2 files changed, 8 insertions(+)

diff --git 
a/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
 
b/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
index 3e74b3c48..41c004af9 100644
--- 
a/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
+++ 
b/client-spark/spark2/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
@@ -104,6 +104,10 @@ public class DelegationRssShuffleManager implements 
ShuffleManager {
               Constants.SORT_SHUFFLE_MANAGER_NAME, sparkConf, true);
       sparkConf.set(RssSparkConfig.RSS_ENABLED.key(), "false");
       sparkConf.set("spark.shuffle.manager", "sort");
+      if (sparkConf.getBoolean(Constants.SPARK_DYNAMIC_ENABLED, false)) {
+        sparkConf.set("spark.shuffle.service.enabled", "true");
+        LOG.info("Auto enable shuffle service while dynamic allocation is 
enabled.");
+      }
       LOG.info("Use SortShuffleManager");
     } catch (Exception e) {
       throw new RssException(e.getMessage());
diff --git 
a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
 
b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
index a2e14456b..d850bcdb3 100644
--- 
a/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
+++ 
b/client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java
@@ -104,6 +104,10 @@ public class DelegationRssShuffleManager implements 
ShuffleManager {
               Constants.SORT_SHUFFLE_MANAGER_NAME, sparkConf, true);
       sparkConf.set(RssSparkConfig.RSS_ENABLED.key(), "false");
       sparkConf.set("spark.shuffle.manager", "sort");
+      if (sparkConf.getBoolean(Constants.SPARK_DYNAMIC_ENABLED, false)) {
+        sparkConf.set("spark.shuffle.service.enabled", "true");
+        LOG.info("Auto enable shuffle service while dynamic allocation is 
enabled.");
+      }
       LOG.info("Use SortShuffleManager");
     } catch (Exception e) {
       throw new RssException(e.getMessage());