(uniffle) branch master updated: [#2549] fix(spark): Invalid remote storage configuration was propagated during application registration (#2550)

[zuston]

This is an automated email from the ASF dual-hosted git repository.

zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new 1713c1f70 [#2549] fix(spark): Invalid remote storage configuration was 
propagated during application registration (#2550)
1713c1f70 is described below

commit 1713c1f707ca3b2d95bdbd1fb7940f6377e31a93
Author: Junfan Zhang <zus...@apache.org>
AuthorDate: Tue Jul 29 10:28:38 2025 +0800

    [#2549] fix(spark): Invalid remote storage configuration was propagated 
during application registration (#2550)
    
    ### What changes were proposed in this pull request?
    
    fix invalid remote storage configuration was propagated during application 
registration
    
    ### Why are the changes needed?
    
    fix #2549
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    Yes. existing tests and internal cluster tests.
---
 .../shuffle/manager/RssShuffleManagerBase.java      |  6 +++---
 .../apache/spark/shuffle/RssShuffleManagerTest.java | 21 +++++++++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git 
a/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
 
b/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
index 11bc1aa4c..e9994aa35 100644
--- 
a/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
+++ 
b/client-spark/common/src/main/java/org/apache/uniffle/shuffle/manager/RssShuffleManagerBase.java
@@ -1443,10 +1443,10 @@ public abstract class RssShuffleManagerBase implements 
RssShuffleManagerInterfac
         "Finish register shuffleId {} with {} ms", shuffleId, 
(System.currentTimeMillis() - start));
   }
 
-  protected RemoteStorageInfo getRemoteStorageInfo() {
+  @VisibleForTesting
+  public RemoteStorageInfo getRemoteStorageInfo() {
     String storageType = sparkConf.get(RssSparkConfig.RSS_STORAGE_TYPE.key());
-    RemoteStorageInfo defaultRemoteStorage =
-        new 
RemoteStorageInfo(sparkConf.get(RssSparkConfig.RSS_REMOTE_STORAGE_PATH.key(), 
""));
+    RemoteStorageInfo defaultRemoteStorage = 
getDefaultRemoteStorageInfo(sparkConf);
     return ClientUtils.fetchRemoteStorage(
         appId, defaultRemoteStorage, dynamicConfEnabled, storageType, 
shuffleWriteClient);
   }
diff --git 
a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/RssShuffleManagerTest.java
 
b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/RssShuffleManagerTest.java
index 2a92b6ed5..877bf6e62 100644
--- 
a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/RssShuffleManagerTest.java
+++ 
b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/RssShuffleManagerTest.java
@@ -26,6 +26,7 @@ import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import org.apache.uniffle.client.util.RssClientConfig;
+import org.apache.uniffle.common.RemoteStorageInfo;
 import org.apache.uniffle.common.ShuffleDataDistributionType;
 import org.apache.uniffle.common.config.ConfigOption;
 import org.apache.uniffle.common.config.RssClientConf;
@@ -96,6 +97,26 @@ public class RssShuffleManagerTest extends 
RssShuffleManagerTestBase {
     }
   }
 
+  @Test
+  public void testGetRemoteStorageInfo() {
+    setupMockedRssShuffleUtils(StatusCode.SUCCESS);
+
+    SparkConf conf = new SparkConf();
+    conf.set(RssSparkConfig.RSS_DYNAMIC_CLIENT_CONF_ENABLED.key(), "false");
+    conf.set(RssSparkConfig.RSS_COORDINATOR_QUORUM.key(), "m1:8001,m2:8002");
+    conf.set("spark.driver.host", "localhost");
+    conf.set("spark.rss.storage.type", StorageType.LOCALFILE.name());
+    conf.set(RssSparkConfig.RSS_TEST_MODE_ENABLE, true);
+
+    // inject some hadoop configs
+    conf.set("spark.rss.hadoop.k1", "v1");
+    conf.set("spark.rss.hadoop.k2", "v2");
+
+    RssShuffleManager shuffleManager = new RssShuffleManager(conf, true);
+    RemoteStorageInfo remoteStorageInfo = 
shuffleManager.getRemoteStorageInfo();
+    assertEquals(2, remoteStorageInfo.getConfItems().size());
+  }
+
   @Test
   public void testCreateShuffleManagerServer() {
     setupMockedRssShuffleUtils(StatusCode.SUCCESS);