[hadoop] branch branch-3.2 updated: YARN-9875. Improve fair scheduler configuration store on HDFS. (#3262)

[aajisaka]

This is an automated email from the ASF dual-hosted git repository.

aajisaka pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new d830e84  YARN-9875. Improve fair scheduler configuration store on 
HDFS. (#3262)
d830e84 is described below

commit d830e8421777181a8e28e0cfc2e33fd0a5cfea1c
Author: Akira Ajisaka <aajis...@apache.org>
AuthorDate: Wed Aug 4 16:12:54 2021 +0900

    YARN-9875. Improve fair scheduler configuration store on HDFS. (#3262)
    
    Contributed by Prabhu Joseph
    
    (cherry picked from commit 155864da006346a500ff35c2f6b69281093195b1)
    
     Conflicts:
        
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
    
    Co-authored-by: Eric Yang <ey...@apache.org>
---
 .../capacity/conf/FSSchedulerConfigurationStore.java         | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index c85506f..e32e725 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -67,7 +67,7 @@ public class FSSchedulerConfigurationStore extends 
YarnConfigurationStore {
   private Path configVersionFile;
 
   @Override
-  public void initialize(Configuration conf, Configuration vSchedConf,
+  public void initialize(Configuration fsConf, Configuration vSchedConf,
       RMContext rmContext) throws Exception {
     this.configFilePathFilter = new PathFilter() {
       @Override
@@ -81,6 +81,7 @@ public class FSSchedulerConfigurationStore extends 
YarnConfigurationStore {
       }
     };
 
+    Configuration conf = new Configuration(fsConf);
     String schedulerConfPathStr = conf.get(
         YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH);
     if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) {
@@ -89,6 +90,15 @@ public class FSSchedulerConfigurationStore extends 
YarnConfigurationStore {
               + " must be set");
     }
     this.schedulerConfDir = new Path(schedulerConfPathStr);
+    String scheme = schedulerConfDir.toUri().getScheme();
+    if (scheme == null) {
+      scheme = FileSystem.getDefaultUri(conf).getScheme();
+    }
+    if (scheme != null) {
+      String disableCacheName = String.format("fs.%s.impl.disable.cache",
+          scheme);
+      conf.setBoolean(disableCacheName, true);
+    }
     this.fileSystem = this.schedulerConfDir.getFileSystem(conf);
     this.maxVersion = conf.getInt(
         YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION,

---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org