This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 717c853 YARN-9755. Fixed RM failing to start when
FileSystemBasedConfigurationProvider is configured. Contributed by
Prabhu Joseph
717c853 is described below
commit 717c853873dd3b9112f5c15059a24655b8654607
Author: Eric Yang <ey...@apache.org>
AuthorDate: Tue Aug 27 13:14:59 2019 -0400
YARN-9755. Fixed RM failing to start when
FileSystemBasedConfigurationProvider is configured.
Contributed by Prabhu Joseph
---
.../yarn/FileSystemBasedConfigurationProvider.java | 14 ++++++--
.../server/resourcemanager/ResourceManager.java | 32 +++++++++----------
.../server/resourcemanager/TestRMAdminService.java | 37 ++++++++++++++++++++++
3 files changed, 65 insertions(+), 18 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
index b6ba660..3532d13 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
@@ -71,10 +71,20 @@ public class FileSystemBasedConfigurationProvider
@Override
public synchronized void initInternal(Configuration bootstrapConf)
throws Exception {
+ Configuration conf = new Configuration(bootstrapConf);
configDir =
- new Path(bootstrapConf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE,
+ new Path(conf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE,
YarnConfiguration.DEFAULT_FS_BASED_RM_CONF_STORE));
- fs = configDir.getFileSystem(bootstrapConf);
+ String scheme = configDir.toUri().getScheme();
+ if (scheme == null) {
+ scheme = FileSystem.getDefaultUri(conf).getScheme();
+ }
+ if (scheme != null) {
+ String disableCacheName = String.format("fs.%s.impl.disable.cache",
+ scheme);
+ conf.setBoolean(disableCacheName, true);
+ }
+ fs = configDir.getFileSystem(conf);
fs.mkdirs(configDir);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index c0a9133..57be468 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -267,6 +267,22 @@ public class ResourceManager extends CompositeService
this.rmContext = new RMContextImpl();
rmContext.setResourceManager(this);
+ // Set HA configuration should be done before login
+ this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf));
+ if (this.rmContext.isHAEnabled()) {
+ HAUtil.verifyAndSetConfiguration(this.conf);
+ }
+
+ // Set UGI and do login
+ // If security is enabled, use login user
+ // If security is not enabled, use current user
+ this.rmLoginUGI = UserGroupInformation.getCurrentUser();
+ try {
+ doSecureLogin();
+ } catch(IOException ie) {
+ throw new YarnRuntimeException("Failed to login", ie);
+ }
+
this.configurationProvider =
ConfigurationProviderFactory.getConfigurationProvider(conf);
this.configurationProvider.init(this.conf);
@@ -285,22 +301,6 @@ public class ResourceManager extends CompositeService
loadConfigurationXml(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
validateConfigs(this.conf);
-
- // Set HA configuration should be done before login
- this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf));
- if (this.rmContext.isHAEnabled()) {
- HAUtil.verifyAndSetConfiguration(this.conf);
- }
-
- // Set UGI and do login
- // If security is enabled, use login user
- // If security is not enabled, use current user
- this.rmLoginUGI = UserGroupInformation.getCurrentUser();
- try {
- doSecureLogin();
- } catch(IOException ie) {
- throw new YarnRuntimeException("Failed to login", ie);
- }
// register the handlers for all AlwaysOn services using setupDispatcher().
rmDispatcher = setupDispatcher();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 6659f10..e235632 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
@@ -50,6 +52,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
@@ -59,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -172,6 +176,39 @@ public class TestRMAdminService {
}
}
+
+ @Test
+ public void testFileSystemCloseWithFileSystemBasedConfigurationProvider()
+ throws Exception {
+ MiniDFSCluster hdfsCluster = null;
+ try {
+ HdfsConfiguration hdfsConfig = new HdfsConfiguration();
+ hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
+ .numDataNodes(1).build();
+ FileSystem fs1 = hdfsCluster.getFileSystem();
+ ConfigurationProvider configurationProvider = new
+ FileSystemBasedConfigurationProvider();
+ configurationProvider.init(hdfsConfig);
+ fs1.close();
+ try {
+ configurationProvider.getConfigurationInputStream(hdfsConfig,
+ "yarn-site.xml");
+ } catch (IOException e) {
+ if (e.getMessage().contains("Filesystem closed")) {
+ fail("FileSystemBasedConfigurationProvider failed to handle " +
+ "FileSystem close");
+ } else {
+ fail("Should not get any exceptions");
+ }
+ }
+ } finally {
+ if (hdfsCluster != null) {
+ hdfsCluster.shutdown();
+ }
+ }
+ }
+
+
@Test
public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider()
throws IOException, YarnException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
