This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 9672b81 YARN-9900. Revert to previous state when Invalid Config is
applied and Refresh Support in SchedulerConfig Format. Contributed by Prabhu
Joseph.
9672b81 is described below
commit 9672b81fa33f6a21e1e34fe188f09905206eca06
Author: Sunil G <sun...@apache.org>
AuthorDate: Wed Oct 16 18:14:31 2019 +0530
YARN-9900. Revert to previous state when Invalid Config is applied and
Refresh Support in SchedulerConfig Format. Contributed by Prabhu Joseph.
(cherry picked from commit 090f73a9aaeb4c03032a78c0a837478b527c27e0)
---
.../scheduler/MutableConfigurationProvider.java | 2 ++
.../conf/MutableCSConfigurationProvider.java | 16 ++++++++++
.../resourcemanager/webapp/RMWebServices.java | 7 +++++
.../TestRMWebServicesConfigurationMutation.java | 35 ++++++++++++++++++++--
4 files changed, 57 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index eff8aa8..03902e3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -73,6 +73,8 @@ public interface MutableConfigurationProvider {
void formatConfigurationInStore(Configuration conf) throws Exception;
+ void revertToOldConfig(Configuration config) throws Exception;
+
/**
* Closes the configuration provider, releasing any required resources.
* @throws IOException on failure to close
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 1f78d09..992eea7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -166,6 +166,7 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
formatLock.writeLock().lock();
try {
confStore.format();
+ oldConf = new Configuration(schedConf);
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
this.schedConf = new Configuration(false);
@@ -185,6 +186,21 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
}
@Override
+ public void revertToOldConfig(Configuration config) throws Exception {
+ formatLock.writeLock().lock();
+ try {
+ schedConf = oldConf;
+ confStore.format();
+ confStore.initialize(config, oldConf, rmContext);
+ confStore.checkVersion();
+ } catch (Exception e) {
+ throw new IOException(e);
+ } finally {
+ formatLock.writeLock().unlock();
+ }
+ }
+
+ @Override
public void confirmPendingMutation(boolean isValid) throws Exception {
formatLock.readLock().lock();
try {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 6ba8f43..4c3af9f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2337,6 +2337,13 @@ public class RMWebServices extends WebServices
implements RMWebServiceProtocol {
MutableConfigurationProvider mutableConfigurationProvider =
((MutableConfScheduler) scheduler).getMutableConfProvider();
mutableConfigurationProvider.formatConfigurationInStore(conf);
+ try {
+ rm.getRMContext().getRMAdminService().refreshQueues();
+ } catch (IOException | YarnException e) {
+ LOG.error("Exception thrown when formatting configuration.", e);
+ mutableConfigurationProvider.revertToOldConfig(conf);
+ throw e;
+ }
return Response.status(Status.OK).entity("Configuration under " +
"store successfully formatted.").build();
} catch (Exception e) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index c717d8b..773b4bf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -191,15 +191,44 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
@Test
public void testFormatSchedulerConf() throws Exception {
- testAddNestedQueue();
+ CapacitySchedulerConfiguration newConf = getSchedulerConf();
+ assertNotNull(newConf);
+ assertEquals(3, newConf.getQueues("root").length);
+
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ Map<String, String> nearEmptyCapacity = new HashMap<>();
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "1E-4");
+ QueueConfigInfo d = new QueueConfigInfo("root.formattest",
+ nearEmptyCapacity);
+ updateInfo.getAddQueueInfo().add(d);
+
+ Map<String, String> stoppedParam = new HashMap<>();
+ stoppedParam.put(CapacitySchedulerConfiguration.STATE,
+ QueueState.STOPPED.toString());
+ QueueConfigInfo stoppedInfo = new QueueConfigInfo("root.formattest",
+ stoppedParam);
+ updateInfo.getUpdateQueueInfo().add(stoppedInfo);
+
+ // Add a queue root.formattest to the existing three queues
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ newConf = getSchedulerConf();
+ assertNotNull(newConf);
+ assertEquals(4, newConf.getQueues("root").length);
+
+ // Format the scheduler config and validate root.formattest is not present
+ response = r.path("ws").path("v1").path("cluster")
.queryParam("user.name", userName)
.path(RMWSConsts.FORMAT_SCHEDULER_CONF)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
- CapacitySchedulerConfiguration orgConf = getSchedulerConf();
- assertEquals(3, orgConf.getQueues("root").length);
+ newConf = getSchedulerConf();
+ assertEquals(3, newConf.getQueues("root").length);
}
private long getConfigVersion() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
