YARN-7251. Misc changes to YARN-5734
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06c8938a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06c8938a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06c8938a Branch: refs/heads/branch-2 Commit: 06c8938ab0db1bac13ffd989a57ea64ee293cbcf Parents: 40bec38 Author: Jonathan Hung <jh...@linkedin.com> Authored: Mon Sep 25 17:52:40 2017 -0700 Committer: Jonathan Hung <jh...@linkedin.com> Committed: Mon Oct 9 11:12:08 2017 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 3 +- .../hadoop/yarn/conf/YarnConfiguration.java | 33 ++++++++++++++- .../hadoop/yarn/client/cli/SchedConfCLI.java | 42 ++++++++++++++------ .../yarn/client/cli/TestSchedConfCLI.java | 20 +++++----- .../hadoop/yarn/webapp/dao/QueueConfigInfo.java | 2 +- .../yarn/webapp/dao/SchedConfUpdateInfo.java | 2 +- .../src/main/resources/yarn-default.xml | 15 ++++--- .../scheduler/capacity/CapacityScheduler.java | 12 +++--- .../CapacitySchedulerConfiguration.java | 12 ------ .../conf/MutableCSConfigurationProvider.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 2 +- .../resourcemanager/TestRMAdminService.java | 4 +- .../conf/TestLeveldbConfigurationStore.java | 2 - .../TestMutableCSConfigurationProvider.java | 3 ++ .../capacity/conf/TestZKConfigurationStore.java | 2 - .../TestRMWebServicesConfigurationMutation.java | 28 ++++++------- .../src/site/markdown/CapacityScheduler.md | 11 ++--- .../src/site/markdown/ResourceManagerRest.md | 40 ++++++++++--------- .../src/site/markdown/YarnCommands.md | 12 +++--- 20 files changed, 146 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 46d0e54..dfcb797 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -84,6 +84,7 @@ function print_usage(){ echo " node prints node report(s)" echo " queue prints queue information" echo " logs dump container logs" + echo " schedulerconf updates scheduler configuration" echo " classpath prints the class path needed to" echo " get the Hadoop jar and the" echo " required libraries" @@ -301,7 +302,7 @@ elif [ "$COMMAND" = "daemonlog" ] ; then elif [ "$COMMAND" = "cluster" ] ; then CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" -elif [ "$COMMAND" = "schedconf" ] ; then +elif [ "$COMMAND" = "schedulerconf" ] ; then CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" elif [ "$COMMAND" = "top" ]; then http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 1da1596..3a650cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -289,7 +289,7 @@ goto :eof set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof -:schedconf +:schedulerconf set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof @@ -338,6 +338,7 @@ goto :eof @echo node prints node report(s) @echo queue prints queue information @echo logs dump container logs + @echo schedulerconf updates scheduler configuration @echo classpath prints the class path needed to get the @echo Hadoop jar and the required libraries @echo daemonlog get/set the log level for each daemon http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d20d4c9..817ada1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -609,33 +609,64 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; + @Private + @Unstable public static final String SCHEDULER_CONFIGURATION_STORE_CLASS = YARN_PREFIX + "scheduler.configuration.store.class"; + @Private + @Unstable + public static final String FILE_CONFIGURATION_STORE = "file"; + @Private + @Unstable public static final String MEMORY_CONFIGURATION_STORE = "memory"; + @Private + @Unstable public static final String LEVELDB_CONFIGURATION_STORE = "leveldb"; + @Private + @Unstable public static final String ZK_CONFIGURATION_STORE = "zk"; + @Private + @Unstable public static final String DEFAULT_CONFIGURATION_STORE = - MEMORY_CONFIGURATION_STORE; + FILE_CONFIGURATION_STORE; + @Private + @Unstable public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX + "scheduler.configuration.leveldb-store.path"; + @Private + @Unstable public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = YARN_PREFIX + "scheduler.configuration.leveldb-store.compaction-interval-secs"; + @Private + @Unstable public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L; + @Private + @Unstable public static final String RM_SCHEDCONF_MAX_LOGS = YARN_PREFIX + "scheduler.configuration.store.max-logs"; + @Private + @Unstable public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000; + @Private + @Unstable public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000; /** Parent znode path under which ZKConfigurationStore will create znodes. */ + @Private + @Unstable public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX + "scheduler.configuration.zk-store.parent-path"; + @Private + @Unstable public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH = "/confstore"; + @Private + @Unstable public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS = YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java index e17062e..11bfdd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -27,7 +27,7 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.MissingArgumentException; import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -48,7 +48,7 @@ import java.util.Map; * CLI for modifying scheduler configuration. */ @Public -@Evolving +@Unstable public class SchedConfCLI extends Configured implements Tool { private static final String ADD_QUEUES_OPTION = "addQueues"; @@ -135,7 +135,7 @@ public class SchedConfCLI extends Configured implements Tool { WebResource webResource = webServiceClient.resource(WebAppUtils. getRMWebAppURLWithScheme(getConf())); ClientResponse response = webResource.path("ws").path("v1").path("cluster") - .path("sched-conf").accept(MediaType.APPLICATION_JSON) + .path("scheduler-conf").accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -170,7 +170,7 @@ public class SchedConfCLI extends Configured implements Tool { if (args == null) { return; } - List<String> queuesToRemove = Arrays.asList(args.split(",")); + List<String> queuesToRemove = Arrays.asList(args.split(";")); updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove)); } @@ -199,11 +199,14 @@ public class SchedConfCLI extends Configured implements Tool { } private QueueConfigInfo getQueueConfigInfo(String arg) { - String[] queueArgs = arg.split(","); - String queuePath = queueArgs[0]; + String[] args = arg.split(":"); + String queuePath = args[0]; Map<String, String> queueConfigs = new HashMap<>(); - for (int i = 1; i < queueArgs.length; ++i) { - putKeyValuePair(queueConfigs, queueArgs[i]); + if (args.length > 1) { + String[] queueArgs = args[1].split(","); + for (int i = 0; i < queueArgs.length; ++i) { + putKeyValuePair(queueConfigs, queueArgs[i]); + } } return new QueueConfigInfo(queuePath, queueConfigs); } @@ -228,11 +231,24 @@ public class SchedConfCLI extends Configured implements Tool { } private void printUsage() { - System.out.println("yarn schedconf [-add queueAddPath1,confKey1=confVal1," - + "confKey2=confVal2;queueAddPath2,confKey3=confVal3] " - + "[-remove queueRemovePath1,queueRemovePath2] " - + "[-update queueUpdatePath1,confKey1=confVal1] " + System.out.println("yarn schedulerconf [-add " + + "\"queueAddPath1:confKey1=confVal1,confKey2=confVal2;" + + "queueAddPath2:confKey3=confVal3\"] " + + "[-remove \"queueRemovePath1;queueRemovePath2\"] " + + "[-update \"queueUpdatePath1:confKey1=confVal1\"] " + "[-global globalConfKey1=globalConfVal1," - + "globalConfKey2=globalConfVal2]"); + + "globalConfKey2=globalConfVal2]\n" + + "Example (adding queues): yarn schedulerconf -add " + + "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0," + + "maximum-capacity=0\"\n" + + "Example (removing queues): yarn schedulerconf -remove \"root.a.a1;" + + "root.a.a2\"\n" + + "Example (updating queues): yarn schedulerconf -update \"root.a.a1" + + ":capacity=25,maximum-capacity=25;root.a.a2:capacity=75," + + "maximum-capacity=75\"\n" + + "Example (global scheduler update): yarn schedulerconf " + + "-global yarn.scheduler.capacity.maximum-applications=10000\n" + + "Note: This is an alpha feature, the syntax/options are subject to " + + "change, please run at your own risk."); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index d2f0639..5364e83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -61,20 +61,20 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testInvalidConf() throws Exception { // conf pair with no key should be invalid - int exitCode = cli.run(new String[] {"-add", "root.a,=confVal"}); + int exitCode = cli.run(new String[] {"-add", "root.a:=confVal"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); - exitCode = cli.run(new String[] {"-update", "root.a,=confVal"}); + exitCode = cli.run(new String[] {"-update", "root.a:=confVal"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); - exitCode = cli.run(new String[] {"-add", "root.a,confKey=confVal=conf"}); + exitCode = cli.run(new String[] {"-add", "root.a:confKey=confVal=conf"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); - exitCode = cli.run(new String[] {"-update", "root.a,confKey=confVal=c"}); + exitCode = cli.run(new String[] {"-update", "root.a:confKey=confVal=c"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); @@ -83,8 +83,7 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testAddQueues() { SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); - cli.addQueues("root.a,a1=aVal1,a2=aVal2," + - "a3=", schedUpdateInfo); + cli.addQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo); QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0); assertEquals("root.a", addInfo.getQueue()); Map<String, String> params = addInfo.getParams(); @@ -94,7 +93,7 @@ public class TestSchedConfCLI { assertNull(params.get("a3")); schedUpdateInfo = new SchedConfUpdateInfo(); - cli.addQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo); + cli.addQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo); assertEquals(2, schedUpdateInfo.getAddQueueInfo().size()); QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0); assertEquals("root.b", bAddInfo.getQueue()); @@ -111,7 +110,7 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testRemoveQueues() { SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); - cli.removeQueues("root.a,root.b,root.c.c1", schedUpdateInfo); + cli.removeQueues("root.a;root.b;root.c.c1", schedUpdateInfo); List<String> removeInfo = schedUpdateInfo.getRemoveQueueInfo(); assertEquals(3, removeInfo.size()); assertEquals("root.a", removeInfo.get(0)); @@ -122,8 +121,7 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testUpdateQueues() { SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); - cli.updateQueues("root.a,a1=aVal1,a2=aVal2," + - "a3=", schedUpdateInfo); + cli.updateQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo); QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); assertEquals("root.a", updateInfo.getQueue()); Map<String, String> params = updateInfo.getParams(); @@ -133,7 +131,7 @@ public class TestSchedConfCLI { assertNull(params.get("a3")); schedUpdateInfo = new SchedConfUpdateInfo(); - cli.updateQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo); + cli.updateQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo); assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size()); QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); assertEquals("root.b", bUpdateInfo.getQueue()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java index d1d91c2..6d4e0cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java @@ -34,7 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement; @XmlAccessorType(XmlAccessType.FIELD) public class QueueConfigInfo { - @XmlElement(name = "queueName") + @XmlElement(name = "queue-name") private String queue; private HashMap<String, String> params = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java index bb84096..4546291 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java @@ -31,7 +31,7 @@ import javax.xml.bind.annotation.XmlRootElement; * Information for making scheduler configuration changes (supports adding, * removing, or updating a queue, as well as global scheduler conf changes). */ -@XmlRootElement(name = "schedConf") +@XmlRootElement(name = "sched-conf") @XmlAccessorType(XmlAccessType.FIELD) public class SchedConfUpdateInfo { http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f415b4b..97de749 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3194,14 +3194,17 @@ <property> <description> - The type of configuration store to use for storing scheduler - configurations, if using a mutable configuration provider. - Keywords such as "memory" map to certain configuration store - implementations. If keyword is not found, try to load this - value as a class. + The type of configuration store to use for scheduler configurations. + Default is "file", which uses file based capacity-scheduler.xml to + retrieve and change scheduler configuration. To enable API based + scheduler configuration, use either "memory" (in memory storage, no + persistence across restarts), "leveldb" (leveldb based storage), or + "zk" (zookeeper based storage). API based configuration is only useful + when using a scheduler which supports mutable configuration. Currently + only capacity scheduler supports this. </description> <name>yarn.scheduler.configuration.store.class</name> - <value>memory</value> + <value>file</value> </property> <property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index ea77b56..6e7b8a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -297,18 +297,20 @@ public class CapacityScheduler extends try { writeLock.lock(); String confProviderStr = configuration.get( - CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER); + YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.DEFAULT_CONFIGURATION_STORE); switch (confProviderStr) { - case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER: + case YarnConfiguration.FILE_CONFIGURATION_STORE: this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext); break; - case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER: + case YarnConfiguration.MEMORY_CONFIGURATION_STORE: + case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: + case YarnConfiguration.ZK_CONFIGURATION_STORE: this.csConfProvider = new MutableCSConfigurationProvider(rmContext); break; default: - throw new IOException("Invalid CS configuration provider: " + + throw new IOException("Invalid configuration store class: " + confProviderStr); } this.csConfProvider.init(configuration); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 40cb893..3a519ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -315,18 +315,6 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur @Private public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; - - public static final String CS_CONF_PROVIDER = PREFIX - + "configuration.provider"; - - @Private - public static final String FILE_CS_CONF_PROVIDER = "file"; - - @Private - public static final String STORE_CS_CONF_PROVIDER = "store"; - - @Private - public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER; AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index b955562..c0fae88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -66,7 +66,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, public void init(Configuration config) throws IOException { String store = config.get( YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, - YarnConfiguration.DEFAULT_CONFIGURATION_STORE); + YarnConfiguration.MEMORY_CONFIGURATION_STORE); switch (store) { case YarnConfiguration.MEMORY_CONFIGURATION_STORE: this.confStore = new InMemoryConfigurationStore(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 90c4814..1410d2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -2417,7 +2417,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } @PUT - @Path("/sched-conf") + @Path("/scheduler-conf") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public synchronized Response updateSchedulerConfiguration( http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 0d409f0..a0ae700 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -194,8 +194,8 @@ public class TestRMAdminService { @Test public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() { - configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + configuration.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); try { rm = new MockRM(configuration); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java index 779208a..324cbee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java @@ -59,8 +59,6 @@ public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest { public void setUp() throws Exception { super.setUp(); FileUtil.fullyDelete(TEST_DIR); - conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, YarnConfiguration.LEVELDB_CONFIGURATION_STORE); conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 9b080cd..5d43ebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -80,6 +81,8 @@ public class TestMutableCSConfigurationProvider { @Test public void testInMemoryBackedProvider() throws Exception { Configuration conf = new Configuration(); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); confProvider.init(conf); assertNull(confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index 3cfa8da..355f741 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -199,8 +199,6 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest { conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_IDS, rmIds); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); - conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, YarnConfiguration.ZK_CONFIGURATION_STORE); conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 5923688..f7cac38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -89,8 +89,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); - conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName); try { if (CONF_FILE.exists()) { @@ -185,7 +185,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getAddQueueInfo().add(d); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -220,7 +220,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -246,7 +246,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getRemoveQueueInfo().add("root.a.a2"); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -271,7 +271,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getRemoveQueueInfo().add("root.c"); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -302,7 +302,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -334,7 +334,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(configInfo); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -362,7 +362,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { } response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -397,7 +397,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { 0.001f); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -415,7 +415,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(aUpdateInfo); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -445,7 +445,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -470,7 +470,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -485,7 +485,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { // Unset maximum-applications. Should be set to default. response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index d70f891..17a6ab9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -295,7 +295,7 @@ The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierach Changing Queue Configuration ---------------------------- -Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API. +Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API. This behavior can be changed via `yarn.scheduler.configuration.store.class` in yarn-site.xml. Possible values are *file*, which allows modifying properties via file; *memory*, which allows modifying properties via API, but does not persist changes across restart; *leveldb*, which allows modifying properties via API and stores changes in leveldb backing store; and *zk*, which allows modifying properties via API and stores changes in zookeeper backing store. The default value is *file*. ### Changing queue configuration via file @@ -308,16 +308,17 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. + **Note:** This feature is in alpha phase and is subject to change. + | Property | Description | |:---- |:---- | - | `yarn.scheduler.capacity.configuration.provider` | The type of configuration provider to use for capacity scheduler. To enable changing queue configuration via API, this should be set to *store*. Default value is *file*, which disables the API and reverts back to changing queue configuration via file. | - | `yarn.scheduler.configuration.store.class` | The type of backing store to use. Default value is *memory*, which stores the scheduler configuration in memory (and does not persist configuration changes across restarts). Other values are *leveldb* (using a leveldb-based implementation), and *zk* (using a zookeeper-based implementation). | + | `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). | | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. | | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. | | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. | | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. | | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. | - **Note:** When enabling backing store for scheduler configuration, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file. + **Note:** When enabling scheduler configuration mutations via `yarn.scheduler.configuration.store.class`, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file. - See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedconf) for examples on how to change scheduler configuration via command line. + See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedulerconf) for examples on how to change scheduler configuration via command line. http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 2ead7a0..397c1f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -4436,16 +4436,18 @@ Scheduler Configuration Mutation API The scheduler configuration mutation API provides a way to modify scheduler/queue configuration and queue hierarchy. +Please note that this feature is currently in the alpha stage and is subject to change. + ### URI - * http://rm-http-address:port/ws/v1/cluster/sched-conf + * http://rm-http-address:port/ws/v1/cluster/scheduler-conf ### HTTP Operations Supported * PUT -### Elements of the *schedConf* object +### Elements of the *sched-conf* object | Item | Data Type | Description | |:---- |:---- |:---- | @@ -4466,7 +4468,7 @@ Request for updating queue configurations. | Item | Data Type | Description | |:---- |:---- |:---- | -| queueName | string | Full path name of the queue to update | +| queue-name | string | Full path name of the queue to update | | params | map | A map of key value configuration pairs to update for this queue | Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example sets *root.default*'s maximum applications to 100 and its minimum user limit percent to 10. @@ -4475,12 +4477,12 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <schedConf> + <sched-conf> <update-queue> - <queueName>root.default</queueName> + <queue-name>root.default</queue-name> <params> <entry> <key>maximum-applications</key> @@ -4492,7 +4494,7 @@ HTTP Request: </entry> </params> </update-queue> - </schedConf> + </sched-conf> ``` @@ -4511,7 +4513,7 @@ Request for adding queues/updating queue configurations. | Item | Data Type | Description | |:---- |:---- |:---- | -| queueName | string | Full path name of the queue to add | +| queue-name | string | Full path name of the queue to add | | params | map | A map of key value configuration pairs to set for this queue | Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example adds a queue *root.a* with capacity/maximum-capacity 10, and adjusts *root.default*'s capacity/maximum-capacity to 90. (More complex examples include adding a queue whose parent is also being added in the same request, or adding multiple sibling queues.) @@ -4520,12 +4522,12 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <schedConf> + <sched-conf> <add-queue> - <queueName>root.a</queueName> + <queue-name>root.a</queue-name> <params> <entry> <key>capacity</key> @@ -4538,7 +4540,7 @@ HTTP Request: </params> </add-queue> <update-queue> - <queueName>root.default</queueName> + <queue-name>root.default</queue-name> <params> <entry> <key>capacity</key> @@ -4550,7 +4552,7 @@ HTTP Request: </entry> </params> </update-queue> - </schedConf> + </sched-conf> ``` @@ -4573,13 +4575,13 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <schedConf> + <sched-conf> <remove-queue>root.a</remove-queue> <remove-queue>root.b</remove-queue> - </schedConf> + </sched-conf> ``` @@ -4597,17 +4599,17 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <schedConf> + <sched-conf> <global-updates> <entry> <key>yarn.scheduler.capacity.queue-mappings-override.enable</key> <value>true</value> </entry> </global-updates> - </schedConf> + </sched-conf> ``` http://git-wip-us.apache.org/repos/asf/hadoop/blob/06c8938a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index b110197..1965612 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -238,18 +238,18 @@ Usage: Runs ResourceManager admin client -### schedconf +### schedulerconf -Usage: `yarn schedconf [options]` +Usage: `yarn schedulerconf [options]` | COMMAND\_OPTIONS | Description | |:---- |:---- | -| -add <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. | -| -remove <queuePath1,queuePath2> | Comma-separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. | -| -update <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. | +| -add <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. | +| -remove <"queuePath1;queuePath2"> | Semicolon separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. | +| -update <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. | | -global <key1=val1,key2=val2> | Update scheduler global configurations. This example sets key1=val1 and key2=val2 for scheduler's global configuration. | -Updates scheduler configuration +Updates scheduler configuration. Note, this feature is in alpha phase and is subject to change. ### scmadmin --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
