Repository: ambari Updated Branches: refs/heads/trunk f4c838436 -> e7d0e781b
AMBARI-19886. Update ambari managed llap queue to set preemption policy. (Siddharth Seth via Swapan Shridhar). Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e7d0e781 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e7d0e781 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e7d0e781 Branch: refs/heads/trunk Commit: e7d0e781b3506d20806f521b30ce4629e67641f5 Parents: f4c8384 Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Fri Feb 10 16:10:19 2017 -0800 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Fri Feb 10 16:10:25 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.5/services/stack_advisor.py | 15 +++++++- .../stacks/2.5/common/test_stack_advisor.py | 36 +++++++++++++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d0e781/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 9fe8fc3..4de9a41 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1532,13 +1532,18 @@ class HDP25StackAdvisor(HDP24StackAdvisor): elif prop == 'yarn.scheduler.capacity.root.default.maximum-capacity': updated_cap_sched_configs_str = updated_cap_sched_configs_str \ + prop + "=" + adjusted_default_queue_cap + "\n" + elif prop == 'yarn.scheduler.capacity.root.ordering-policy': + # Don't put this in again. We're re-writing the llap section. + pass elif prop.startswith('yarn.') and '.llap.' not in prop: updated_cap_sched_configs_str = updated_cap_sched_configs_str + prop + "=" + val + "\n" # Now, append the 'llap' queue related properties - updated_cap_sched_configs_str += """yarn.scheduler.capacity.root.{0}.user-limit-factor=1 + updated_cap_sched_configs_str += """yarn.scheduler.capacity.root.ordering-policy=priority-utilization +yarn.scheduler.capacity.root.{0}.user-limit-factor=1 yarn.scheduler.capacity.root.{0}.state=RUNNING yarn.scheduler.capacity.root.{0}.ordering-policy=fifo +yarn.scheduler.capacity.root.{0}.priority=10 yarn.scheduler.capacity.root.{0}.minimum-user-limit-percent=100 yarn.scheduler.capacity.root.{0}.maximum-capacity={1} yarn.scheduler.capacity.root.{0}.capacity={1} @@ -1560,13 +1565,18 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu putCapSchedProperty(prop, adjusted_default_queue_cap) elif prop == 'yarn.scheduler.capacity.root.default.maximum-capacity': putCapSchedProperty(prop, adjusted_default_queue_cap) + elif prop == 'yarn.scheduler.capacity.root.ordering-policy': + # Don't put this in again. We're re-writing the llap section. + pass elif prop.startswith('yarn.') and '.llap.' not in prop: putCapSchedProperty(prop, val) # Add new 'llap' queue related configs. + putCapSchedProperty("yarn.scheduler.capacity.root.ordering-policy", "priority-utilization") putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".user-limit-factor", "1") putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".state", "RUNNING") putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".ordering-policy", "fifo") + putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".priority", "10") putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".minimum-user-limit-percent", "100") putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".maximum-capacity", llap_queue_cap_perc) putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".capacity", llap_queue_cap_perc) @@ -1634,6 +1644,9 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu # Set 'default' max. capacity back to maximum val updated_default_queue_configs = updated_default_queue_configs \ + prop + "="+DEFAULT_MAX_CAPACITY + "\n" + elif prop == 'yarn.scheduler.capacity.root.ordering-policy': + # Don't set this property. The default will be picked up. + pass elif prop.startswith('yarn.'): updated_default_queue_configs = updated_default_queue_configs + prop + "=" + val + "\n" else: # Update 'llap' related configs in 'updated_llap_queue_configs' http://git-wip-us.apache.org/repos/asf/ambari/blob/e7d0e781/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index ef83bca..176dd99 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -158,9 +158,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=20\n' @@ -188,9 +190,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -220,6 +224,7 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=STOPPED\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=0\n' 'yarn.scheduler.capacity.root.llap.capacity=0\n' @@ -749,9 +754,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -922,6 +929,7 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=STOPPED\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -1092,9 +1100,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=20\n' 'yarn.scheduler.capacity.root.llap.capacity=20\n' @@ -1292,9 +1302,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=0\n' 'yarn.scheduler.capacity.root.llap.capacity=0\n' @@ -1486,9 +1498,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -1689,9 +1703,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -1887,9 +1903,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -1962,7 +1980,7 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'minimum': '1', 'maximum': '4'}) - self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=66.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=66.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mi nimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=34.0\nyarn.scheduler.capacity.root.llap.capacity=34.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) + self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=66.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=66.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.r oot.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=34.0\nyarn.scheduler.capacity.root.llap.capacity=34.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties']) @@ -2087,9 +2105,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -2360,7 +2380,7 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mini mum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) + self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.roo t.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3) @@ -2488,9 +2508,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -2563,7 +2585,7 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=2.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=2.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mini mum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=98.0\nyarn.scheduler.capacity.root.llap.capacity=98.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) + self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=2.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=2.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.roo t.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=98.0\nyarn.scheduler.capacity.root.llap.capacity=98.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1.0') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) @@ -2688,9 +2710,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -2762,7 +2786,7 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=80.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=80.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mi nimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=20.0\nyarn.scheduler.capacity.root.llap.capacity=20.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) + self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=80.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=80.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.r oot.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=20.0\nyarn.scheduler.capacity.root.llap.capacity=20.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1') self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'}) @@ -2886,9 +2910,11 @@ class TestHDP25StackAdvisor(TestCase): 'yarn.scheduler.capacity.root.acl_administer_queue=*\n' 'yarn.scheduler.capacity.node-locality-delay=40\n' 'yarn.scheduler.capacity.queue-mappings-override.enable=false\n' + 'yarn.scheduler.capacity.root.ordering-policy=priority-utilization\n' 'yarn.scheduler.capacity.root.llap.user-limit-factor=1\n' 'yarn.scheduler.capacity.root.llap.state=RUNNING\n' 'yarn.scheduler.capacity.root.llap.ordering-policy=fifo\n' + 'yarn.scheduler.capacity.root.llap.priority=10\n' 'yarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\n' 'yarn.scheduler.capacity.root.llap.maximum-capacity=40\n' 'yarn.scheduler.capacity.root.llap.capacity=40\n' @@ -2958,7 +2984,7 @@ class TestHDP25StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts) - self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mini mum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) + self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.roo t.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'}) self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'}) self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)