Repository: ambari Updated Branches: refs/heads/trunk 2975e5108 -> 025ca257a
AMBARI-11515. Configs: increasing 'Memory allocated for all YARN containers on a node' slightly makes 'Maximum Container Size' go out of range (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/025ca257 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/025ca257 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/025ca257 Branch: refs/heads/trunk Commit: 025ca257a4598ce05c4e1295a57ffa9cc03fea2b Parents: 2975e51 Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Thu May 28 14:58:48 2015 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Thu May 28 14:58:55 2015 -0700 ---------------------------------------------------------------------- .../resources/stacks/HDP/2.2/services/stack_advisor.py | 10 ++++++++++ .../resources/stacks/HDPWIN/2.2/services/stack_advisor.py | 10 ++++++++++ .../test/python/stacks/2.2/common/test_stack_advisor.py | 10 ++++++++++ 3 files changed, 30 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/025ca257/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 5e78ee9..ec84818 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -59,6 +59,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) + # Above is the default calculated 'maximum' values derived purely from hosts. + # However, there are 'maximum' and other attributes that actually change based on the values + # of other configs. We need to update those values. + if ("yarn-site" in services["configurations"]): + if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]): + putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) + putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) + if ("yarn.nodemanager.resource.cpu-vcores" in services["configurations"]["yarn-site"]["properties"]): + putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) + putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) if "yarn-env" in services["configurations"] and "yarn_cgroups_enabled" in services["configurations"]["yarn-env"]["properties"]: yarn_cgroups_enabled = services["configurations"]["yarn-env"]["properties"]["yarn_cgroups_enabled"].lower() == "true" http://git-wip-us.apache.org/repos/asf/ambari/blob/025ca257/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index 9ab1470..ce00471 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -196,6 +196,16 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) + # Above is the default calculated 'maximum' values derived purely from hosts. + # However, there are 'maximum' and other attributes that actually change based on the values + # of other configs. We need to update those values. + if ("yarn-site" in services["configurations"]): + if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]): + putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) + putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) + if ("yarn.nodemanager.resource.cpu-vcores" in services["configurations"]["yarn-site"]["properties"]): + putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) + putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) if "yarn-env" in services["configurations"] and "yarn_cgroups_enabled" in services["configurations"]["yarn-env"]["properties"]: yarn_cgroups_enabled = services["configurations"]["yarn-env"]["properties"]["yarn_cgroups_enabled"].lower() == "true" http://git-wip-us.apache.org/repos/asf/ambari/blob/025ca257/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index d2867e1..f4727a0 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -903,6 +903,16 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + # Test - with no 'changed-configurations', we should get updated 'maximum's. + services.pop("changed-configurations", None) + services.pop("configurations", None) + services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}} + expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9' + expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9' + expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321' + expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-mb"]["maximum"] = '4321' + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) def test_recommendHiveConfigurationAttributes(self):
