AMBARI-19855 Stack advisor issues encountered (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b0c93e9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b0c93e9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b0c93e9 Branch: refs/heads/branch-dev-patch-upgrade Commit: 4b0c93e96edc1f34b69a5c33fae22c08e94f21d2 Parents: d222f57 Author: Dmytro Sen <d...@apache.org> Authored: Fri Feb 3 17:14:17 2017 +0200 Committer: Nate Cole <nc...@hortonworks.com> Committed: Mon Feb 13 15:45:35 2017 -0500 ---------------------------------------------------------------------- .../common-services/YARN/3.0.0.3.0/service_advisor.py | 8 ++++---- .../main/resources/stacks/HDP/2.2/services/stack_advisor.py | 8 ++++---- .../resources/stacks/HDPWIN/2.2/services/stack_advisor.py | 8 ++++---- .../src/test/python/stacks/2.2/common/test_stack_advisor.py | 7 ++++--- 4 files changed, 16 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4b0c93e9/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py index 44bdf80..30dd474 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py @@ -335,10 +335,10 @@ class YARNRecommender(service_advisor.ServiceAdvisor): putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site") nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts) if (nodeManagerHost is not None): - cpuPercentageLimit = 0.8 - if "yarn.nodemanager.resource.percentage-physical-cpu-limit" in configurations["yarn-site"]["properties"]: - cpuPercentageLimit = float(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"]) - cpuLimit = max(1, int(floor(nodeManagerHost["Hosts"]["cpu_count"] * cpuPercentageLimit))) + cpuPercentageLimit = 80.0 + if "yarn-site" in services["configurations"] and "yarn.nodemanager.resource.percentage-physical-cpu-limit" in services["configurations"]["yarn-site"]["properties"]: + cpuPercentageLimit = float(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"]) + cpuLimit = max(1, int(floor(nodeManagerHost["Hosts"]["cpu_count"] * (cpuPercentageLimit / 100.0)))) putYarnProperty('yarn.nodemanager.resource.cpu-vcores', str(cpuLimit)) putYarnProperty('yarn.scheduler.maximum-allocation-vcores', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) putYarnPropertyAttribute('yarn.nodemanager.resource.memory-mb', 'maximum', int(nodeManagerHost["Hosts"]["total_mem"] / 1024)) # total_mem in kb http://git-wip-us.apache.org/repos/asf/ambari/blob/4b0c93e9/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index cba611c..421d3d4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -141,10 +141,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site") nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts) if (nodeManagerHost is not None): - cpuPercentageLimit = 0.8 - if "yarn.nodemanager.resource.percentage-physical-cpu-limit" in configurations["yarn-site"]["properties"]: - cpuPercentageLimit = float(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"]) - cpuLimit = max(1, int(floor(nodeManagerHost["Hosts"]["cpu_count"] * cpuPercentageLimit))) + cpuPercentageLimit = 80.0 + if "yarn-site" in services["configurations"] and "yarn.nodemanager.resource.percentage-physical-cpu-limit" in services["configurations"]["yarn-site"]["properties"]: + cpuPercentageLimit = float(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"]) + cpuLimit = max(1, int(floor(nodeManagerHost["Hosts"]["cpu_count"] * (cpuPercentageLimit / 100.0)))) putYarnProperty('yarn.nodemanager.resource.cpu-vcores', str(cpuLimit)) putYarnProperty('yarn.scheduler.maximum-allocation-vcores', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) putYarnPropertyAttribute('yarn.nodemanager.resource.memory-mb', 'maximum', int(nodeManagerHost["Hosts"]["total_mem"] / 1024)) # total_mem in kb http://git-wip-us.apache.org/repos/asf/ambari/blob/4b0c93e9/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index b72f046..a3d9598 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -221,10 +221,10 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site") nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts) if (nodeManagerHost is not None): - cpuPercentageLimit = 0.8 - if "yarn.nodemanager.resource.percentage-physical-cpu-limit" in configurations["yarn-site"]["properties"]: - cpuPercentageLimit = float(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"]) - cpuLimit = max(1, int(math.floor(nodeManagerHost["Hosts"]["cpu_count"] * cpuPercentageLimit))) + cpuPercentageLimit = 80.0 + if "yarn-site" in services["configurations"] and "yarn.nodemanager.resource.percentage-physical-cpu-limit" in services["configurations"]["yarn-site"]["properties"]: + cpuPercentageLimit = float(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"]) + cpuLimit = max(1, int(floor(nodeManagerHost["Hosts"]["cpu_count"] * (cpuPercentageLimit / 100.0)))) putYarnProperty('yarn.nodemanager.resource.cpu-vcores', str(cpuLimit)) putYarnProperty('yarn.scheduler.maximum-allocation-vcores', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) putYarnPropertyAttribute('yarn.nodemanager.resource.memory-mb', 'maximum', int(nodeManagerHost["Hosts"]["total_mem"] / 1024)) # total_mem in kb http://git-wip-us.apache.org/repos/asf/ambari/blob/4b0c93e9/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index a26b661..0c7af1a 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1152,7 +1152,7 @@ class TestHDP22StackAdvisor(TestCase): # Test host NodeManager CPU cores and 'yarn.nodemanager.resource.percentage-physical-cpu-limit' hosts["items"][2]["Hosts"]["cpu_count"] = 10 - configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"] = '0.5' + services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"] = '50' services["changed-configurations"].append({ "type": "yarn-site", "name": "yarn.nodemanager.resource.percentage-physical-cpu-limit", @@ -1161,7 +1161,7 @@ class TestHDP22StackAdvisor(TestCase): expected["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"] = '5' expected["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-vcores"] = '1' expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-vcores"] = '5' - expected["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"] = '0.5' + expected["yarn-site"]["properties"]["yarn.nodemanager.resource.percentage-physical-cpu-limit"] = '50' expected["yarn-site"]["property_attributes"]["yarn.nodemanager.resource.cpu-vcores"]["maximum"] = '20' expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '5' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '5' @@ -1175,7 +1175,8 @@ class TestHDP22StackAdvisor(TestCase): "old_value": "1280" }] services.pop("configurations", None) - services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321'}}} + services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', + "yarn.nodemanager.resource.percentage-physical-cpu-limit" : '50'}}} expected["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] = '4321' expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"] = '4321'