Repository: ambari Updated Branches: refs/heads/trunk 1f1a9c792 -> 0ebdf4e66
AMBARI-10916. Recommended value is greater then maximum for yarn.nodemanager.resource.memory-mb (dsen via srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0ebdf4e6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0ebdf4e6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0ebdf4e6 Branch: refs/heads/trunk Commit: 0ebdf4e660bf7ff97b623e2bbca76055e3c9956b Parents: 1f1a9c7 Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Mon May 4 16:22:55 2015 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Mon May 4 16:25:02 2015 -0700 ---------------------------------------------------------------------- .../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py | 5 ++++- .../src/main/resources/stacks/HDP/2.2/services/stack_advisor.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0ebdf4e6/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index d56b201..c67c39d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -123,7 +123,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor): def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): putYarnProperty = self.putProperty(configurations, "yarn-site", services) putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services) - putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) + nodemanagerMinRam = 1048576 # 1TB in mb + for nodemanager in self.getHostsWithComponent("YARN", "NODEMANAGER", services, hosts): + nodemanagerMinRam = min(nodemanager["Hosts"]["total_mem"]/1024, nodemanagerMinRam) + putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(min(clusterData['containers'] * clusterData['ramPerContainer'], nodemanagerMinRam)))) putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) http://git-wip-us.apache.org/repos/asf/ambari/blob/0ebdf4e6/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 1854946..6eea9b7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -516,10 +516,13 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts): self.recommendYARNConfigurations(configurations, clusterData, services, hosts) putMapredProperty = self.putProperty(configurations, "mapred-site", services) + nodemanagerMinRam = 1048576 # 1TB in mb + for nodemanager in self.getHostsWithComponent("YARN", "NODEMANAGER", services, hosts): + nodemanagerMinRam = min(nodemanager["Hosts"]["total_mem"]/1024, nodemanagerMinRam) putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}") putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) - putMapredProperty('mapreduce.reduce.memory.mb', int(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))) + putMapredProperty('mapreduce.reduce.memory.mb', min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam))) putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', str(min(int(0.7*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])), 2047)))
