AMBARI-10828. Ambari memory calculation for Tez for config property: tez.runtime.io.sort.mb (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/798f1072 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/798f1072 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/798f1072 Branch: refs/heads/trunk Commit: 798f1072dd1db98a1980f8c245a10b14883235ce Parents: 1ba28aa Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Wed Apr 29 14:32:19 2015 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Wed Apr 29 14:32:19 2015 +0300 ---------------------------------------------------------------------- .../resources/stacks/HDP/2.3/services/stack_advisor.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/798f1072/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 8b56eb9..c3a9945 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -29,7 +29,15 @@ class HDP23StackAdvisor(HDP22StackAdvisor): def recommendTezConfigurations(self, configurations, clusterData, services, hosts): super(HDP23StackAdvisor, self).recommendTezConfigurations(configurations, clusterData, services, hosts) + + putTezProperty = self.putProperty(configurations, "tez-site") + # remove 2gb limit for tez.runtime.io.sort.mb + # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case comment calculation code below + taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory']) + taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory) + putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4)) + if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]: if services["configurations"]["tez-site"]["properties"]["tez.runtime.sorter.class"] == "LEGACY": putTezAttribute = self.putPropertyAttribute(configurations, "tez-site") - putTezAttribute("tez.runtime.io.sort.mb","maximum", 2047) \ No newline at end of file + putTezAttribute("tez.runtime.io.sort.mb", "maximum", 2047)
