Repository: ambari Updated Branches: refs/heads/trunk d4d256de4 -> 61e339211
AMBARI-11937. 'tez.task.launch.cmd-opts' and 'tez.am.launch.cmd-opts' should have JDK8 specific GC params (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/61e33921 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/61e33921 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/61e33921 Branch: refs/heads/trunk Commit: 61e3392113329f1e1a44b3247e9e1baab2bc7985 Parents: d4d256d Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Mon Jun 15 16:37:26 2015 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Mon Jun 15 16:46:36 2015 -0700 ---------------------------------------------------------------------- .../stacks/HDP/2.3/services/stack_advisor.py | 17 +- .../stacks/2.3/common/test_stack_advisor.py | 187 ++++++++++++++++++- 2 files changed, 201 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/61e33921/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index e97dcce..d65d711 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -76,12 +76,25 @@ class HDP23StackAdvisor(HDP22StackAdvisor): pass pass - if latest_tez_jar_version: tez_url = 'http://{0}:{1}/views/TEZ/{2}/TEZ_CLUSTER_INSTANCE'.format(server_host, server_port, latest_tez_jar_version) putTezProperty("tez.tez-ui.history-url.base", tez_url) pass + # TEZ JVM options + jvmGCParams = "-XX:+UseParallelGC" + if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]: + # JDK8 needs different parameters + match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"]) + if match and len(match.groups()) > 0: + # Is version >= 1.8 + versionSplits = re.split("\.", match.group(1)) + if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7: + jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB" + putTezProperty('tez.am.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams) + putTezProperty('tez.task.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams) + + def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): super(HDP23StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts) putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services) @@ -155,7 +168,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): if match and len(match.groups()) > 0: # Is version >= 1.8 versionSplits = re.split("\.", match.group(1)) - if versionSplits and len(versionSplits)>1 and int(versionSplits[0])>0 and int(versionSplits[1])>7: + if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7: jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB" putHiveSiteProperty('hive.tez.java.opts', "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA " + jvmGCParams + " -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps") http://git-wip-us.apache.org/repos/asf/ambari/blob/61e33921/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 614644f..3089dfa 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -475,12 +475,197 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - # Test JDK1.8 + # Test JDK1.9 services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.9.2_44'} expected['hive-site']['properties']['hive.tez.java.opts'] = "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps" self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + def test_recommendTezConfigurations(self): + self.maxDiff = None + configurations = { + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "256", + "yarn.scheduler.maximum-allocation-mb": "8192", + }, + }, + "capacity-scheduler": { + "properties": { + "yarn.scheduler.capacity.root.queues": "queue1,queue2" + } + } + } + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + "capacity-scheduler": { + "properties": { + "yarn.scheduler.capacity.root.queues": "queue1,queue2" + } + }, + "tez-site": { + "properties": { + "tez.task.resource.memory.mb": "768", + "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC", + "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC", + "tez.runtime.io.sort.mb": "307", + "tez.session.am.dag.submit.timeout.secs": "600", + "tez.runtime.unordered.output.buffer.size-mb": "57", + "tez.am.resource.memory.mb": "4000" + } + }, + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "256", + "yarn.scheduler.maximum-allocation-mb": "8192" + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN", + "StackServices": { + "service_name": "YARN", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.2" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "APP_TIMELINE_SERVER", + "display_name": "App Timeline Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6403.ambari.apache.org" + ] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1-2", + "component_category": "MASTER", + "component_name": "RESOURCEMANAGER", + "display_name": "ResourceManager", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "CLIENT", + "component_name": "YARN_CLIENT", + "display_name": "YARN Client", + "is_client": "true", + "is_master": "false", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": configurations, + "changed-configurations": [ ], + "ambari-server-properties": {} + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6402.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6402.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6402.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6403.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6403.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6403.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + # Test JDK1.7 + services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'} + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + # Test JDK1.8 + services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.8_44'} + expected['tez-site']['properties']['tez.am.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB" + expected['tez-site']['properties']['tez.task.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB" + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + # Test JDK1.9 + services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.9.2_44'} + expected['tez-site']['properties']['tez.am.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB" + expected['tez-site']['properties']['tez.task.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB" + self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + def test_validateHiveConfigurations(self): properties = {"hive_security_authorization": "None", "hive.exec.orc.default.stripe.size": "8388608",
