Repository: ambari Updated Branches: refs/heads/trunk 7aecacbf7 -> c3690fecf
AMBARI-11060 Pig-HA tests fails with java.lang.OutOfMemoryError: Java heap space (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c3690fec Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c3690fec Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c3690fec Branch: refs/heads/trunk Commit: c3690fecf23eb787901a70f077add6b1df54fd5b Parents: 7aecacb Author: Dmytro Sen <d...@apache.org> Authored: Tue May 12 12:12:08 2015 +0300 Committer: Dmytro Sen <d...@apache.org> Committed: Tue May 12 12:12:08 2015 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.2/services/stack_advisor.py | 10 +- .../stacks/2.2/common/test_stack_advisor.py | 258 +++++++++++++++++++ 2 files changed, 266 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c3690fec/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 24248f2..a7aa973 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -576,10 +576,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor): nodemanagerMinRam = min(nodemanager["Hosts"]["total_mem"]/1024, nodemanagerMinRam) putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}") - putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) putMapredProperty('mapreduce.reduce.memory.mb', min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam))) + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + min_mapreduce_map_memory_mb = 0 + min_mapreduce_map_java_opts = 0 + if ("PIG" in servicesList): + min_mapreduce_map_memory_mb = 1500 + min_mapreduce_map_java_opts = 1024 + putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))) mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])); - putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(mapredMapXmx) + "m") + putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts, mapredMapXmx)) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', str(min(int(0.7*mapredMapXmx), 2047))) # Property Attributes http://git-wip-us.apache.org/repos/asf/ambari/blob/c3690fec/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 51574f3..05bded6 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1144,6 +1144,264 @@ class TestHDP22StackAdvisor(TestCase): self.assertEquals(configurations, expected) + def test_recommendMapredConfigurationAttributesWithPigService(self): + configurations = { + "mapred-site": { + "properties": { + "mapreduce.map.memory.mb": "1024", + "mapreduce.reduce.memory.mb": "682", + "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}", + "mapreduce.reduce.java.opts": "-Xmx546m", + "yarn.app.mapreduce.am.resource.mb": "682", + "mapreduce.map.java.opts": "-Xmx546m", + "mapreduce.task.io.sort.mb": "273" + } + }, + "yarn-site": { + "properties": { + "yarn.nodemanager.resource.memory-mb": "1280", + "yarn.scheduler.minimum-allocation-mb": "100", + "yarn.scheduler.maximum-allocation-mb": "1280", + "yarn.nodemanager.resource.cpu-vcores": "2" + }, + } + } + clusterData = { + "cpu": 4, + "containers" : 5, + "ramPerContainer": 256 + } + expected = { + "yarn-env": { + "properties": { + "min_user_id": "500" + } + }, + "mapred-site": { + "properties": { + "mapreduce.map.memory.mb": "1500", + "mapreduce.reduce.memory.mb": "200", + "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}", + "mapreduce.reduce.java.opts": "-Xmx160m", + "yarn.app.mapreduce.am.resource.mb": "100", + "mapreduce.map.java.opts": "-Xmx1200m", + "mapreduce.task.io.sort.mb": "840" + }, + "property_attributes": { + 'mapreduce.task.io.sort.mb': {'maximum': '2047'}, + 'yarn.app.mapreduce.am.resource.mb': {'maximum': '1280', + 'minimum': '100'}, + 'mapreduce.map.memory.mb': {'maximum': '1280', + 'minimum': '100'}, + 'mapreduce.reduce.memory.mb': {'maximum': '1280', + 'minimum': '100'} + } + }, + "yarn-site": { + "properties": { + "yarn.nodemanager.resource.memory-mb": "1280", + "yarn.scheduler.minimum-allocation-mb": "100", + "yarn.scheduler.maximum-allocation-vcores": "1", + "yarn.scheduler.minimum-allocation-vcores": "1", + "yarn.scheduler.maximum-allocation-mb": "1280", + "yarn.nodemanager.resource.cpu-vcores": "1" + }, + "property_attributes": { + 'yarn.nodemanager.resource.memory-mb': {'maximum': '1877'}, + 'yarn.nodemanager.resource.cpu-vcores': {'maximum': '2'}, + 'yarn.scheduler.minimum-allocation-vcores': {'maximum': '1'}, + 'yarn.scheduler.maximum-allocation-vcores': {'maximum': '1'}, + 'yarn.scheduler.minimum-allocation-mb': {'maximum': '1280'}, + 'yarn.scheduler.maximum-allocation-mb': {'maximum': '1280'} + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/PIG", + "StackServices": { + "service_name": "PIG", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.2" + }, "components": [ + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "0+", + "component_category": "CLIENT", + "component_name": "PIG", + "display_name": "Pig", + "is_client": "true", + "is_master": "false", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2", + "StackServices" : { + "service_name" : "MAPREDUCE2", + "service_version" : "2.6.0.2.2", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER", + "StackServiceComponents" : { + "advertise_version" : "true", + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "HISTORYSERVER", + "custom_commands" : [ ], + "display_name" : "History Server", + "is_client" : "false", + "is_master" : "true", + "service_name" : "MAPREDUCE2", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ "c6402.ambari.apache.org" ] + }, + "auto_deploy" : { + "enabled" : "true", + "location" : "YARN/RESOURCEMANAGER" + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "HISTORYSERVER", + "dependent_service_name" : "MAPREDUCE2", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + }]}, + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN", + "StackServices": { + "service_name": "YARN", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.2" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "APP_TIMELINE_SERVER", + "display_name": "App Timeline Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6403.ambari.apache.org" + ] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1-2", + "component_category": "MASTER", + "component_name": "RESOURCEMANAGER", + "display_name": "ResourceManager", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "CLIENT", + "component_name": "YARN_CLIENT", + "display_name": "YARN Client", + "is_client": "true", + "is_master": "false", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": configurations, + "changed-configurations": [ + { + "type": "yarn-site", + "name": "yarn.scheduler.minimum-allocation-mb" + }, + ] + + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6402.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6402.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6402.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6403.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6403.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6403.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + + self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + def test_recommendMapredConfigurationAttributes(self): configurations = { "mapred-site": {