Repository: ambari Updated Branches: refs/heads/branch-2.2 395bca848 -> 328102511
AMBARI-15428. Fix HBase master heap recommendations.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/32810251 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/32810251 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/32810251 Branch: refs/heads/branch-2.2 Commit: 328102511791af6ba73a12a3f2275e794a9b8f28 Parents: 395bca8 Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Thu Mar 17 11:06:23 2016 +0200 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Thu Mar 17 11:06:23 2016 +0200 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 23 +++++++++++-- .../stacks/2.0.6/common/test_stack_advisor.py | 35 ++++++++++---------- .../stacks/2.1/common/test_stack_advisor.py | 33 +++++++++--------- .../stacks/2.2/common/test_stack_advisor.py | 4 +-- .../stacks/2.3/common/test_stack_advisor.py | 4 +-- 5 files changed, 58 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/32810251/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 8bd89d1..6132c2c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -258,9 +258,25 @@ class HDP206StackAdvisor(DefaultStackAdvisor): def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts): # recommendations for HBase env config + + # If cluster size is < 100, hbase master heap = 2G + # else If cluster size is < 500, hbase master heap = 4G + # else hbase master heap = 8G + # for small test clusters use 1 gb + hostsCount = 0 + if hosts and "items" in hosts: + hostsCount = len(hosts["items"]) + + hbaseMasterRam = { + hostsCount < 20: 1, + 20 <= hostsCount < 100: 2, + 100 <= hostsCount < 500: 4, + 500 <= hostsCount: 8 + }[True] + putHbaseProperty = self.putProperty(configurations, "hbase-env", services) putHbaseProperty('hbase_regionserver_heapsize', int(clusterData['hbaseRam']) * 1024) - putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024) + putHbaseProperty('hbase_master_heapsize', hbaseMasterRam * 1024) # recommendations for HBase site config putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services) @@ -759,7 +775,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): {"os":12, "hbase":16}, {"os":24, "hbase":24}, {"os":32, "hbase":32}, - {"os":64, "hbase":64} + {"os":64, "hbase":32} ] index = { cluster["ram"] <= 4: 0, @@ -774,9 +790,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor): 128 < cluster["ram"] <= 256: 9, 256 < cluster["ram"]: 10 }[1] + + cluster["reservedRam"] = ramRecommendations[index]["os"] cluster["hbaseRam"] = ramRecommendations[index]["hbase"] + cluster["minContainerSize"] = { cluster["ram"] <= 4: 256, 4 < cluster["ram"] <= 8: 512, http://git-wip-us.apache.org/repos/asf/ambari/blob/32810251/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 2f1959a..6dd0b44 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -665,23 +665,22 @@ class TestHDP206StackAdvisor(TestCase): servicesList = ["HBASE"] configurations = {} components = [] + host_item = { + "Hosts" : { + "cpu_count" : 6, + "total_mem" : 50331648, + "disk_info" : [ + {"mountpoint" : "/"}, + {"mountpoint" : "/dev/shm"}, + {"mountpoint" : "/vagrant"}, + {"mountpoint" : "/"}, + {"mountpoint" : "/dev/shm"}, + {"mountpoint" : "/vagrant"} + ] + } + } hosts = { - "items" : [ - { - "Hosts" : { - "cpu_count" : 6, - "total_mem" : 50331648, - "disk_info" : [ - {"mountpoint" : "/"}, - {"mountpoint" : "/dev/shm"}, - {"mountpoint" : "/vagrant"}, - {"mountpoint" : "/"}, - {"mountpoint" : "/dev/shm"}, - {"mountpoint" : "/vagrant"} - ] - } - } - ] + "items" : [host_item for i in range(1, 300)] } services = { "services" : [ @@ -707,7 +706,7 @@ class TestHDP206StackAdvisor(TestCase): }, "hbase-env": { "properties": { - "hbase_master_heapsize": "8192", + "hbase_master_heapsize": "4096", "hbase_regionserver_heapsize": "8192", } } @@ -716,7 +715,7 @@ class TestHDP206StackAdvisor(TestCase): clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, None) self.assertEquals(clusterData['hbaseRam'], 8) - self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, None) + self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/32810251/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py index 63d34af..076687d 100644 --- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py @@ -189,23 +189,22 @@ class TestHDP21StackAdvisor(TestCase): servicesList = ["HBASE"] configurations = {} components = [] + host_item = { + "Hosts" : { + "cpu_count" : 6, + "total_mem" : 50331648, + "disk_info" : [ + {"mountpoint" : "/"}, + {"mountpoint" : "/dev/shm"}, + {"mountpoint" : "/vagrant"}, + {"mountpoint" : "/"}, + {"mountpoint" : "/dev/shm"}, + {"mountpoint" : "/vagrant"} + ] + } + } hosts = { - "items" : [ - { - "Hosts" : { - "cpu_count" : 6, - "total_mem" : 50331648, - "disk_info" : [ - {"mountpoint" : "/"}, - {"mountpoint" : "/dev/shm"}, - {"mountpoint" : "/vagrant"}, - {"mountpoint" : "/"}, - {"mountpoint" : "/dev/shm"}, - {"mountpoint" : "/vagrant"} - ] - } - } - ] + "items" : [host_item for i in range(1, 600)] } services = { "services" : [ @@ -240,7 +239,7 @@ class TestHDP21StackAdvisor(TestCase): clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, None) self.assertEquals(clusterData['hbaseRam'], 8) - self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, None) + self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) def test_recommendHDFSConfigurations(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/32810251/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 0aa776b..7a6c512 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -2266,7 +2266,7 @@ class TestHDP22StackAdvisor(TestCase): }, "hbase-env": { "properties": { - "hbase_master_heapsize": "8192", + "hbase_master_heapsize": "1024", "hbase_regionserver_heapsize": "8192", } } @@ -2503,7 +2503,7 @@ class TestHDP22StackAdvisor(TestCase): }, "hbase-env": { "properties": { - "hbase_master_heapsize": "8192", + "hbase_master_heapsize": "1024", "hbase_regionserver_heapsize": "8192", }, "property_attributes": { http://git-wip-us.apache.org/repos/asf/ambari/blob/32810251/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 3858d42..107b39c 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -845,7 +845,7 @@ class TestHDP23StackAdvisor(TestCase): }, "hbase-env": { "properties": { - "hbase_master_heapsize": "114688", + "hbase_master_heapsize": "1024", "hbase_max_direct_memory_size": "94208", "hbase_regionserver_heapsize": "20480" } @@ -975,7 +975,7 @@ class TestHDP23StackAdvisor(TestCase): expected["hbase-site"]["property_attributes"]["hbase.bucketcache.ioengine"] = {"delete": "true"} expected["hbase-site"]["property_attributes"]["hbase.bucketcache.percentage.in.combinedcache"] = {"delete": "true"} expected["hbase-env"]["property_attributes"] = {"hbase_max_direct_memory_size" : {"delete": "true"}} - expected["hbase-env"]["properties"]["hbase_master_heapsize"] = "4096" + expected["hbase-env"]["properties"]["hbase_master_heapsize"] = "1024" expected["hbase-env"]["properties"]["hbase_regionserver_heapsize"] = "4096" self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations, expected)
