Repository: ambari Updated Branches: refs/heads/trunk e866f0426 -> 54be9c4e6
AMBARI-10837. HDFS Review: Multiple recommendation API updates for HDFS configs (mpapirkovskyy via srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/54be9c4e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/54be9c4e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/54be9c4e Branch: refs/heads/trunk Commit: 54be9c4e67a035a4316ff8784c5686fd84f04141 Parents: e866f04 Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Wed Apr 29 16:45:44 2015 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Wed Apr 29 16:45:49 2015 -0700 ---------------------------------------------------------------------- .../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 12 +++++++ .../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 18 +++------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 9 +++++ .../HDP/2.2/services/HDFS/themes/theme.json | 7 +++- .../stacks/HDP/2.2/services/stack_advisor.py | 37 ++++++++++++-------- .../stacks/2.2/common/test_stack_advisor.py | 22 ++++++------ 6 files changed, 65 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml index 99e1130..5d363b5 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml @@ -59,6 +59,12 @@ <value>200</value> <description>NameNode new generation size</description> <display-name>NameNode new generation size</display-name> + <depends-on> + <property> + <type>hadoop-env</type> + <name>namenode_heapsize</name> + </property> + </depends-on> <value-attributes> <type>int</type> <minimum>0</minimum> @@ -72,6 +78,12 @@ <value>200</value> <description>NameNode maximum new generation size</description> <display-name>NameNode maximum new generation size</display-name> + <depends-on> + <property> + <type>hadoop-env</type> + <name>namenode_heapsize</name> + </property> + </depends-on> <value-attributes> <type>int</type> <minimum>0</minimum> http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml index 0108b27..68bd0f7 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml @@ -167,7 +167,7 @@ <property> <name>dfs.namenode.safemode.threshold-pct</name> - <value>1.0f</value> + <value>0.999</value> <description> Specifies the percentage of blocks that should satisfy the minimal replication requirement defined by dfs.namenode.replication.min. @@ -176,18 +176,10 @@ </description> <display-name>Minimum replicated blocks %ge</display-name> <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>0.99</value> - <label>0.99f</label> - </entry> - <entry> - <value>1.0</value> - <label>1.0f</label> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> + <type>float</type> + <minimum>0.990</minimum> + <maximum>1.000</maximum> + <increment-step>0.001</increment-step> </value-attributes> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index afefa5d..32678a8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -224,6 +224,15 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return componentHosts[0] return None + def getHostComponentsByCategories(self, hostname, categories, services, hosts): + components = [] + if services is not None and hosts is not None: + for service in services["services"]: + components.extend([componentEntry for componentEntry in service["components"] + if componentEntry["StackServiceComponents"]["component_category"] in categories + and hostname in componentEntry["StackServiceComponents"]["hostnames"]]) + return components + def getConfigurationClusterSummary(self, servicesList, hosts, components, services): hBaseInstalled = False http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json index 4ff39eb..389c70c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json @@ -137,7 +137,12 @@ { "config": "hdfs-site/dfs.namenode.safemode.threshold-pct", "widget": { - "type": "toggle" + "type": "slider", + "units": [ + { + "unit-name": "percent" + } + ] } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index c3af09b..005d275 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -72,8 +72,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts): putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services) putHdfsSiteProperty("dfs.datanode.max.transfer.threads", 16384 if clusterData["hBaseInstalled"] else 4096) + dataDirsCount = 1 - if "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"]: + # Use users 'dfs.datanode.data.dir' first + if "hdfs-site" in services["configurations"] and "dfs.datanode.data.dir" in services["configurations"]["hdfs-site"]["properties"]: + dataDirsCount = len(str(services["configurations"]["hdfs-site"]["properties"]["dfs.datanode.data.dir"]).split(",")) + elif "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"]: dataDirsCount = len(str(configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"]).split(",")) if dataDirsCount <= 2: failedVolumesTolerated = 0 @@ -97,31 +101,32 @@ class HDP22StackAdvisor(HDP21StackAdvisor): if ("RANGER" in servicesList) and (rangerPluginEnabled.lower() == 'Yes'.lower()): putHdfsSiteProperty("dfs.permissions.enabled",'true') - putHdfsSiteProperty("dfs.namenode.safemode.threshold-pct", "0.99f" if len(namenodeHosts) > 1 else "1.0f") + putHdfsSiteProperty("dfs.namenode.safemode.threshold-pct", "0.999" if len(namenodeHosts) > 1 else "1.000") putHdfsEnvProperty = self.putProperty(configurations, "hadoop-env", services) putHdfsEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hadoop-env") putHdfsEnvProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024)) - putHdfsEnvProperty('namenode_opt_newsize', max(int(clusterData['totalAvailableRam'] / 8), 128)) - putHdfsEnvProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256)) - nn_max_heapsize=None + nn_heapsize_limit = None if (namenodeHosts is not None and len(namenodeHosts) > 0): if len(namenodeHosts) > 1: nn_max_heapsize = min(int(namenodeHosts[0]["Hosts"]["total_mem"]), int(namenodeHosts[1]["Hosts"]["total_mem"])) / 1024 + masters_at_host = max(self.getHostComponentsByCategories(namenodeHosts[0]["Hosts"]["host_name"], ["MASTER"], services, hosts), + self.getHostComponentsByCategories(namenodeHosts[1]["Hosts"]["host_name"], ["MASTER"], services, hosts)) else: nn_max_heapsize = int(namenodeHosts[0]["Hosts"]["total_mem"] / 1024) # total_mem in kb + masters_at_host = self.getHostComponentsByCategories(namenodeHosts[0]["Hosts"]["host_name"], ["MASTER"], services, hosts) putHdfsEnvPropertyAttribute('namenode_heapsize', 'maximum', max(nn_max_heapsize, 1024)) - nn_heapsize = nn_max_heapsize - nn_heapsize -= clusterData["reservedRam"] - if clusterData["hBaseInstalled"]: - nn_heapsize -= clusterData["hbaseRam"] - putHdfsEnvProperty('namenode_heapsize', max(int(nn_heapsize / 2), 1024)) - putHdfsEnvProperty('namenode_opt_newsize', max(int(nn_heapsize / 8), 128)) - putHdfsEnvProperty('namenode_opt_maxnewsize', max(int(nn_heapsize / 8), 256)) + nn_heapsize_limit = nn_max_heapsize + nn_heapsize_limit -= clusterData["reservedRam"] + if len(masters_at_host) > 1: + nn_heapsize_limit = int(nn_heapsize_limit/2) + + putHdfsEnvProperty('namenode_heapsize', max(nn_heapsize_limit, 1024)) + datanodeHosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts) if datanodeHosts is not None and len(datanodeHosts) > 0: @@ -163,13 +168,15 @@ class HDP22StackAdvisor(HDP21StackAdvisor): nn_memory_config = nn_memory_configs[index] #override with new values if applicable - if nn_max_heapsize is not None and nn_memory_config['nn_heap'] <= nn_max_heapsize: + if nn_heapsize_limit is not None and nn_memory_config['nn_heap'] <= nn_heapsize_limit: putHdfsEnvProperty('namenode_heapsize', nn_memory_config['nn_heap']) - putHdfsEnvProperty('namenode_opt_newsize', nn_memory_config['nn_opt']) - putHdfsEnvProperty('namenode_opt_maxnewsize', nn_memory_config['nn_opt']) putHdfsEnvPropertyAttribute('dtnode_heapsize', 'maximum', int(min_datanode_ram_kb/1024)) + nn_heapsize = int(configurations["hadoop-env"]["properties"]["namenode_heapsize"]) + putHdfsEnvProperty('namenode_opt_newsize', max(int(nn_heapsize / 8), 128)) + putHdfsEnvProperty('namenode_opt_maxnewsize', max(int(nn_heapsize / 8), 128)) + putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site") putHdfsSitePropertyAttribute('dfs.datanode.failed.volumes.tolerated', 'maximum', dataDirsCount) http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 53439bf..bfc35f9 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -25,7 +25,7 @@ class TestHDP22StackAdvisor(TestCase): def setUp(self): import imp - + self.maxDiff = None self.testDirectory = os.path.dirname(os.path.abspath(__file__)) stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') hdp206StackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py') @@ -1500,7 +1500,7 @@ class TestHDP22StackAdvisor(TestCase): clusterData = { "totalAvailableRam": 2048, "hBaseInstalled": True, - "hbaseRam": 111, + "hbaseRam": 112, "reservedRam": 128 } expected = { @@ -1518,7 +1518,7 @@ class TestHDP22StackAdvisor(TestCase): 'hdfs-site': { 'properties': { 'dfs.datanode.max.transfer.threads': '16384', - 'dfs.namenode.safemode.threshold-pct': '1.0f', + 'dfs.namenode.safemode.threshold-pct': '1.000', 'dfs.datanode.failed.volumes.tolerated': '1', 'dfs.namenode.handler.count': '25', 'dfs.datanode.data.dir': '/path/1,/path/2,/path/3,/path/4' @@ -1680,8 +1680,8 @@ class TestHDP22StackAdvisor(TestCase): ) self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "3072") - self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "512") - self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "512") + self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "384") + self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "384") # Test 2 - add more datanodes for i in xrange(11,30): hostname = "datanode" + `i` @@ -1705,8 +1705,8 @@ class TestHDP22StackAdvisor(TestCase): configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"] = "/path1,/path2,/path3,/path4" self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "9984") - self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1280") - self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1280") + self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1248") + self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1248") # Test 3 - more datanodes than host can handle for i in xrange(31, 90): hostname = "datanode" + `i` @@ -1727,10 +1727,10 @@ class TestHDP22StackAdvisor(TestCase): } ) self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "5000") - self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1250") - self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1250") - + self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "10112") + self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1264") + self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1264") + # Test 4 - KMS empty test from previous call self.assertTrue("dfs.encryption.key.provider.uri" not in configurations["hdfs-site"]["properties"])