Repository: ambari Updated Branches: refs/heads/trunk 9963ac819 -> dcdf95b28
AMBARI-19737. Add a validation check in HSI about Capacity Scheduler preemption requirements. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dcdf95b2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dcdf95b2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dcdf95b2 Branch: refs/heads/trunk Commit: dcdf95b28a98e47062727f270adf2b4c69816f93 Parents: 9963ac8 Author: Swapan Shridhar <sshrid...@hortonworks.com> Authored: Fri Jan 27 11:02:48 2017 -0800 Committer: Swapan Shridhar <sshrid...@hortonworks.com> Committed: Fri Jan 27 11:02:48 2017 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.5/services/stack_advisor.py | 12 ++++++++++++ .../2.5/common/services-normal-his-2-hosts.json | 8 +++++++- .../2.5/common/services-normal-his-valid.json | 8 +++++++- .../stacks/2.5/common/test_stack_advisor.py | 18 +++++++++++------- 4 files changed, 37 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 8e377da..ca4a968 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -388,8 +388,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor): def validateHiveInteractiveEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): hive_site_env_properties = self.getSiteProperties(configurations, "hive-interactive-env") + yarn_site_properties = self.getSiteProperties(configurations, "yarn-site") validationItems = [] hsi_hosts = self.getHostsForComponent(services, "HIVE", "HIVE_SERVER_INTERACTIVE") + + # Check for expecting 'enable_hive_interactive' is ON given that there is HSI on atleast one host present. if len(hsi_hosts) > 0: # HIVE_SERVER_INTERACTIVE is mapped to a host if 'enable_hive_interactive' not in hive_site_env_properties or ( @@ -407,6 +410,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor): "item": self.getErrorItem( "enable_hive_interactive in hive-interactive-env should be set to false.")}) + # Check for 'yarn.resourcemanager.scheduler.monitor.enable' config to be true if HSI is ON. + if yarn_site_properties and 'yarn.resourcemanager.scheduler.monitor.enable' in yarn_site_properties: + scheduler_monitor_enabled = yarn_site_properties['yarn.resourcemanager.scheduler.monitor.enable'] + if scheduler_monitor_enabled.lower() == 'false' and hive_site_env_properties and 'enable_hive_interactive' in hive_site_env_properties and \ + hive_site_env_properties['enable_hive_interactive'].lower() == 'true': + validationItems.append({"config-name": "enable_hive_interactive", + "item": self.getWarnItem( + "When enabling LLAP, set 'yarn.resourcemanager.scheduler.monitor.enable' to true to ensure that LLAP gets the full allocated capacity.")}) + validationProblems = self.toConfigurationValidationProblems(validationItems, "hive-interactive-env") return validationProblems http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json index 44698ed..0044bf1 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json +++ b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json @@ -1108,7 +1108,8 @@ "hive.llap.daemon.queue.name": "llap", "hive.server2.enable.doAs": "true", "hive.llap.daemon.num.executors": "0", - "hive.server2.tez.sessions.per.default.queue": "32" + "hive.server2.tez.sessions.per.default.queue": "32", + "hive.tez.container.size" : "341" } }, "hive-interactive-env" : { @@ -1159,6 +1160,11 @@ "properties": { "tez.am.resource.memory.mb" : "341" } + }, + "tez-interactive-site": { + "properties": { + "tez.am.resource.memory.mb" : "341" + } } }, "changed-configurations" : [ ], http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json index 7955333..fad4a55 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json +++ b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json @@ -1049,7 +1049,8 @@ "configurations" : { "hive-interactive-site" : { "properties": { - "hive.llap.daemon.queue.name": "llap" + "hive.llap.daemon.queue.name": "llap", + "hive.tez.container.size" : "341" } }, "capacity-scheduler" : { @@ -1095,6 +1096,11 @@ "properties": { "tez.am.resource.memory.mb" : "341" } + }, + "tez-interactive-site": { + "properties": { + "tez.am.resource.memory.mb" : "341" + } } }, "changed-configurations" : [ ], http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index a53cb25..ac00bab 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -492,7 +492,8 @@ class TestHDP25StackAdvisor(TestCase): "properties": {"hive_security_authorization": "None"} }, "yarn-site": { - "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} + "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true", + "yarn.resourcemanager.scheduler.monitor.enable": "false"} } } configurations2 = { @@ -506,17 +507,22 @@ class TestHDP25StackAdvisor(TestCase): "properties": {"hive_security_authorization": "None"} }, "yarn-site": { - "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"} + "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true", + "yarn.resourcemanager.scheduler.monitor.enable": "true"} } } services = self.load_json("services-normal-his-valid.json") + # Checks for WARN message that 'yarn.resourcemanager.scheduler.monitor.enable' should be true. res_expected = [ + {'config-type': 'hive-interactive-env', 'message': "When enabling LLAP, set 'yarn.resourcemanager.scheduler.monitor.enable' to true to ensure that LLAP gets the full allocated capacity.", 'type': 'configuration', 'config-name': 'enable_hive_interactive', 'level': 'WARN'} ] # the above error is not what we are checking for - just to keep test happy without having to test res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations, services, {}) self.assertEquals(res, res_expected) + # (1). Checks for ERROR message for 'enable_hive_interactive' to be true. + # (2). Further, no message regarding 'yarn.resourcemanager.scheduler.monitor.enable' as it is true already. res_expected = [ {'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires enable_hive_interactive in hive-interactive-env set to true.', 'type': 'configuration', 'config-name': 'enable_hive_interactive', 'level': 'ERROR'} ] @@ -525,7 +531,6 @@ class TestHDP25StackAdvisor(TestCase): pass - ''' TODO: Commenting, Need to fix validations in 2.5/stack_advisor and then fix the test code """ Tests validations for Hive Server Interactive site. """ @@ -558,7 +563,7 @@ class TestHDP25StackAdvisor(TestCase): # Expected : Error telling about the current size compared to minimum required size. services1 = self.load_json("services-normal-his-valid.json") res_expected1 = [ - {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49%) is less than minimum required " + {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49.0%) is less than minimum required " "capacity (50%) for LLAP app to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'}, ] res1 = self.stackAdvisor.validateHiveInteractiveSiteConfigurations({}, {}, {}, services1, hosts) @@ -575,7 +580,7 @@ class TestHDP25StackAdvisor(TestCase): # than 50% of queue capacity for LLAP. services2 = self.load_json("services-normal-his-2-hosts.json") res_expected2 = [ - {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49%) is less than minimum required " + {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49.0%) is less than minimum required " "capacity (50%) for LLAP app to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'}, {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' current state is : 'STOPPED'. It is required to be in " "'RUNNING' state for LLAP to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'}, @@ -636,7 +641,7 @@ class TestHDP25StackAdvisor(TestCase): }, } res_expected4 = [ - {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49%) is less than minimum required capacity (200%) for LLAP app to run", + {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49.0%) is less than minimum required capacity (200%) for LLAP app to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'}, {'config-type': 'hive-interactive-site', 'message': "Capacity used by 'llap' queue is '250.88'. Service checks may not run as remaining available capacity " "(261.12) in cluster is less than 512 MB.", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'WARN'}] @@ -645,7 +650,6 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(res4, res_expected4) pass - '''