AMBARI-21154: Add JAAS config properties for Atlas Hive hook in HiveCli to use kerberos ticket-cache.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9aa786f7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9aa786f7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9aa786f7 Branch: refs/heads/branch-feature-logsearch-ui Commit: 9aa786f7ea4c21159e6a014b4cbb6a6de155b22c Parents: 2f40250 Author: Vishal Suvagia <vishalsuva...@yahoo.com> Authored: Fri Jun 23 17:41:50 2017 +0530 Committer: Mugdha Varadkar <mug...@apache.org> Committed: Wed Jun 28 11:32:26 2017 +0530 ---------------------------------------------------------------------- .../HIVE/2.1.0.3.0/service_advisor.py | 19 +++ .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 10 +- .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 7 +- .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 5 +- .../stacks/HDP/2.6/services/stack_advisor.py | 21 ++++ .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 9 ++ .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 6 + .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 1 + .../stacks/2.6/common/test_stack_advisor.py | 123 ++++++++++++++----- 9 files changed, 165 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py index 6d3e13d..48058f7 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py @@ -683,6 +683,25 @@ class HiveRecommender(service_advisor.ServiceAdvisor): else: self.logger.info("Not setting Hive Repo user for Ranger.") + security_enabled = self.isSecurityEnabled(services) + enable_atlas_hook = False + + if 'hive-env' in configurations and 'hive.atlas.hook' in configurations['hive-env']['properties']: + enable_atlas_hook = configurations['hive-env']['properties']['hive.atlas.hook'].lower() == 'true' + elif 'hive-env' in services['configurations'] and 'hive.atlas.hook' in services['configurations']['hive-env']['properties']: + enable_atlas_hook = services['configurations']['hive-env']['properties']['hive.atlas.hook'].lower() == 'true' + + if 'hive-atlas-application.properties' in services['configurations']: + putHiveAtlasHookProperty = self.putProperty(configurations, "hive-atlas-application.properties", services) + putHiveAtlasHookPropertyAttribute = self.putPropertyAttribute(configurations,"hive-atlas-application.properties") + if security_enabled and enable_atlas_hook: + putHiveAtlasHookProperty('atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag', 'required') + putHiveAtlasHookProperty('atlas.jaas.ticketBased-KafkaClient.loginModuleName', 'com.sun.security.auth.module.Krb5LoginModule') + putHiveAtlasHookProperty('atlas.jaas.ticketBased-KafkaClient.option.useTicketCache', 'true') + else: + putHiveAtlasHookPropertyAttribute('atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag', 'delete', 'true') + putHiveAtlasHookPropertyAttribute('atlas.jaas.ticketBased-KafkaClient.loginModuleName', 'delete', 'true') + putHiveAtlasHookPropertyAttribute('atlas.jaas.ticketBased-KafkaClient.option.useTicketCache', 'delete', 'true') def getDBDriver(self, databaseType): driverDict = { http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index a29f74b..30796cc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -434,7 +434,15 @@ <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/> <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/> </definition> - + <definition xsi:type="configure" id="hdp_2_6_maint_jaas_config_for_hive_hook" summary="Updating hive atlas application properties"> + <type>hive-atlas-application.properties</type> + <set key ="atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag" value="required" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + <set key ="atlas.jaas.ticketBased-KafkaClient.loginModuleName" value="com.sun.security.auth.module.Krb5LoginModule" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + <set key ="atlas.jaas.ticketBased-KafkaClient.option.useTicketCache" value="true" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + </definition> <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"> <type>hive-site</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 8c659ee..1f37389 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -621,7 +621,12 @@ <summary>Removing atlas.cluster.name property</summary> </task> </execute-stage> - + + <execute-stage service="HIVE" component="HIVE_SERVER" title="Updating hive atlas application properties"> + <task xsi:type="configure" id="hdp_2_6_maint_jaas_config_for_hive_hook"> + <summary>Updating hive atlas application properties</summary> + </task> + </execute-stage> <!-- SPARK --> <execute-stage service="SPARK" component="LIVY_SERVER" title="Apply config changes for Livy Server"> <task xsi:type="configure" id="hdp_2_5_0_0_rename_spark_livy_configs"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 3054ca3..22c9a8d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -825,10 +825,11 @@ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" /> <task xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive" /> <task xsi:type="configure" id="hdp_2_5_0_0_remove_atlas_cluster_name" /> + <task xsi:type="configure" id="hdp_2_6_maint_jaas_config_for_hive_hook"/> </pre-upgrade> - + <pre-downgrade /> - + <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index f8bbca5..82656aa 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -547,6 +547,9 @@ class HDP26StackAdvisor(HDP25StackAdvisor): def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts): super(HDP26StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts) + putHiveAtlasHookProperty = self.putProperty(configurations, "hive-atlas-application.properties", services) + putHiveAtlasHookPropertyAttribute = self.putPropertyAttribute(configurations,"hive-atlas-application.properties") + if 'hive-env' in services['configurations'] and 'hive_user' in services['configurations']['hive-env']['properties']: hive_user = services['configurations']['hive-env']['properties']['hive_user'] else: @@ -566,6 +569,24 @@ class HDP26StackAdvisor(HDP25StackAdvisor): else: self.logger.info("Not setting Hive Repo user for Ranger.") + security_enabled = self.isSecurityEnabled(services) + enable_atlas_hook = False + + if 'hive-env' in configurations and 'hive.atlas.hook' in configurations['hive-env']['properties']: + enable_atlas_hook = configurations['hive-env']['properties']['hive.atlas.hook'].lower() == 'true' + elif 'hive-env' in services['configurations'] and 'hive.atlas.hook' in services['configurations']['hive-env']['properties']: + enable_atlas_hook = services['configurations']['hive-env']['properties']['hive.atlas.hook'].lower() == 'true' + + if 'hive-atlas-application.properties' in services['configurations']: + if security_enabled and enable_atlas_hook: + putHiveAtlasHookProperty('atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag', 'required') + putHiveAtlasHookProperty('atlas.jaas.ticketBased-KafkaClient.loginModuleName', 'com.sun.security.auth.module.Krb5LoginModule') + putHiveAtlasHookProperty('atlas.jaas.ticketBased-KafkaClient.option.useTicketCache', 'true') + else: + putHiveAtlasHookPropertyAttribute('atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag', 'delete', 'true') + putHiveAtlasHookPropertyAttribute('atlas.jaas.ticketBased-KafkaClient.loginModuleName', 'delete', 'true') + putHiveAtlasHookPropertyAttribute('atlas.jaas.ticketBased-KafkaClient.option.useTicketCache', 'delete', 'true') + def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): super(HDP26StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts) if 'hbase-env' in services['configurations'] and 'hbase_user' in services['configurations']['hbase-env']['properties']: http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 1cbd78b..6dd2129 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -66,6 +66,15 @@ <set key="ranger.plugin.hive.urlauth.filesystem.schemes" value="hdfs:,file:,wasb:,adl:" if-type="ranger-hive-security" if-key="ranger.plugin.hive.service.name" if-key-state="present"/> </definition> + <definition xsi:type="configure" id="hdp_2_6_maint_jaas_config_for_hive_hook" summary="Updating hive atlas application properties"> + <type>hive-atlas-application.properties</type> + <set key ="atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag" value="required" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + <set key ="atlas.jaas.ticketBased-KafkaClient.loginModuleName" value="com.sun.security.auth.module.Krb5LoginModule" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + <set key ="atlas.jaas.ticketBased-KafkaClient.option.useTicketCache" value="true" + if-type="cluster-env" if-key="security_enabled" if-value="true"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index ede267a..e262971 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -307,6 +307,12 @@ <task xsi:type="configure" id="hdp_2_6_maint_ranger_hive_plugin_urlauth_filesystem_schemes"/> </execute-stage> + <execute-stage service="HIVE" component="HIVE_SERVER" title="Updating hive atlas application properties"> + <task xsi:type="configure" id="hdp_2_6_maint_jaas_config_for_hive_hook"> + <summary>Updating hive atlas application properties</summary> + </task> + </execute-stage> + <!-- HBASE --> <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for Ranger Hbase plugin"> <task xsi:type="configure" id="hdp_2_6_maint_ranger_hbase_plugin_cluster_name"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index b70943b..6b01ce9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -792,6 +792,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_6_maint_ranger_hive_plugin_cluster_name"/> <task xsi:type="configure" id="hdp_2_6_maint_ranger_hive_plugin_urlauth_filesystem_schemes"/> + <task xsi:type="configure" id="hdp_2_6_maint_jaas_config_for_hive_hook"/> </pre-upgrade> <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> <upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/9aa786f7/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py index d4d28c9..3ba18d8 100644 --- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py @@ -966,19 +966,33 @@ class TestHDP26StackAdvisor(TestCase): def test_recommendHiveConfigurations(self): configurations = { - "ranger-hive-plugin-properties": { - "properties": { - "ranger-hive-plugin-enabled": "Yes", - "REPOSITORY_CONFIG_USERNAME":"hive" + "hive-env" : { + "properties" : { + "hive.atlas.hook" : "false", + "hive_user": "custom_hive", + "hive_security_authorization": "Ranger" } }, - "hive-env":{ - "properties":{ - "hive_security_authorization":"ranger", - "hive_user":"custom_hive" + "ranger-env" : { + "properties" : { + "ranger-hive-plugin-enabled" : "Yes" + } + }, + "cluster-env" : { + "properties" : { + "security_enabled" : "false" + } + }, + "ranger-hive-plugin-properties" : { + "properties" : { + "REPOSITORY_CONFIG_USERNAME": "hive" } + }, + "hive-atlas-application.properties" : { + "properties": {} } } + clusterData = { "cpu": 4, "mapMemory": 3000, @@ -1012,31 +1026,44 @@ class TestHDP26StackAdvisor(TestCase): services = { "services": - [{ - "StackServices": { - "service_name": "YARN" - }, "components": [] - }, - { - "StackServices": { + [ + { + "StackServices" : { + "service_name" : "YARN" + }, + "components" : [] + }, + { + "StackServices" : { "service_name" : "HIVE", "service_version" : "1.2.1.2.6" }, - "components": [ - ] + "components": [] + }, + { + "StackServices" : { + "service_name" : "ATLAS", + "service_version": "0.8.0" + }, + "components": [] + }, + { + "StackServices" : { + "service_name" : "RANGER", + "service_version": "0.7.0" + }, + "components": [] } ], "Versions": { "stack_name" : "HDP", "stack_version": "2.6" }, - "changed-configurations": [ - ], + "changed-configurations": [], "configurations": configurations, "ambari-server-properties": {"ambari-server.user":"ambari_user"} } - expected = { 'yarn-env': { 'properties': { @@ -1047,7 +1074,6 @@ class TestHDP26StackAdvisor(TestCase): }, 'ranger-hive-plugin-properties': { 'properties': { - 'ranger-hive-plugin-enabled': 'Yes', 'REPOSITORY_CONFIG_USERNAME': 'custom_hive' } }, @@ -1068,12 +1094,11 @@ class TestHDP26StackAdvisor(TestCase): }, 'hive-env': { 'properties': { - 'hive.atlas.hook': 'false', - 'hive_security_authorization': 'ranger', + 'hive.atlas.hook': 'true', + 'hive_security_authorization': 'Ranger', 'hive_exec_orc_storage_strategy': 'SPEED', 'hive_timeline_logging_enabled': 'true', - 'hive_txn_acid': 'off', - 'hive_user': 'custom_hive' + 'hive_txn_acid': 'off' } }, 'hiveserver2-site': { @@ -1098,7 +1123,7 @@ class TestHDP26StackAdvisor(TestCase): 'hive.exec.orc.encoding.strategy': 'SPEED', 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.security.authorization.enabled': 'true', - 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', + 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.atlas.hive.hook.HiveHook', 'hive.server2.tez.default.queues': 'default', 'hive.prewarm.enabled': 'false', 'hive.exec.orc.compression.strategy': 'SPEED', @@ -1144,6 +1169,21 @@ class TestHDP26StackAdvisor(TestCase): }, 'atlas.rest.address': { 'delete': 'true' + }, + 'hive.server2.authentication.pam.services': { + 'delete': 'true' + }, + 'hive.server2.custom.authentication.class': { + 'delete': 'true' + }, + 'hive.server2.authentication.kerberos.principal': { + 'delete': 'true' + }, + 'hive.server2.authentication.kerberos.keytab': { + 'delete': 'true' + }, + 'hive.server2.authentication.ldap.url': { + 'delete': 'true' } } }, @@ -1174,16 +1214,35 @@ class TestHDP26StackAdvisor(TestCase): 'delete': 'true' } } + }, + 'hive-atlas-application.properties' : { + 'properties' : {}, + 'property_attributes' : { + 'atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag': {'delete': 'true'}, + 'atlas.jaas.ticketBased-KafkaClient.loginModuleName': {'delete': 'true'}, + 'atlas.jaas.ticketBased-KafkaClient.option.useTicketCache': {'delete': 'true'} + } } } - self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations,expected) - configurations['hive-env']['properties']['hive_user'] = 'hive' - expected['hive-env']['properties']['hive_user'] = 'hive' + recommendedConfigurations = {} + self.stackAdvisor.recommendHIVEConfigurations(recommendedConfigurations, clusterData, services, hosts) + self.assertEquals(recommendedConfigurations, expected) + + services['configurations']['hive-env']['properties']['hive_user'] = 'hive' expected['ranger-hive-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'hive' - self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) - self.assertEquals(configurations,expected) + services['configurations']['cluster-env']['properties']['security_enabled'] = 'true' + expected['hive-atlas-application.properties']['properties']['atlas.jaas.ticketBased-KafkaClient.loginModuleControlFlag'] = 'required' + expected['hive-atlas-application.properties']['properties']['atlas.jaas.ticketBased-KafkaClient.loginModuleName'] = 'com.sun.security.auth.module.Krb5LoginModule' + expected['hive-atlas-application.properties']['properties']['atlas.jaas.ticketBased-KafkaClient.option.useTicketCache'] = 'true' + del expected['hive-atlas-application.properties']['property_attributes'] + expected['core-site'] = { + 'properties': {} + } + + recommendedConfigurations = {} + self.stackAdvisor.recommendHIVEConfigurations(recommendedConfigurations, clusterData, services, hosts) + self.assertEquals(recommendedConfigurations, expected) def test_recommendHBASEConfigurations(self):