Repository: ambari Updated Branches: refs/heads/trunk 12b59857e -> aa033d0c1
AMBARI-15587. improve ranger kms install integeration (alexantonenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aa033d0c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aa033d0c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aa033d0c Branch: refs/heads/trunk Commit: aa033d0c152acdc8d1f7640ccd60f77b9710ed80 Parents: 12b5985 Author: Alex Antonenko <hiv...@gmail.com> Authored: Fri Mar 25 18:49:59 2016 +0200 Committer: Alex Antonenko <hiv...@gmail.com> Committed: Mon Mar 28 11:49:57 2016 +0300 ---------------------------------------------------------------------- .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 10 ++ .../0.5.0.2.3/package/scripts/params.py | 7 +- .../stacks/HDP/2.3/services/stack_advisor.py | 14 ++- .../stacks/2.3/common/test_stack_advisor.py | 104 +++++++++++++++++++ 4 files changed, 132 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py index 92fe529..11a705a 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py @@ -256,6 +256,16 @@ def kms(upgrade_type=None): content=params.kms_log4j, mode=0644 ) + if params.stack_is_hdp23_or_further and params.security_enabled: + # core-site.xml linking required by setup for HDFS encryption + XmlConfig("core-site.xml", + conf_dir=params.kms_conf_dir, + configurations=params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner=params.kms_user, + group=params.kms_group, + mode=0644 + ) def copy_jdbc_connector(stack_version=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py index 30eda0b..ae4591e 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py @@ -18,6 +18,7 @@ limitations under the License. """ import os +from resource_management.libraries.functions import conf_select from resource_management.libraries.script import Script from resource_management.libraries.functions.version import format_stack_version, compare_versions from resource_management.libraries.functions.format import format @@ -33,11 +34,13 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version']) stack_version_formatted = format_stack_version(stack_version_unformatted) stack_is_hdp23_or_further = Script.is_stack_greater_or_equal("2.3") +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +security_enabled = config['configurations']['cluster-env']['security_enabled'] if stack_is_hdp23_or_further: kms_home = '/usr/hdp/current/ranger-kms' kms_conf_dir = '/usr/hdp/current/ranger-kms/conf' - + kms_log_dir = default("/configurations/kms-env/kms_log_dir", "/var/log/ranger/kms") java_home = config['hostLevelParams']['java_home'] kms_user = default("/configurations/kms-env/kms_user", "kms") @@ -203,4 +206,4 @@ if current_host in ranger_kms_hosts: check_db_connection_jar_name = "DBConnectionVerification.jar" check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") ranger_kms_jdbc_connection_url = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.url'] -ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver'] \ No newline at end of file +ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver'] http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 92d18e2..741011c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -460,6 +460,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor): servicesList = [service["StackServices"]["service_name"] for service in services["services"]] putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services) putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services) + kmsEnvProperties = getSiteProperties(services['configurations'], 'kms-env') + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) + putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site") if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']): @@ -492,6 +495,15 @@ class HDP23StackAdvisor(HDP22StackAdvisor): for key in rangerKmsDbProperties: putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key)) + if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList: + kmsUser = kmsEnvProperties['kms_user'] + kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user') + putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*') + if kmsUserOld is not None and kmsUser != kmsUserOld: + putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true') + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)}) + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)}) + def recommendRangerConfigurations(self, configurations, clusterData, services, hosts): super(HDP23StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts) servicesList = [service["StackServices"]["service_name"] for service in services["services"]] @@ -1025,7 +1037,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): validationItems.append({"config-name": PROP_NAME, "item": self.getWarnItem(message.format(PROP_NAME, str(limit)))}) return self.toConfigurationValidationProblems(validationItems, "hdfs-client") - + def isComponentUsingCardinalityForLayout(self, componentName): return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER'] http://git-wip-us.apache.org/repos/asf/ambari/blob/aa033d0c/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 04c69c4..d415b6f 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -2215,3 +2215,107 @@ class TestHDP23StackAdvisor(TestCase): problems = self.stackAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts) self.assertEqual(len(problems), 1) self.assertEqual(problems[0], expected) + + def test_recommendRangerKMSConfigurations(self): + clusterData = {} + services = { + "Versions": { + "stack_version" : "2.3", + }, + "services": [ + { + "StackServices": { + "service_name": "RANGER", + "service_version": "0.5.0.2.3" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "RANGER_ADMIN", + "hostnames": ["host1"] + } + } + ] + } + ], + "configurations": { + "kms-env": { + "properties": { + "kms_user": "kmsname" + } + }, + "core-site": { + "properties": { + } + } + }, + "forced-configurations": [] + } + expected = { + 'kms-properties': { + 'properties': {} + }, + 'dbks-site': { + 'properties': {} + }, + 'core-site': { + 'properties': { + } + } + } + + # non kerberized cluster. There should be no proxyuser configs + recommendedConfigurations = {} + self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None) + self.assertEquals(recommendedConfigurations, expected) + + # kerberized cluster + services['services'].append({ + "StackServices": { + "service_name": "KERBEROS" + } + }) + + expected = { + 'kms-properties': { + 'properties': {} + }, + 'dbks-site': { + 'properties': {} + }, + 'core-site': { + 'properties': { + 'hadoop.proxyuser.kmsname.groups': '*' + } + } + } + + # on kerberized cluster property should be recommended + recommendedConfigurations = {} + self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None) + self.assertEquals(recommendedConfigurations, expected) + + recommendedConfigurations = {} + services['changed-configurations'] = [ + { + 'type': 'kms-env', + 'name': 'kms_user', + 'old_value': 'kmsname' + } + ] + services['configurations']['kms-env']['properties']['kms_user'] = 'kmsnew' + + expected['core-site'] = { + 'properties': { + 'hadoop.proxyuser.kmsnew.groups': '*' + }, + 'property_attributes': { + 'hadoop.proxyuser.kmsname.groups': { + 'delete': 'true' + } + } + } + + # kms_user was changed, old property should be removed + self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None) + self.assertEquals(recommendedConfigurations, expected)