AMBARI-18671. Ranger KMS should add proxy users for yarn and livy (Mugdha Varadkar via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2a0c9414 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2a0c9414 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2a0c9414 Branch: refs/heads/branch-feature-AMBARI-18634 Commit: 2a0c9414b4c1a6244ff3af29227072b6c9d6f43e Parents: f33dd9e Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Wed Nov 9 13:06:32 2016 -0800 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Wed Nov 9 13:09:18 2016 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.3/services/stack_advisor.py | 119 +++++++++++++++---- .../stacks/HDP/2.5/services/stack_advisor.py | 15 ++- .../stacks/2.0.6/common/test_stack_advisor.py | 29 +++++ .../stacks/2.3/common/test_stack_advisor.py | 28 +++++ .../stacks/2.5/common/test_stack_advisor.py | 9 +- 5 files changed, 164 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index d65248b..4e2a994 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -436,6 +436,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor): putCoreSiteProperty = self.putProperty(configurations, "core-site", services) putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site") putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services) + security_enabled = self.isSecurityEnabled(services) + putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services) + putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']): @@ -477,6 +480,95 @@ class HDP23StackAdvisor(HDP22StackAdvisor): default_fs = services['configurations']['core-site']['properties']['fs.defaultFS'] putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit')) + required_services = [{'service' : 'YARN', 'config-type': 'yarn-env', 'property-name': 'yarn_user', 'proxy-category': ['hosts', 'users', 'groups']}, + {'service' : 'SPARK', 'config-type': 'livy-env', 'property-name': 'livy_user', 'proxy-category': ['hosts', 'users', 'groups']}] + + required_services_for_secure = [{'service' : 'HIVE', 'config-type': 'hive-env', 'property-name': 'hive_user', 'proxy-category': ['hosts', 'users']}, + {'service' : 'OOZIE', 'config-type': 'oozie-env', 'property-name': 'oozie_user', 'proxy-category': ['hosts', 'users']}] + + if security_enabled: + required_services.extend(required_services_for_secure) + + # recommendations for kms proxy related properties + self.recommendKMSProxyUsers(configurations, services, hosts, required_services) + + ambari_user = self.getAmbariUser(services) + if security_enabled: + # adding for ambari user + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), '*') + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), '*') + # adding for HTTP + putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.users', '*') + putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.hosts', '*') + else: + self.deleteKMSProxyUsers(configurations, services, hosts, required_services_for_secure) + # deleting ambari user proxy properties + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), 'delete', 'true') + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), 'delete', 'true') + # deleting HTTP proxy properties + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.hosts', 'delete', 'true') + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.users', 'delete', 'true') + + def recommendKMSProxyUsers(self, configurations, services, hosts, requiredServices): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services) + putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") + + if 'forced-configurations' not in services: + services["forced-configurations"] = [] + + for index in range(len(requiredServices)): + service = requiredServices[index]['service'] + config_type = requiredServices[index]['config-type'] + property_name = requiredServices[index]['property-name'] + proxy_category = requiredServices[index]['proxy-category'] + + if service in servicesList: + if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']: + service_user = services['configurations'][config_type]['properties'][property_name] + service_old_user = getOldValue(self, services, config_type, property_name) + + if 'groups' in proxy_category: + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(service_user), '*') + if 'hosts' in proxy_category: + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), '*') + if 'users' in proxy_category: + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(service_user), '*') + + if service_old_user is not None and service_user != service_old_user: + if 'groups' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_old_user), 'delete', 'true') + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_old_user)}) + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_user)}) + if 'hosts' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_old_user), 'delete', 'true') + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_old_user)}) + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_user)}) + if 'users' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_old_user), 'delete', 'true') + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_old_user)}) + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_user)}) + + def deleteKMSProxyUsers(self, configurations, services, hosts, requiredServices): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") + + for index in range(len(requiredServices)): + service = requiredServices[index]['service'] + config_type = requiredServices[index]['config-type'] + property_name = requiredServices[index]['property-name'] + proxy_category = requiredServices[index]['proxy-category'] + + if service in servicesList: + if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']: + service_user = services['configurations'][config_type]['properties'][property_name] + + if 'groups' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_user), 'delete', 'true') + if 'hosts' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), 'delete', 'true') + if 'users' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_user), 'delete', 'true') def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName): connection_string = self.getDBConnectionHostPort(db_type, db_host) @@ -696,16 +788,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor): else: putYarnSitePropertyAttributes('yarn.authorization-provider', 'delete', 'true') - if 'yarn-site' in services["configurations"] and 'yarn.resourcemanager.proxy-user-privileges.enabled' in services["configurations"]["yarn-site"]["properties"]: - if self.isSecurityEnabled(services): - # enable proxy-user privileges for secure clusters for long-running services (spark streaming etc) - putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'true') - if 'RANGER_KMS' in servicesList: - # disable proxy-user privileges on secure clusters as it does not work with TDE - putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false') - else: - putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false') - def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts): putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services) @@ -753,8 +835,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations, "hive-site": self.validateHiveConfigurations}, "HBASE": {"hbase-site": self.validateHBASEConfigurations}, - "KAFKA": {"kafka-broker": self.validateKAFKAConfigurations}, - "YARN": {"yarn-site": self.validateYARNConfigurations}, + "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations}, "RANGER": {"admin-properties": self.validateRangerAdminConfigurations, "ranger-env": self.validateRangerConfigurationsEnv} } @@ -950,20 +1031,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "kafka-broker") - def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - yarn_site = properties - validationItems = [] - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - if 'RANGER_KMS' in servicesList and 'KERBEROS' in servicesList: - yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled'] - if yarn_resource_proxy_enabled.lower() == 'true': - validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled', - "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled " \ - "property value as false under yarn-site" - )}) - - return self.toConfigurationValidationProblems(validationItems, "yarn-site") - def isComponentUsingCardinalityForLayout(self, componentName): return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER'] http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 9ceebb9..dfb7b0c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1699,16 +1699,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor): def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor, self).recommendRangerKMSConfigurations(configurations, clusterData, services, hosts) - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services) - if 'ranger-env' in services['configurations'] and 'ranger_user' in services['configurations']['ranger-env']['properties']: - rangerUser = services['configurations']['ranger-env']['properties']['ranger_user'] + security_enabled = self.isSecurityEnabled(services) + required_services = [{'service' : 'RANGER', 'config-type': 'ranger-env', 'property-name': 'ranger_user', 'proxy-category': ['hosts', 'users', 'groups']}] - if 'kms-site' in services['configurations'] and 'KERBEROS' in servicesList: - putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(rangerUser), '*') - putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(rangerUser), '*') - putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(rangerUser), '*') + if security_enabled: + # recommendations for kms proxy related properties + self.recommendKMSProxyUsers(configurations, services, hosts, required_services) + else: + self.deleteKMSProxyUsers(configurations, services, hosts, required_services) def recommendRangerConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts) http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 0a73c55..125c9ca 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -2648,3 +2648,32 @@ class TestHDP206StackAdvisor(TestCase): self.assertFalse(recommendations['blueprint']['host_groups'][0]['components']) # Assert that DATANODE is placed on host-group-2 self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][0]['name'], 'DATANODE') + + def test_validateYARNConfigurations(self): + configurations = { + "cluster-env": { + "properties": { + "user_group": "hadoop", + } + }, + "yarn-site": { + "properties": { + 'yarn.nodemanager.resource.memory-mb' : '12288', + 'yarn.scheduler.minimum-allocation-mb' : '3072', + 'yarn.nodemanager.linux-container-executor.group': 'hadoop', + 'yarn.scheduler.maximum-allocation-mb': '12288' + } + } + } + services = {'configurations': {} } + recommendedDefaults = {'yarn.nodemanager.resource.memory-mb' : '12288', + 'yarn.scheduler.minimum-allocation-mb' : '3072', + 'yarn.nodemanager.linux-container-executor.group': 'hadoop', + 'yarn.scheduler.maximum-allocation-mb': '12288'} + properties = {'yarn.nodemanager.resource.memory-mb' : '12288', + 'yarn.scheduler.minimum-allocation-mb' : '3072', + 'yarn.nodemanager.linux-container-executor.group': 'hadoop', + 'yarn.scheduler.maximum-allocation-mb': '12288'} + + res = self.stackAdvisor.validateYARNConfigurations(properties, recommendedDefaults, configurations, services, {}) + self.assertFalse(res) http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index ad6842f..39cbbce 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1543,6 +1543,9 @@ class TestHDP23StackAdvisor(TestCase): def test_recommendRangerKMSConfigurations(self): clusterData = {} services = { + "ambari-server-properties": { + "ambari-server.user": "root" + }, "Versions": { "stack_version" : "2.3", }, @@ -1583,6 +1586,11 @@ class TestHDP23StackAdvisor(TestCase): 'db_host' : 'c6401.ambari.apache.org:1521:XE', 'db_name' : "XE" } + }, + 'cluster-env': { + 'properties': { + 'security_enabled': 'false' + } } }, "forced-configurations": [] @@ -1604,6 +1612,16 @@ class TestHDP23StackAdvisor(TestCase): 'ranger-kms-audit': { 'properties': { } + }, + 'kms-site': { + 'properties': { + }, + 'property_attributes': { + 'hadoop.kms.proxyuser.HTTP.hosts': {'delete': 'true'}, + 'hadoop.kms.proxyuser.HTTP.users': {'delete': 'true'}, + 'hadoop.kms.proxyuser.root.hosts': {'delete': 'true'}, + 'hadoop.kms.proxyuser.root.users': {'delete': 'true'} + } } } @@ -1618,6 +1636,8 @@ class TestHDP23StackAdvisor(TestCase): "service_name": "KERBEROS" } }) + services['configurations']['cluster-env']['properties']['security_enabled'] = "true" + services['configurations']['cluster-env']['properties']['ambari_principal_name'] = "ambari-...@example.com" expected = { 'kms-properties': { @@ -1637,6 +1657,14 @@ class TestHDP23StackAdvisor(TestCase): 'ranger-kms-audit': { 'properties': { } + }, + 'kms-site': { + 'properties': { + 'hadoop.kms.proxyuser.HTTP.hosts': '*', + 'hadoop.kms.proxyuser.HTTP.users': '*', + 'hadoop.kms.proxyuser.ambari-cl1.hosts': '*', + 'hadoop.kms.proxyuser.ambari-cl1.users': '*' + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/2a0c9414/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 6fe357a..7124307 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -446,9 +446,9 @@ class TestHDP25StackAdvisor(TestCase): def test_validateYarnConfigurations(self): properties = {'enable_hive_interactive': 'true', 'hive_server_interactive_host': 'c6401.ambari.apache.org', - 'hive.tez.container.size': '2048'} + 'hive.tez.container.size': '2048', "yarn.nodemanager.linux-container-executor.group": "hadoop"} recommendedDefaults = {'enable_hive_interactive': 'true', - "hive_server_interactive_host": "c6401.ambari.apache.org"} + "hive_server_interactive_host": "c6401.ambari.apache.org", "yarn.nodemanager.linux-container-executor.group": "hadoop"} configurations = { "hive-interactive-env": { "properties": {'enable_hive_interactive': 'true', "hive_server_interactive_host": "c6401.ambari.apache.org"} @@ -461,6 +461,11 @@ class TestHDP25StackAdvisor(TestCase): }, "yarn-site": { "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "false"} + }, + "cluster-env": { + "properties": { + "user_group": "hadoop", + } } } services = self.load_json("services-normal-his-valid.json")
