Repository: ambari Updated Branches: refs/heads/branch-2.5 6b1d05ae8 -> a8aa8ab17
AMBARI-19360 : Ranger policies not syncing for storm, kafka and knox (Vishal Suvagia via mugdha) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a8aa8ab1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a8aa8ab1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a8aa8ab1 Branch: refs/heads/branch-2.5 Commit: a8aa8ab174185f5ecd0859db187f57891bf84a8f Parents: 6b1d05a Author: Vishal Suvagia <vishalsuva...@yahoo.com> Authored: Fri Jan 13 11:13:46 2017 +0530 Committer: Mugdha Varadkar <mug...@apache.org> Committed: Fri Jan 13 12:41:10 2017 +0530 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../functions/setup_ranger_plugin_xml.py | 12 +++++++ .../KAFKA/0.8.1/package/scripts/params.py | 1 + .../0.8.1/package/scripts/setup_ranger_kafka.py | 6 ++++ .../0.5.0.2.2/package/scripts/params_linux.py | 1 + .../package/scripts/setup_ranger_knox.py | 9 +++++ .../STORM/0.9.1/package/scripts/params_linux.py | 1 + .../0.9.1/package/scripts/setup_ranger_storm.py | 35 +++++++++++++++++++- .../HDP/2.0.6/properties/stack_features.json | 5 +++ 9 files changed, 70 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index f4837fb..6fba224 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -108,3 +108,4 @@ class StackFeature: RANGER_TAGSYNC_SSL_XML_SUPPORT="ranger_tagsync_ssl_xml_support" RANGER_SOLR_CONFIG_SUPPORT='ranger_solr_config_support' HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED="hive_interactive_atlas_hook_required" + CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT='core_site_for_ranger_plugins' http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index ba6061a..6561928 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -33,6 +33,7 @@ from resource_management.core.source import DownloadSource, InlineTemplate from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2 from resource_management.core.utils import PasswordString from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.format import format def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, component_downloaded_custom_connector, component_driver_curl_source, @@ -215,3 +216,14 @@ def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_versio group = component_group, mode = 0640 ) + + +def setup_core_site_for_required_plugins(component_user, component_group, create_core_site_path, config): + XmlConfig('core-site.xml', + conf_dir=create_core_site_path, + configurations=config['configurations']['core-site'], + configuration_attributes=config['configuration_attributes']['core-site'], + owner=component_user, + group=component_group, + mode=0644 + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 41970f8..82849c8 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -58,6 +58,7 @@ version_for_stack_feature_checks = get_stack_feature_version(config) stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) +stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, version_for_stack_feature_checks) # When downgrading the 'version' and 'current_version' are both pointing to the downgrade-target version # downgrade_from_version provides the source-version the downgrade is happening from http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py index 3a9f51e..528dec2 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py @@ -17,6 +17,7 @@ limitations under the License. from resource_management.core.logger import Logger from resource_management.core.resources import File, Execute from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_core_site_for_required_plugins def setup_ranger_kafka(): import params @@ -80,5 +81,10 @@ def setup_ranger_kafka(): group = params.user_group, mode = 0755 ) + if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.security_enabled: + Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions") + setup_core_site_for_required_plugins(component_user=params.kafka_user,component_group=params.user_group,create_core_site_path = params.conf_dir, config = params.config) + else: + Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations") else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index 4a23bca..d245178 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -60,6 +60,7 @@ version_for_stack_feature_checks = get_stack_feature_version(config) stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) +stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, version_for_stack_feature_checks) # This is the version whose state is CURRENT. During an RU, this is the source version. # DO NOT format it since we need the build number too. http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py index d4eb6b9..7601dfa 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py @@ -18,6 +18,9 @@ limitations under the License. """ from resource_management.core.logger import Logger +from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_core_site_for_required_plugins + + def setup_ranger_knox(upgrade_type=None): import params @@ -95,5 +98,11 @@ def setup_ranger_knox(upgrade_type=None): credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble) + if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.has_namenode and params.security_enabled: + Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating core-site.xml from namenode configuraitions") + setup_core_site_for_required_plugins(component_user=params.knox_user, component_group=params.knox_group,create_core_site_path = params.knox_conf_dir, config = params.config) + else: + Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations") + else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 67d5769..c167d7c 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -66,6 +66,7 @@ stack_version_formatted = status_params.stack_version_formatted stack_supports_ru = stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted) stack_supports_storm_kerberos = stack_version_formatted and check_stack_feature(StackFeature.STORM_KERBEROS, stack_version_formatted) stack_supports_storm_ams = stack_version_formatted and check_stack_feature(StackFeature.STORM_AMS, stack_version_formatted) +stack_supports_core_site_for_ranger_plugin = check_stack_feature(StackFeature.CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT, stack_version_formatted) # get the correct version to use for checking stack features version_for_stack_feature_checks = get_stack_feature_version(config) http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py index caafd55..e81d62a 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py @@ -18,6 +18,10 @@ limitations under the License. """ from resource_management.core.logger import Logger +from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_core_site_for_required_plugins +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.core.resources import File, Directory def setup_ranger_storm(upgrade_type=None): """ @@ -93,8 +97,37 @@ def setup_ranger_storm(upgrade_type=None): plugin_security_properties=params.config['configurations']['ranger-storm-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-storm-security'], plugin_policymgr_ssl_properties=params.config['configurations']['ranger-storm-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-storm-policymgr-ssl'], component_list=['storm-client', 'storm-nimbus'], audit_db_is_enabled=params.xa_audit_db_is_enabled, - credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble) + + + site_files_create_path = format('{storm_component_home_dir}/extlib-daemon/ranger-storm-plugin-impl/conf') + Directory(site_files_create_path, + owner = params.storm_user, + group = params.user_group, + mode=0775, + create_parents = True, + cd_access = 'a' + ) + + if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_storm and params.has_namenode and params.security_enabled: + Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions") + setup_core_site_for_required_plugins(component_user=params.storm_user,component_group=params.user_group,create_core_site_path = site_files_create_path, config = params.config) + if len(params.namenode_hosts) > 1: + Logger.info('Ranger Storm plugin is enabled along with security and NameNode is HA , creating hdfs-site.xml') + XmlConfig("hdfs-site.xml", + conf_dir=site_files_create_path, + configurations=params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner=params.storm_user, + group=params.user_group, + mode=0644 + ) + else: + Logger.info('Ranger Storm plugin is not enabled or security is disabled, removing hdfs-site.xml') + File(format('{site_files_create_path}/hdfs-site.xml'), action="delete") + else: + Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations") else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/a8aa8ab1/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index c3044af..a753f89 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -343,6 +343,11 @@ "name": "hive_interactive_atlas_hook_required", "description": "Registering Atlas Hook for Hive Interactive.", "min_version": "2.6.0.0" + }, + { + "name": "core_site_for_ranger_plugins", + "description": "Adding core-site.xml in when Ranger plugin is enabled for Storm, Kafka, and Knox.", + "min_version": "2.6.0.0" } ] }
