Repository: ambari Updated Branches: refs/heads/branch-2.5 ecd504c92 -> f8497d2c2
AMBARI-20636 Allow users to add custom configs for Ranger service in all plugins (mugdha) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8497d2c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8497d2c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8497d2c Branch: refs/heads/branch-2.5 Commit: f8497d2c2e9f7d0de095b7f039d066e553d9cc36 Parents: ecd504c Author: Mugdha Varadkar <mug...@apache.org> Authored: Fri Mar 31 14:40:06 2017 +0530 Committer: Mugdha Varadkar <mug...@apache.org> Committed: Mon Apr 3 11:33:38 2017 +0530 ---------------------------------------------------------------------- .../libraries/functions/setup_ranger_plugin_xml.py | 16 ++++++++++++++-- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 5 +++++ .../0.96.0.2.0/package/scripts/params_linux.py | 6 +++++- .../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 6 +++++- .../HIVE/0.12.0.2.0/package/scripts/params_linux.py | 6 +++++- .../KAFKA/0.8.1/package/scripts/params.py | 6 +++++- .../KNOX/0.5.0.2.2/package/scripts/params_linux.py | 8 ++++++-- .../RANGER_KMS/0.5.0.2.3/package/scripts/params.py | 5 +++++ .../STORM/0.9.1/package/scripts/params_linux.py | 6 +++++- .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 6 +++++- 10 files changed, 60 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 04a5bb1..c510dac 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -__all__ = ["setup_ranger_plugin", "get_audit_configs"] +__all__ = ["setup_ranger_plugin", "get_audit_configs", "generate_ranger_service_config"] import os import ambari_simplejson as json @@ -279,4 +279,16 @@ def get_audit_configs(config): audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" - return jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver \ No newline at end of file + return jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver + +def generate_ranger_service_config(ranger_plugin_properties): + custom_service_config_dict = {} + ranger_plugin_properties_copy = {} + ranger_plugin_properties_copy.update(ranger_plugin_properties) + + for key, value in ranger_plugin_properties_copy.iteritems(): + if key.startswith("ranger.service.config.param."): + modify_key_name = key.replace("ranger.service.config.param.","") + custom_service_config_dict[modify_key_name] = value + + return custom_service_config_dict http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 6d46d7c..e243662 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -31,6 +31,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config def configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled, metadata_protocol): @@ -393,6 +394,10 @@ if stack_supports_atlas_ranger_plugin and enable_ranger_atlas: 'ambari.service.check.user' : policy_user } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + atlas_repository_configuration.update(custom_ranger_service_config) + if security_enabled: atlas_repository_configuration['policy.download.auth.users'] = metadata_user atlas_repository_configuration['tag.download.auth.users'] = metadata_user http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index ab8a4d9..bae6161 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -41,7 +41,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.script.script import Script from resource_management.libraries.functions.expect import expect from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -366,6 +366,10 @@ if enable_ranger_hbase: 'assetType': '2' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hbase_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hbase_ranger_plugin_config['policy.download.auth.users'] = hbase_user hbase_ranger_plugin_config['tag.download.auth.users'] = hbase_user http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 07cb409..35ad895 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -44,7 +44,7 @@ from resource_management.libraries.functions.get_lzo_packages import get_lzo_pac from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -516,6 +516,10 @@ if enable_ranger_hdfs: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hdfs_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index c5d2d13..5f2a36d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -43,7 +43,7 @@ from resource_management.libraries.functions.expect import expect from resource_management.libraries import functions from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config from resource_management.libraries.functions.get_architecture import get_architecture from resource_management.core.utils import PasswordString @@ -785,6 +785,10 @@ if enable_ranger_hive: 'assetType': '3' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hive_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hive_ranger_plugin_config['policy.download.auth.users'] = hive_user hive_ranger_plugin_config['tag.download.auth.users'] = hive_user http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 69d801a..f56c2b7 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -34,7 +34,7 @@ from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -255,6 +255,10 @@ if enable_ranger_kafka and is_supported_kafka_ranger: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: ranger_plugin_config['policy.download.auth.users'] = kafka_user ranger_plugin_config['tag.download.auth.users'] = kafka_user http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index f461f41..22f00c8 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -39,7 +39,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions.constants import StackFeature from resource_management.libraries.functions import is_empty -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -362,7 +362,11 @@ if enable_ranger_knox: 'name': repo_name, 'repositoryType': 'knox', 'assetType': '5', - } + } + + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + knox_ranger_plugin_config.update(custom_ranger_service_config) if stack_supports_ranger_kerberos and security_enabled: knox_ranger_plugin_config['policy.download.auth.users'] = knox_user http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py index db59973..f2abe80 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py @@ -28,6 +28,7 @@ from resource_management.libraries.functions.stack_features import get_stack_fea from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.get_bare_principal import get_bare_principal from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -260,6 +261,10 @@ if stack_supports_ranger_kerberos: rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower()) kms_plugin_config['policy.download.auth.users'] = format('keyadmin,{rangerkms_bare_principal}') +custom_ranger_service_config = generate_ranger_service_config(config['configurations']['kms-properties']) +if len(custom_ranger_service_config) > 0: + kms_plugin_config.update(custom_ranger_service_config) + kms_ranger_plugin_repo = { 'isEnabled' : 'true', 'configs' : kms_plugin_config, http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 5d8a5f3..44b256e 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -41,7 +41,7 @@ from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.functions import is_empty from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -344,6 +344,10 @@ if enable_ranger_storm: 'assetType': '6' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + storm_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: policy_user = format('{storm_user},{storm_bare_jaas_principal}') storm_ranger_plugin_config['policy.download.auth.users'] = policy_user http://git-wip-us.apache.org/repos/asf/ambari/blob/f8497d2c/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index f9228be..d7868d3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -35,7 +35,7 @@ from resource_management.libraries.functions.default import default from resource_management.libraries import functions from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config import status_params @@ -452,6 +452,10 @@ if enable_ranger_yarn and is_supported_yarn_ranger: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos: ranger_plugin_config['ambari.service.check.user'] = policy_user ranger_plugin_config['hadoop.security.authentication'] = 'kerberos' if security_enabled else 'simple'