Repository: ambari Updated Branches: refs/heads/trunk b88512e71 -> 3bab2125f
AMBARI-20636 Allow users to add custom configs for Ranger service in all plugins (mugdha) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3bab2125 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3bab2125 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3bab2125 Branch: refs/heads/trunk Commit: 3bab2125fc9570398ed1514aa11d9b1d910ecde1 Parents: b88512e Author: Mugdha Varadkar <mug...@apache.org> Authored: Fri Mar 31 14:45:55 2017 +0530 Committer: Mugdha Varadkar <mug...@apache.org> Committed: Mon Apr 3 11:40:04 2017 +0530 ---------------------------------------------------------------------- .../libraries/functions/setup_ranger_plugin_xml.py | 16 ++++++++++++++-- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 5 +++++ .../0.96.0.2.0/package/scripts/params_linux.py | 6 +++++- .../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 6 +++++- .../HDFS/3.0.0.3.0/package/scripts/params_linux.py | 6 +++++- .../HIVE/0.12.0.2.0/package/scripts/params_linux.py | 6 +++++- .../HIVE/2.1.0.3.0/package/scripts/params_linux.py | 6 +++++- .../KAFKA/0.8.1/package/scripts/params.py | 6 +++++- .../KNOX/0.5.0.2.2/package/scripts/params_linux.py | 8 ++++++-- .../RANGER_KMS/0.5.0.2.3/package/scripts/params.py | 5 +++++ .../STORM/0.9.1/package/scripts/params_linux.py | 6 +++++- .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 6 +++++- .../YARN/3.0.0.3.0/package/scripts/params_linux.py | 6 +++++- 13 files changed, 75 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 04a5bb1..c510dac 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -__all__ = ["setup_ranger_plugin", "get_audit_configs"] +__all__ = ["setup_ranger_plugin", "get_audit_configs", "generate_ranger_service_config"] import os import ambari_simplejson as json @@ -279,4 +279,16 @@ def get_audit_configs(config): audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" - return jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver \ No newline at end of file + return jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver + +def generate_ranger_service_config(ranger_plugin_properties): + custom_service_config_dict = {} + ranger_plugin_properties_copy = {} + ranger_plugin_properties_copy.update(ranger_plugin_properties) + + for key, value in ranger_plugin_properties_copy.iteritems(): + if key.startswith("ranger.service.config.param."): + modify_key_name = key.replace("ranger.service.config.param.","") + custom_service_config_dict[modify_key_name] = value + + return custom_service_config_dict http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 21d172b..bf4848b 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -33,6 +33,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config def configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled, metadata_protocol): @@ -395,6 +396,10 @@ if stack_supports_atlas_ranger_plugin and enable_ranger_atlas: 'ambari.service.check.user' : policy_user } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + atlas_repository_configuration.update(custom_ranger_service_config) + if security_enabled: atlas_repository_configuration['policy.download.auth.users'] = metadata_user atlas_repository_configuration['tag.download.auth.users'] = metadata_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 268d81c..e78bfc2 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -41,7 +41,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge from resource_management.libraries.script.script import Script from resource_management.libraries.functions.expect import expect from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -366,6 +366,10 @@ if enable_ranger_hbase: 'assetType': '2' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hbase_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hbase_ranger_plugin_config['policy.download.auth.users'] = hbase_user hbase_ranger_plugin_config['tag.download.auth.users'] = hbase_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 41f44c3..b8785f3 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -44,7 +44,7 @@ from resource_management.libraries.functions.get_lzo_packages import get_lzo_pac from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -515,6 +515,10 @@ if enable_ranger_hdfs: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hdfs_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py index 62a5edd..4fa6f0c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py @@ -43,7 +43,7 @@ from resource_management.libraries.functions.format_jvm_option import format_jvm from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -511,6 +511,10 @@ if enable_ranger_hdfs: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hdfs_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 880ddc5..84bac38 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -43,7 +43,7 @@ from resource_management.libraries.functions.expect import expect from resource_management.libraries import functions from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config from resource_management.libraries.functions.get_architecture import get_architecture from resource_management.core.utils import PasswordString @@ -786,6 +786,10 @@ if enable_ranger_hive: 'assetType': '3' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hive_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hive_ranger_plugin_config['policy.download.auth.users'] = hive_user hive_ranger_plugin_config['tag.download.auth.users'] = hive_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py index 880ddc5..84bac38 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py @@ -43,7 +43,7 @@ from resource_management.libraries.functions.expect import expect from resource_management.libraries import functions from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config from resource_management.libraries.functions.get_architecture import get_architecture from resource_management.core.utils import PasswordString @@ -786,6 +786,10 @@ if enable_ranger_hive: 'assetType': '3' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + hive_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: hive_ranger_plugin_config['policy.download.auth.users'] = hive_user hive_ranger_plugin_config['tag.download.auth.users'] = hive_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index b338add..5b9db89 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -34,7 +34,7 @@ from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -255,6 +255,10 @@ if enable_ranger_kafka and is_supported_kafka_ranger: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: ranger_plugin_config['policy.download.auth.users'] = kafka_user ranger_plugin_config['tag.download.auth.users'] = kafka_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index 8beb2c0..5c07fa4 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -38,7 +38,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions.constants import StackFeature from resource_management.libraries.functions import is_empty -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -356,7 +356,11 @@ if enable_ranger_knox: 'name': repo_name, 'repositoryType': 'knox', 'assetType': '5', - } + } + + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + knox_ranger_plugin_config.update(custom_ranger_service_config) if stack_supports_ranger_kerberos and security_enabled: knox_ranger_plugin_config['policy.download.auth.users'] = knox_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py index db59973..f2abe80 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py @@ -28,6 +28,7 @@ from resource_management.libraries.functions.stack_features import get_stack_fea from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.get_bare_principal import get_bare_principal from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -260,6 +261,10 @@ if stack_supports_ranger_kerberos: rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower()) kms_plugin_config['policy.download.auth.users'] = format('keyadmin,{rangerkms_bare_principal}') +custom_ranger_service_config = generate_ranger_service_config(config['configurations']['kms-properties']) +if len(custom_ranger_service_config) > 0: + kms_plugin_config.update(custom_ranger_service_config) + kms_ranger_plugin_repo = { 'isEnabled' : 'true', 'configs' : kms_plugin_config, http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 5d8a5f3..44b256e 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -41,7 +41,7 @@ from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster from resource_management.libraries.functions import is_empty from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config # server configurations config = Script.get_config() @@ -344,6 +344,10 @@ if enable_ranger_storm: 'assetType': '6' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + storm_ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos and security_enabled: policy_user = format('{storm_user},{storm_bare_jaas_principal}') storm_ranger_plugin_config['policy.download.auth.users'] = policy_user http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 03d3fb2..a4de8fa 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -35,7 +35,7 @@ from resource_management.libraries.functions.default import default from resource_management.libraries import functions from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config import status_params @@ -459,6 +459,10 @@ if enable_ranger_yarn and is_supported_yarn_ranger: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos: ranger_plugin_config['ambari.service.check.user'] = policy_user ranger_plugin_config['hadoop.security.authentication'] = 'kerberos' if security_enabled else 'simple' http://git-wip-us.apache.org/repos/asf/ambari/blob/3bab2125/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py index d069722..a8cdda9 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py @@ -35,7 +35,7 @@ from resource_management.libraries.functions.default import default from resource_management.libraries import functions from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture -from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config import status_params # a map of the Ambari role to the component name @@ -456,6 +456,10 @@ if enable_ranger_yarn and is_supported_yarn_ranger: 'assetType': '1' } + custom_ranger_service_config = generate_ranger_service_config(ranger_plugin_properties) + if len(custom_ranger_service_config) > 0: + ranger_plugin_config.update(custom_ranger_service_config) + if stack_supports_ranger_kerberos: ranger_plugin_config['ambari.service.check.user'] = policy_user ranger_plugin_config['hadoop.security.authentication'] = 'kerberos' if security_enabled else 'simple'