AMBARI-10924. Stack changes for supporting modifications done in Ranger for HDP 2.3 (Gautam Borad via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e50a2ac3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e50a2ac3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e50a2ac3 Branch: refs/heads/trunk Commit: e50a2ac31b28e90ba1cfee70bf337c65910b7ee7 Parents: 9c570b8 Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Thu May 14 10:30:50 2015 -0700 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Thu May 14 10:31:10 2015 -0700 ---------------------------------------------------------------------- .../libraries/functions/setup_ranger_plugin.py | 8 +- .../functions/setup_ranger_plugin_xml.py | 162 ++++++++ .../0.96.0.2.0/package/scripts/params_linux.py | 33 +- .../package/scripts/setup_ranger_hbase.py | 17 +- .../2.1.0.2.0/package/scripts/params_linux.py | 30 +- .../package/scripts/setup_ranger_hdfs.py | 25 +- .../0.12.0.2.0/package/scripts/params_linux.py | 29 +- .../package/scripts/setup_ranger_hive.py | 25 +- .../0.5.0.2.2/package/scripts/params_linux.py | 23 +- .../package/scripts/setup_ranger_knox.py | 17 +- .../RANGER/0.4.0/configuration/ranger-env.xml | 16 +- .../RANGER/0.4.0/package/scripts/params.py | 42 +- .../0.4.0/package/scripts/ranger_admin.py | 19 +- .../0.4.0/package/scripts/ranger_usersync.py | 8 +- .../0.4.0/package/scripts/setup_ranger.py | 7 + .../0.4.0/package/scripts/setup_ranger_xml.py | 195 +++++++++ .../0.9.1.2.1/package/scripts/params_linux.py | 26 +- .../package/scripts/setup_ranger_storm.py | 25 +- .../2.1.0.2.0/package/scripts/params_linux.py | 23 +- .../package/scripts/setup_ranger_yarn.py | 20 +- .../HBASE/configuration/ranger-hbase-audit.xml | 270 ++++++++++++ .../ranger-hbase-plugin-properties.xml | 130 +++++- .../ranger-hbase-policymgr-ssl.xml | 59 +++ .../configuration/ranger-hbase-security.xml | 64 +++ .../HDFS/configuration/ranger-hdfs-audit.xml | 270 ++++++++++++ .../ranger-hdfs-plugin-properties.xml | 125 +++++- .../configuration/ranger-hdfs-policymgr-ssl.xml | 59 +++ .../HDFS/configuration/ranger-hdfs-security.xml | 64 +++ .../HIVE/configuration/ranger-hive-audit.xml | 270 ++++++++++++ .../ranger-hive-plugin-properties.xml | 130 +++++- .../configuration/ranger-hive-policymgr-ssl.xml | 59 +++ .../HIVE/configuration/ranger-hive-security.xml | 65 +++ .../KNOX/configuration/ranger-knox-audit.xml | 270 ++++++++++++ .../ranger-knox-plugin-properties.xml | 125 +++++- .../configuration/ranger-knox-policymgr-ssl.xml | 59 +++ .../KNOX/configuration/ranger-knox-security.xml | 59 +++ .../RANGER/configuration/admin-properties.xml | 60 ++- .../RANGER/configuration/ranger-admin-site.xml | 227 ++++++++++ .../RANGER/configuration/ranger-env.xml | 34 ++ .../RANGER/configuration/ranger-site.xml | 66 +++ .../RANGER/configuration/ranger-ugsync-site.xml | 281 +++++++++++++ .../configuration/usersync-properties.xml | 108 +++++ .../stacks/HDP/2.3/services/RANGER/metainfo.xml | 8 + .../STORM/configuration/ranger-storm-audit.xml | 270 ++++++++++++ .../ranger-storm-plugin-properties.xml | 125 +++++- .../ranger-storm-policymgr-ssl.xml | 59 +++ .../configuration/ranger-storm-security.xml | 59 +++ .../YARN/configuration/ranger-yarn-audit.xml | 270 ++++++++++++ .../configuration/ranger-yarn-policymgr-ssl.xml | 59 +++ .../YARN/configuration/ranger-yarn-security.xml | 59 +++ .../test/python/stacks/2.2/configs/default.json | 3 +- .../2.2/configs/ranger-admin-upgrade.json | 4 +- .../2.2/configs/ranger-usersync-upgrade.json | 3 +- .../test/python/stacks/2.2/configs/secured.json | 3 +- .../test/python/stacks/2.3/configs/default.json | 5 +- ambari-web/app/data/HDP2.3/site_properties.js | 414 ++++++++++++++++++- 56 files changed, 4773 insertions(+), 172 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py index eb22926..35d4953 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py @@ -20,9 +20,9 @@ limitations under the License. __all__ = ["setup_ranger_plugin"] import os +from datetime import datetime from resource_management.libraries.functions.ranger_functions import Rangeradmin -from resource_management.core.resources import File -from resource_management.core.resources import Execute +from resource_management.core.resources import File, Execute from resource_management.libraries.functions.format import format from resource_management.libraries.functions.get_hdp_version import get_hdp_version from resource_management.core.logger import Logger @@ -37,7 +37,7 @@ def setup_ranger_plugin(component_select_name, service_name, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, - plugin_enabled,api_version=None): + plugin_enabled,api_version=None, **kwargs): File(downloaded_custom_connector, content = DownloadSource(driver_curl_source) ) @@ -77,4 +77,4 @@ def setup_ranger_plugin(component_select_name, service_name, environment=cmd_env, logoutput=True, sudo=True, - ) + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py new file mode 100644 index 0000000..494b22f --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +__all__ = ["setup_ranger_plugin"] + + +import os +from datetime import datetime +from resource_management.libraries.functions.ranger_functions import Rangeradmin +from resource_management.core.resources import File, Directory, Execute +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.get_hdp_version import get_hdp_version +from resource_management.core.logger import Logger +from resource_management.core.source import DownloadSource, InlineTemplate +from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2 + + +def setup_ranger_plugin(component_select_name, service_name, + component_downloaded_custom_connector, component_driver_curl_source, + component_driver_curl_target, java_home, + repo_name, plugin_repo_dict, + ranger_env_properties, plugin_properties, + policy_user, policymgr_mgr_url, + plugin_enabled, conf_dict, component_user, component_group, + cache_service_list, plugin_audit_properties, plugin_audit_attributes, + plugin_security_properties, plugin_security_attributes, + plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, + component_list, audit_db_is_enabled, credential_file, + xa_audit_db_password, ssl_truststore_password, + ssl_keystore_password, api_version=None): + + File(component_downloaded_custom_connector, + content = DownloadSource(component_driver_curl_source) + ) + + Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target), + not_if=format("test -f {component_driver_curl_target}"), + sudo=True + ) + + hdp_version = get_hdp_version(component_select_name) + component_conf_dir = conf_dict + + if plugin_enabled: + + if api_version == 'v2' and api_version is not None: + ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url) + else: + ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url) + + ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, + ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], + ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], + policy_user) + + current_datetime = datetime.now() + + File(format('{component_conf_dir}/ranger-security.xml'), + owner = component_user, + group = component_group, + mode = 0644, + content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')) + ) + + Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')], + owner = component_user, + group = component_group, + mode=0775, + recursive = True + ) + + for cache_service in cache_service_list: + File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')), + owner = component_user, + group = component_group, + mode = 0644 + ) + + XmlConfig(format('ranger-{service_name}-audit.xml'), + conf_dir=component_conf_dir, + configurations=plugin_audit_properties, + configuration_attributes=plugin_audit_attributes, + owner = component_user, + group = component_group, + mode=0744) + + XmlConfig(format('ranger-{service_name}-security.xml'), + conf_dir=component_conf_dir, + configurations=plugin_security_properties, + configuration_attributes=plugin_security_attributes, + owner = component_user, + group = component_group, + mode=0744) + + XmlConfig("ranger-policymgr-ssl.xml", + conf_dir=component_conf_dir, + configurations=plugin_policymgr_ssl_properties, + configuration_attributes=plugin_policymgr_ssl_attributes, + owner = component_user, + group = component_group, + mode=0744) + + setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list) + + setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, + xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, + component_user, component_group) + + else: + File(format('{component_conf_dir}/ranger-security.xml'), + action="delete" + ) + + +def setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list): + + jar_files = os.listdir(format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/lib')) + + for jar_file in jar_files: + for component in component_list: + Execute(('ln','-sf',format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/lib/{jar_file}'),format('/usr/hdp/current/{component}/lib/{jar_file}')), + not_if=format('ls /usr/hdp/current/{component}/lib/{jar_file}'), + only_if=format('ls /usr/hdp/{hdp_version}/ranger-{service_name}-plugin/lib/{jar_file}'), + sudo=True) + +def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password, + ssl_truststore_password, ssl_keystore_password, component_user, component_group): + + cred_lib_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install/lib/*') + cred_setup_prefix = format('python /usr/hdp/{hdp_version}/ranger-{service_name}-plugin/ranger_credential_helper.py -l "{cred_lib_path}"') + + if audit_db_is_enabled: + cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "auditDBCred" -v "{xa_audit_db_password}" -c 1') + Execute(cred_setup, logoutput=True) + + cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "sslKeyStore" -v "{ssl_keystore_password}" -c 1') + Execute(cred_setup, logoutput=True) + + cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "sslTrustStore" -v "{ssl_truststore_password}" -c 1') + Execute(cred_setup, logoutput=True) + + File(credential_file, + owner = component_user, + group = component_group + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index c077f54..500d1ec 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -186,17 +186,15 @@ HdfsResource = functools.partial( # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 - +has_ranger_admin = not len(ranger_admin_hosts) == 0 +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] # ranger hbase properties policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR'] -xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] -xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_hbase' @@ -209,7 +207,6 @@ hbase_security_authentication = config['configurations']['hbase-site']['hbase.se hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication'] repo_config_username = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] -repo_config_password = config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-hbase-plugin-properties'] @@ -218,24 +215,27 @@ policy_user = config['configurations']['ranger-hbase-plugin-properties']['policy #For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' + if has_ranger_admin: enable_ranger_hbase = (config['configurations']['ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled'].lower() == 'yes') - - if xa_audit_db_flavor.lower() == 'mysql': + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) + repo_config_password = unicode(config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + + if xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" - elif xa_audit_db_flavor.lower() == 'oracle': + elif xa_audit_db_flavor == 'oracle': jdbc_jar_name = "ojdbc6.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'postgres': + elif xa_audit_db_flavor == 'postgres': jdbc_jar_name = "postgresql.jar" jdbc_symlink_name = "postgres-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'sqlserver': + elif xa_audit_db_flavor == 'sqlserver': jdbc_jar_name = "sqljdbc4.jar" jdbc_symlink_name = "mssql-jdbc-driver.jar" downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}") - driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") @@ -250,7 +250,7 @@ if has_ranger_admin: 'commonNameForCertificate': common_name_for_certificate, 'hbase.master.kerberos.principal': master_jaas_princ if security_enabled else '' } - + hbase_ranger_plugin_repo = { 'isActive': 'true', 'config': json.dumps(hbase_ranger_plugin_config), @@ -260,5 +260,10 @@ if has_ranger_admin: 'assetType': '2' } - - + if xml_configurations_supported: + xa_audit_db_is_enabled = config['configurations']['ranger-hbase-audit']['xasecure.audit.db.is.enabled'] + ssl_keystore_file_path = config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore'] + ssl_truststore_file_path = config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore'] + ssl_keystore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) + ssl_truststore_password = unicode(config['configurations']['ranger-hbase-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py index e767d32..88a6686 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py @@ -17,19 +17,32 @@ See the License for the specific language governing permissions and limitations under the License. """ -from resource_management import * +from resource_management.core.logger import Logger def setup_ranger_hbase(): import params if params.has_ranger_admin: + + if params.xml_configurations_supported: + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + else: + from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + setup_ranger_plugin('hbase-client', 'hbase', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, params.repo_name, params.hbase_ranger_plugin_repo, params.ranger_env, params.ranger_plugin_properties, params.policy_user, params.policymgr_mgr_url, - params.enable_ranger_hbase + params.enable_ranger_hbase, conf_dict=params.hbase_conf_dir, + component_user=params.hbase_user, component_group=params.user_group, cache_service_list=['hbaseMaster', 'hbaseRegional'], + plugin_audit_properties=params.config['configurations']['ranger-hbase-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hbase-audit'], + plugin_security_properties=params.config['configurations']['ranger-hbase-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hbase-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hbase-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hbase-policymgr-ssl'], + component_list=['hbase-client', 'hbase-master', 'hbase-regionserver'], audit_db_is_enabled=params.xa_audit_db_password, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password ) else: Logger.info('Ranger admin not installed') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 18aede8..9cc2831 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -332,16 +332,14 @@ mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefi # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 - +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] #ranger hdfs properties policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR'] -xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] -xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_hadoop' @@ -353,7 +351,6 @@ hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties' common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate'] repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] -repo_config_password = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] if security_enabled: sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/_h...@example.com") @@ -366,24 +363,27 @@ policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_ #For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' + if has_ranger_admin: enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes') - - if xa_audit_db_flavor.lower() == 'mysql': + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) + repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + + if xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" - elif xa_audit_db_flavor.lower() == 'oracle': + elif xa_audit_db_flavor == 'oracle': jdbc_jar_name = "ojdbc6.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'postgres': + elif xa_audit_db_flavor == 'postgres': jdbc_jar_name = "postgresql.jar" jdbc_symlink_name = "postgres-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'sqlserver': + elif xa_audit_db_flavor == 'sqlserver': jdbc_jar_name = "sqljdbc4.jar" jdbc_symlink_name = "mssql-jdbc-driver.jar" downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") - driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") @@ -400,7 +400,7 @@ if has_ranger_admin: 'dfs.namenode.kerberos.principal': nn_principal_name if security_enabled else '', 'dfs.secondary.namenode.kerberos.principal': sn_principal_name if security_enabled else '' } - + hdfs_ranger_plugin_repo = { 'isActive': 'true', 'config': json.dumps(hdfs_ranger_plugin_config), @@ -409,3 +409,11 @@ if has_ranger_admin: 'repositoryType': 'hdfs', 'assetType': '1' } + + if xml_configurations_supported: + xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.db.is.enabled'] + ssl_keystore_file_path = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore'] + ssl_truststore_file_path = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore'] + ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) + ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py index e73398f..4226e51 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py @@ -17,19 +17,32 @@ See the License for the specific language governing permissions and limitations under the License. """ -from resource_management import * +from resource_management.core.logger import Logger def setup_ranger_hdfs(): import params - + if params.has_ranger_admin: - setup_ranger_plugin('hadoop-client', 'hdfs', + + if params.xml_configurations_supported: + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + else: + from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + + setup_ranger_plugin('hadoop-client', 'hdfs', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java_home, params.repo_name, params.hdfs_ranger_plugin_repo, params.ranger_env, params.ranger_plugin_properties, params.policy_user, params.policymgr_mgr_url, - params.enable_ranger_hdfs - ) + params.enable_ranger_hdfs, conf_dict=params.hadoop_conf_dir, + component_user=params.hdfs_user, component_group=params.user_group, cache_service_list=['hdfs'], + plugin_audit_properties=params.config['configurations']['ranger-hdfs-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hdfs-audit'], + plugin_security_properties=params.config['configurations']['ranger-hdfs-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hdfs-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hdfs-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hdfs-policymgr-ssl'], + component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_password, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password + ) else: - Logger.info('Ranger admin not installed') \ No newline at end of file + Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index c1b6d24..fb90b57 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -385,14 +385,13 @@ ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 if Script.is_hdp_stack_greater_or_equal("2.2"): enable_ranger_hive = (config['configurations']['ranger-hive-plugin-properties']['ranger-hive-plugin-enabled'].lower() == 'yes') +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] #ranger hive properties policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR'] -xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] -xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_hive' @@ -400,7 +399,6 @@ jdbc_driver_class_name = config['configurations']['ranger-hive-plugin-properties common_name_for_certificate = config['configurations']['ranger-hive-plugin-properties']['common.name.for.certificate'] repo_config_username = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] -repo_config_password = config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-hive-plugin-properties'] @@ -408,19 +406,22 @@ policy_user = config['configurations']['ranger-hive-plugin-properties']['policy_ if security_enabled: hive_principal = hive_server_principal.replace('_HOST',hostname.lower()) - + #For curl command in ranger plugin to get db connector if has_ranger_admin: - if xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'mysql': + repo_config_password = unicode(config['configurations']['ranger-hive-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + + if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql': ranger_jdbc_symlink_name = "mysql-jdbc-driver.jar" ranger_jdbc_jar_name = "mysql-connector-java.jar" - elif xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'oracle': + elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle': ranger_jdbc_jar_name = "ojdbc6.jar" ranger_jdbc_symlink_name = "oracle-jdbc-driver.jar" - elif xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'postgres': + elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres': ranger_jdbc_jar_name = "postgresql.jar" ranger_jdbc_symlink_name = "postgres-jdbc-driver.jar" - elif xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'sqlserver': + elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqlserver': ranger_jdbc_jar_name = "sqljdbc4.jar" ranger_jdbc_symlink_name = "mssql-jdbc-driver.jar" @@ -428,7 +429,7 @@ if has_ranger_admin: ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_symlink_name}") ranger_driver_curl_target = format("{java_share_dir}/{ranger_jdbc_jar_name}") - + hive_ranger_plugin_config = { 'username': repo_config_username, 'password': repo_config_password, @@ -445,3 +446,13 @@ if has_ranger_admin: 'repositoryType': 'hive', 'assetType': '3' } + + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) + + if xml_configurations_supported: + xa_audit_db_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.db.is.enabled'] + ssl_keystore_file_path = config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore'] + ssl_truststore_file_path = config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore'] + ssl_keystore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) + ssl_truststore_password = unicode(config['configurations']['ranger-hive-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py index f6b6f94..12c2894 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py @@ -17,19 +17,32 @@ See the License for the specific language governing permissions and limitations under the License. """ -from resource_management import * +from resource_management.core.logger import Logger def setup_ranger_hive(): import params - + if params.has_ranger_admin: + + if params.xml_configurations_supported: + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + else: + from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + setup_ranger_plugin('hive-server2', 'hive', - params.downloaded_custom_connector, params.driver_curl_source, - params.driver_curl_target, params.java64_home, + params.ranger_downloaded_custom_connector, params.ranger_driver_curl_source, + params.ranger_driver_curl_target, params.java64_home, params.repo_name, params.hive_ranger_plugin_repo, params.ranger_env, params.ranger_plugin_properties, params.policy_user, params.policymgr_mgr_url, - params.enable_ranger_hive + params.enable_ranger_hive, conf_dict=params.hive_server_conf_dir, + component_user=params.hive_user, component_group=params.user_group, cache_service_list=['hiveServer2'], + plugin_audit_properties=params.config['configurations']['ranger-hive-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hive-audit'], + plugin_security_properties=params.config['configurations']['ranger-hive-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hive-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hive-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hive-policymgr-ssl'], + component_list=['hive-client', 'hive-metastore', 'hive-server2'], audit_db_is_enabled=params.xa_audit_db_password, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password ) else: - Logger.info('Ranger admin not installed') \ No newline at end of file + Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index 5dfc2d7..99c6b48 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -155,16 +155,15 @@ if security_enabled: # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] # ranger knox properties policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR'] -xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] -xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_knox' @@ -172,7 +171,6 @@ knox_home = config['configurations']['ranger-knox-plugin-properties']['KNOX_HOME common_name_for_certificate = config['configurations']['ranger-knox-plugin-properties']['common.name.for.certificate'] repo_config_username = config['configurations']['ranger-knox-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] -repo_config_password = config['configurations']['ranger-knox-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-knox-plugin-properties'] @@ -183,17 +181,20 @@ jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' if has_ranger_admin: enable_ranger_knox = (config['configurations']['ranger-knox-plugin-properties']['ranger-knox-plugin-enabled'].lower() == 'yes') + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) + repo_config_password = unicode(config['configurations']['ranger-knox-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() - if xa_audit_db_flavor.lower() == 'mysql': + if xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" - elif xa_audit_db_flavor.lower() == 'oracle': + elif xa_audit_db_flavor == 'oracle': jdbc_jar_name = "ojdbc6.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'postgres': + elif xa_audit_db_flavor == 'postgres': jdbc_jar_name = "postgresql.jar" jdbc_symlink_name = "postgres-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'sqlserver': + elif xa_audit_db_flavor == 'sqlserver': jdbc_jar_name = "sqljdbc4.jar" jdbc_symlink_name = "mssql-jdbc-driver.jar" @@ -217,3 +218,11 @@ if has_ranger_admin: 'repositoryType': 'knox', 'assetType': '5', } + + if xml_configurations_supported: + xa_audit_db_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.db.is.enabled'] + ssl_keystore_file_path = config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.keystore'] + ssl_truststore_file_path = config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.truststore'] + ssl_keystore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) + ssl_truststore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py index 5b79fd9..ee818b6 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py @@ -17,19 +17,32 @@ See the License for the specific language governing permissions and limitations under the License. """ -from resource_management import * +from resource_management.core.logger import Logger def setup_ranger_knox(): import params if params.has_ranger_admin: + + if params.xml_configurations_supported: + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + else: + from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + setup_ranger_plugin('knox-server', 'knox', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java_home, params.repo_name, params.knox_ranger_plugin_repo, params.ranger_env, params.ranger_plugin_properties, params.policy_user, params.policymgr_mgr_url, - params.enable_ranger_knox + params.enable_ranger_knox, conf_dict=params.knox_conf_dir, + component_user=params.knox_user, component_group=params.knox_group, cache_service_list=['knox'], + plugin_audit_properties=params.config['configurations']['ranger-knox-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-knox-audit'], + plugin_security_properties=params.config['configurations']['ranger-knox-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-knox-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-knox-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-knox-policymgr-ssl'], + component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_password, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password ) else: Logger.info('Ranger admin not installed') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml index 50ed09d..e47a251 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml @@ -78,9 +78,15 @@ <value>-</value> <property-type>TEXT</property-type> <description>Oracle Home needs to be set to path where oracle is installed, this will help install Ranger Admin when used with Oracle as database.</description> - </property> + </property> + + <property> + <name>xml_configurations_supported</name> + <value>false</value> + <description></description> + </property> - <property><!-- #for DB Test connection command --> + <property> <name>ranger_jdbc_connection_url</name> <value>jdbc:mysql://localhost</value> <description>Ranger JDBC connection url, mainly used for database test connection</description> @@ -92,4 +98,10 @@ <description>Ranger JDBC driver, mainly used for database test connection</description> </property> + <property> + <name>ranger_pid_dir</name> + <value>/var/run/ranger</value> + <description></description> + </property> + </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py index befad8d..8a241da 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from resource_management.libraries.functions import format +import os from resource_management.libraries.script import Script from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.format import format @@ -42,6 +42,8 @@ host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] + stack_is_hdp22_or_further = Script.is_hdp_stack_greater_or_equal("2.2") stack_is_hdp23_or_further = Script.is_hdp_stack_greater_or_equal("2.3") @@ -53,23 +55,32 @@ if stack_is_hdp22_or_further: usersync_home = '/usr/hdp/current/ranger-usersync' usersync_start = '/usr/bin/ranger-usersync-start' usersync_stop = '/usr/bin/ranger-usersync-stop' + ranger_ugsync_conf = '/etc/ranger/usersync/conf' usersync_services_file = "/usr/hdp/current/ranger-usersync/ranger-usersync-services.sh" java_home = config['hostLevelParams']['java_home'] unix_user = config['configurations']['ranger-env']['ranger_user'] unix_group = config['configurations']['ranger-env']['ranger_group'] +ranger_pid_dir = config['configurations']['ranger-env']['ranger_pid_dir'] +usersync_log_dir = config['configurations']['ranger-env']['ranger_usersync_log_dir'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] -db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] - +db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() usersync_exturl = config['configurations']['admin-properties']['policymgr_external_url'] +ranger_host = config['clusterHostInfo']['ranger_admin_hosts'][0] +ranger_external_url = config['configurations']['admin-properties']['policymgr_external_url'] +ranger_db_name = config['configurations']['admin-properties']['db_name'] +ranger_auditdb_name = config['configurations']['admin-properties']['audit_db_name'] sql_command_invoker = config['configurations']['admin-properties']['SQL_COMMAND_INVOKER'] db_root_user = config['configurations']['admin-properties']['db_root_user'] db_root_password = unicode(config['configurations']['admin-properties']['db_root_password']) db_host = config['configurations']['admin-properties']['db_host'] +ranger_db_user = config['configurations']['admin-properties']['db_user'] +ranger_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] +ranger_db_password = unicode(config['configurations']['admin-properties']['db_password']) #ranger-env properties oracle_home = default("/configurations/ranger-env/oracle_home", "-") @@ -93,4 +104,27 @@ elif db_flavor and db_flavor.lower() == 'sqlserver': downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") -driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") \ No newline at end of file +driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") + +#for db connection +check_db_connection_jar_name = "DBConnectionVerification.jar" +check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +ranger_jdbc_connection_url = config["configurations"]["ranger-env"]["ranger_jdbc_connection_url"] +ranger_jdbc_driver = config["configurations"]["ranger-env"]["ranger_jdbc_driver"] + +ranger_credential_provider_path = config["configurations"]["ranger-admin-site"]["ranger.credential.provider.path"] +ranger_jpa_jdbc_credential_alias = config["configurations"]["ranger-admin-site"]["ranger.jpa.jdbc.credential.alias"] +ranger_ambari_db_password = unicode(config["configurations"]["admin-properties"]["db_password"]) + +ranger_jpa_audit_jdbc_credential_alias = config["configurations"]["ranger-admin-site"]["ranger.jpa.audit.jdbc.credential.alias"] +ranger_ambari_audit_db_password = unicode(config["configurations"]["admin-properties"]["audit_db_password"]) + +ugsync_jceks_path = config["configurations"]["ranger-ugsync-site"]["ranger.usersync.credstore.filename"] +cred_lib_path = os.path.join(ranger_home,"cred","lib","*") +cred_setup_prefix = format('python {ranger_home}/ranger_credential_helper.py -l "{cred_lib_path}"') +ranger_audit_source_type = config["configurations"]["ranger-admin-site"]["ranger.audit.source.type"] +if xml_configurations_supported: + ranger_usersync_keystore_password = unicode(config["configurations"]["ranger-ugsync-site"]["ranger.usersync.keystore.password"]) + ranger_usersync_ldap_ldapbindpassword = unicode(config["configurations"]["ranger-ugsync-site"]["ranger.usersync.ldap.ldapbindpassword"]) + ranger_usersync_truststore_password = unicode(config["configurations"]["ranger-ugsync-site"]["ranger.usersync.truststore.password"]) + http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py index f88625e..f56f860 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py @@ -23,7 +23,6 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.libraries.functions.format import format from resource_management.core.logger import Logger from resource_management.core import shell -from setup_ranger import setup_ranger_admin from ranger_service import ranger_service import upgrade @@ -34,8 +33,18 @@ class RangerAdmin(Script): def install(self, env): self.install_packages(env) + import params + env.set_params(params) + if params.xml_configurations_supported: + from setup_ranger_xml import setup_ranger_db + setup_ranger_db() + self.configure(env) + if params.xml_configurations_supported: + from setup_ranger_xml import setup_java_patch + setup_java_patch() + def stop(self, env, rolling_restart=False): import params @@ -66,8 +75,12 @@ class RangerAdmin(Script): def configure(self, env): import params env.set_params(params) - - setup_ranger_admin() + if params.xml_configurations_supported: + from setup_ranger_xml import ranger + else: + from setup_ranger import ranger + + ranger('ranger_admin') if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py index a31a369..4ef9377 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py @@ -24,7 +24,6 @@ from resource_management.libraries.functions.format import format from resource_management.core.logger import Logger from resource_management.core import shell from ranger_service import ranger_service -from setup_ranger import setup_usersync import upgrade class RangerUsersync(Script): @@ -36,8 +35,13 @@ class RangerUsersync(Script): def configure(self, env): import params env.set_params(params) + + if params.xml_configurations_supported: + from setup_ranger_xml import ranger + else: + from setup_ranger import ranger - setup_usersync() + ranger('ranger_usersync') def start(self, env, rolling_restart=False): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py index cc85b90..36a5f29 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py @@ -23,6 +23,13 @@ import os from resource_management import * from resource_management.core.logger import Logger +def ranger(name=None): + if name == 'ranger_admin': + setup_ranger_admin() + + if name == 'ranger_usersync': + setup_usersync() + def setup_ranger_admin(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py new file mode 100644 index 0000000..0230b60 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +from resource_management.core.logger import Logger +from resource_management.core.resources.system import File, Directory, Execute +from resource_management.core.source import DownloadSource +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.resources.modify_properties_file import ModifyPropertiesFile +from resource_management.core.exceptions import Fail +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.is_empty import is_empty + +# This file contains functions used for setup/configure of Ranger Admin and Ranger Usersync. +# The design is to mimic what is done by the setup.sh script bundled by Ranger component currently. + +def ranger(name=None): + """ + parameter name: name of ranger service component + """ + if name == 'ranger_admin': + setup_ranger_admin() + + if name == 'ranger_usersync': + setup_usersync() + +def setup_ranger_admin(): + import params + + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + ) + + db_connection_check_command = format( + "{java_home}/bin/java -cp {check_db_connection_jar}:{driver_curl_target} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}") + + Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10) + + Execute(('ln','-sf', format('{ranger_home}/ews/webapp/WEB-INF/classes/conf'), format('{ranger_home}/conf')), + not_if=format("ls {ranger_home}/conf"), + only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"), + sudo=True) + + Execute(('chown','-R',format('{unix_user}:{unix_group}'), format('{ranger_home}/')), sudo=True) + + Execute(('ln','-sf', format('{ranger_home}/ews/ranger-admin-services.sh'),'/usr/bin/ranger-admin'), + not_if=format("ls /usr/bin/ranger-admin"), + only_if=format("ls {ranger_home}/ews/ranger-admin-services.sh"), + sudo=True) + + XmlConfig("ranger-admin-site.xml", + conf_dir=params.ranger_conf, + configurations=params.config['configurations']['ranger-admin-site'], + configuration_attributes=params.config['configuration_attributes']['ranger-admin-site'], + owner=params.unix_user, + group=params.unix_group, + mode=0644) + + Directory(os.path.join(params.ranger_conf,'ranger_jaas'), + mode=0700, + owner=params.unix_user, + group=params.unix_group, + ) + + do_keystore_setup() + + +def setup_ranger_db(): + import params + + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source) + ) + + Directory(params.java_share_dir, + mode=0755 + ) + + if not os.path.isfile(params.driver_curl_target): + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target), + path=["/bin", "/usr/bin/"], + not_if=format("test -f {driver_curl_target}"), + sudo=True) + + if not os.path.isfile(os.path.join(params.ranger_home, 'ews', 'lib',params.jdbc_jar_name)): + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(params.ranger_home, 'ews', 'lib')), + path=["/bin", "/usr/bin/"], + sudo=True) + + ModifyPropertiesFile(format("{ranger_home}/install.properties"), + properties = params.config['configurations']['admin-properties'] + ) + + dba_setup = format('python {ranger_home}/dba_script.py -q') + db_setup = format('python {ranger_home}/db_setup.py') + + Execute(dba_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + Execute(db_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + + +def setup_java_patch(): + import params + + setup_java_patch = format('python {ranger_home}/db_setup.py -javapatch') + Execute(setup_java_patch, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True) + + +def do_keystore_setup(): + import params + + if not is_empty(params.ranger_credential_provider_path): + jceks_path = params.ranger_credential_provider_path + cred_setup = format('{cred_setup_prefix} -f {jceks_path} -k "{ranger_jpa_jdbc_credential_alias}" -v "{ranger_ambari_db_password}" -c 1') + + Execute(cred_setup, logoutput=True) + + if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password): + jceks_path = params.ranger_credential_provider_path + cred_setup = format('{cred_setup_prefix} -f {jceks_path} -k "{ranger_jpa_audit_jdbc_credential_alias}" -v "{ranger_ambari_audit_db_password}" -c 1') + + Execute(cred_setup, logoutput=True) + + File(params.ranger_credential_provider_path, + owner = params.unix_user, + group = params.unix_group + ) + + +def setup_usersync(): + import params + + Directory(params.ranger_pid_dir, + mode=0750, + owner = params.unix_user, + group = params.unix_group + ) + + Directory(params.usersync_log_dir, + owner = params.unix_user, + group = params.unix_group + ) + + XmlConfig("ranger-ugsync-site.xml", + conf_dir=params.ranger_ugsync_conf, + configurations=params.config['configurations']['ranger-ugsync-site'], + configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'], + owner=params.unix_user, + group=params.unix_group, + mode=0644) + + cred_lib = os.path.join(params.usersync_home,"lib","*") + + cred_setup = format('python {ranger_home}/ranger_credential_helper.py -l "{cred_lib}" -f {ugsync_jceks_path} -k "usersync_ssl_key_password" -v "{ranger_usersync_keystore_password}" -c 1') + Execute(cred_setup, logoutput=True) + + cred_setup = format('python {ranger_home}/ranger_credential_helper.py -l "{cred_lib}" -f {ugsync_jceks_path} -k "ranger.usersync.ldap.bindalias" -v "{ranger_usersync_ldap_ldapbindpassword}" -c 1') + Execute(cred_setup, logoutput=True) + + cred_setup = format('python {ranger_home}/ranger_credential_helper.py -l "{cred_lib}" -f {ugsync_jceks_path} -k "usersync.ssl.truststore.password" -v "{ranger_usersync_truststore_password}" -c 1') + Execute(cred_setup, logoutput=True) + + File(params.ugsync_jceks_path, + owner = params.unix_user, + group = params.unix_group + ) + + File([params.usersync_start, params.usersync_stop], + owner = params.unix_user, + group = params.unix_group + ) + + File(params.usersync_services_file, + mode = 0755, + ) + + Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'), + not_if=format("ls /usr/bin/ranger-usersync"), + only_if=format("ls {usersync_services_file}"), + sudo=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py index 729e383..cc639ba 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py @@ -140,16 +140,14 @@ metric_collector_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink*.jar" # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 - +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] #ranger storm properties policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR'] -xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] -xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_storm' @@ -158,8 +156,6 @@ common_name_for_certificate = config['configurations']['ranger-storm-plugin-prop storm_ui_port = config['configurations']['storm-site']['ui.port'] repo_config_username = config['configurations']['ranger-storm-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] -repo_config_password = config['configurations']['ranger-storm-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] - ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-storm-plugin-properties'] policy_user = config['configurations']['ranger-storm-plugin-properties']['policy_user'] @@ -167,19 +163,23 @@ policy_user = config['configurations']['ranger-storm-plugin-properties']['policy #For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' + if has_ranger_admin: enable_ranger_storm = (config['configurations']['ranger-storm-plugin-properties']['ranger-storm-plugin-enabled'].lower() == 'yes') + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) + repo_config_password = unicode(config['configurations']['ranger-storm-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() - if xa_audit_db_flavor.lower() == 'mysql': + if xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" - elif xa_audit_db_flavor.lower() == 'oracle': + elif xa_audit_db_flavor == 'oracle': jdbc_jar_name = "ojdbc6.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'postgres': + elif xa_audit_db_flavor == 'postgres': jdbc_jar_name = "postgresql.jar" jdbc_symlink_name = "postgres-jdbc-driver.jar" - elif xa_audit_db_flavor.lower() == 'sqlserver': + elif xa_audit_db_flavor == 'sqlserver': jdbc_jar_name = "sqljdbc4.jar" jdbc_symlink_name = "mssql-jdbc-driver.jar" @@ -203,3 +203,11 @@ if has_ranger_admin: 'repositoryType': 'storm', 'assetType': '6' } + + if xml_configurations_supported: + xa_audit_db_is_enabled = config['configurations']['ranger-storm-audit']['xasecure.audit.db.is.enabled'] + ssl_keystore_file_path = config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.keystore'] + ssl_truststore_file_path = config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.truststore'] + ssl_keystore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) + ssl_truststore_password = unicode(config['configurations']['ranger-storm-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py index a91874a..2d2c9c1 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py @@ -17,19 +17,32 @@ See the License for the specific language governing permissions and limitations under the License. """ -from resource_management import * +from resource_management.core.logger import Logger def setup_ranger_storm(): import params - + if params.has_ranger_admin and params.security_enabled: - setup_ranger_plugin('storm-nimbus', 'storm', + + if params.xml_configurations_supported: + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + else: + from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + + setup_ranger_plugin('storm-nimbus', 'storm', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, params.repo_name, params.storm_ranger_plugin_repo, params.ranger_env, params.ranger_plugin_properties, params.policy_user, params.policymgr_mgr_url, - params.enable_ranger_storm - ) + params.enable_ranger_storm, conf_dict=params.conf_dir, + component_user=params.storm_user, component_group=params.user_group, cache_service_list=['storm'], + plugin_audit_properties=params.config['configurations']['ranger-storm-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-storm-audit'], + plugin_security_properties=params.config['configurations']['ranger-storm-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-storm-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-storm-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-storm-policymgr-ssl'], + component_list=['storm-client', 'storm-nimbus'], audit_db_is_enabled=params.xa_audit_db_password, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password + ) else: - Logger.info('Ranger admin not installed') \ No newline at end of file + Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 5dd21b1..faac4ed 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -258,6 +258,7 @@ cgroups_dir = "/cgroups_test/cpu" # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] # hostname of the active HDFS HA Namenode (only used when HA is enabled) dfs_ha_namenode_active = default("/configurations/hadoop-env/dfs_ha_initial_namenode_active", None) @@ -275,10 +276,10 @@ if has_ranger_admin: enable_ranger_yarn = (config['configurations']['ranger-yarn-plugin-properties']['ranger-yarn-plugin-enabled'].lower() == 'yes') policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR'] - xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] - xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_yarn' @@ -288,7 +289,7 @@ if has_ranger_admin: ranger_plugin_config = { 'username' : config['configurations']['ranger-yarn-plugin-properties']['REPOSITORY_CONFIG_USERNAME'], - 'password' : config['configurations']['ranger-yarn-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'], + 'password' : unicode(config['configurations']['ranger-yarn-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']), 'yarn.url' : config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'], 'commonNameForCertificate' : config['configurations']['ranger-yarn-plugin-properties']['common.name.for.certificate'] } @@ -305,16 +306,16 @@ if has_ranger_admin: #For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' - if xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'mysql': + if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" - elif xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'oracle': + elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle': jdbc_jar_name = "ojdbc6.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar" - elif xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'postgres': + elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres': jdbc_jar_name = "postgresql.jar" jdbc_symlink_name = "postgres-jdbc-driver.jar" - elif xa_audit_db_flavor and xa_audit_db_flavor.lower() == 'sqlserver': + elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqlserver': jdbc_jar_name = "sqljdbc4.jar" jdbc_symlink_name = "mssql-jdbc-driver.jar" @@ -322,3 +323,11 @@ if has_ranger_admin: driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") + + if xml_configurations_supported: + xa_audit_db_is_enabled = config['configurations']['ranger-yarn-audit']['xasecure.audit.db.is.enabled'] + ssl_keystore_file_path = config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.keystore'] + ssl_truststore_file_path = config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.truststore'] + ssl_keystore_password = unicode(config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) + ssl_truststore_password = unicode(config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') http://git-wip-us.apache.org/repos/asf/ambari/blob/e50a2ac3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py index 0e3ed98..4d53180 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py @@ -7,30 +7,38 @@ regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - """ from resource_management.core.logger import Logger -from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin def setup_ranger_yarn(): import params if params.has_ranger_admin: - api_version = 'v2' + + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + + setup_ranger_plugin('hadoop-yarn-resourcemanager', 'yarn', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, params.repo_name, params.yarn_ranger_plugin_repo, params.ranger_env, params.ranger_plugin_properties, params.policy_user, params.policymgr_mgr_url, - params.enable_ranger_yarn, api_version) + params.enable_ranger_yarn, conf_dict=params.hadoop_conf_dir, + component_user=params.yarn_user, component_group=params.user_group, cache_service_list=['yarn'], + plugin_audit_properties=params.config['configurations']['ranger-yarn-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-yarn-audit'], + plugin_security_properties=params.config['configurations']['ranger-yarn-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-yarn-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-yarn-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-yarn-policymgr-ssl'], + component_list=['hadoop-yarn-resourcemanager'], audit_db_is_enabled=params.xa_audit_db_password, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, + api_version = 'v2' + ) else: Logger.info('Ranger admin not installed') \ No newline at end of file
