Repository: ambari Updated Branches: refs/heads/trunk 87eef6e97 -> 67c6f5561
AMBARI-19044 ADDENDUM Install & configure Ranger plugin components independently of Ranger admin components (mugdha) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/67c6f556 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/67c6f556 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/67c6f556 Branch: refs/heads/trunk Commit: 67c6f5561aec7e51472aee521ae5b84c82ae4353 Parents: 87eef6e Author: Mugdha Varadkar <mug...@apache.org> Authored: Wed Jan 18 18:26:34 2017 +0530 Committer: Mugdha Varadkar <mug...@apache.org> Committed: Thu Jan 19 15:42:23 2017 +0530 ---------------------------------------------------------------------- .../3.0.0.3.0/package/scripts/params_linux.py | 162 +++++++------- .../package/scripts/setup_ranger_hdfs.py | 43 ++-- .../3.0.0.3.0/package/scripts/params_linux.py | 213 ++++++++++--------- .../package/scripts/resourcemanager.py | 2 +- .../package/scripts/setup_ranger_yarn.py | 4 +- 5 files changed, 220 insertions(+), 204 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/67c6f556/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py index 512ca27..59ae815 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py @@ -43,7 +43,7 @@ from resource_management.libraries.functions.format_jvm_option import format_jvm from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs from resource_management.libraries.functions import is_empty - +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -389,92 +389,99 @@ dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") -# ranger host -ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 -xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] -ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] - -#ranger hdfs properties -policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] -if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): - policymgr_mgr_url = policymgr_mgr_url.rstrip('/') -xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') -xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') -xa_db_host = config['configurations']['admin-properties']['db_host'] -repo_name = str(config['clusterName']) + '_hadoop' - hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication'] hadoop_security_authorization = config['configurations']['core-site']['hadoop.security.authorization'] fs_default_name = config['configurations']['core-site']['fs.defaultFS'] hadoop_security_auth_to_local = config['configurations']['core-site']['hadoop.security.auth_to_local'] -hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection'] -common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate'] - -repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] if security_enabled: sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/_h...@example.com") sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower()) -ranger_env = config['configurations']['ranger-env'] -ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties'] -policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user'] - -#For curl command in ranger plugin to get db connector +# for curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' is_https_enabled = is_https_enabled_in_hdfs(config['configurations']['hdfs-site']['dfs.http.policy'], config['configurations']['hdfs-site']['dfs.https.enable']) -if has_ranger_admin: - enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes') +# ranger hdfs plugin section start + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 + +# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature +xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) + +# ambari-server hostname +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +# ranger hdfs plugin enabled property +enable_ranger_hdfs = default("/configurations/ranger-hdfs-plugin-properties/ranger-hdfs-plugin-enabled", "No") +enable_ranger_hdfs = True if enable_ranger_hdfs.lower() == 'yes' else False + +# get ranger hdfs properties if enable_ranger_hdfs is True +if enable_ranger_hdfs: + # ranger policy url + policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + if xml_configurations_supported: + policymgr_mgr_url = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.policy.rest.url'] + + if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + + # ranger audit db user + xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + + # ranger hdfs service name + repo_name = str(config['clusterName']) + '_hadoop' + repo_name_value = config['configurations']['ranger-hdfs-security']['ranger.plugin.hdfs.service.name'] + if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": + repo_name = repo_name_value + + hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection'] + common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate'] + repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] + + # ranger-env config + ranger_env = config['configurations']['ranger-env'] + # create ranger-env config having external ranger credential properties + if not has_ranger_admin and enable_ranger_hdfs: + external_admin_username = default('/configurations/ranger-hdfs-plugin-properties/external_admin_username', 'admin') + external_admin_password = default('/configurations/ranger-hdfs-plugin-properties/external_admin_password', 'admin') + external_ranger_admin_username = default('/configurations/ranger-hdfs-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin') + external_ranger_admin_password = default('/configurations/ranger-hdfs-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin') + ranger_env = {} + ranger_env['admin_username'] = external_admin_username + ranger_env['admin_password'] = external_admin_password + ranger_env['ranger_admin_username'] = external_ranger_admin_username + ranger_env['ranger_admin_password'] = external_ranger_admin_password + + ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties'] + policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user'] + repo_config_password = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'] + xa_audit_db_password = '' - if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: - xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) - repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) - xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + + downloaded_custom_connector = None previous_jdbc_jar_name = None + driver_curl_source = None + driver_curl_target = None + previous_jdbc_jar = None + + # to get db connector related properties + if has_ranger_admin and stack_supports_ranger_audit_db: + xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] + jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config) - if stack_supports_ranger_audit_db: - - if xa_audit_db_flavor == 'mysql': - jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) - audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "com.mysql.jdbc.Driver" - elif xa_audit_db_flavor == 'oracle': - jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) - colon_count = xa_db_host.count(':') - if colon_count == 2 or colon_count == 0: - audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') - else: - audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') - jdbc_driver = "oracle.jdbc.OracleDriver" - elif xa_audit_db_flavor == 'postgres': - jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) - audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "org.postgresql.Driver" - elif xa_audit_db_flavor == 'mssql': - jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') - jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - elif xa_audit_db_flavor == 'sqla': - jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') - jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" - - downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - driver_curl_target = format("{hadoop_lib_home}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None - previous_jdbc_jar = format("{hadoop_lib_home}/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None - - sql_connector_jar = '' + downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_target = format("{hadoop_lib_home}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + previous_jdbc_jar = format("{hadoop_lib_home}/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + sql_connector_jar = '' hdfs_ranger_plugin_config = { 'username': repo_config_username, @@ -498,6 +505,7 @@ if has_ranger_admin: 'repositoryType': 'hdfs', 'assetType': '1' } + if stack_supports_ranger_kerberos and security_enabled: hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user @@ -514,14 +522,16 @@ if has_ranger_admin: } xa_audit_db_is_enabled = False - ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] if xml_configurations_supported and stack_supports_ranger_audit_db: xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.db'] - xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None - ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None - ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None - credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None - #For SQLA explicitly disable audit to DB for Ranger - if xa_audit_db_flavor == 'sqla': + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False + ssl_keystore_password = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None + ssl_truststore_password = config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') + + # for SQLA explicitly disable audit to DB for Ranger + if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla': xa_audit_db_is_enabled = False + +# ranger hdfs plugin section end \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/67c6f556/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/setup_ranger_hdfs.py index e3aff9d..d0d92d7 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/setup_ranger_hdfs.py @@ -29,7 +29,7 @@ from resource_management.libraries.functions.format import format def setup_ranger_hdfs(upgrade_type=None): import params - if params.has_ranger_admin: + if params.enable_ranger_hdfs: stack_version = None @@ -93,29 +93,28 @@ def setup_ranger_hdfs(upgrade_type=None): target_file = source_file + ".bak" Execute(("mv", source_file, target_file), sudo=True, only_if=format("test -f {source_file}")) else: - Logger.info('Ranger admin not installed') + Logger.info('Ranger Hdfs plugin is not enabled') def create_ranger_audit_hdfs_directories(): import params - if params.has_ranger_admin: - if params.xml_configurations_supported and params.enable_ranger_hdfs and params.xa_audit_hdfs_is_enabled: - params.HdfsResource("/ranger/audit", - type="directory", - action="create_on_execute", - owner=params.hdfs_user, - group=params.hdfs_user, - mode=0755, - recursive_chmod=True, - ) - params.HdfsResource("/ranger/audit/hdfs", - type="directory", - action="create_on_execute", - owner=params.hdfs_user, - group=params.hdfs_user, - mode=0700, - recursive_chmod=True, - ) - params.HdfsResource(None, action="execute") + if params.enable_ranger_hdfs and params.xml_configurations_supported and params.xa_audit_hdfs_is_enabled: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True, + ) + params.HdfsResource("/ranger/audit/hdfs", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0700, + recursive_chmod=True, + ) + params.HdfsResource(None, action="execute") else: - Logger.info('Ranger admin not installed') + Logger.info('Skipping creation of audit directory for Ranger Hdfs Plugin.') http://git-wip-us.apache.org/repos/asf/ambari/blob/67c6f556/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py index 96b227b..4d47925 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py @@ -35,7 +35,7 @@ from resource_management.libraries.functions.default import default from resource_management.libraries import functions from resource_management.libraries.functions import is_empty from resource_management.libraries.functions.get_architecture import get_architecture - +from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs import status_params # a map of the Ambari role to the component name @@ -301,9 +301,6 @@ tez_lib_uris = default("/configurations/tez-site/tez.lib.uris", None) #for create_hdfs_directory hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] - - - hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] is_webhdfs_enabled = hdfs_site['dfs.webhdfs.enabled'] @@ -348,12 +345,6 @@ node_label_enable = config['configurations']['yarn-site']['yarn.node-labels.enab cgroups_dir = "/cgroups_test/cpu" -# *********************** RANGER PLUGIN CHANGES *********************** -# ranger host -ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -has_ranger_admin = not len(ranger_admin_hosts) == 0 -xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] -ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] # hostname of the active HDFS HA Namenode (only used when HA is enabled) dfs_ha_namenode_active = default("/configurations/hadoop-env/dfs_ha_initial_namenode_active", None) if dfs_ha_namenode_active is not None: @@ -384,103 +375,119 @@ if rm_ha_enabled: rm_webapp_address = config['configurations']['yarn-site'][rm_webapp_address_property] rm_webapp_addresses_list.append(rm_webapp_address) -#ranger yarn properties -if has_ranger_admin: - is_supported_yarn_ranger = config['configurations']['yarn-env']['is_supported_yarn_ranger'] - - if is_supported_yarn_ranger: - enable_ranger_yarn = (config['configurations']['ranger-yarn-plugin-properties']['ranger-yarn-plugin-enabled'].lower() == 'yes') - policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] - if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): - policymgr_mgr_url = policymgr_mgr_url.rstrip('/') - xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() - xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') - xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') - xa_audit_db_password = '' - if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: - xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) - xa_db_host = config['configurations']['admin-properties']['db_host'] - repo_name = str(config['clusterName']) + '_yarn' - - ranger_env = config['configurations']['ranger-env'] - ranger_plugin_properties = config['configurations']['ranger-yarn-plugin-properties'] - policy_user = config['configurations']['ranger-yarn-plugin-properties']['policy_user'] - yarn_rest_url = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] - - ranger_plugin_config = { - 'username' : config['configurations']['ranger-yarn-plugin-properties']['REPOSITORY_CONFIG_USERNAME'], - 'password' : unicode(config['configurations']['ranger-yarn-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']), - 'yarn.url' : format('{scheme}://{yarn_rest_url}'), - 'commonNameForCertificate' : config['configurations']['ranger-yarn-plugin-properties']['common.name.for.certificate'] - } - - yarn_ranger_plugin_repo = { - 'isEnabled': 'true', - 'configs': ranger_plugin_config, - 'description': 'yarn repo', - 'name': repo_name, - 'repositoryType': 'yarn', - 'type': 'yarn', - 'assetType': '1' - } - - if stack_supports_ranger_kerberos: - ranger_plugin_config['ambari.service.check.user'] = policy_user - ranger_plugin_config['hadoop.security.authentication'] = 'kerberos' if security_enabled else 'simple' - - if stack_supports_ranger_kerberos and security_enabled: - ranger_plugin_config['policy.download.auth.users'] = yarn_user - ranger_plugin_config['tag.download.auth.users'] = yarn_user - - #For curl command in ranger plugin to get db connector - jdk_location = config['hostLevelParams']['jdk_location'] - java_share_dir = '/usr/share/java' - previous_jdbc_jar_name = None - if stack_supports_ranger_audit_db: - if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql': - jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) - audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "com.mysql.jdbc.Driver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle': - jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) - colon_count = xa_db_host.count(':') - if colon_count == 2 or colon_count == 0: - audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') - else: - audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') - jdbc_driver = "oracle.jdbc.OracleDriver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres': - jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) - audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') - jdbc_driver = "org.postgresql.Driver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'mssql': - jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') - jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqla': - jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) - previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) - audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') - jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" +# for curl command in ranger plugin to get db connector +jdk_location = config['hostLevelParams']['jdk_location'] + +# ranger yarn plugin section start + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 + +# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature +xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) + +# ambari-server hostname +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +# ranger yarn plugin enabled property +enable_ranger_yarn = default("/configurations/ranger-yarn-plugin-properties/ranger-yarn-plugin-enabled", "No") +enable_ranger_yarn = True if enable_ranger_yarn.lower() == 'yes' else False + +# ranger yarn-plugin supported flag, instead of using is_supported_yarn_ranger/yarn-env, using stack feature +is_supported_yarn_ranger = check_stack_feature(StackFeature.YARN_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks) + +# get ranger yarn properties if enable_ranger_yarn is True +if enable_ranger_yarn and is_supported_yarn_ranger: + # get ranger policy url + policymgr_mgr_url = config['configurations']['ranger-yarn-security']['ranger.plugin.yarn.policy.rest.url'] + + if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + + # ranger audit db user + xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + + xa_audit_db_password = '' + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db and has_ranger_admin: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + + # ranger yarn service/repository name + repo_name = str(config['clusterName']) + '_yarn' + repo_name_value = config['configurations']['ranger-yarn-security']['ranger.plugin.yarn.service.name'] + if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": + repo_name = repo_name_value + + # ranger-env config + ranger_env = config['configurations']['ranger-env'] + + # create ranger-env config having external ranger credential properties + if not has_ranger_admin and enable_ranger_yarn: + external_admin_username = default('/configurations/ranger-yarn-plugin-properties/external_admin_username', 'admin') + external_admin_password = default('/configurations/ranger-yarn-plugin-properties/external_admin_password', 'admin') + external_ranger_admin_username = default('/configurations/ranger-yarn-plugin-properties/external_ranger_admin_username', 'amb_ranger_admin') + external_ranger_admin_password = default('/configurations/ranger-yarn-plugin-properties/external_ranger_admin_password', 'amb_ranger_admin') + ranger_env = {} + ranger_env['admin_username'] = external_admin_username + ranger_env['admin_password'] = external_admin_password + ranger_env['ranger_admin_username'] = external_ranger_admin_username + ranger_env['ranger_admin_password'] = external_ranger_admin_password + + ranger_plugin_properties = config['configurations']['ranger-yarn-plugin-properties'] + policy_user = config['configurations']['ranger-yarn-plugin-properties']['policy_user'] + yarn_rest_url = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] + + ranger_plugin_config = { + 'username' : config['configurations']['ranger-yarn-plugin-properties']['REPOSITORY_CONFIG_USERNAME'], + 'password' : unicode(config['configurations']['ranger-yarn-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']), + 'yarn.url' : format('{scheme}://{yarn_rest_url}'), + 'commonNameForCertificate' : config['configurations']['ranger-yarn-plugin-properties']['common.name.for.certificate'] + } + + yarn_ranger_plugin_repo = { + 'isEnabled': 'true', + 'configs': ranger_plugin_config, + 'description': 'yarn repo', + 'name': repo_name, + 'repositoryType': 'yarn', + 'type': 'yarn', + 'assetType': '1' + } + + if stack_supports_ranger_kerberos: + ranger_plugin_config['ambari.service.check.user'] = policy_user + ranger_plugin_config['hadoop.security.authentication'] = 'kerberos' if security_enabled else 'simple' + + if stack_supports_ranger_kerberos and security_enabled: + ranger_plugin_config['policy.download.auth.users'] = yarn_user + ranger_plugin_config['tag.download.auth.users'] = yarn_user + + downloaded_custom_connector = None + previous_jdbc_jar_name = None + driver_curl_source = None + driver_curl_target = None + previous_jdbc_jar = None + + if has_ranger_admin and stack_supports_ranger_audit_db: + xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] + jdbc_jar_name, previous_jdbc_jar_name, audit_jdbc_url, jdbc_driver = get_audit_configs(config) downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None driver_curl_target = format("{hadoop_yarn_home}/lib/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None previous_jdbc_jar = format("{hadoop_yarn_home}/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + xa_audit_db_is_enabled = False + if xml_configurations_supported and stack_supports_ranger_audit_db: + xa_audit_db_is_enabled = config['configurations']['ranger-yarn-audit']['xasecure.audit.destination.db'] + + xa_audit_hdfs_is_enabled = config['configurations']['ranger-yarn-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else False + ssl_keystore_password = config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password'] if xml_configurations_supported else None + ssl_truststore_password = config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password'] if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') + + # for SQLA explicitly disable audit to DB for Ranger + if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False - ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] - if xml_configurations_supported and stack_supports_ranger_audit_db: - xa_audit_db_is_enabled = config['configurations']['ranger-yarn-audit']['xasecure.audit.destination.db'] - xa_audit_hdfs_is_enabled = config['configurations']['ranger-yarn-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None - ssl_keystore_password = unicode(config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None - ssl_truststore_password = unicode(config['configurations']['ranger-yarn-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None - credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None - - #For SQLA explicitly disable audit to DB for Ranger - if xa_audit_db_flavor == 'sqla': - xa_audit_db_is_enabled = False + +# ranger yarn plugin end section \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/67c6f556/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py index 657a020..4d8d95e 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py @@ -128,7 +128,7 @@ class ResourcemanagerDefault(Resourcemanager): env.set_params(params) self.configure(env) # FOR SECURITY - if params.has_ranger_admin and params.is_supported_yarn_ranger: + if params.enable_ranger_yarn and params.is_supported_yarn_ranger: setup_ranger_yarn() #Ranger Yarn Plugin related calls # wait for active-dir and done-dir to be created by ATS if needed http://git-wip-us.apache.org/repos/asf/ambari/blob/67c6f556/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/setup_ranger_yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/setup_ranger_yarn.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/setup_ranger_yarn.py index 6ea7f82..d29e4dc 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/setup_ranger_yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/setup_ranger_yarn.py @@ -19,7 +19,7 @@ from resource_management.core.logger import Logger def setup_ranger_yarn(): import params - if params.has_ranger_admin: + if params.enable_ranger_yarn: from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin @@ -68,4 +68,4 @@ def setup_ranger_yarn(): component_user_keytab=params.rm_keytab if params.security_enabled else None ) else: - Logger.info('Ranger admin not installed') + Logger.info('Ranger Yarn plugin is not enabled')
