http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml new file mode 100644 index 0000000..d328d89 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/metainfo.xml @@ -0,0 +1,111 @@ +<?xml version="1.0"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>RANGER_KMS</name> + <displayName>Ranger KMS</displayName> + <comment>Key Management Server</comment> + <version>1.0.0.3.0</version> + <components> + + <component> + <name>RANGER_KMS_SERVER</name> + <displayName>Ranger KMS Server</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/kms_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <logs> + <log> + <logId>ranger_kms</logId> + <primary>true</primary> + </log> + </logs> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + </component> + </components> + + + <osSpecifics> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>ranger_${stack_version}-kms</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>ranger-${stack_version}-kms</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <configuration-dependencies> + <config-type>kms-properties</config-type> + <config-type>kms-site</config-type> + <config-type>kms-log4j</config-type> + <config-type>dbks-site</config-type> + <config-type>ranger-kms-site</config-type> + <config-type>ranger-kms-audit</config-type> + <config-type>ranger-kms-policymgr-ssl</config-type> + <config-type>ranger-kms-security</config-type> + </configuration-dependencies> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>RANGER</service> + <service>HDFS</service> + </requiredServices> + + <themes> + <theme> + <fileName>theme_version_1.json</fileName> + <default>true</default> + </theme> + </themes> + + </service> + </services> +</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py new file mode 100755 index 0000000..6e4a171 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms.py @@ -0,0 +1,675 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +import fileinput +import os +import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. +import urllib2, base64, httplib +from StringIO import StringIO as BytesIO +from datetime import datetime +from resource_management.core.resources.system import File, Directory, Execute +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.resources.modify_properties_file import ModifyPropertiesFile +from resource_management.core.source import DownloadSource, InlineTemplate +from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.ranger_functions import Rangeradmin +from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2 +from resource_management.libraries.functions.decorator import safe_retry +from resource_management.core.utils import PasswordString +from resource_management.core.shell import as_sudo +import re +import time +import socket + +def password_validation(password, key): + import params + if password.strip() == "": + raise Fail("Blank password is not allowed for {0} property. Please enter valid password.".format(key)) + if re.search("[\\\`'\"]",password): + raise Fail("{0} password contains one of the unsupported special characters like \" ' \ `".format(key)) + else: + Logger.info("Password validated") + +def setup_kms_db(stack_version=None): + import params + + if params.has_ranger_admin: + + kms_home = params.kms_home + version = params.version + if stack_version is not None: + kms_home = format("{stack_root}/{stack_version}/ranger-kms") + version = stack_version + + password_validation(params.kms_master_key_password, 'KMS master key') + + copy_jdbc_connector(stack_version=version) + + env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home} + if params.db_flavor.lower() == 'sqla': + env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path} + + dba_setup = format('ambari-python-wrap {kms_home}/dba_script.py -q') + db_setup = format('ambari-python-wrap {kms_home}/db_setup.py') + + if params.create_db_user: + Logger.info('Setting up Ranger KMS DB and DB User') + Execute(dba_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10) + else: + Logger.info('Separate DBA property not set. Assuming Ranger KMS DB and DB User exists!') + Execute(db_setup, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10) + +def setup_java_patch(): + import params + + if params.has_ranger_admin: + + kms_home = params.kms_home + setup_java_patch = format('ambari-python-wrap {kms_home}/db_setup.py -javapatch') + + env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home} + if params.db_flavor.lower() == 'sqla': + env_dict = {'RANGER_KMS_HOME':kms_home, 'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH':params.ld_library_path} + + Execute(setup_java_patch, environment=env_dict, logoutput=True, user=params.kms_user, tries=5, try_sleep=10) + + kms_lib_path = format('{kms_home}/ews/webapp/lib/') + files = os.listdir(kms_lib_path) + hadoop_jar_files = [] + + for x in files: + if x.startswith('hadoop-common') and x.endswith('.jar'): + hadoop_jar_files.append(x) + + if len(hadoop_jar_files) != 0: + for f in hadoop_jar_files: + Execute((format('{java_home}/bin/jar'),'-uf', format('{kms_home}/ews/webapp/lib/{f}'), format('{kms_home}/ews/webapp/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory')), + user=params.kms_user) + + File(format('{kms_home}/ews/webapp/lib/{f}'), owner=params.kms_user, group=params.kms_group) + +def do_keystore_setup(cred_provider_path, credential_alias, credential_password): + import params + + if cred_provider_path is not None: + java_bin = format('{java_home}/bin/java') + file_path = format('jceks://file{cred_provider_path}') + cmd = (java_bin, '-cp', params.cred_lib_path, 'org.apache.ranger.credentialapi.buildks', 'create', credential_alias, '-value', PasswordString(credential_password), '-provider', file_path) + Execute(cmd, + environment={'JAVA_HOME': params.java_home}, + logoutput=True, + sudo=True, + ) + + File(cred_provider_path, + owner = params.kms_user, + group = params.kms_group, + mode = 0640 + ) + +def kms(upgrade_type=None): + import params + + if params.has_ranger_admin: + + Directory(params.kms_conf_dir, + owner = params.kms_user, + group = params.kms_group, + create_parents = True + ) + + Directory("/etc/security/serverKeys", + create_parents = True, + cd_access = "a" + ) + + Directory("/etc/ranger/kms", + create_parents = True, + cd_access = "a" + ) + + copy_jdbc_connector() + + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + mode = 0644, + ) + + cp = format("{check_db_connection_jar}") + if params.db_flavor.lower() == 'sqla': + cp = cp + os.pathsep + format("{kms_home}/ews/webapp/lib/sajdbc4.jar") + else: + path_to_jdbc = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") + if not os.path.isfile(path_to_jdbc): + path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + \ + params.default_connectors_map[params.db_flavor.lower()] if params.db_flavor.lower() in params.default_connectors_map else None + if not os.path.isfile(path_to_jdbc): + path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*" + error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.db_flavor] + \ + " in ranger kms lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" + Logger.error(error_message) + + cp = cp + os.pathsep + path_to_jdbc + + db_connection_check_command = format( + "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}") + + env_dict = {} + if params.db_flavor.lower() == 'sqla': + env_dict = {'LD_LIBRARY_PATH':params.ld_library_path} + + Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10, environment=env_dict) + + if params.xa_audit_db_is_enabled and params.driver_source is not None and not params.driver_source.endswith("/None"): + if params.xa_previous_jdbc_jar and os.path.isfile(params.xa_previous_jdbc_jar): + File(params.xa_previous_jdbc_jar, action='delete') + + File(params.downloaded_connector_path, + content = DownloadSource(params.driver_source), + mode = 0644 + ) + + Execute(('cp', '--remove-destination', params.downloaded_connector_path, params.driver_target), + path=["/bin", "/usr/bin/"], + sudo=True) + + File(params.driver_target, mode=0644) + + Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF', 'classes', 'lib'), + mode=0755, + owner=params.kms_user, + group=params.kms_group + ) + + Execute(('cp',format('{kms_home}/ranger-kms-initd'),'/etc/init.d/ranger-kms'), + not_if=format('ls /etc/init.d/ranger-kms'), + only_if=format('ls {kms_home}/ranger-kms-initd'), + sudo=True) + + File('/etc/init.d/ranger-kms', + mode = 0755 + ) + + Directory(format('{kms_home}/'), + owner = params.kms_user, + group = params.kms_group, + recursive_ownership = True, + ) + + Directory(params.ranger_kms_pid_dir, + mode=0755, + owner = params.kms_user, + group = params.user_group, + cd_access = "a", + create_parents=True + ) + + if params.stack_supports_pid: + File(format('{kms_conf_dir}/ranger-kms-env-piddir.sh'), + content = format("export RANGER_KMS_PID_DIR_PATH={ranger_kms_pid_dir}\nexport KMS_USER={kms_user}"), + owner = params.kms_user, + group = params.kms_group, + mode=0755 + ) + + Directory(params.kms_log_dir, + owner = params.kms_user, + group = params.kms_group, + cd_access = 'a', + create_parents=True, + mode=0755 + ) + + File(format('{kms_conf_dir}/ranger-kms-env-logdir.sh'), + content = format("export RANGER_KMS_LOG_DIR={kms_log_dir}"), + owner = params.kms_user, + group = params.kms_group, + mode=0755 + ) + + Execute(('ln','-sf', format('{kms_home}/ranger-kms'),'/usr/bin/ranger-kms'), + not_if=format('ls /usr/bin/ranger-kms'), + only_if=format('ls {kms_home}/ranger-kms'), + sudo=True) + + File('/usr/bin/ranger-kms', mode = 0755) + + Execute(('ln','-sf', format('{kms_home}/ranger-kms'),'/usr/bin/ranger-kms-services.sh'), + not_if=format('ls /usr/bin/ranger-kms-services.sh'), + only_if=format('ls {kms_home}/ranger-kms'), + sudo=True) + + File('/usr/bin/ranger-kms-services.sh', mode = 0755) + + Execute(('ln','-sf', format('{kms_home}/ranger-kms-initd'),format('{kms_home}/ranger-kms-services.sh')), + not_if=format('ls {kms_home}/ranger-kms-services.sh'), + only_if=format('ls {kms_home}/ranger-kms-initd'), + sudo=True) + + File(format('{kms_home}/ranger-kms-services.sh'), mode = 0755) + + Directory(params.kms_log_dir, + owner = params.kms_user, + group = params.kms_group, + mode = 0775 + ) + + do_keystore_setup(params.credential_provider_path, params.jdbc_alias, params.db_password) + do_keystore_setup(params.credential_provider_path, params.masterkey_alias, params.kms_master_key_password) + if params.stack_support_kms_hsm and params.enable_kms_hsm: + do_keystore_setup(params.credential_provider_path, params.hms_partition_alias, unicode(params.hms_partition_passwd)) + if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled: + do_keystore_setup(params.ranger_kms_cred_ssl_path, params.ranger_kms_ssl_keystore_alias, params.ranger_kms_ssl_passwd) + + # remove plain-text password from xml configs + dbks_site_copy = {} + dbks_site_copy.update(params.config['configurations']['dbks-site']) + + for prop in params.dbks_site_password_properties: + if prop in dbks_site_copy: + dbks_site_copy[prop] = "_" + + XmlConfig("dbks-site.xml", + conf_dir=params.kms_conf_dir, + configurations=dbks_site_copy, + configuration_attributes=params.config['configuration_attributes']['dbks-site'], + owner=params.kms_user, + group=params.kms_group, + mode=0644 + ) + + ranger_kms_site_copy = {} + ranger_kms_site_copy.update(params.config['configurations']['ranger-kms-site']) + if params.stack_supports_ranger_kms_ssl: + # remove plain-text password from xml configs + for prop in params.ranger_kms_site_password_properties: + if prop in ranger_kms_site_copy: + ranger_kms_site_copy[prop] = "_" + + XmlConfig("ranger-kms-site.xml", + conf_dir=params.kms_conf_dir, + configurations=ranger_kms_site_copy, + configuration_attributes=params.config['configuration_attributes']['ranger-kms-site'], + owner=params.kms_user, + group=params.kms_group, + mode=0644 + ) + + XmlConfig("kms-site.xml", + conf_dir=params.kms_conf_dir, + configurations=params.config['configurations']['kms-site'], + configuration_attributes=params.config['configuration_attributes']['kms-site'], + owner=params.kms_user, + group=params.kms_group, + mode=0644 + ) + + File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"), + owner=params.kms_user, + group=params.kms_group, + content=InlineTemplate(params.kms_log4j), + mode=0644 + ) + if params.security_enabled: + # core-site.xml linking required by setup for HDFS encryption + XmlConfig("core-site.xml", + conf_dir=params.kms_conf_dir, + configurations=params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner=params.kms_user, + group=params.kms_group, + mode=0644 + ) + else: + File(format('{kms_conf_dir}/core-site.xml'), action="delete") + +def copy_jdbc_connector(stack_version=None): + import params + + if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"): + error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \ + Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location) + Logger.error(error_message) + + if params.driver_curl_source and not params.driver_curl_source.endswith("/None"): + if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar): + File(params.previous_jdbc_jar, action='delete') + + kms_home = params.kms_home + if stack_version is not None: + kms_home = format("{stack_root}/{stack_version}/ranger-kms") + + driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") + + File(params.downloaded_custom_connector, + content = DownloadSource(params.driver_curl_source), + mode = 0644 + ) + + Directory(os.path.join(kms_home, 'ews', 'lib'), + mode=0755 + ) + + if params.db_flavor.lower() == 'sqla': + Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True) + + Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')), + path=["/bin", "/usr/bin/"], + sudo=True) + + Directory(params.jdbc_libs_dir, + cd_access="a", + create_parents=True) + + Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False), + path=["/bin", "/usr/bin/"]) + + File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644) + else: + Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')), + path=["/bin", "/usr/bin/"], + sudo=True) + + File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644) + + ModifyPropertiesFile(format("{kms_home}/install.properties"), + properties = params.config['configurations']['kms-properties'], + owner = params.kms_user + ) + + if params.db_flavor.lower() == 'sqla': + ModifyPropertiesFile(format("{kms_home}/install.properties"), + properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')}, + owner = params.kms_user, + ) + else: + ModifyPropertiesFile(format("{kms_home}/install.properties"), + properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')}, + owner = params.kms_user, + ) + +def enable_kms_plugin(): + + import params + + if params.has_ranger_admin: + + ranger_flag = False + + if params.stack_supports_ranger_kerberos and params.security_enabled: + if not is_empty(params.rangerkms_principal) and params.rangerkms_principal != '': + ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal) + else: + ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.spengo_keytab, params.spnego_principal) + else: + ranger_flag = check_ranger_service() + + if not ranger_flag: + Logger.error('Error in Get/Create service for Ranger Kms.') + + current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + File(format('{kms_conf_dir}/ranger-security.xml'), + owner = params.kms_user, + group = params.kms_group, + mode = 0644, + content = format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>') + ) + + Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')], + owner = params.kms_user, + group = params.kms_group, + mode=0775, + create_parents = True + ) + + File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')), + owner = params.kms_user, + group = params.kms_group, + mode = 0644 + ) + + # remove plain-text password from xml configs + plugin_audit_properties_copy = {} + plugin_audit_properties_copy.update(params.config['configurations']['ranger-kms-audit']) + + if params.plugin_audit_password_property in plugin_audit_properties_copy: + plugin_audit_properties_copy[params.plugin_audit_password_property] = "crypted" + + XmlConfig("ranger-kms-audit.xml", + conf_dir=params.kms_conf_dir, + configurations=plugin_audit_properties_copy, + configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'], + owner=params.kms_user, + group=params.kms_group, + mode=0744) + + XmlConfig("ranger-kms-security.xml", + conf_dir=params.kms_conf_dir, + configurations=params.config['configurations']['ranger-kms-security'], + configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'], + owner=params.kms_user, + group=params.kms_group, + mode=0744) + + # remove plain-text password from xml configs + ranger_kms_policymgr_ssl_copy = {} + ranger_kms_policymgr_ssl_copy.update(params.config['configurations']['ranger-kms-policymgr-ssl']) + + for prop in params.kms_plugin_password_properties: + if prop in ranger_kms_policymgr_ssl_copy: + ranger_kms_policymgr_ssl_copy[prop] = "crypted" + + XmlConfig("ranger-policymgr-ssl.xml", + conf_dir=params.kms_conf_dir, + configurations=ranger_kms_policymgr_ssl_copy, + configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'], + owner=params.kms_user, + group=params.kms_group, + mode=0744) + + if params.xa_audit_db_is_enabled: + cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1') + Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) + + cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1') + Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) + + cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1') + Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) + + File(params.credential_file, + owner = params.kms_user, + group = params.kms_group, + mode = 0640 + ) + + # create ranger kms audit directory + if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True + ) + params.HdfsResource("/ranger/audit/kms", + type="directory", + action="create_on_execute", + owner=params.kms_user, + group=params.kms_group, + mode=0750, + recursive_chmod=True + ) + params.HdfsResource(None, action="execute") + + if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1: + Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml') + XmlConfig("hdfs-site.xml", + conf_dir=params.kms_conf_dir, + configurations=params.config['configurations']['hdfs-site'], + configuration_attributes=params.config['configuration_attributes']['hdfs-site'], + owner=params.kms_user, + group=params.kms_group, + mode=0644 + ) + else: + File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete") + +def setup_kms_jce(): + import params + + if params.jce_name is not None: + Directory(params.jce_source_dir, + create_parents = True + ) + + jce_target = format('{jce_source_dir}/{jce_name}') + + File(jce_target, + content = DownloadSource(format('{jdk_location}/{jce_name}')), + mode = 0644, + ) + + File([format("{java_home}/jre/lib/security/local_policy.jar"), format("{java_home}/jre/lib/security/US_export_policy.jar")], + action = "delete", + ) + + unzip_cmd = ("unzip", "-o", "-j", "-q", jce_target, "-d", format("{java_home}/jre/lib/security")) + + Execute(unzip_cmd, + only_if = format("test -e {java_home}/jre/lib/security && test -f {jce_target}"), + path = ['/bin/','/usr/bin'], + sudo = True + ) + else: + Logger.warning("Required jce policy zip is not available, need to setup manually") + +def check_ranger_service(): + import params + + policymgr_mgr_url = params.policymgr_mgr_url + if policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url) + ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}") + response_code = ranger_adm_obj.check_ranger_login_urllib2(policymgr_mgr_url) + + if response_code is not None and response_code == 200: + user_resp_code = ranger_adm_obj.create_ambari_admin_user(params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password) + if user_resp_code is not None and user_resp_code == 200: + get_repo_flag = get_repo(policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger) + if not get_repo_flag: + return create_repo(policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger) + else: + return True + else: + return False + else: + Logger.error('Ranger service is not reachable') + return False + +@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False) +def create_repo(url, data, usernamepassword): + try: + base_url = url + '/service/public/v2/api/service' + base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') + headers = { + 'Accept': 'application/json', + "Content-Type": "application/json" + } + request = urllib2.Request(base_url, data, headers) + request.add_header("Authorization", "Basic {0}".format(base64string)) + result = urllib2.urlopen(request, timeout=20) + response_code = result.getcode() + response = json.loads(json.JSONEncoder().encode(result.read())) + if response_code == 200: + Logger.info('Repository created Successfully') + return True + else: + Logger.info('Repository not created') + return False + except urllib2.URLError, e: + if isinstance(e, urllib2.HTTPError): + raise Fail("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read())) + else: + raise Fail("Error creating service. Reason - {0}.".format(e.reason)) + except socket.timeout as e: + raise Fail("Error creating service. Reason - {0}".format(e)) + +@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False) +def get_repo(url, name, usernamepassword): + try: + base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true' + request = urllib2.Request(base_url) + base64string = base64.encodestring(usernamepassword).replace('\n', '') + request.add_header("Content-Type", "application/json") + request.add_header("Accept", "application/json") + request.add_header("Authorization", "Basic {0}".format(base64string)) + result = urllib2.urlopen(request, timeout=20) + response_code = result.getcode() + response = json.loads(result.read()) + if response_code == 200 and len(response) > 0: + for repo in response: + if repo.get('name').lower() == name.lower() and repo.has_key('name'): + Logger.info('KMS repository exist') + return True + else: + Logger.info('KMS repository doesnot exist') + return False + else: + Logger.info('KMS repository doesnot exist') + return False + except urllib2.URLError, e: + if isinstance(e, urllib2.HTTPError): + raise Fail("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read())) + else: + raise Fail("Error getting {0} service. Reason - {1}.".format(name, e.reason)) + except socket.timeout as e: + raise Fail("Error creating service. Reason - {0}".format(e)) + +def check_ranger_service_support_kerberos(user, keytab, principal): + import params + + policymgr_mgr_url = params.policymgr_mgr_url + if policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url) + response_code = ranger_adm_obj.check_ranger_login_curl(user, keytab, principal, policymgr_mgr_url, True) + + if response_code is not None and response_code[0] == 200: + get_repo_name_response = ranger_adm_obj.get_repository_by_name_curl(user, keytab, principal, params.repo_name, 'kms', 'true', is_keyadmin = True) + if get_repo_name_response is not None: + Logger.info('KMS repository {0} exist'.format(get_repo_name_response['name'])) + return True + else: + create_repo_response = ranger_adm_obj.create_repository_curl(user, keytab, principal, params.repo_name, json.dumps(params.kms_ranger_plugin_repo), None, is_keyadmin = True) + if create_repo_response is not None and len(create_repo_response) > 0: + return True + else: + return False + else: + Logger.error('Ranger service is not reachable') + return False http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py new file mode 100755 index 0000000..613931b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_server.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.core.exceptions import Fail +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions import stack_select +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute, File +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.libraries.functions.format import format +from resource_management.core.logger import Logger +from resource_management.core import shell +from resource_management.libraries.functions.default import default +from kms import kms, setup_kms_db, setup_java_patch, enable_kms_plugin, setup_kms_jce +from kms_service import kms_service +import upgrade + +class KmsServer(Script): + + def get_component_name(self): + return "ranger-kms" + + def install(self, env): + self.install_packages(env) + import params + env.set_params(params) + + setup_kms_db() + self.configure(env) + setup_java_patch() + + def stop(self, env, upgrade_type=None): + import params + + env.set_params(params) + kms_service(action = 'stop', upgrade_type=upgrade_type) + if params.stack_supports_pid: + File(params.ranger_kms_pid_file, + action = "delete" + ) + + def start(self, env, upgrade_type=None): + import params + + env.set_params(params) + self.configure(env) + enable_kms_plugin() + setup_kms_jce() + kms_service(action = 'start', upgrade_type=upgrade_type) + + def status(self, env): + import status_params + env.set_params(status_params) + + if status_params.stack_supports_pid: + check_process_status(status_params.ranger_kms_pid_file) + return + + cmd = 'ps -ef | grep proc_rangerkms | grep -v grep' + code, output = shell.call(cmd, timeout=20) + if code != 0: + Logger.debug('KMS process not running') + raise ComponentIsNotRunning() + pass + + def configure(self, env): + import params + + env.set_params(params) + kms() + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + upgrade.prestart(env, "ranger-kms") + kms(upgrade_type=upgrade_type) + setup_java_patch() + + def setup_ranger_kms_database(self, env): + import params + env.set_params(params) + + upgrade_stack = stack_select._get_upgrade_stack() + if upgrade_stack is None: + raise Fail('Unable to determine the stack and stack version') + + stack_version = upgrade_stack[1] + Logger.info(format('Setting Ranger KMS database schema, using version {stack_version}')) + setup_kms_db(stack_version=stack_version) + + def get_log_folder(self): + import params + return params.kms_log_dir + + def get_user(self): + import params + return params.kms_user + +if __name__ == "__main__": + KmsServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py new file mode 100644 index 0000000..2ff48c3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/kms_service.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute, File +from resource_management.core import shell +from resource_management.libraries.functions.format import format +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +from resource_management.libraries.functions.show_logs import show_logs +from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from resource_management.libraries.functions.constants import Direction +import os + +def kms_service(action='start', upgrade_type=None): + import params + + env_dict = {'JAVA_HOME': params.java_home} + if params.db_flavor.lower() == 'sqla': + env_dict = {'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH': params.ld_library_path} + + if action == 'start': + no_op_test = format('ps -ef | grep proc_rangerkms | grep -v grep') + cmd = format('{kms_home}/ranger-kms start') + try: + Execute(cmd, not_if=no_op_test, environment=env_dict, user=format('{kms_user}')) + except: + show_logs(params.kms_log_dir, params.kms_user) + raise + elif action == 'stop': + if upgrade_type == UPGRADE_TYPE_NON_ROLLING and params.upgrade_direction == Direction.UPGRADE: + if os.path.isfile(format('{kms_home}/ranger-kms')): + File(format('{kms_home}/ranger-kms'), + owner=params.kms_user, + group = params.kms_group + ) + cmd = format('{kms_home}/ranger-kms stop') + try: + Execute(cmd, environment=env_dict, user=format('{kms_user}')) + except: + show_logs(params.kms_log_dir, params.kms_user) + raise http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py new file mode 100755 index 0000000..003eee1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/params.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +from resource_management.libraries.functions import conf_select +from resource_management.libraries.script import Script +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.stack_features import get_stack_feature_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.get_bare_principal import get_bare_principal +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import get_kinit_path + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() +stack_root = Script.get_stack_root() + +stack_name = default("/hostLevelParams/stack_name", None) +version = default("/commandParams/version", None) +upgrade_direction = default("/commandParams/upgrade_direction", None) + +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) + +# get the correct version to use for checking stack features +version_for_stack_feature_checks = get_stack_feature_version(config) + +stack_supports_config_versioning = check_stack_feature(StackFeature.CONFIG_VERSIONING, version_for_stack_feature_checks) +stack_support_kms_hsm = check_stack_feature(StackFeature.RANGER_KMS_HSM_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) +stack_supports_pid = check_stack_feature(StackFeature.RANGER_KMS_PID_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_kms_ssl = check_stack_feature(StackFeature.RANGER_KMS_SSL, version_for_stack_feature_checks) + +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +security_enabled = config['configurations']['cluster-env']['security_enabled'] + +if stack_supports_config_versioning: + kms_home = format('{stack_root}/current/ranger-kms') + kms_conf_dir = format('{stack_root}/current/ranger-kms/conf') + +kms_log_dir = default("/configurations/kms-env/kms_log_dir", "/var/log/ranger/kms") +java_home = config['hostLevelParams']['java_home'] +kms_user = default("/configurations/kms-env/kms_user", "kms") +kms_group = default("/configurations/kms-env/kms_group", "kms") + +ranger_kms_audit_log_maxfilesize = default('/configurations/kms-log4j/ranger_kms_audit_log_maxfilesize',256) +ranger_kms_audit_log_maxbackupindex = default('/configurations/kms-log4j/ranger_kms_audit_log_maxbackupindex',20) +ranger_kms_log_maxfilesize = default('/configurations/kms-log4j/ranger_kms_log_maxfilesize',256) +ranger_kms_log_maxbackupindex = default('/configurations/kms-log4j/ranger_kms_log_maxbackupindex',20) + +jdk_location = config['hostLevelParams']['jdk_location'] +kms_log4j = config['configurations']['kms-log4j']['content'] + +# ranger host +ranger_admin_hosts = config['clusterHostInfo']['ranger_admin_hosts'][0] +has_ranger_admin = len(ranger_admin_hosts) > 0 +kms_host = config['clusterHostInfo']['ranger_kms_server_hosts'][0] +kms_port = config['configurations']['kms-env']['kms_port'] + +create_db_user = config['configurations']['kms-env']['create_db_user'] + +#kms properties +db_flavor = (config['configurations']['kms-properties']['DB_FLAVOR']).lower() +db_host = config['configurations']['kms-properties']['db_host'] +db_name = config['configurations']['kms-properties']['db_name'] +db_user = config['configurations']['kms-properties']['db_user'] +db_password = unicode(config['configurations']['kms-properties']['db_password']) +kms_master_key_password = unicode(config['configurations']['kms-properties']['KMS_MASTER_KEY_PASSWD']) +credential_provider_path = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.credential.provider.path'] +jdbc_alias = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.credential.alias'] +masterkey_alias = config['configurations']['dbks-site']['ranger.ks.masterkey.credential.alias'] +repo_name = str(config['clusterName']) + '_kms' +repo_name_value = config['configurations']['ranger-kms-security']['ranger.plugin.kms.service.name'] +if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": + repo_name = repo_name_value +cred_lib_path = os.path.join(kms_home,"cred","lib","*") +cred_setup_prefix = (format('{kms_home}/ranger_credential_helper.py'), '-l', cred_lib_path) +credential_file = format('/etc/ranger/{repo_name}/cred.jceks') + +if has_ranger_admin: + policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') + xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + xa_audit_db_password = '' + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + xa_db_host = config['configurations']['admin-properties']['db_host'] + + admin_uname = config['configurations']['ranger-env']['admin_username'] + admin_password = config['configurations']['ranger-env']['admin_password'] + ambari_ranger_admin = config['configurations']['ranger-env']['ranger_admin_username'] + ambari_ranger_password = config['configurations']['ranger-env']['ranger_admin_password'] + admin_uname_password = format("{admin_uname}:{admin_password}") + ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] + +default_connectors_map = { "mssql":"sqljdbc4.jar", + "mysql":"mysql-connector-java.jar", + "postgres":"postgresql-jdbc.jar", + "oracle":"ojdbc.jar", + "sqla":"sajdbc4.jar"} + +java_share_dir = '/usr/share/java' +jdbc_jar_name = None +previous_jdbc_jar_name = None +if db_flavor == 'mysql': + jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + db_jdbc_url = format('jdbc:log4jdbc:mysql://{db_host}/{db_name}') + db_jdbc_driver = "com.mysql.jdbc.Driver" + jdbc_dialect = "org.eclipse.persistence.platform.database.MySQLPlatform" +elif db_flavor == 'oracle': + jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = db_host.count(':') + if colon_count == 2 or colon_count == 0: + db_jdbc_url = format('jdbc:oracle:thin:@{db_host}') + else: + db_jdbc_url = format('jdbc:oracle:thin:@//{db_host}') + db_jdbc_driver = "oracle.jdbc.OracleDriver" + jdbc_dialect = "org.eclipse.persistence.platform.database.OraclePlatform" +elif db_flavor == 'postgres': + jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + db_jdbc_url = format('jdbc:postgresql://{db_host}/{db_name}') + db_jdbc_driver = "org.postgresql.Driver" + jdbc_dialect = "org.eclipse.persistence.platform.database.PostgreSQLPlatform" +elif db_flavor == 'mssql': + jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + db_jdbc_url = format('jdbc:sqlserver://{db_host};databaseName={db_name}') + db_jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + jdbc_dialect = "org.eclipse.persistence.platform.database.SQLServerPlatform" +elif db_flavor == 'sqla': + jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + db_jdbc_url = format('jdbc:sqlanywhere:database={db_name};host={db_host}') + db_jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + jdbc_dialect = "org.eclipse.persistence.platform.database.SQLAnywherePlatform" + +downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") + +driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") +driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") +previous_jdbc_jar = format("{kms_home}/ews/webapp/lib/{previous_jdbc_jar_name}") +ews_lib_jar_path = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") + +if db_flavor == 'sqla': + downloaded_custom_connector = format("{tmp_dir}/sqla-client-jdbc.tar.gz") + jar_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/java/sajdbc4.jar") + libs_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/native/lib64/*") + jdbc_libs_dir = format("{kms_home}/native/lib64") + ld_library_path = format("{jdbc_libs_dir}") + +if has_ranger_admin: + xa_previous_jdbc_jar_name = None + if stack_supports_ranger_audit_db: + if xa_audit_db_flavor == 'mysql': + jdbc_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "com.mysql.jdbc.Driver" + elif xa_audit_db_flavor == 'oracle': + jdbc_jar = default("/hostLevelParams/custom_oracle_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = xa_db_host.count(':') + if colon_count == 2 or colon_count == 0: + audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') + else: + audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') + jdbc_driver = "oracle.jdbc.OracleDriver" + elif xa_audit_db_flavor == 'postgres': + jdbc_jar = default("/hostLevelParams/custom_postgres_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "org.postgresql.Driver" + elif xa_audit_db_flavor == 'mssql': + jdbc_jar = default("/hostLevelParams/custom_mssql_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') + jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + elif xa_audit_db_flavor == 'sqla': + jdbc_jar = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') + jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + + downloaded_connector_path = format("{tmp_dir}/{jdbc_jar}") if stack_supports_ranger_audit_db else None + driver_source = format("{jdk_location}/{jdbc_jar}") if stack_supports_ranger_audit_db else None + driver_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar}") if stack_supports_ranger_audit_db else None + xa_previous_jdbc_jar = format("{kms_home}/ews/webapp/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + +repo_config_username = config['configurations']['kms-properties']['REPOSITORY_CONFIG_USERNAME'] +repo_config_password = unicode(config['configurations']['kms-properties']['REPOSITORY_CONFIG_PASSWORD']) + +kms_plugin_config = { + 'username' : repo_config_username, + 'password' : repo_config_password, + 'provider' : format('kms://http@{kms_host}:{kms_port}/kms') +} + +xa_audit_db_is_enabled = False +if stack_supports_ranger_audit_db: + xa_audit_db_is_enabled = config['configurations']['ranger-kms-audit']['xasecure.audit.destination.db'] +ssl_keystore_password = unicode(config['configurations']['ranger-kms-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) +ssl_truststore_password = unicode(config['configurations']['ranger-kms-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + +#For SQLA explicitly disable audit to DB for Ranger +if xa_audit_db_flavor == 'sqla': + xa_audit_db_is_enabled = False + +current_host = config['hostname'] +ranger_kms_hosts = config['clusterHostInfo']['ranger_kms_server_hosts'] +if current_host in ranger_kms_hosts: + kms_host = current_host + +check_db_connection_jar_name = "DBConnectionVerification.jar" +check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +ranger_kms_jdbc_connection_url = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.url'] +ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver'] + +jce_name = default("/hostLevelParams/jce_name", None) +jce_source_dir = format('{tmp_dir}/jce_dir') + +#kms hsm support +enable_kms_hsm = default("/configurations/dbks-site/ranger.ks.hsm.enabled", False) +hms_partition_alias = default("/configurations/dbks-site/ranger.ks.hsm.partition.password.alias", "ranger.kms.hsm.partition.password") +hms_partition_passwd = default("/configurations/kms-env/hsm_partition_password", None) + +# kms kerberos from stack 2.5 onward +rangerkms_bare_principal = 'rangerkms' + +if stack_supports_ranger_kerberos: + if security_enabled: + rangerkms_principal = config['configurations']['dbks-site']['ranger.ks.kerberos.principal'] + rangerkms_keytab = config['configurations']['dbks-site']['ranger.ks.kerberos.keytab'] + if not is_empty(rangerkms_principal) and rangerkms_principal != '': + rangerkms_bare_principal = get_bare_principal(rangerkms_principal) + rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower()) + kms_plugin_config['policy.download.auth.users'] = format('keyadmin,{rangerkms_bare_principal}') + +custom_ranger_service_config = generate_ranger_service_config(config['configurations']['kms-properties']) +if len(custom_ranger_service_config) > 0: + kms_plugin_config.update(custom_ranger_service_config) + +kms_ranger_plugin_repo = { + 'isEnabled' : 'true', + 'configs' : kms_plugin_config, + 'description' : 'kms repo', + 'name' : repo_name, + 'type' : 'kms' +} + +# ranger kms pid +user_group = config['configurations']['cluster-env']['user_group'] +ranger_kms_pid_dir = default("/configurations/kms-env/ranger_kms_pid_dir", "/var/run/ranger_kms") +ranger_kms_pid_file = format('{ranger_kms_pid_dir}/rangerkms.pid') + +if security_enabled: + spengo_keytab = config['configurations']['kms-site']['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab'] + spnego_principal = config['configurations']['kms-site']['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal'] + spnego_principal = spnego_principal.replace('_HOST', current_host.lower()) + +plugin_audit_password_property = 'xasecure.audit.destination.db.password' +kms_plugin_password_properties = ['xasecure.policymgr.clientssl.keystore.password', 'xasecure.policymgr.clientssl.truststore.password'] +dbks_site_password_properties = ['ranger.db.encrypt.key.password', 'ranger.ks.jpa.jdbc.password', 'ranger.ks.hsm.partition.password'] +ranger_kms_site_password_properties = ['ranger.service.https.attrib.keystore.pass'] +ranger_kms_cred_ssl_path = config['configurations']['ranger-kms-site']['ranger.credential.provider.path'] +ranger_kms_ssl_keystore_alias = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.credential.alias'] +ranger_kms_ssl_passwd = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.pass'] +ranger_kms_ssl_enabled = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.ssl.enabled'] + +xa_audit_hdfs_is_enabled = default("/configurations/ranger-kms-audit/xasecure.audit.destination.hdfs", False) +namenode_host = default("/clusterHostInfo/namenode_host", []) + +# need this to capture cluster name from where ranger kms plugin is enabled +cluster_name = config['clusterName'] + +has_namenode = len(namenode_host) > 0 + +hdfs_user = default("/configurations/hadoop-env/hdfs_user", None) +hdfs_user_keytab = default("/configurations/hadoop-env/hdfs_user_keytab", None) +hdfs_principal_name = default("/configurations/hadoop-env/hdfs_principal_name", None) +default_fs = default("/configurations/core-site/fs.defaultFS", None) +hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +import functools +# create partial functions with common arguments for every HdfsResource call +# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs +) + +local_component_list = default("/localComponents", []) +has_hdfs_client_on_node = 'HDFS_CLIENT' in local_component_list \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py new file mode 100644 index 0000000..84e4e73 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/service_check.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +from resource_management.core.logger import Logger +from resource_management.core import shell +from resource_management.core.exceptions import ComponentIsNotRunning + + +class KmsServiceCheck(Script): + def service_check(self, env): + import params + + env.set_params(params) + cmd = 'ps -ef | grep proc_rangerkms | grep -v grep' + code, output = shell.call(cmd, timeout=20) + if code == 0: + Logger.info('KMS process up and running') + else: + Logger.debug('KMS process not running') + raise ComponentIsNotRunning() + +if __name__ == "__main__": + KmsServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py new file mode 100644 index 0000000..34d0082 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/status_params.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_name = default("/hostLevelParams/stack_name", None) +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) +stack_supports_pid = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KMS_PID_SUPPORT, stack_version_formatted) +ranger_kms_pid_dir = default("/configurations/kms-env/ranger_kms_pid_dir", "/var/run/ranger_kms") +ranger_kms_pid_file = format('{ranger_kms_pid_dir}/rangerkms.pid') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py new file mode 100644 index 0000000..8478bb8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/scripts/upgrade.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.format import format + +def prestart(env, stack_component): + import params + + if params.version and params.stack_supports_config_versioning: + conf_select.select(params.stack_name, stack_component, params.version) + stack_select.select(stack_component, params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2 b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2 new file mode 100644 index 0000000..306fade --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/package/templates/input.config-ranger-kms.json.j2 @@ -0,0 +1,48 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')}}/kms.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json new file mode 100644 index 0000000..7ddab41 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/role_command_order.json @@ -0,0 +1,7 @@ +{ + "general_deps" : { + "_comment" : "dependencies for RANGER-KMS", + "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"], + "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/3dc51b0c/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py new file mode 100644 index 0000000..b81e05b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py @@ -0,0 +1,358 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +import imp +import os +import traceback +import re +import socket +import fnmatch + + +from resource_management.core.logger import Logger + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"} + +class Ranger_KMSServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(Ranger_KMSServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + # Nothing to do + pass + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return [] + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + """ + Entry point. + Must be overriden in child class. + """ + # Logger.info("Class: %s, Method: %s. Recommending Service Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + recommender = RangerKMSRecommender() + recommender.recommendRangerKMSConfigurationsFromHDP23(configurations, clusterData, services, hosts) + recommender.recommendRangerKMSConfigurationsFromHDP25(configurations, clusterData, services, hosts) + recommender.recommendRangerKMSConfigurationsFromHDP26(configurations, clusterData, services, hosts) + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + # Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + validator = RangerKMSValidator() + # Calls the methods of the validator using arguments, + # method(siteProperties, siteRecommendations, configurations, services, hosts) + return validator.validateListOfConfigUsingMethod(configurations, recommendedDefaults, services, hosts, validator.validators) + +class RangerKMSRecommender(service_advisor.ServiceAdvisor): + """ + RangerKMS Recommender suggests properties when adding the service for the first time or modifying configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(RangerKMSRecommender, self) + self.as_super.__init__(*args, **kwargs) + + def recommendRangerKMSConfigurationsFromHDP23(self, configurations, clusterData, services, hosts): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services) + putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services) + kmsEnvProperties = self.getSiteProperties(services['configurations'], 'kms-env') + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) + putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site") + putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services) + security_enabled = self.isSecurityEnabled(services) + putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services) + putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") + + if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"] + + if ('db_host' in services['configurations']['kms-properties']['properties']) and ('db_name' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"] + rangerKmsDbName = services['configurations']["kms-properties"]["properties"]["db_name"] + + ranger_kms_db_url_dict = { + 'MYSQL': {'ranger.ks.jpa.jdbc.driver': 'com.mysql.jdbc.Driver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName}, + 'ORACLE': {'ranger.ks.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost, rangerKmsDbName)}, + 'POSTGRES': {'ranger.ks.jpa.jdbc.driver': 'org.postgresql.Driver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName}, + 'MSSQL': {'ranger.ks.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';databaseName=' + rangerKmsDbName}, + 'SQLA': {'ranger.ks.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver', + 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';database=' + rangerKmsDbName} + } + + rangerKmsDbProperties = ranger_kms_db_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_url_dict['MYSQL']) + for key in rangerKmsDbProperties: + putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key)) + + if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList: + kmsUser = kmsEnvProperties['kms_user'] + kmsUserOld = self.getOldValue(services, 'kms-env', 'kms_user') + self.put_proxyuser_value(kmsUser, '*', is_groups=True, services=services, configurations=configurations, put_function=putCoreSiteProperty) + if kmsUserOld is not None and kmsUser != kmsUserOld: + putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true') + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)}) + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)}) + + if "HDFS" in servicesList: + if 'core-site' in services['configurations'] and ('fs.defaultFS' in services['configurations']['core-site']['properties']): + default_fs = services['configurations']['core-site']['properties']['fs.defaultFS'] + putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit')) + + required_services = [{'service' : 'YARN', 'config-type': 'yarn-env', 'property-name': 'yarn_user', 'proxy-category': ['hosts', 'users', 'groups']}, + {'service' : 'SPARK', 'config-type': 'livy-env', 'property-name': 'livy_user', 'proxy-category': ['hosts', 'users', 'groups']}] + + required_services_for_secure = [{'service' : 'HIVE', 'config-type': 'hive-env', 'property-name': 'hive_user', 'proxy-category': ['hosts', 'users']}, + {'service' : 'OOZIE', 'config-type': 'oozie-env', 'property-name': 'oozie_user', 'proxy-category': ['hosts', 'users']}] + + if security_enabled: + required_services.extend(required_services_for_secure) + + # recommendations for kms proxy related properties + self.recommendKMSProxyUsers(configurations, services, hosts, required_services) + + ambari_user = self.getAmbariUser(services) + if security_enabled: + # adding for ambari user + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), '*') + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), '*') + # adding for HTTP + putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.users', '*') + putRangerKmsSiteProperty('hadoop.kms.proxyuser.HTTP.hosts', '*') + else: + self.deleteKMSProxyUsers(configurations, services, hosts, required_services_for_secure) + # deleting ambari user proxy properties + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(ambari_user), 'delete', 'true') + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(ambari_user), 'delete', 'true') + # deleting HTTP proxy properties + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.hosts', 'delete', 'true') + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.HTTP.users', 'delete', 'true') + + def recommendRangerKMSConfigurationsFromHDP25(self, configurations, clusterData, services, hosts): + + security_enabled = self.isSecurityEnabled(services) + required_services = [{'service' : 'RANGER', 'config-type': 'ranger-env', 'property-name': 'ranger_user', 'proxy-category': ['hosts', 'users', 'groups']}] + + if security_enabled: + # recommendations for kms proxy related properties + self.recommendKMSProxyUsers(configurations, services, hosts, required_services) + else: + self.deleteKMSProxyUsers(configurations, services, hosts, required_services) + + def recommendRangerKMSConfigurationsFromHDP26(self, configurations, clusterData, services, hosts): + putRangerKmsEnvProperty = self.putProperty(configurations, "kms-env", services) + + ranger_kms_ssl_enabled = False + ranger_kms_ssl_port = "9393" + if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.attrib.ssl.enabled' in services['configurations']['ranger-kms-site']['properties']: + ranger_kms_ssl_enabled = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.attrib.ssl.enabled'].lower() == "true" + + if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.port' in services['configurations']['ranger-kms-site']['properties']: + ranger_kms_ssl_port = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.port'] + + if ranger_kms_ssl_enabled: + putRangerKmsEnvProperty("kms_port", ranger_kms_ssl_port) + else: + putRangerKmsEnvProperty("kms_port", "9292") + + def getDBConnectionHostPort(self, db_type, db_host): + connection_string = "" + if db_type is None or db_type == "": + return connection_string + else: + colon_count = db_host.count(':') + if colon_count == 0: + if DB_TYPE_DEFAULT_PORT_MAP.has_key(db_type): + connection_string = db_host + ":" + DB_TYPE_DEFAULT_PORT_MAP[db_type] + else: + connection_string = db_host + elif colon_count == 1: + connection_string = db_host + elif colon_count == 2: + connection_string = db_host + + return connection_string + + def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName): + connection_string = self.getDBConnectionHostPort(db_type, db_host) + colon_count = db_host.count(':') + if colon_count == 1 and '/' in db_host: + connection_string = "//" + connection_string + elif colon_count == 0 or colon_count == 1: + connection_string = "//" + connection_string + "/" + rangerDbName if rangerDbName else "//" + connection_string + + return connection_string + + def recommendKMSProxyUsers(self, configurations, services, hosts, requiredServices): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services) + putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") + + if 'forced-configurations' not in services: + services["forced-configurations"] = [] + + for index in range(len(requiredServices)): + service = requiredServices[index]['service'] + config_type = requiredServices[index]['config-type'] + property_name = requiredServices[index]['property-name'] + proxy_category = requiredServices[index]['proxy-category'] + + if service in servicesList: + if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']: + service_user = services['configurations'][config_type]['properties'][property_name] + service_old_user = self.getOldValue(services, config_type, property_name) + + if 'groups' in proxy_category: + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.groups'.format(service_user), '*') + if 'hosts' in proxy_category: + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), '*') + if 'users' in proxy_category: + putRangerKmsSiteProperty('hadoop.kms.proxyuser.{0}.users'.format(service_user), '*') + + if service_old_user is not None and service_user != service_old_user: + if 'groups' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_old_user), 'delete', 'true') + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_old_user)}) + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.groups".format(service_user)}) + if 'hosts' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_old_user), 'delete', 'true') + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_old_user)}) + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.hosts".format(service_user)}) + if 'users' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_old_user), 'delete', 'true') + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_old_user)}) + services["forced-configurations"].append({"type" : "kms-site", "name" : "hadoop.kms.proxyuser.{0}.users".format(service_user)}) + + def deleteKMSProxyUsers(self, configurations, services, hosts, requiredServices): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") + + for index in range(len(requiredServices)): + service = requiredServices[index]['service'] + config_type = requiredServices[index]['config-type'] + property_name = requiredServices[index]['property-name'] + proxy_category = requiredServices[index]['proxy-category'] + + if service in servicesList: + if config_type in services['configurations'] and property_name in services['configurations'][config_type]['properties']: + service_user = services['configurations'][config_type]['properties'][property_name] + + if 'groups' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.groups'.format(service_user), 'delete', 'true') + if 'hosts' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.hosts'.format(service_user), 'delete', 'true') + if 'users' in proxy_category: + putRangerKmsSitePropertyAttribute('hadoop.kms.proxyuser.{0}.users'.format(service_user), 'delete', 'true') + +class RangerKMSValidator(service_advisor.ServiceAdvisor): + """ + RangerKMS Validator checks the correctness of properties whenever the service is first added or the user attempts to + change configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(RangerKMSValidator, self) + self.as_super.__init__(*args, **kwargs) + + self.validators = []