AMBARI-13421. Blueprints: install for Ranger Components (ranger-admin, ranger-usersync, ranger-kms) (Sebastian Toader via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c4c83384 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c4c83384 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c4c83384 Branch: refs/heads/branch-dev-patch-upgrade Commit: c4c833842978d321c0d08c319b2f308d5861e323 Parents: 1ff22df Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Thu Oct 22 10:17:07 2015 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Thu Oct 22 10:17:07 2015 -0700 ---------------------------------------------------------------------- .../libraries/functions/ranger_functions.py | 8 +- .../libraries/functions/ranger_functions_v2.py | 39 +++++-- .../libraries/functions/setup_ranger_plugin.py | 6 +- .../functions/setup_ranger_plugin_xml.py | 6 +- .../java/org/apache/ambari/server/Role.java | 6 ++ .../HBASE/0.96.0.2.0/package/scripts/params.py | 3 +- .../package/scripts/setup_ranger_hbase.py | 7 +- .../HDFS/2.1.0.2.0/package/scripts/params.py | 1 + .../package/scripts/setup_ranger_hdfs.py | 7 +- .../HIVE/0.12.0.2.0/package/scripts/params.py | 1 + .../package/scripts/setup_ranger_hive.py | 9 +- .../KAFKA/0.8.1.2.2/package/scripts/params.py | 1 + .../package/scripts/setup_ranger_kafka.py | 8 +- .../KNOX/0.5.0.2.2/package/scripts/params.py | 3 +- .../package/scripts/setup_ranger_knox.py | 9 +- .../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 1 - .../STORM/0.9.1.2.1/package/scripts/params.py | 2 +- .../package/scripts/setup_ranger_storm.py | 7 +- .../YARN/2.1.0.2.0/package/scripts/params.py | 2 +- .../package/scripts/setup_ranger_yarn.py | 10 +- .../stacks/HDP/2.2/role_command_order.json | 13 ++- .../ambari/server/stack/StackManagerTest.java | 105 +++++++++++++++++++ 22 files changed, 218 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py index de58976..dcf59c1 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py @@ -30,7 +30,7 @@ import re class Rangeradmin: sInstance = None - def __init__(self, url='http://localhost:6080'): + def __init__(self, url='http://localhost:6080', skip_if_rangeradmin_down = True): self.baseUrl = url self.urlLogin = self.baseUrl + '/login.jsp' @@ -41,6 +41,10 @@ class Rangeradmin: self.urlGroups = self.baseUrl + '/service/xusers/groups' self.urlUsers = self.baseUrl + '/service/xusers/users' self.urlSecUsers = self.baseUrl + '/service/xusers/secure/users' + self.skip_if_rangeradmin_down = skip_if_rangeradmin_down + + if self.skip_if_rangeradmin_down: + Logger.info("Rangeradmin: Skip ranger admin if it's down !") def get_repository_by_name_urllib2(self, name, component, status, usernamepassword): """ @@ -121,6 +125,8 @@ class Rangeradmin: raise Fail('{0} Repository creation failed in Ranger admin'.format(component.title())) else: raise Fail('Ambari admin user creation failed') + elif not self.skip_if_rangeradmin_down: + raise Fail("Connection failed to Ranger Admin !") def create_repository_urllib2(self, data, usernamepassword, policy_user): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py index 81658bf..b79f6d8 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py @@ -27,12 +27,10 @@ from resource_management.core.exceptions import Fail from resource_management.libraries.functions.format import format import re - class RangeradminV2: sInstance = None - def __init__(self, url='http://localhost:6080'): - + def __init__(self, url='http://localhost:6080', skip_if_rangeradmin_down = True): self.base_url = url self.url_login = self.base_url + '/login.jsp' self.url_login_post = self.base_url + '/j_spring_security_check' @@ -42,6 +40,10 @@ class RangeradminV2: self.url_groups = self.base_url + '/service/xusers/groups' self.url_users = self.base_url + '/service/xusers/users' self.url_sec_users = self.base_url + '/service/xusers/secure/users' + self.skip_if_rangeradmin_down = skip_if_rangeradmin_down + + if self.skip_if_rangeradmin_down: + Logger.info("RangeradminV2: Skip ranger admin if it's down !") def get_repository_by_name_urllib2(self, name, component, status, usernamepassword): """ @@ -88,21 +90,36 @@ class RangeradminV2: ambari_ranger_password = unicode(ambari_ranger_password) admin_password = unicode(admin_password) ambari_username_password_for_ranger = format('{ambari_ranger_admin}:{ambari_ranger_password}') + if response_code is not None and response_code == 200: user_resp_code = self.create_ambari_admin_user(ambari_ranger_admin, ambari_ranger_password, format("{admin_uname}:{admin_password}")) if user_resp_code is not None and user_resp_code == 200: - repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger) - if repo is not None: - Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name'])) - else: - response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger) - if response is not None: - Logger.info('{0} Repository created in Ranger admin'.format(component.title())) + retryCount = 0 + while retryCount <= 5: + repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger) + if repo is not None: + Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name'])) + break else: - Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) + response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger) + if response is not None: + Logger.info('{0} Repository created in Ranger admin'.format(component.title())) + break + else: + if retryCount < 5: + Logger.info("Retry Repository Creation is being called") + time.sleep(30) # delay for 30 seconds + retryCount += 1 + else: + Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) + raise Fail('{0} Repository creation failed in Ranger admin'.format(component.title())) else: Logger.error('Ambari admin user creation failed') + raise Fail('Ambari admin user creation failed') + elif not self.skip_if_rangeradmin_down: + raise Fail("Connection failed to Ranger Admin !") + def create_repository_urllib2(self, data, usernamepassword): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py index e4a19aa..e5e4266 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py @@ -37,7 +37,7 @@ def setup_ranger_plugin(component_select_name, service_name, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, - plugin_enabled, component_user, component_group, api_version=None, **kwargs): + plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs): File(downloaded_custom_connector, content = DownloadSource(driver_curl_source), mode = 0644 @@ -68,9 +68,9 @@ def setup_ranger_plugin(component_select_name, service_name, if plugin_enabled: cmd = (format('enable-{service_name}-plugin.sh'),) if api_version == 'v2' and api_version is not None: - ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url) + ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down) else: - ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url) + ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 0b404a9..29ffe0d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -44,7 +44,7 @@ def setup_ranger_plugin(component_select_name, service_name, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, - ssl_keystore_password, api_version=None, hdp_version_override = None): + ssl_keystore_password, api_version=None, hdp_version_override = None, skip_if_rangeradmin_down = True): if audit_db_is_enabled: File(component_downloaded_custom_connector, @@ -68,9 +68,9 @@ def setup_ranger_plugin(component_select_name, service_name, if plugin_enabled: if api_version == 'v2' and api_version is not None: - ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url) + ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) else: - ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url) + ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/java/org/apache/ambari/server/Role.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/Role.java b/ambari-server/src/main/java/org/apache/ambari/server/Role.java index df60988..f72cc5b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/Role.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/Role.java @@ -113,6 +113,12 @@ public class Role { public static final Role METRICS_MONITOR = valueOf("METRICS_MONITOR"); public static final Role AMS_SERVICE_CHECK = valueOf("AMBARI_METRICS_SERVICE_CHECK"); public static final Role ACCUMULO_CLIENT = valueOf("ACCUMULO_CLIENT"); + public static final Role RANGER_ADMIN = valueOf("RANGER_ADMIN"); + public static final Role RANGER_USERSYNC = valueOf("RANGER_USERSYNC"); + public static final Role KNOX_GATEWAY = valueOf("KNOX_GATEWAY"); + public static final Role KAFKA_BROKER = valueOf("KAFKA_BROKER"); + public static final Role NIMBUS = valueOf("NIMBUS"); + public static final Role RANGER_KMS_SERVER = valueOf("RANGER_KMS_SERVER"); public static final Role INSTALL_PACKAGES = valueOf("install_packages"); public static final Role UPDATE_REPO = valueOf("update_repo"); http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py index 36dd07f..f3208ce 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py @@ -25,4 +25,5 @@ if OSCheck.is_windows_family(): else: from params_linux import * -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) \ No newline at end of file +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) +retryAble = default("/commandParams/command_retry_enabled", False) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py index 6b4dfaa..8f4a6d0 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py @@ -34,6 +34,11 @@ def setup_ranger_hbase(rolling_upgrade = False): if rolling_upgrade: hdp_version = params.version + if params.retryAble: + Logger.info("HBase: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("HBase: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + setup_ranger_plugin('hbase-client', 'hbase', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, @@ -48,6 +53,6 @@ def setup_ranger_hbase(rolling_upgrade = False): component_list=['hbase-client', 'hbase-master', 'hbase-regionserver'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, - hdp_version_override = hdp_version) + hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble) else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py index b89eefd..7514918 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py @@ -26,3 +26,4 @@ else: host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize'] +retryAble = default("/commandParams/command_retry_enabled", False) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py index 6a64b2f..bd158ec 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py @@ -34,6 +34,11 @@ def setup_ranger_hdfs(upgrade_type=None): if upgrade_type is not None: hdp_version = params.version + if params.retryAble: + Logger.info("HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + setup_ranger_plugin('hadoop-client', 'hdfs', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java_home, @@ -48,6 +53,6 @@ def setup_ranger_hdfs(upgrade_type=None): component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, - hdp_version_override = hdp_version) + hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble) else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py index 36f7983..f10a3f3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py @@ -26,3 +26,4 @@ else: from params_linux import * host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) +retryAble = default("/commandParams/command_retry_enabled", False) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py index b0f0c3f..5fdaa70 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py @@ -32,7 +32,12 @@ def setup_ranger_hive(rolling_upgrade = False): hdp_version = None if rolling_upgrade: hdp_version = params.version - + + if params.retryAble: + Logger.info("Hive: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("Hive: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + setup_ranger_plugin('hive-server2', 'hive', params.ranger_downloaded_custom_connector, params.ranger_driver_curl_source, params.ranger_driver_curl_target, params.java64_home, @@ -47,6 +52,6 @@ def setup_ranger_hive(rolling_upgrade = False): component_list=['hive-client', 'hive-metastore', 'hive-server2'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, - hdp_version_override = hdp_version) + hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble) else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py index dc0c087..da8333a 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py @@ -32,6 +32,7 @@ import status_params config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_name = default("/hostLevelParams/stack_name", None) +retryAble = default("/commandParams/command_retry_enabled", False) version = default("/commandParams/version", None) host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py index 540bb9a..c210791 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py @@ -24,6 +24,12 @@ def setup_ranger_kafka(): if params.has_ranger_admin: from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + + if params.retryAble: + Logger.info("Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + setup_ranger_plugin('kafka-broker', 'kafka', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, @@ -38,7 +44,7 @@ def setup_ranger_kafka(): component_list=['kafka-broker'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, - api_version = 'v2') + api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble) if params.enable_ranger_kafka: Execute(('cp', '--remove-destination', params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target), http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py index 8fe1028..14e021d 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py @@ -26,4 +26,5 @@ if OSCheck.is_windows_family(): else: from params_linux import * -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) \ No newline at end of file +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) +retryAble = default("/commandParams/command_retry_enabled", False) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py index f1319b3..1efe9e0 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py @@ -33,7 +33,12 @@ def setup_ranger_knox(rolling_upgrade = False): if rolling_upgrade: hdp_version = params.version - setup_ranger_plugin('knox-server', 'knox', + if params.retryAble: + Logger.info("Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + + setup_ranger_plugin('knox-server', 'knox', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java_home, params.repo_name, params.knox_ranger_plugin_repo, @@ -47,6 +52,6 @@ def setup_ranger_knox(rolling_upgrade = False): component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, - hdp_version_override = hdp_version) + hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble) else: Logger.info('Ranger admin not installed') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py index 570b2b7..4c5bd1a 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py @@ -264,7 +264,6 @@ def enable_kms_plugin(): import params if params.has_ranger_admin: - ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url) ambari_username_password_for_ranger = format("{ambari_ranger_admin}:{ambari_ranger_password}") response_code = ranger_adm_obj.check_ranger_login_urllib2(params.policymgr_mgr_url) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py index 1e591f4..f10a3f3 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py @@ -26,4 +26,4 @@ else: from params_linux import * host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) - +retryAble = default("/commandParams/command_retry_enabled", False) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py index d874ba3..5d90f5b 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py @@ -33,6 +33,11 @@ def setup_ranger_storm(rolling_upgrade = False): if rolling_upgrade: hdp_version = params.version + if params.retryAble: + Logger.info("Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + setup_ranger_plugin('storm-nimbus', 'storm', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, @@ -47,6 +52,6 @@ def setup_ranger_storm(rolling_upgrade = False): component_list=['storm-client', 'storm-nimbus'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, - hdp_version_override = hdp_version) + hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble) else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py index 5695e83..073e84f 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py @@ -28,4 +28,4 @@ else: from params_linux import * host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) - +retryAble = default("/commandParams/command_retry_enabled", False) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py index c8b12df..5db65d0d 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/setup_ranger_yarn.py @@ -22,8 +22,12 @@ def setup_ranger_yarn(): if params.has_ranger_admin: from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin - - + + if params.retryAble: + Logger.info("YARN: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("YARN: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + setup_ranger_plugin('hadoop-yarn-resourcemanager', 'yarn', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, @@ -38,7 +42,7 @@ def setup_ranger_yarn(): component_list=['hadoop-yarn-resourcemanager'], audit_db_is_enabled=params.xa_audit_db_is_enabled, credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, - api_version = 'v2' + api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble ) else: Logger.info('Ranger admin not installed') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json index 3571a6c..3beed16 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json @@ -21,7 +21,16 @@ "FLUME_SERVICE_CHECK-SERVICE_CHECK": ["FLUME_HANDLER-START"], "FALCON_SERVICE_CHECK-SERVICE_CHECK": ["FALCON_SERVER-START"], "SLIDER_SERVICE_CHECK-SERVICE_CHECK" : ["NODEMANAGER-START", "RESOURCEMANAGER-START"], - "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START"], - "KAFKA_SERVICE_CHECK-SERVICE_CHECK": ["KAFKA_BROKER-START"] + "KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START"], + "KAFKA_SERVICE_CHECK-SERVICE_CHECK": ["KAFKA_BROKER-START"], + "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"], + "ZOOKEEPER_SERVER-START" : ["RANGER_USERSYNC-START"], + "DATANODE-START" : ["RANGER_USERSYNC-START"], + "NAMENODE-START" : ["RANGER_USERSYNC-START"], + "KNOX_GATEWAY-START": ["RANGER_USERSYNC-START"], + "RESOURCEMANAGER-START" : ["RANGER_USERSYNC-START"], + "NIMBUS-START": ["RANGER_USERSYNC-START"], + "HBASE_MASTER-START": ["RANGER_USERSYNC-START"], + "HIVE_SERVER-START" : ["RANGER_USERSYNC-START"] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c4c83384/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java index be8b073..6a13ab2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java @@ -41,6 +41,8 @@ import java.util.Map; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.Role; +import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.MetainfoDAO; @@ -54,6 +56,7 @@ import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.stack.MetricDefinition; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.state.stack.StackRoleCommandOrder; import org.apache.commons.lang.StringUtils; import org.junit.BeforeClass; import org.junit.Test; @@ -667,5 +670,107 @@ public class StackManagerTest { } } + @Test + public void testServicesWithRangerPluginRoleCommandOrder() throws AmbariException { + // Given + String stackRoot = ClassLoader.getSystemClassLoader().getResource("stacks").getPath().replace("test-classes","classes"); + String commonServices = ClassLoader.getSystemClassLoader().getResource("common-services").getPath().replace("test-classes","classes"); + + MetainfoDAO metaInfoDao = createNiceMock(MetainfoDAO.class); + StackDAO stackDao = createNiceMock(StackDAO.class); + ActionMetadata actionMetadata = createNiceMock(ActionMetadata.class); + Configuration config = createNiceMock(Configuration.class); + + expect(config.getSharedResourcesDirPath()).andReturn( + ClassLoader.getSystemClassLoader().getResource("").getPath()).anyTimes(); + + replay(config, metaInfoDao, stackDao, actionMetadata); + + OsFamily osFamily = new OsFamily(config); + + StackManager stackManager = new StackManager(new File(stackRoot), new File(commonServices), osFamily, metaInfoDao, actionMetadata, stackDao); + + String rangerUserSyncRoleCommand = Role.RANGER_USERSYNC + "-" + RoleCommand.START; + String rangerAdminRoleCommand = Role.RANGER_ADMIN + "-" + RoleCommand.START; + + // When + StackInfo hdp = stackManager.getStack("HDP", "2.3"); + Map<String, Object> rco = hdp.getRoleCommandOrder().getContent(); + + // Then + // verify that services that have ranger plugin are after ranger admin in the role command order sequence + // as these services require ranger admin and ranger user sync to up upfront + Map<String, Object> generalDeps = (Map<String, Object>)rco.get("general_deps"); + + // HDFS + String nameNodeRoleCommand = Role.NAMENODE + "-" + RoleCommand.START; + ArrayList<String> nameNodeBlockers = (ArrayList<String>)generalDeps.get(nameNodeRoleCommand); + + assertTrue(nameNodeRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, nameNodeBlockers.contains(rangerUserSyncRoleCommand)); + + String dataNodeRoleCommand = Role.DATANODE + "-" + RoleCommand.START; + ArrayList<String> dataNodeBlockers = (ArrayList<String>)generalDeps.get(dataNodeRoleCommand); + + assertTrue(dataNodeRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, dataNodeBlockers.contains(rangerUserSyncRoleCommand)); + + // YARN + String resourceManagerCommandRoleCommand = Role.RESOURCEMANAGER + "-" + RoleCommand.START; + ArrayList<String> resourceManagerBlockers = (ArrayList<String>)generalDeps.get(resourceManagerCommandRoleCommand); + + assertTrue(resourceManagerCommandRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, resourceManagerBlockers.contains(rangerUserSyncRoleCommand)); + + + // HBase + String hbaseRoleCommand = Role.HBASE_MASTER + "-" + RoleCommand.START; + ArrayList<String> hbaseBlockers = (ArrayList<String>)generalDeps.get(hbaseRoleCommand); + + assertTrue(hbaseRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, hbaseBlockers.contains(rangerUserSyncRoleCommand)); + + // Knox + String knoxRoleCommand = Role.KNOX_GATEWAY + "-" + RoleCommand.START; + ArrayList<String> knoxBlockers = (ArrayList<String>)generalDeps.get(knoxRoleCommand); + + assertTrue(knoxRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, knoxBlockers.contains(rangerUserSyncRoleCommand)); + + // Kafka + String kafkaRoleCommand = Role.KAFKA_BROKER + "-" + RoleCommand.START; + ArrayList<String> kafkaBlockers = (ArrayList<String>)generalDeps.get(kafkaRoleCommand); + + assertTrue(Role.KAFKA_BROKER + "-" + RoleCommand.START + " should be dependent of " + rangerUserSyncRoleCommand, kafkaBlockers.contains(rangerUserSyncRoleCommand)); + + // Hive + String hiveRoleCommand = Role.HIVE_SERVER + "-" + RoleCommand.START; + ArrayList<String> hiveBlockers = (ArrayList<String>)generalDeps.get(hiveRoleCommand); + + assertTrue(hiveRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, hiveBlockers.contains(rangerUserSyncRoleCommand)); + + // Storm + String stormRoleCommand = Role.NIMBUS + "-" + RoleCommand.START; + ArrayList<String> stormBlockers = (ArrayList<String>)generalDeps.get(stormRoleCommand); + + assertTrue(stormRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, stormBlockers.contains(rangerUserSyncRoleCommand)); + + // Ranger KMS + String kmsRoleCommand = Role.RANGER_KMS_SERVER + "-" + RoleCommand.START; + ArrayList<String> rangerKmsBlockers = (ArrayList<String>)generalDeps.get(kmsRoleCommand); + + assertTrue(kmsRoleCommand + " should be dependent of " + rangerAdminRoleCommand, rangerKmsBlockers.contains(rangerAdminRoleCommand)); + + // Ranger User Sync + ArrayList<String> rangerUserSyncBlockers = (ArrayList<String>)generalDeps.get(rangerUserSyncRoleCommand); + + assertTrue(rangerUserSyncRoleCommand + " should be dependent of " + rangerAdminRoleCommand, rangerUserSyncBlockers.contains(rangerAdminRoleCommand)); + assertTrue(rangerUserSyncRoleCommand + " should be dependent of " + kmsRoleCommand, rangerUserSyncBlockers.contains(kmsRoleCommand)); + + // Zookeeper Server + String zookeeperServerRoleCommand = Role.ZOOKEEPER_SERVER + "-" + RoleCommand.START; + ArrayList<String> zookeeperBlockers = (ArrayList<String>)generalDeps.get(zookeeperServerRoleCommand); + + assertTrue(zookeeperServerRoleCommand + " should be dependent of " + rangerUserSyncRoleCommand, zookeeperBlockers.contains(rangerUserSyncRoleCommand)); + + + } + + //todo: component override assertions }