Repository: ambari Updated Branches: refs/heads/branch-2.1 64d7b1dd9 -> dcf1a161b
AMBARI-13583: Express Upgrade: Handle Flume, Spark, Ranger, Slider in all upgrade paths and downgrades (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dcf1a161 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dcf1a161 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dcf1a161 Branch: refs/heads/branch-2.1 Commit: dcf1a161bb6e8a64ae635251ed248370bababd9b Parents: 64d7b1d Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Wed Oct 28 12:39:22 2015 -0700 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Wed Oct 28 12:40:58 2015 -0700 ---------------------------------------------------------------------- .../state/stack/upgrade/ClusterGrouping.java | 76 ++++++-------------- .../1.4.0.2.0/package/scripts/flume_handler.py | 74 +++++++++---------- .../FLUME/1.4.0.2.0/package/scripts/params.py | 2 +- .../0.4.0/package/scripts/ranger_admin.py | 18 ++--- .../0.4.0/package/scripts/ranger_usersync.py | 8 +-- .../0.4.0/package/scripts/service_check.py | 2 +- .../0.4.0/package/scripts/setup_ranger_xml.py | 32 ++++----- .../0.5.0.2.3/package/scripts/kms_server.py | 6 +- .../SLIDER/0.60.0.2.2/package/scripts/params.py | 2 +- .../0.60.0.2.2/package/scripts/slider_client.py | 27 ++++--- .../package/scripts/job_history_server.py | 7 +- .../SPARK/1.2.0.2.2/package/scripts/params.py | 2 +- .../1.2.0.2.2/package/scripts/spark_client.py | 3 +- .../package/scripts/spark_thrift_server.py | 7 +- .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 10 +-- .../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml | 16 ++--- .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 44 +++++++++--- .../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml | 35 ++++++--- .../AmbariManagementControllerTest.java | 2 +- .../ambari/server/state/UpgradeHelperTest.java | 34 +++++++++ .../stack/upgrade/ClusterGroupingTest.java | 75 ------------------- .../python/stacks/2.0.6/FLUME/test_flume.py | 4 +- .../stacks/2.2/RANGER/test_ranger_admin.py | 4 +- .../stacks/2.2/SLIDER/test_slider_client.py | 8 +-- .../stacks/2.2/SPARK/test_job_history_server.py | 4 +- .../stacks/2.2/SPARK/test_spark_client.py | 4 +- .../2.3/SPARK/test_spark_thrift_server.py | 4 +- .../upgrades/upgrade_server_action_test.xml | 52 ++++++++++++++ 28 files changed, 287 insertions(+), 275 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java index df416f2..d3af7ac 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java @@ -138,27 +138,9 @@ public class ClusterGrouping extends Grouping { switch (task.getType()) { case MANUAL: - wrapper = getManualStageWrapper(upgradeContext, execution); - break; - case SERVER_ACTION: - wrapper = new StageWrapper( - StageWrapper.Type.SERVER_SIDE_ACTION, - execution.title, - new TaskWrapper(null, null, Collections.<String>emptySet(), task)); - break; - case CONFIGURE: - Set<String> matchingHosts = getHostsForExecuteStage(upgradeContext, execution); - if (matchingHosts == null || matchingHosts.isEmpty()) { // No target services, don't create stage - wrapper = null; - } else { - // We don't care about real host list - wrapper = new StageWrapper( - StageWrapper.Type.SERVER_SIDE_ACTION, - execution.title, - new TaskWrapper(null, null, Collections.<String>emptySet(), execution.task)); - } + wrapper = getServerActionStageWrapper(upgradeContext, execution); break; case EXECUTE: @@ -180,20 +162,19 @@ public class ClusterGrouping extends Grouping { } /** - * Return a Stage Wrapper for a manual task that runs on the server. + * Return a Stage Wrapper for a server side action that runs on the server. * @param ctx Upgrade Context * @param execution Execution Stage * @return Returns a Stage Wrapper */ - private StageWrapper getManualStageWrapper(UpgradeContext ctx, ExecuteStage execution) { + private StageWrapper getServerActionStageWrapper(UpgradeContext ctx, ExecuteStage execution) { String service = execution.service; String component = execution.component; String id = execution.id; Task task = execution.task; - if (null != id && id.equals("unhealthy-hosts")) { - + if ( Task.Type.MANUAL == task.getType() && null != id && id.equals("unhealthy-hosts")) { // !!! this specific task is used ONLY when there are unhealthy if (ctx.getUnhealthy().isEmpty()) { return null; @@ -203,44 +184,31 @@ public class ClusterGrouping extends Grouping { fillHostDetails(mt, ctx.getUnhealthy()); } - Set<String> realHosts = getHostsForExecuteStage(ctx, execution); - - if (realHosts == null) { // No target services, don't create stage - return null; - } else { - return new StageWrapper( - StageWrapper.Type.SERVER_SIDE_ACTION, - execution.title, - new TaskWrapper(service, component, realHosts, task)); - } - } - - /** - * Looks for real hosts that match service/component restrictions of - * an Execution Stage - * @param ctx Upgrade Context - * @param execution Execution Stage - * @return <ul> - * <li> empty Set if Execution Stage has no restrictions; - * <li> set of real hosts that match restrictions if Execution Stage has - * restrictions and some hosts match - * <li> null if Execution Stage has restrictions, but no any host matches - */ - static Set<String> getHostsForExecuteStage(UpgradeContext ctx, ExecuteStage execution) { - String service = execution.service; - String component = execution.component; - Set<String> realHosts = Collections.emptySet(); + if (null != service && !service.isEmpty() && null != component && !component.isEmpty()) { + HostsType hosts = ctx.getResolver().getMasterAndHosts(service, component); - if (null == hosts) { - realHosts = null; + + if (null == hosts || hosts.hosts.isEmpty()) { + return null; } else { - realHosts = new LinkedHashSet<>(hosts.hosts); + realHosts = new LinkedHashSet<String>(hosts.hosts); } } - return realHosts; + + if (Task.Type.MANUAL == task.getType()) { + return new StageWrapper( + StageWrapper.Type.SERVER_SIDE_ACTION, + execution.title, + new TaskWrapper(service, component, realHosts, task)); + } else { + return new StageWrapper( + StageWrapper.Type.SERVER_SIDE_ACTION, + execution.title, + new TaskWrapper(null, null, Collections.<String>emptySet(), task)); + } } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py index e1fe577..937547c 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py @@ -33,57 +33,36 @@ from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl class FlumeHandler(Script): + def configure(self, env): + import params + env.set_params(params) + flume(action='config') - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class FlumeHandlerLinux(FlumeHandler): def get_stack_to_component(self): return {"HDP": "flume-server"} - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def install(self, env): import params self.install_packages(env) env.set_params(params) - @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) - def install(self, env): - if not check_windows_service_exists(service_mapping.flume_win_service_name): - self.install_packages(env) - self.configure(env) - - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) flume(action='start') - @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) - def start(self, env): - import params - env.set_params(params) - self.configure(env) - Service(service_mapping.flume_win_service_name, action="start") - - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def stop(self, env, rolling_restart=False): + def stop(self, env, upgrade_type=None): import params env.set_params(params) flume(action='stop') # only backup data on upgrade - if rolling_restart and params.upgrade_direction == Direction.UPGRADE: + if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE: flume_upgrade.post_stop_backup() - @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) - def stop(self, env): - Service(service_mapping.flume_win_service_name, action="stop") - - def configure(self, env): - import params - env.set_params(params) - flume(action='config') - - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def status(self, env): import params env.set_params(params) @@ -94,8 +73,8 @@ class FlumeHandler(Script): json['processes'] = processes self.put_structured_out(json) - # only throw an exception if there are agents defined and there is a - # problem with the processes; if there are no agents defined, then + # only throw an exception if there are agents defined and there is a + # problem with the processes; if there are no agents defined, then # the service should report STARTED (green) ONLY if the desired state is started. otherwise, INSTALLED (red) if len(expected_agents) > 0: for proc in processes: @@ -104,14 +83,7 @@ class FlumeHandler(Script): elif len(expected_agents) == 0 and 'INSTALLED' == get_desired_state(): raise ComponentIsNotRunning() - - @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) - def status(self, env): - import params - check_windows_service_status(service_mapping.flume_win_service_name) - - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) @@ -120,7 +92,7 @@ class FlumeHandler(Script): if not params.version or Script.is_hdp_stack_less_than("2.2"): return - Logger.info("Executing Flume Rolling Upgrade pre-restart") + Logger.info("Executing Flume Stack Upgrade pre-restart") conf_select.select(params.stack_name, "flume", params.version) hdp_select.select("flume-server", params.version) @@ -128,5 +100,25 @@ class FlumeHandler(Script): if params.upgrade_direction == Direction.UPGRADE: flume_upgrade.pre_start_restore() +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class FlumeHandlerWindows(FlumeHandler): + def install(self, env): + if not check_windows_service_exists(service_mapping.flume_win_service_name): + self.install_packages(env) + self.configure(env) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + Service(service_mapping.flume_win_service_name, action="start") + + def stop(self, env, upgrade_type=None): + Service(service_mapping.flume_win_service_name, action="stop") + + def status(self, env): + import params + check_windows_service_status(service_mapping.flume_win_service_name) + if __name__ == "__main__": FlumeHandler().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py index 9a6d5b6..d2f2d02 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py @@ -33,7 +33,7 @@ config = Script.get_config() stack_name = default("/hostLevelParams/stack_name", None) host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade version = default("/commandParams/version", None) user_group = config['configurations']['cluster-env']['user_group'] http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py index 23648e1..a0007b3 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py @@ -48,14 +48,14 @@ class RangerAdmin(Script): from setup_ranger_xml import setup_java_patch setup_java_patch() - def stop(self, env, rolling_restart=False): + def stop(self, env, upgrade_type=None): import params env.set_params(params) Execute(format('{params.ranger_stop}'), environment={'JAVA_HOME': params.java_home}, user=params.unix_user) - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) @@ -63,17 +63,17 @@ class RangerAdmin(Script): if params.xml_configurations_supported: from setup_ranger_xml import ranger, setup_ranger_db, setup_java_patch - ranger('ranger_admin', rolling_upgrade=True) - setup_ranger_db(rolling_upgrade=True) - setup_java_patch(rolling_upgrade=True) + ranger('ranger_admin', upgrade_type=upgrade_type) + setup_ranger_db(upgrade_type=upgrade_type) + setup_java_patch(upgrade_type=upgrade_type) self.set_ru_rangeradmin_in_progress() - def post_rolling_restart(self,env): + def post_upgrade_restart(self,env, upgrade_type=None): if os.path.isfile(RangerAdmin.upgrade_marker_file): os.remove(RangerAdmin.upgrade_marker_file) - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) @@ -86,7 +86,7 @@ class RangerAdmin(Script): if code != 0: if self.is_ru_rangeradmin_in_progress(): - Logger.info('Ranger admin process not running - skipping as rolling upgrade is in progress') + Logger.info('Ranger admin process not running - skipping as stack upgrade is in progress') else: Logger.debug('Ranger admin process not running') raise ComponentIsNotRunning() @@ -105,7 +105,7 @@ class RangerAdmin(Script): def set_ru_rangeradmin_in_progress(self): config_dir = os.path.dirname(RangerAdmin.upgrade_marker_file) try: - msg = "Starting RU" + msg = "Starting Upgrade" if (not os.path.exists(config_dir)): os.makedirs(config_dir) ofp = open(RangerAdmin.upgrade_marker_file, 'w') http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py index 6a2d4a1..72133bc 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py @@ -43,14 +43,14 @@ class RangerUsersync(Script): ranger('ranger_usersync') - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) ranger_service('ranger_usersync') - def stop(self, env, rolling_restart=False): + def stop(self, env, upgrade_type=None): import params env.set_params(params) @@ -65,14 +65,14 @@ class RangerUsersync(Script): raise ComponentIsNotRunning() pass - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) upgrade.prestart(env, "ranger-usersync") if params.xml_configurations_supported: from setup_ranger_xml import ranger - ranger('ranger_usersync', rolling_upgrade=True) + ranger('ranger_usersync', upgrade_type=upgrade_type) def get_stack_to_component(self): return {"HDP": "ranger-usersync"} http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/service_check.py index c7a89fa..699e3c4 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/service_check.py @@ -37,7 +37,7 @@ class RangerServiceCheck(Script): def check_ranger_admin_service(self, ranger_external_url): if (self.is_ru_rangeradmin_in_progress()): - Logger.info('Ranger admin process not running - skipping as rolling upgrade is in progress') + Logger.info('Ranger admin process not running - skipping as stack upgrade is in progress') else: Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {ranger_external_url}/login.jsp | grep 200"), tries = 10, http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index f6a9cb8..611fde7 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -33,17 +33,17 @@ from resource_management.core.shell import as_sudo # This file contains functions used for setup/configure of Ranger Admin and Ranger Usersync. # The design is to mimic what is done by the setup.sh script bundled by Ranger component currently. -def ranger(name=None, rolling_upgrade=False): +def ranger(name=None, upgrade_type=None): """ parameter name: name of ranger service component """ if name == 'ranger_admin': - setup_ranger_admin(rolling_upgrade=rolling_upgrade) + setup_ranger_admin(upgrade_type=upgrade_type) if name == 'ranger_usersync': - setup_usersync(rolling_upgrade=rolling_upgrade) + setup_usersync(upgrade_type=upgrade_type) -def setup_ranger_admin(rolling_upgrade=False): +def setup_ranger_admin(upgrade_type=None): import params ranger_home = params.ranger_home @@ -55,7 +55,7 @@ def setup_ranger_admin(rolling_upgrade=False): recursive = True ) - if rolling_upgrade: + if upgrade_type is not None: ranger_home = format("/usr/hdp/{version}/ranger-admin") ranger_conf = format("/usr/hdp/{version}/ranger-admin/conf") @@ -85,7 +85,7 @@ def setup_ranger_admin(rolling_upgrade=False): only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"), sudo=True) - if rolling_upgrade: + if upgrade_type is not None: src_file = format('{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml') dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) @@ -127,10 +127,10 @@ def setup_ranger_admin(rolling_upgrade=False): group=params.unix_group, ) - do_keystore_setup(rolling_upgrade=rolling_upgrade) + do_keystore_setup(upgrade_type=upgrade_type) -def setup_ranger_db(rolling_upgrade=False): +def setup_ranger_db(upgrade_type=None): import params File(params.downloaded_custom_connector, @@ -152,7 +152,7 @@ def setup_ranger_db(rolling_upgrade=False): File(params.driver_curl_target, mode=0644) ranger_home = params.ranger_home - if rolling_upgrade: + if upgrade_type is not None: ranger_home = format("/usr/hdp/{version}/ranger-admin") if params.db_flavor.lower() == 'sqla': @@ -210,11 +210,11 @@ def setup_ranger_db(rolling_upgrade=False): ) -def setup_java_patch(rolling_upgrade=False): +def setup_java_patch(upgrade_type=None): import params ranger_home = params.ranger_home - if rolling_upgrade: + if upgrade_type is not None: ranger_home = format("/usr/hdp/{version}/ranger-admin") env_dict = {'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME':params.java_home} @@ -229,14 +229,14 @@ def setup_java_patch(rolling_upgrade=False): ) -def do_keystore_setup(rolling_upgrade=False): +def do_keystore_setup(upgrade_type=None): import params ranger_home = params.ranger_home cred_lib_path = params.cred_lib_path cred_setup_prefix = params.cred_setup_prefix - if rolling_upgrade: + if upgrade_type is not None: ranger_home = format("/usr/hdp/{version}/ranger-admin") cred_lib_path = os.path.join(ranger_home,"cred","lib","*") cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib_path) @@ -280,7 +280,7 @@ def password_validation(password): else: Logger.info("password validated") -def setup_usersync(rolling_upgrade=False): +def setup_usersync(upgrade_type=None): import params usersync_home = params.usersync_home @@ -289,7 +289,7 @@ def setup_usersync(rolling_upgrade=False): if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': password_validation(params.ranger_usersync_ldap_ldapbindpassword) - if rolling_upgrade: + if upgrade_type is not None: usersync_home = format("/usr/hdp/{version}/ranger-usersync") ranger_ugsync_conf = format("/usr/hdp/{version}/ranger-usersync/conf") @@ -308,7 +308,7 @@ def setup_usersync(rolling_upgrade=False): owner = params.unix_user ) - if rolling_upgrade: + if upgrade_type is not None: src_file = format('{usersync_home}/conf.dist/ranger-ugsync-default.xml') dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_server.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_server.py index 18d1ba1..4ffc8eb 100755 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_server.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_server.py @@ -41,13 +41,13 @@ class KmsServer(Script): self.configure(env) setup_java_patch() - def stop(self, env, rolling_restart=False): + def stop(self, env, upgrade_type=None): import params env.set_params(params) kms_service(action = 'stop') - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) @@ -69,7 +69,7 @@ class KmsServer(Script): env.set_params(params) kms() - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params.py index 61033c0..7b6a490 100644 --- a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params.py @@ -37,7 +37,7 @@ config = Script.get_config() stack_name = default("/hostLevelParams/stack_name", None) -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade version = default("/commandParams/version", None) stack_version_unformatted = str(config['hostLevelParams']['stack_version']) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider_client.py b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider_client.py index e0ec7d4..5865048 100644 --- a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider_client.py +++ b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider_client.py @@ -26,13 +26,15 @@ from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl class SliderClient(Script): + def status(self, env): + raise ClientComponentHasNoStatus() - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SliderClientLinux(SliderClient): def get_stack_to_component(self): return {"HDP": "slider-client"} - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) @@ -46,27 +48,22 @@ class SliderClient(Script): conf_select.select(params.stack_name, "hadoop", params.version) hdp_select.select("hadoop-client", params.version) - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def install(self, env): self.install_packages(env) self.configure(env) - @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) - def install(self, env): - import params - if params.slider_home is None: - self.install_packages(env) - self.configure(env) - - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def configure(self, env): import params env.set_params(params) slider() - def status(self, env): - raise ClientComponentHasNoStatus() - +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SliderClientWindows(SliderClient): + def install(self, env): + import params + if params.slider_home is None: + self.install_packages(env) + self.configure(env) if __name__ == "__main__": SliderClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py index 4923383..943664a 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py @@ -47,14 +47,14 @@ class JobHistoryServer(Script): setup_spark(env, 'server', action = 'config') - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) spark_service('jobhistoryserver', action='start') - def stop(self, env, rolling_restart=False): + def stop(self, env, upgrade_type=None): import params env.set_params(params) @@ -70,11 +70,12 @@ class JobHistoryServer(Script): def get_stack_to_component(self): return {"HDP": "spark-historyserver"} - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + Logger.info("Executing Spark Job History Server Stack Upgrade pre-restart") conf_select.select(params.stack_name, "spark", params.version) hdp_select.select("spark-historyserver", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py index 50b241f..b96d21f 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py @@ -51,7 +51,7 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) -# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade version = default("/commandParams/version", None) # TODO! FIXME! Version check is not working as of today : http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py index ce4cdcc..d4b8f85 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py @@ -46,11 +46,12 @@ class SparkClient(Script): def get_stack_to_component(self): return {"HDP": "spark-client"} - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + Logger.info("Executing Spark Client Stack Upgrade pre-restart") conf_select.select(params.stack_name, "spark", params.version) hdp_select.select("spark-client", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py index 170fbca..d2145e2 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py @@ -46,14 +46,14 @@ class SparkThriftServer(Script): env.set_params(params) setup_spark(env, 'server', action = 'config') - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) spark_service('sparkthriftserver',action='start') - def stop(self, env, rolling_restart=False): + def stop(self, env, upgrade_type=None): import params env.set_params(params) spark_service('sparkthriftserver',action='stop') @@ -66,11 +66,12 @@ class SparkThriftServer(Script): def get_stack_to_component(self): return {"HDP": "spark-thriftserver"} - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if params.version and compare_versions(format_hdp_stack_version(params.version), '2.3.2.0') >= 0: + Logger.info("Executing Spark Thrift Server Stack Upgrade pre-restart") conf_select.select(params.stack_name, "spark", params.version) hdp_select.select("spark-thriftserver", params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml index b893f83..d4dba00 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml @@ -117,7 +117,7 @@ </execute-stage> </group> - <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services (HDFS, HBase, ZooKeeper and Ranger)"> + <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services"> <direction>UPGRADE</direction> <skippable>true</skippable> <service-check>false</service-check> @@ -148,26 +148,26 @@ <!-- If the user attempts a downgrade after this point, they will need to restore backups before starting any of the services. --> - <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Restore Oozie Database"> <task xsi:type="manual"> <message>Before continuing, please restore the Oozie Server database on {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Restore Hive Metastore"> <task xsi:type="manual"> <message>Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Restore HBASE Snapshot"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> <function>restore_snapshot</function> </task> </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <execute-stage service="HDFS" component="NAMENODE" title="Restore HDFS Snapshot"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> <function>restore_snapshot</function> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml index a2f1c94..d15f547 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml @@ -135,14 +135,14 @@ </task> </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger Database"> <task xsi:type="manual"> <message>Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> </group> - <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services (HDFS, HBase, ZooKeeper and Ranger)"> + <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services"> <direction>UPGRADE</direction> <skippable>true</skippable> <service-check>false</service-check> @@ -178,39 +178,39 @@ <!-- If the user attempts a downgrade after this point, they will need to restore backups before starting any of the services. --> - <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Restore Oozie Database"> <task xsi:type="manual"> <message>Before continuing, please restore the Oozie Server database on {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Restore Hive Metastore"> <task xsi:type="manual"> <message>Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data"> <task xsi:type="manual"> <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /etc/knox/data/security/" on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Restore HBASE Snapshot"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> <function>restore_snapshot</function> <!-- TODO, this function name is new. --> </task> </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <execute-stage service="HDFS" component="NAMENODE" title="Restore HDFS Snapshot"> <task xsi:type="execute" hosts="master"> <!-- TODO, this can be any NameNode, not just the active. --> <script>scripts/namenode.py</script> <function>restore_snapshot</function> <!-- TODO, this function doesn't exist yet. --> </task> </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Restore Ranger Database"> <task xsi:type="manual"> <message>Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> </task> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml index 44413d3..2e0dff4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml @@ -158,7 +158,7 @@ </task> </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger Database"> <task xsi:type="manual"> <message>Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> </task> @@ -167,7 +167,7 @@ <!--TODO: Ranger KMS seems to be absent at 2.2 stack, not backing it up--> </group> - <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services (HDFS, HBase, ZooKeeper and Ranger)"> + <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services"> <direction>UPGRADE</direction> <skippable>true</skippable> <service-check>false</service-check> @@ -203,39 +203,39 @@ <!-- If the user attempts a downgrade after this point, they will need to restore backups before starting any of the services. --> - <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Restore Oozie Database"> <task xsi:type="manual"> <message>Before continuing, please restore the Oozie Server database on {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Restore Hive Metastore"> <task xsi:type="manual"> <message>Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data"> <task xsi:type="manual"> <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /etc/knox/data/security/" on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Restore HBASE Snapshot"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> <function>restore_snapshot</function> </task> </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <execute-stage service="HDFS" component="NAMENODE" title="Restore HDFS Snapshot"> <task xsi:type="execute" hosts="master"> <script>scripts/namenode.py</script> <function>restore_snapshot</function> </task> </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Restore Ranger Database"> <task xsi:type="manual"> <message>Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> </task> @@ -256,7 +256,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_admin"/> </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger"> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Calculating Ranger Properties"> <task xsi:type="server_action" summary="Calculating Ranger Properties" class="org.apache.ambari.server.serveraction.upgrades.RangerConfigCalculation" /> </execute-stage> @@ -500,6 +500,13 @@ <service name="RANGER"> <component>RANGER_ADMIN</component> <component>RANGER_USERSYNC</component> + </service> + </group> + + <group xsi:type="restart" name="RANGER_KMS" title="Ranger KMS"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="RANGER_KMS"> <component>RANGER_KMS_SERVER</component> </service> </group> @@ -517,7 +524,7 @@ </service> </group> - <group xsi:type="cluster" name="HDFS_LEAFE_SAFEMODE" title="HDFS - Wait to leave Safemode"> + <group xsi:type="cluster" name="HDFS_LEAVE_SAFEMODE" title="HDFS - Wait to leave Safemode"> <service-check>false</service-check> <skippable>true</skippable> <direction>UPGRADE</direction> @@ -595,6 +602,15 @@ </service> </group> + <group xsi:type="restart" name="SPARK" title="Spark"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="SPARK"> + <component>SPARK_JOBHISTORYSERVER</component> + <component>SPARK_CLIENT</component> <!-- TODO, parallelize --> + </service> + </group> + <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. --> <group name="Upgrade Oozie" title="Upgrade Oozie Database"> <direction>UPGRADE</direction> @@ -671,6 +687,14 @@ </execute-stage> </group> + <group xsi:type="restart" name="SLIDER" title="Slider"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="SLIDER"> + <component>SLIDER</component> + </service> + </group> + <group xsi:type="restart" name="FLUME" title="Flume"> <service-check>false</service-check> <skippable>true</skippable> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml index 407b22b..e6dea75 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml @@ -136,20 +136,20 @@ </task> </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger Database"> <task xsi:type="manual"> <message>Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Pre Upgrade Ranger KMS"> + <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Backup Ranger KMS Database"> <task xsi:type="manual"> <message>Before continuing, please backup Ranger KMS database</message> </task> </execute-stage> </group> - <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services (HDFS, HBase, ZooKeeper and Ranger)"> + <group xsi:type="stop" name="STOP_LOW_LEVEL_SERVICE_COMPONENTS" title="Stop Components for Core Services"> <direction>UPGRADE</direction> <skippable>true</skippable> <service-check>false</service-check> @@ -172,6 +172,9 @@ <service name="RANGER"> <component>RANGER_USERSYNC</component> <component>RANGER_ADMIN</component> + </service> + + <service name="RANGER_KMS"> <component>RANGER_KMS_SERVER</component> </service> @@ -187,43 +190,50 @@ <!-- If the user attempts a downgrade after this point, they will need to restore backups before starting any of the services. --> - <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Backup Oozie Database"> + <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Restore Oozie Database"> <task xsi:type="manual"> <message>Before continuing, please restore the Oozie Server database on {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore"> + <execute-stage service="HIVE" component="HIVE_METASTORE" title="Restore Hive Metastore"> <task xsi:type="manual"> <message>Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Backup Knox Data"> + <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Restore Knox Data"> <task xsi:type="manual"> <message>Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /etc/knox/data/security/" on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> - <execute-stage service="HBASE" component="HBASE_MASTER" title="Snapshot HBASE"> + <execute-stage service="HBASE" component="HBASE_MASTER" title="Restore HBASE Snapshot"> <task xsi:type="execute" hosts="master"> <script>scripts/hbase_upgrade.py</script> <function>restore_snapshot</function> <!-- TODO (Alejandro), this function name is new. --> </task> </execute-stage> - <execute-stage service="HDFS" component="NAMENODE" title="Snapshot HDFS"> + <execute-stage service="HDFS" component="NAMENODE" title="Restore HDFS Snapshot"> <task xsi:type="execute" hosts="master"> <!-- TODO (Alejandro), this can be any NameNode, not just the active. --> <script>scripts/namenode.py</script> <function>restore_snapshot</function> <!-- TODO (Alejandro), this function doesn't exist yet. --> </task> </execute-stage> - <execute-stage service="RANGER" component="RANGER_ADMIN" title="Backup Ranger"> + <execute-stage service="RANGER" component="RANGER_ADMIN" title="Restore Ranger Database"> <task xsi:type="manual"> <message>Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}.</message> </task> </execute-stage> + + <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Restore Ranger KMS Database"> + <task xsi:type="manual"> + <message>Before continuing, please restore Ranger KMS database</message> + </task> + </execute-stage> + </group> <!-- After processing this group, will change the effective Stack of the UpgradeContext object. --> @@ -262,6 +272,13 @@ <service name="RANGER"> <component>RANGER_ADMIN</component> <component>RANGER_USERSYNC</component> + </service> + </group> + + <group name="RANGER_KMS" title="Ranger KMS"> + <service-check>false</service-check> + <skippable>true</skippable> + <service name="RANGER_KMS"> <component>RANGER_KMS_SERVER</component> </service> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 36d1174..394bae4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -7216,7 +7216,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(1, responsesWithParams.size()); StackVersionResponse resp = responsesWithParams.iterator().next(); assertNotNull(resp.getUpgradePacks()); - assertEquals(6, resp.getUpgradePacks().size()); + assertEquals(7, resp.getUpgradePacks().size()); assertTrue(resp.getUpgradePacks().contains("upgrade_test")); } http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index 6099e16..45e7da1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -242,6 +242,40 @@ public class UpgradeHelperTest { assertEquals(8, groups.get(3).items.size()); } + @Test + public void testUpgradeServerActionOrchestration() throws Exception { + Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); + + ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); + si.setDisplayName("Zk"); + ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); + ci.setDisplayName("ZooKeeper1 Server2"); + + assertTrue(upgrades.containsKey("upgrade_server_action_test")); + UpgradePack upgrade = upgrades.get("upgrade_server_action_test"); + assertNotNull(upgrade); + + makeCluster(); + + UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21, + HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING); + + List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); + + assertEquals(1, groups.size()); + UpgradeGroupHolder group = groups.get(0); + assertEquals("CLUSTER_SERVER_ACTIONS", group.name); + List<StageWrapper> stageWrappers = group.items; + assertEquals(6, stageWrappers.size()); + assertEquals("Pre Upgrade", stageWrappers.get(0).getText()); + assertEquals("Pre Upgrade Zookeeper", stageWrappers.get(1).getText()); + assertEquals("Configuring", stageWrappers.get(2).getText()); + assertEquals("Configuring HDFS", stageWrappers.get(3).getText()); + assertEquals("Calculating Properties", stageWrappers.get(4).getText()); + assertEquals("Calculating HDFS Properties", stageWrappers.get(5).getText()); + + } + /** * Verify that a Rolling Upgrades restarts the NameNodes in the following order: standby, active. * @throws Exception http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/ClusterGroupingTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/ClusterGroupingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/ClusterGroupingTest.java deleted file mode 100644 index 0729e2a..0000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/ClusterGroupingTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.state.stack.upgrade; - -import org.apache.ambari.server.stack.HostsType; -import org.apache.ambari.server.stack.MasterHostResolver; -import org.apache.ambari.server.state.UpgradeContext; -import org.junit.Test; - -import java.util.LinkedHashSet; -import java.util.Set; - -import static org.easymock.EasyMock.*; -import static org.junit.Assert.*; - -public class ClusterGroupingTest { - - @Test - public void testGetHostsForExecuteStage() throws Exception { - UpgradeContext ctx = createNiceMock(UpgradeContext.class); - ClusterGrouping.ExecuteStage execution = new ClusterGrouping.ExecuteStage(); - MasterHostResolver resolverMock = createMock(MasterHostResolver.class); - expect(ctx.getResolver()).andReturn(resolverMock).anyTimes(); - replay(ctx); - - // Check case when execution stage defines no service/component requirements - replay(resolverMock); - Set<String> realHosts = ClusterGrouping.getHostsForExecuteStage(ctx, execution); - assertNotNull(realHosts); - assertTrue(realHosts.isEmpty()); - verify(resolverMock); - - // Check case when execution stage defines service/component requirements, - // but no hosts match them - execution.service = "HBASE"; - execution.component = "HBASE_MASTER"; - reset(resolverMock); - expect(resolverMock.getMasterAndHosts(anyString(), anyString())).andReturn(null).once(); - replay(resolverMock); - realHosts = ClusterGrouping.getHostsForExecuteStage(ctx, execution); - assertNull(realHosts); - verify(resolverMock); - - // Check case when execution stage defines service/component requirements, - // and some hosts match them - execution.service = "HBASE"; - execution.component = "HBASE_MASTER"; - reset(resolverMock); - HostsType hostsType = new HostsType(); - hostsType.hosts.add("host1"); - hostsType.hosts.add("host2"); - expect(resolverMock.getMasterAndHosts(anyString(), anyString())).andReturn(hostsType).once(); - replay(resolverMock); - realHosts = ClusterGrouping.getHostsForExecuteStage(ctx, execution); - assertNotNull(realHosts); - assertArrayEquals(new String [] {"host1", "host2"}, realHosts.toArray()); - verify(resolverMock); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py b/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py index 5ce39af..ee9377f 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py +++ b/ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py @@ -513,10 +513,10 @@ class TestFlumeHandler(RMFTestCase): owner="flume", content=content) - def test_pre_rolling_restart(self): + def test_pre_upgrade_restart(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/flume_handler.py", classname = "FlumeHandler", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_file="flume_22.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/python/stacks/2.2/RANGER/test_ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/RANGER/test_ranger_admin.py b/ambari-server/src/test/python/stacks/2.2/RANGER/test_ranger_admin.py index bce7475..66a20a8 100644 --- a/ambari-server/src/test/python/stacks/2.2/RANGER/test_ranger_admin.py +++ b/ambari-server/src/test/python/stacks/2.2/RANGER/test_ranger_admin.py @@ -192,7 +192,7 @@ class TestRangerAdmin(RMFTestCase): ) - def test_pre_rolling_upgrade_23(self, ): + def test_pre_upgrade_restart_23(self, ): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/ranger-admin-upgrade.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -201,7 +201,7 @@ class TestRangerAdmin(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/ranger_usersync.py", classname = "RangerAdmin", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py b/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py index cb95291..503cf22 100644 --- a/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py +++ b/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py @@ -115,10 +115,10 @@ class TestSliderClient(RMFTestCase): self.assertNoMoreResources() - def test_pre_rolling_restart(self): + def test_pre_upgrade_restart(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/slider_client.py", classname = "SliderClient", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_file="default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) @@ -128,7 +128,7 @@ class TestSliderClient(RMFTestCase): self.assertNoMoreResources() - def test_pre_rolling_restart_23(self): + def test_pre_upgrade_restart_23(self): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -137,7 +137,7 @@ class TestSliderClient(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/slider_client.py", classname = "SliderClient", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py index cb03a76..c6b2ec6 100644 --- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py +++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py @@ -280,7 +280,7 @@ class TestJobHistoryServer(RMFTestCase): ) @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") - def test_pre_rolling_restart_23(self, copy_to_hdfs_mock): + def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -291,7 +291,7 @@ class TestJobHistoryServer(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py", classname = "JobHistoryServer", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py index 043d3bd..131aa6b 100644 --- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py +++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py @@ -124,7 +124,7 @@ class TestSparkClient(RMFTestCase): group = 'spark', ) - def test_pre_rolling_restart_23(self): + def test_pre_upgrade_restart_23(self): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -134,7 +134,7 @@ class TestSparkClient(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_client.py", classname = "SparkClient", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py index d6ac468..3a154f0 100644 --- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py +++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py @@ -147,7 +147,7 @@ class TestSparkThriftServer(RMFTestCase): ) @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") - def test_pre_rolling_restart_23(self, copy_to_hdfs_mock): + def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -158,7 +158,7 @@ class TestSparkThriftServer(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_thrift_server.py", classname = "SparkThriftServer", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/dcf1a161/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml new file mode 100644 index 0000000..ee43d1c --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml @@ -0,0 +1,52 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <target>2.2.*.*</target> + <target-stack>HDP-2.2.0</target-stack> + <type>ROLLING</type> + <order> + <group xsi:type="cluster" name="CLUSTER_SERVER_ACTIONS" title="Cluster Server Actions"> + + <execute-stage title="Pre Upgrade"> + <task xsi:type="manual"> + <message>Foo</message> + </task> + </execute-stage> + <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Pre Upgrade Zookeeper"> + <task xsi:type="manual"> + <message>Back stuff up.</message> + </task> + </execute-stage> + + <execute-stage title="Configuring"> + <task xsi:type="configure" id="hdp_2_2_0_nn_pre_upgrade"/> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Configuring HDFS"> + <task xsi:type="configure" id="hdp_2_2_0_nn_pre_upgrade"/> + </execute-stage> + + <execute-stage title="Calculating Properties"> + <task xsi:type="server_action" summary="Calculating Properties" class="org.apache.ambari.server.serveraction.upgrades.HBaseConfigCalculation" /> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Calculating HDFS Properties"> + <task xsi:type="server_action" summary="Calculating Properties" class="org.apache.ambari.server.serveraction.upgrades.HBaseConfigCalculation" /> + </execute-stage> + + </group> + </order> +</upgrade> \ No newline at end of file