AMBARI-18842: Provide support for removing an mpack on the command line (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bd2cd4ab Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bd2cd4ab Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bd2cd4ab Branch: refs/heads/branch-feature-AMBARI-18634 Commit: bd2cd4ab61798a44a670797a9348b6e818b7701b Parents: a8d6f03 Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Wed Nov 23 10:30:00 2016 -0800 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Wed Nov 23 10:30:00 2016 -0800 ---------------------------------------------------------------------- ambari-server/sbin/ambari-server | 6 +- ambari-server/src/main/python/ambari-server.py | 137 ++++++++++++++++--- .../main/python/ambari_server/setupActions.py | 1 + .../main/python/ambari_server/setupMpacks.py | 62 +++++++-- ambari-server/src/test/python/TestMpacks.py | 10 +- 5 files changed, 177 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bd2cd4ab/ambari-server/sbin/ambari-server ---------------------------------------------------------------------- diff --git a/ambari-server/sbin/ambari-server b/ambari-server/sbin/ambari-server index f08db13..8afabb1 100755 --- a/ambari-server/sbin/ambari-server +++ b/ambari-server/sbin/ambari-server @@ -188,13 +188,17 @@ case "${1:-}" in echo -e "Installing management pack" $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ ;; + uninstall-mpack) + echo -e "Uninstalling management pack" + $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ + ;; upgrade-mpack) echo -e "Upgrading management pack" $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ ;; *) echo "Usage: $AMBARI_EXECUTABLE - {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-cleanup|install-mpack|upgrade-mpack} [options] + {start|stop|reset|restart|upgrade|status|upgradestack|setup|setup-jce|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore|update-host-names|check-database|enable-stack|setup-sso|db-cleanup|install-mpack|uninstall-mpack|upgrade-mpack} [options] Use $AMBARI_PYTHON_EXECUTABLE <action> --help to get details on options available. Or, simply invoke ambari-server.py --help to print the options." exit 1 http://git-wip-us.apache.org/repos/asf/ambari/blob/bd2cd4ab/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index d43e0f2..41b2234 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -39,7 +39,7 @@ from ambari_server.serverUtils import is_server_runing, refresh_stack_hash from ambari_server.serverSetup import reset, setup, setup_jce_policy from ambari_server.serverUpgrade import upgrade, upgrade_stack, set_current from ambari_server.setupHttps import setup_https, setup_truststore -from ambari_server.setupMpacks import install_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \ +from ambari_server.setupMpacks import install_mpack, uninstall_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \ SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME from ambari_server.setupSso import setup_sso from ambari_server.dbCleanup import db_cleanup @@ -52,7 +52,7 @@ from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SY SETUP_ACTION, SETUP_SECURITY_ACTION,START_ACTION, STATUS_ACTION, STOP_ACTION, RESTART_ACTION, UPGRADE_ACTION, \ UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \ UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \ - DB_CLEANUP_ACTION, INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION + DB_CLEANUP_ACTION, INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION, PAM_SETUP_ACTION from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam from ambari_server.userInput import get_validated_string_input @@ -316,6 +316,35 @@ def restore(args): BackupRestore_main(restore_command) +_action_option_dependence_map = { +} + +def add_parser_options(*args, **kwargs): + required_for_actions = kwargs.pop("required_for_actions", []) + optional_for_actions = kwargs.pop("optional_for_actions", []) + parser = kwargs.pop("parser") + for action in required_for_actions: + if not action in _action_option_dependence_map: + _action_option_dependence_map[action] = ([], []) + _action_option_dependence_map[action][0].append((args[0], kwargs["dest"])) + for action in optional_for_actions: + if not action in _action_option_dependence_map: + _action_option_dependence_map[action] = ([], []) + _action_option_dependence_map[action][1].append((args[0], kwargs["dest"])) + parser.add_option(*args, **kwargs) + +def print_action_arguments_help(action): + if action in _action_option_dependence_map: + required_options = _action_option_dependence_map[action][0] + optional_options = _action_option_dependence_map[action][1] + if required_options or optional_options: + print "Options used by action {0}:".format(action) + if required_options: + print " required:{0}".format( + ";".join([print_opt for print_opt, _ in required_options])) + if optional_options: + print " optional:{0}".format( + ";".join([print_opt for print_opt, _ in optional_options])) @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) def init_parser_options(parser): @@ -361,18 +390,45 @@ def init_parser_options(parser): help="Specifies the path to the JDBC driver JAR file") parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip properties file validation", dest="skip_properties_validation") parser.add_option('--skip-database-check', action="store_true", default=False, help="Skip database consistency check", dest="skip_database_check") - parser.add_option('--mpack', default=None, - help="Specified the path for management pack to be installed/upgraded", - dest="mpack_path") - parser.add_option('--purge', action="store_true", default=False, - help="Purge existing resources specified in purge-list", - dest="purge") + add_parser_options('--mpack', + default=None, + help="Specify the path for management pack to be installed/upgraded", + dest="mpack_path", + parser=parser, + required_for_actions=[INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION] + ) + add_parser_options('--mpack-name', + default=None, + help="Specify the management pack name to be uninstalled", + dest="mpack_name", + parser=parser, + required_for_actions=[UNINSTALL_MPACK_ACTION] + ) + add_parser_options('--purge', + action="store_true", + default=False, + help="Purge existing resources specified in purge-list", + dest="purge", + parser=parser, + optional_for_actions=[INSTALL_MPACK_ACTION] + ) purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME]) default_purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME]) - parser.add_option('--purge-list', default=default_purge_resources, - help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources), - dest="purge_list") - parser.add_option('--force', action="store_true", default=False, help="Force install management pack", dest="force") + add_parser_options('--purge-list', + default=default_purge_resources, + help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources), + dest="purge_list", + parser=parser, + optional_for_actions=[INSTALL_MPACK_ACTION] + ) + add_parser_options('--force', + action="store_true", + default=False, + help="Force install management pack", + dest="force", + parser=parser, + optional_for_actions=[INSTALL_MPACK_ACTION] + ) # -b and -i the remaining available short options # -h reserved for help @@ -441,18 +497,45 @@ def init_parser_options(parser): parser.add_option('--stack', dest="stack_name", default=None, type="string", help="Specify stack name for the stack versions that needs to be enabled") parser.add_option("-d", "--from-date", dest="cleanup_from_date", default=None, type="string", help="Specify date for the cleanup process in 'yyyy-MM-dd' format") - parser.add_option('--mpack', default=None, - help="Specified the path for management pack to be installed/upgraded", - dest="mpack_path") - parser.add_option('--purge', action="store_true", default=False, - help="Purge existing resources specified in purge-list", - dest="purge") + add_parser_options('--mpack', + default=None, + help="Specify the path for management pack to be installed/upgraded", + dest="mpack_path", + parser=parser, + required_for_actions=[INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION] + ) + add_parser_options('--mpack-name', + default=None, + help="Specify the management pack name to be uninstalled", + dest="mpack_name", + parser=parser, + required_for_actions=[UNINSTALL_MPACK_ACTION] + ) + add_parser_options('--purge', + action="store_true", + default=False, + help="Purge existing resources specified in purge-list", + dest="purge", + parser=parser, + optional_for_actions=[INSTALL_MPACK_ACTION] + ) purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME]) default_purge_resources = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME]) - parser.add_option('--purge-list', default=default_purge_resources, - help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources), - dest="purge_list") - parser.add_option('--force', action="store_true", default=False, help="Force install management pack", dest="force") + add_parser_options('--purge-list', + default=default_purge_resources, + help="Comma separated list of resources to purge ({0}). By default ({1}) will be purged.".format(purge_resources, default_purge_resources), + dest="purge_list", + parser=parser, + optional_for_actions=[INSTALL_MPACK_ACTION] + ) + add_parser_options('--force', + action="store_true", + default=False, + help="Force install management pack", + dest="force", + parser=parser, + optional_for_actions=[INSTALL_MPACK_ACTION] + ) parser.add_option('--ldap-url', default=None, help="Primary url for LDAP", dest="ldap_url") parser.add_option('--ldap-secondary-url', default=None, help="Secondary url for LDAP", dest="ldap_secondary_url") @@ -622,6 +705,7 @@ def create_user_action_map(args, options): REFRESH_STACK_HASH_ACTION: UserAction(refresh_stack_hash_action), SETUP_SSO_ACTION: UserActionRestart(setup_sso, options), INSTALL_MPACK_ACTION: UserAction(install_mpack, options), + UNINSTALL_MPACK_ACTION: UserAction(uninstall_mpack, options), UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options) } return action_map @@ -651,6 +735,7 @@ def create_user_action_map(args, options): SETUP_SSO_ACTION: UserActionRestart(setup_sso, options), DB_CLEANUP_ACTION: UserAction(db_cleanup, options), INSTALL_MPACK_ACTION: UserAction(install_mpack, options), + UNINSTALL_MPACK_ACTION: UserAction(uninstall_mpack, options), UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options), PAM_SETUP_ACTION: UserAction(setup_pam) } @@ -743,6 +828,14 @@ def main(options, args, parser): options.exit_code = None try: + if action in _action_option_dependence_map: + required, optional = _action_option_dependence_map[action] + for opt_str, opt_dest in required: + if hasattr(options, opt_dest) and getattr(options, opt_dest) is None: + print "Missing option {0} for action {1}".format(opt_str, action) + print_action_arguments_help(action) + print "Run ambari-server.py --help to see detailed description of each option" + raise FatalException(1, "Missing option") action_obj.execute() if action_obj.need_restart: http://git-wip-us.apache.org/repos/asf/ambari/blob/bd2cd4ab/ambari-server/src/main/python/ambari_server/setupActions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupActions.py b/ambari-server/src/main/python/ambari_server/setupActions.py index c87e0b2..7ea0752 100644 --- a/ambari-server/src/main/python/ambari_server/setupActions.py +++ b/ambari-server/src/main/python/ambari_server/setupActions.py @@ -45,5 +45,6 @@ SETUP_JCE_ACTION = "setup-jce" ENABLE_STACK_ACTION = "enable-stack" DB_CLEANUP_ACTION = "db-cleanup" INSTALL_MPACK_ACTION = "install-mpack" +UNINSTALL_MPACK_ACTION = "uninstall-mpack" UPGRADE_MPACK_ACTION = "upgrade-mpack" PAM_SETUP_ACTION = "setup-pam" http://git-wip-us.apache.org/repos/asf/ambari/blob/bd2cd4ab/ambari-server/src/main/python/ambari_server/setupMpacks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupMpacks.py b/ambari-server/src/main/python/ambari_server/setupMpacks.py index 131e14d..04b96f5 100755 --- a/ambari-server/src/main/python/ambari_server/setupMpacks.py +++ b/ambari-server/src/main/python/ambari_server/setupMpacks.py @@ -34,7 +34,7 @@ from ambari_server.serverConfiguration import get_ambari_properties, get_ambari_ get_common_services_location, get_mpacks_staging_location, get_server_temp_location, get_extension_location, \ get_java_exe_path, read_ambari_user, parse_properties_file, JDBC_DATABASE_PROPERTY, get_dashboard_location from ambari_server.setupSecurity import ensure_can_start_under_current_user, generate_env -from ambari_server.setupActions import INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION +from ambari_server.setupActions import INSTALL_MPACK_ACTION, UNINSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION from ambari_server.userInput import get_YN_input from ambari_server.dbConfiguration import ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST @@ -582,7 +582,7 @@ def search_mpacks(mpack_name, max_mpack_version=None): results.append((staged_mpack_name, staged_mpack_version)) return results -def uninstall_mpacks(mpack_name, max_mpack_version=None): +def _uninstall_mpacks(mpack_name, max_mpack_version=None): """ Uninstall all "mpack_name" management packs. If "max_mpack_version" is specified uninstall only management packs < max_mpack_version @@ -591,9 +591,9 @@ def uninstall_mpacks(mpack_name, max_mpack_version=None): """ results = search_mpacks(mpack_name, max_mpack_version) for result in results: - uninstall_mpack(result[0], result[1]) + _uninstall_mpack(result[0], result[1]) -def uninstall_mpack(mpack_name, mpack_version): +def _uninstall_mpack(mpack_name, mpack_version): """ Uninstall specific management pack :param mpack_name: Management pack name @@ -818,27 +818,43 @@ def get_replay_log_file(): replay_log_file = os.path.join(mpacks_staging_location, MPACKS_REPLAY_LOG_FILENAME) return replay_log_file -def add_replay_log(mpack_command, mpack_archive_path, purge, force, verbose): +def add_replay_log(mpack_command, mpack_archive_path, purge, purge_list, force, verbose): """ Helper function to add mpack replay log entry :param mpack_command: mpack command :param mpack_archive_path: mpack archive path (/var/lib/ambari-server/resources/mpacks/mpack.tar.gz) :param purge: purge command line option + :param purge: purge list command line option :param force: force command line option :param verbose: verbose command line option """ replay_log_file = get_replay_log_file() - log = { 'mpack_command' : mpack_command, 'mpack_path' : mpack_archive_path, 'purge' : purge, 'force' : force, 'verbose' : verbose } + log = { 'mpack_command' : mpack_command, 'mpack_path' : mpack_archive_path, 'purge' : purge, 'purge_list': purge_list, 'force' : force, 'verbose' : verbose } with open(replay_log_file, "a") as replay_log: replay_log.write("{0}\n".format(log)) +def remove_replay_logs(mpack_name): + replay_log_file = get_replay_log_file() + logs = [] + if os.path.exists(replay_log_file): + with open(replay_log_file, "r") as f: + for log in f: + log = log.strip() + options = _named_dict(ast.literal_eval(log)) + (name, version) = _get_mpack_name_version(options.mpack_path) + if mpack_name != name: + logs.append(log) + with open(replay_log_file, "w") as replay_log: + for log in logs: + replay_log.write("{0}\n".format(log)) + def install_mpack(options, replay_mode=False): - logger.info("Install mpack.") """ Install management pack :param options: Command line options :param replay_mode: Flag to indicate if executing command in replay mode """ + logger.info("Install mpack.") # Force install when replaying logs if replay_mode: options.force = True @@ -848,15 +864,39 @@ def install_mpack(options, replay_mode=False): _execute_hook(mpack_metadata, AFTER_INSTALL_HOOK_NAME, mpack_staging_dir) if not replay_mode: - add_replay_log(INSTALL_MPACK_ACTION, mpack_archive_path, options.purge, options.force, options.verbose) + add_replay_log(INSTALL_MPACK_ACTION, mpack_archive_path, options.purge, options.purge_list, options.force, options.verbose) + +def uninstall_mpack(options, replay_mode=False): + """ + Uninstall management pack + :param options: Command line options + :param replay_mode: Flag to indicate if executing command in replay mode + """ + logger.info("Uninstall mpack.") + mpack_name = options.mpack_name + + if not mpack_name: + print_error_msg("Management pack name not specified!") + raise FatalException(-1, 'Management pack name not specified!') + + results = search_mpacks(mpack_name) + if not results: + print_error_msg("No management packs found that can be uninstalled!") + raise FatalException(-1, 'No management packs found that can be uninstalled!') + + _uninstall_mpacks(mpack_name) + + print_info_msg("Management pack {0} successfully uninstalled!".format(mpack_name)) + if not replay_mode: + remove_replay_logs(mpack_name) def upgrade_mpack(options, replay_mode=False): - logger.info("Upgrade mpack.") """ Upgrade management pack :param options: command line options :param replay_mode: Flag to indicate if executing command in replay mode """ + logger.info("Upgrade mpack.") mpack_path = options.mpack_path if options.purge: print_error_msg("Purge is not supported with upgrade_mpack action!") @@ -879,14 +919,14 @@ def upgrade_mpack(options, replay_mode=False): (mpack_metadata, mpack_name, mpack_version, mpack_staging_dir, mpack_archive_path) = _install_mpack(options, replay_mode, is_upgrade=True) # Uninstall old management packs - uninstall_mpacks(mpack_name, mpack_version) + _uninstall_mpacks(mpack_name, mpack_version) # Execute post upgrade hook _execute_hook(mpack_metadata, AFTER_UPGRADE_HOOK_NAME, mpack_staging_dir) print_info_msg("Management pack {0}-{1} successfully upgraded!".format(mpack_name, mpack_version)) if not replay_mode: - add_replay_log(UPGRADE_MPACK_ACTION, mpack_archive_path, options.purge, options.force, options.verbose) + add_replay_log(UPGRADE_MPACK_ACTION, mpack_archive_path, options.purge, options.purge_list, options.force, options.verbose) def replay_mpack_logs(): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/bd2cd4ab/ambari-server/src/test/python/TestMpacks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestMpacks.py b/ambari-server/src/test/python/TestMpacks.py index 8329cf0..ba37856 100644 --- a/ambari-server/src/test/python/TestMpacks.py +++ b/ambari-server/src/test/python/TestMpacks.py @@ -48,7 +48,7 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur serverConfiguration.search_file = _search_file from ambari_server.setupMpacks import install_mpack, upgrade_mpack, replay_mpack_logs, \ - purge_stacks_and_mpacks, validate_purge, read_mpack_metadata, uninstall_mpack, \ + purge_stacks_and_mpacks, validate_purge, read_mpack_metadata, _uninstall_mpack, \ STACK_DEFINITIONS_RESOURCE_NAME, EXTENSION_DEFINITIONS_RESOURCE_NAME, \ SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME, GRAFANA_DASHBOARDS_DIRNAME, \ DASHBOARDS_DIRNAME, SERVICE_METRICS_DIRNAME @@ -546,13 +546,13 @@ class TestMpacks(TestCase): @patch("ambari_server.setupMpacks.get_ambari_version") @patch("ambari_server.setupMpacks.get_ambari_properties") @patch("ambari_server.setupMpacks.add_replay_log") - @patch("ambari_server.setupMpacks.uninstall_mpack") + @patch("ambari_server.setupMpacks._uninstall_mpack") @patch("ambari_server.setupMpacks.purge_stacks_and_mpacks") @patch("ambari_server.setupMpacks.expand_mpack") @patch("ambari_server.setupMpacks.download_mpack") @patch("ambari_server.setupMpacks.run_os_command") def test_upgrade_stack_mpack(self, run_os_command_mock, download_mpack_mock, expand_mpack_mock, purge_stacks_and_mpacks_mock, - uninstall_mpack_mock, add_replay_log_mock, get_ambari_properties_mock, + _uninstall_mpack_mock, add_replay_log_mock, get_ambari_properties_mock, get_ambari_version_mock, create_symlink_mock, os_mkdir_mock, shutil_move_mock, os_path_exists_mock, create_symlink_using_path_mock): options = self._create_empty_options_mock() @@ -727,7 +727,7 @@ class TestMpacks(TestCase): self.assertEqual(18, create_symlink_mock.call_count) create_symlink_using_path_mock.assert_has_calls(create_symlink_using_path_calls) self.assertEqual(1, create_symlink_using_path_mock.call_count) - uninstall_mpack_mock.assert_has_calls([call("mystack-ambari-mpack", "1.0.0.0")]) + _uninstall_mpack_mock.assert_has_calls([call("mystack-ambari-mpack", "1.0.0.0")]) self.assertTrue(add_replay_log_mock.called) @@ -786,7 +786,7 @@ class TestMpacks(TestCase): common_services_directory = fake_configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY] dashboard_directory = fake_configs[serverConfiguration.DASHBOARD_PATH_PROPERTY] - uninstall_mpack("mystack-ambari-mpack", "1.0.0.1") + _uninstall_mpack("mystack-ambari-mpack", "1.0.0.1") self.assertEqual(1, sudo_rmtree_mock.call_count) self.assertEqual(6, sudo_unlink_mock.call_count)
