Repository: ambari Updated Branches: refs/heads/trunk 6fe7f8327 -> 9404dee98
AMBARI-15663: Ambari Management Packs - V0 version (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9404dee9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9404dee9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9404dee9 Branch: refs/heads/trunk Commit: 9404dee98b626e9066977459955b4737a530a644 Parents: 6fe7f83 Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Thu Mar 31 23:44:34 2016 -0700 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Thu Mar 31 23:44:34 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/tar_archive.py | 30 +- ambari-server/conf/unix/ambari.properties | 4 +- ambari-server/conf/windows/ambari.properties | 2 + ambari-server/sbin/ambari-server | 8 + .../server/configuration/Configuration.java | 10 + ambari-server/src/main/python/ambari-server.py | 29 +- .../python/ambari_server/serverConfiguration.py | 30 +- .../main/python/ambari_server/setupActions.py | 4 +- .../main/python/ambari_server/setupMpacks.py | 611 +++++++++++++++++++ .../src/test/python/TestAmbariServer.py | 3 +- 10 files changed, 720 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py index 71dc4a4..c682c3e 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py @@ -18,6 +18,10 @@ See the License for the specific language governing permissions and limitations under the License. ''' +import os +import tarfile +import zipfile +from contextlib import closing from resource_management.core.resources.system import Execute def archive_dir(output_filename, input_dir): @@ -50,4 +54,28 @@ def untar_archive(archive, directory): sudo = True, tries = 3, try_sleep = 1, - ) \ No newline at end of file + ) + +def extract_archive(archive, directory): + if archive.endswith('.tar.gz') or path.endswith('.tgz'): + mode = 'r:gz' + elif archive.endswith('.tar.bz2') or path.endswith('.tbz'): + mode = 'r:bz2' + else: + raise ValueError, "Could not extract `%s` as no appropriate extractor is found" % path + with closing(tarfile.open(archive, mode)) as tar: + tar.extractall(directory) + +def get_archive_root_dir(archive): + if archive.endswith('.tar.gz') or path.endswith('.tgz'): + mode = 'r:gz' + elif archive.endswith('.tar.bz2') or path.endswith('.tbz'): + mode = 'r:bz2' + else: + raise ValueError, "Could not extract `%s` as no appropriate extractor is found" % path + root_dir = None + with closing(tarfile.open(archive, mode)) as tar: + names = tar.getnames() + if names: + root_dir = os.path.commonprefix(names) + return root_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index 5f05c2e..8c91923 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -113,4 +113,6 @@ http.x-frame-options=DENY # HTTP Header settings for Ambari Views views.http.strict-transport-security=max-age=31536000 views.http.x-xss-protection=1; mode=block -views.http.x-frame-options=SAMEORIGIN \ No newline at end of file +views.http.x-frame-options=SAMEORIGIN + +mpacks.staging.path=$ROOT/var/lib/ambari-server/resources/mpacks http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/conf/windows/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties index 803f351..64cce3b 100644 --- a/ambari-server/conf/windows/ambari.properties +++ b/ambari-server/conf/windows/ambari.properties @@ -98,3 +98,5 @@ http.x-frame-options=DENY views.http.strict-transport-security=max-age=31536000 views.http.x-xss-protection=1; mode=block views.http.x-frame-options=SAMEORIGIN + +mpacks.staging.path=resources\\mpacks http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/sbin/ambari-server ---------------------------------------------------------------------- diff --git a/ambari-server/sbin/ambari-server b/ambari-server/sbin/ambari-server index 5b5bd91..83a52c6 100755 --- a/ambari-server/sbin/ambari-server +++ b/ambari-server/sbin/ambari-server @@ -162,6 +162,14 @@ case "$1" in echo -e "Cleanup database..." $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ ;; + install-mpack) + echo -e "Installing management pack" + $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ + ;; + upgrade-mpack) + echo -e "Upgrading management pack" + $PYTHON "$AMBARI_PYTHON_EXECUTABLE" $@ + ;; *) echo "Usage: $AMBARI_PYTHON_EXECUTABLE {start|stop|restart|setup|setup-jce|upgrade|status|upgradestack|setup-ldap|sync-ldap|set-current|setup-security|setup-sso|refresh-stack-hash|backup|restore|update-host-names|enable-stack|check-database|db-cleanup} [options] http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 7767fb2..9a78dc2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -132,6 +132,7 @@ public class Configuration { public static final String RESOURCES_DIR_KEY = "resources.dir"; public static final String METADATA_DIR_PATH = "metadata.path"; public static final String COMMON_SERVICES_DIR_PATH = "common.services.path"; + public static final String MPACKS_STAGING_DIR_PATH = "mpacks.staging.path"; public static final String SERVER_VERSION_FILE = "server.version.file"; public static final String SERVER_VERSION_KEY = "version"; public static final String JAVA_HOME_KEY = "java.home"; @@ -1356,6 +1357,15 @@ public class Configuration { return properties.getProperty(COMMON_SERVICES_DIR_PATH); } + /** + * Gets ambari management packs staging directory + * @return String + */ + public String getMpacksStagingPath() { + return properties.getProperty(MPACKS_STAGING_DIR_PATH); + } + + public String getServerVersionFilePath() { return properties.getProperty(SERVER_VERSION_FILE); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/src/main/python/ambari-server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py index bc86d32..e0ce37e 100755 --- a/ambari-server/src/main/python/ambari-server.py +++ b/ambari-server/src/main/python/ambari-server.py @@ -36,6 +36,7 @@ from ambari_server.serverUtils import is_server_runing, refresh_stack_hash from ambari_server.serverSetup import reset, setup, setup_jce_policy from ambari_server.serverUpgrade import upgrade, upgrade_stack, set_current from ambari_server.setupHttps import setup_https, setup_truststore +from ambari_server.setupMpacks import install_mpack, upgrade_mpack from ambari_server.setupSso import setup_sso from ambari_server.dbCleanup import db_cleanup from ambari_server.hostUpdate import update_host_names @@ -45,8 +46,9 @@ from ambari_server.enableStack import enable_stack_version from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SYNC_ACTION, PSTART_ACTION, \ REFRESH_STACK_HASH_ACTION, RESET_ACTION, RESTORE_ACTION, UPDATE_HOST_NAMES_ACTION, CHECK_DATABASE_ACTION, \ SETUP_ACTION, SETUP_SECURITY_ACTION,START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, \ - SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, \ - SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, DB_CLEANUP_ACTION + SETUP_JCE_ACTION, SET_CURRENT_ACTION, START_ACTION, STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, \ + UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT_ACTION, ENABLE_STACK_ACTION, SETUP_SSO_ACTION, \ + DB_CLEANUP_ACTION, INSTALL_MPACK_ACTION, UPGRADE_MPACK_ACTION from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas from ambari_server.userInput import get_validated_string_input @@ -322,6 +324,13 @@ def init_parser_options(parser): help="Specifies the path to the JDBC driver JAR file") parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip properties file validation", dest="skip_properties_validation") parser.add_option('--skip-database-validation', action="store_true", default=False, help="Skip database consistency validation", dest="skip_database_validation") + parser.add_option('--mpack', default=None, + help="Specified the path for management pack to be installed/upgraded", + dest="mpack_path") + parser.add_option('--purge', action="store_true", default=False, + help="Purge existing stack definitions and previously installed management packs", + dest="purge") + parser.add_option('--force', action="store_true", default=False, help="Force install management pack", dest="force") # -b and -i the remaining available short options # -h reserved for help @@ -390,7 +399,13 @@ def init_parser_options(parser): parser.add_option('--stack', dest="stack_name", default=None, type="string", help="Specify stack name for the stack versions that needs to be enabled") parser.add_option("-d", "--from-date", dest="cleanup_from_date", default=None, type="string", help="Specify date for the cleanup process in 'yyyy-MM-dd' format") - + parser.add_option('--mpack', default=None, + help="Specified the path for management pack to be installed/upgraded", + dest="mpack_path") + parser.add_option('--purge', action="store_true", default=False, + help="Purge existing stack definitions and previously installed management packs", + dest="purge") + parser.add_option('--force', action="store_true", default=False, help="Force install management pack", dest="force") @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) def are_cmd_line_db_args_blank(options): @@ -520,7 +535,9 @@ def create_user_action_map(args, options): LDAP_SETUP_ACTION: UserAction(setup_ldap), SETUP_SECURITY_ACTION: UserActionRestart(setup_security, options), REFRESH_STACK_HASH_ACTION: UserAction(refresh_stack_hash_action), - SETUP_SSO_ACTION: UserActionRestart(setup_sso, options) + SETUP_SSO_ACTION: UserActionRestart(setup_sso, options), + INSTALL_MPACK_ACTION: UserAction(install_mpack, options), + UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options) } return action_map @@ -546,7 +563,9 @@ def create_user_action_map(args, options): CHECK_DATABASE_ACTION: UserAction(check_database, options), ENABLE_STACK_ACTION: UserAction(enable_stack, options, args), SETUP_SSO_ACTION: UserActionRestart(setup_sso, options), - DB_CLEANUP_ACTION: UserAction(db_cleanup, options) + DB_CLEANUP_ACTION: UserAction(db_cleanup, options), + INSTALL_MPACK_ACTION: UserAction(install_mpack, options), + UPGRADE_MPACK_ACTION: UserAction(upgrade_mpack, options) } return action_map http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 2d49b9a..a259a1f 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -184,6 +184,7 @@ DEFAULT_DB_NAME = "ambari" SECURITY_KEYS_DIR = "security.server.keys_dir" COMMON_SERVICES_PATH_PROPERTY = 'common.services.path' +MPACKS_STAGING_PATH_PROPERTY = 'mpacks.staging.path' WEBAPP_DIR_PROPERTY = 'webapp.dir' SHARED_RESOURCES_DIR = 'shared.resources.dir' BOOTSTRAP_SCRIPT = 'bootstrap.script' @@ -195,7 +196,8 @@ REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PAT WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY, SECURITY_KEYS_DIR, JDBC_DATABASE_NAME_PROPERTY, NR_USER_PROPERTY, JAVA_HOME_PROPERTY, JDBC_PASSWORD_PROPERTY, SHARED_RESOURCES_DIR, JDBC_USER_NAME_PROPERTY, BOOTSTRAP_SCRIPT, RESOURCES_DIR_PROPERTY, CUSTOM_ACTION_DEFINITIONS, - BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY, PID_DIR_PROPERTY] + BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY, PID_DIR_PROPERTY, + MPACKS_STAGING_PATH_PROPERTY] def get_conf_dir(): try: @@ -366,6 +368,8 @@ class ServerConfigDefaults(object): self.DEFAULT_DB_NAME = "ambari" self.STACK_LOCATION_DEFAULT = "" + self.COMMON_SERVICES_LOCATION_DEFAULT = "" + self.MPACKS_STAGING_LOCATION_DEFAULT = "" self.DEFAULT_VIEWS_DIR = "" @@ -421,6 +425,8 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults): self.SERVER_RESOURCES_DIR = "resources" self.STACK_LOCATION_DEFAULT = "resources\\stacks" + self.COMMON_SERVICES_LOCATION_DEFAULT = "resources\\common-services" + self.MPACKS_STAGING_LOCATION_DEFAULT = "resources\\mpacks" self.DEFAULT_VIEWS_DIR = "resources\\views" @@ -503,6 +509,8 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults): self.SERVER_RESOURCES_DIR = AmbariPath.get("/var/lib/ambari-server/resources") self.STACK_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/stacks") + self.COMMON_SERVICES_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/common-services") + self.MPACKS_STAGING_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/mpacks") self.DEFAULT_VIEWS_DIR = AmbariPath.get("/var/lib/ambari-server/resources/views") @@ -1328,7 +1336,7 @@ def get_resources_location(properties): return resources_dir # -# Stack upgrade +# Stack location # def get_stack_location(properties): stack_location = properties[STACK_LOCATION_KEY] @@ -1336,6 +1344,24 @@ def get_stack_location(properties): stack_location = configDefaults.STACK_LOCATION_DEFAULT return stack_location +# +# Common services location +# +def get_common_services_location(properties): + common_services_location = properties[COMMON_SERVICES_PATH_PROPERTY] + if common_services_location is None: + common_services_location = configDefaults.COMMON_SERVICES_LOCATION_DEFAULT + return common_services_location + +# +# Management packs staging location +# +def get_mpacks_staging_location(properties): + mpacks_staging_location = properties[MPACKS_STAGING_PATH_PROPERTY] + if mpacks_staging_location is None: + mpacks_staging_location = configDefaults.MPACKS_STAGING_LOCATION_DEFAULT + return mpacks_staging_location + def get_missing_properties(properties): missing_propertiers = [] for property in REQUIRED_PROPERTIES: http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/src/main/python/ambari_server/setupActions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupActions.py b/ambari-server/src/main/python/ambari_server/setupActions.py index 6c75844..182c11e 100644 --- a/ambari-server/src/main/python/ambari_server/setupActions.py +++ b/ambari-server/src/main/python/ambari_server/setupActions.py @@ -42,4 +42,6 @@ BACKUP_ACTION = "backup" RESTORE_ACTION = "restore" SETUP_JCE_ACTION = "setup-jce" ENABLE_STACK_ACTION = "enable-stack" -DB_CLEANUP_ACTION = "db-cleanup" \ No newline at end of file +DB_CLEANUP_ACTION = "db-cleanup" +INSTALL_MPACK_ACTION = "install-mpack" +UPGRADE_MPACK_ACTION = "upgrade-mpack" http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/src/main/python/ambari_server/setupMpacks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupMpacks.py b/ambari-server/src/main/python/ambari_server/setupMpacks.py new file mode 100644 index 0000000..45bb675 --- /dev/null +++ b/ambari-server/src/main/python/ambari_server/setupMpacks.py @@ -0,0 +1,611 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import os +import shutil +import tempfile +import json + +from ambari_commons.exceptions import FatalException +from ambari_commons.inet_utils import download_file +from ambari_commons.logging_utils import print_info_msg, print_error_msg +from ambari_commons.os_utils import copy_file +from ambari_server.serverConfiguration import get_ambari_properties, get_ambari_version, get_stack_location, \ + get_common_services_location, get_mpacks_staging_location + +from resource_management.core import sudo +from resource_management.libraries.functions.tar_archive import extract_archive, get_archive_root_dir +from resource_management.libraries.functions.version import compare_versions + +class _named_dict(dict): + """ + Allow to get dict items using attribute notation, eg dict.attr == dict['attr'] + """ + def __init__(self, _dict): + + def repl_list(_list): + for i, e in enumerate(_list): + if isinstance(e, list): + _list[i] = repl_list(e) + if isinstance(e, dict): + _list[i] = _named_dict(e) + return _list + + dict.__init__(self, _dict) + for key, value in self.iteritems(): + if isinstance(value, dict): + self[key] = _named_dict(value) + if isinstance(value, list): + self[key] = repl_list(value) + + def __getattr__(self, item): + if item in self: + return self[item] + else: + dict.__getattr__(self, item) + +def download_mpack(mpack_path): + """ + Download management pack + :param mpack_path: Path to management pack + :return: Path where the management pack was downloaded + """ + # Download management pack to a temp location + tmpdir = tempfile.gettempdir() + archive_filename = os.path.basename(mpack_path) + tmp_archive_path = os.path.join(tmpdir, archive_filename) + + print_info_msg("Download management pack to temp location {0}".format(tmp_archive_path)) + if os.path.exists(tmp_archive_path): + os.remove(tmp_archive_path) + if os.path.exists(mpack_path): + # local path + copy_file(mpack_path, tmp_archive_path) + else: + # remote path + download_file(mpack_path, tmp_archive_path) + return tmp_archive_path + +def expand_mpack(archive_path): + """ + Expand management pack + :param archive_path: Local path to management pack + :return: Path where the management pack was expanded + """ + tmpdir = tempfile.gettempdir() + archive_root_dir = get_archive_root_dir(archive_path) + if not archive_root_dir: + print_error_msg("Malformed management pack. Root directory missing!") + raise FatalException(-1, 'Malformed management pack. Root directory missing!') + + # Expand management pack in temp directory + tmp_root_dir = os.path.join(tmpdir, archive_root_dir) + print_info_msg("Expand management pack at temp location {0}".format(tmp_root_dir)) + if os.path.exists(tmp_root_dir): + sudo.rmtree(tmp_root_dir) + extract_archive(archive_path, tmpdir) + + if not os.path.exists(tmp_root_dir): + print_error_msg("Malformed management pack. Failed to expand management pack!") + raise FatalException(-1, 'Malformed management pack. Failed to expand management pack!') + return tmp_root_dir + +def read_mpack_metadata(mpack_dir): + """ + Read management pack metadata + :param mpack_dir: Path where the expanded management pack is location + :return: Management pack metadata + """ + # Read mpack metadata + mpack_metafile = os.path.join(mpack_dir, "mpack.json") + if not os.path.exists(mpack_metafile): + print_error_msg("Malformed management pack. Metadata file missing!") + return None + + mpack_metadata = _named_dict(json.load(open(mpack_metafile, "r"))) + return mpack_metadata + +def get_mpack_properties(): + """ + Read ambari properties required for management packs + :return: (stack_location, service_definitions_location, mpacks_staging_location) + """ + # Get ambari config properties + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + stack_location = get_stack_location(properties) + service_definitions_location = get_common_services_location(properties) + mpacks_staging_location = get_mpacks_staging_location(properties) + ambari_version = get_ambari_version(properties) + return stack_location, service_definitions_location, mpacks_staging_location + +def create_symlink(src_dir, dest_dir, file_name, force=False): + """ + Helper function to create symbolic link (dest_dir/file_name -> src_dir/file_name) + :param src_dir: Source directory + :param dest_dir: Destination directory + :param file_name: File name + :param force: Remove existing symlink + """ + src_path = os.path.join(src_dir, file_name) + dest_link = os.path.join(dest_dir, file_name) + if force and os.path.islink(dest_link): + sudo.unlink(dest_link) + sudo.symlink(src_path, dest_link) + +def remove_symlinks(stack_location, service_definitions_location, staged_mpack_dir): + """ + Helper function to remove all symbolic links pointed to a management pack + :param stack_location: Path to stacks folder + (/var/lib/ambari-server/resources/stacks) + :param service_definitions_location: Path to service_definitions folder + (/var/lib/ambari-server/resources/common-services) + :param staged_mpack_dir: Path to management pack staging location + (/var/lib/ambari-server/resources/mpacks/mpack_name-mpack_version) + """ + for location in [stack_location, service_definitions_location]: + for root, dirs, files in os.walk(location): + for name in files: + file = os.path.join(root, name) + if os.path.islink(file) and staged_mpack_dir in os.path.realpath(file): + print_info_msg("Removing symlink {0}".format(file)) + sudo.unlink(file) + for name in dirs: + dir = os.path.join(root, name) + if os.path.islink(dir) and staged_mpack_dir in os.path.realpath(dir): + print_info_msg("Removing symlink {0}".format(dir)) + sudo.unlink(dir) + +def purge_stacks_and_mpacks(): + """ + Purge all stacks and management packs + """ + # Get ambari mpacks config properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + + print_info_msg("Purging existing stack definitions and management packs") + + if os.path.exists(stack_location): + print_info_msg("Purging stack location: " + stack_location) + sudo.rmtree(stack_location) + + if os.path.exists(service_definitions_location): + print_info_msg("Purging service definitions location: " + service_definitions_location) + sudo.rmtree(service_definitions_location) + + if os.path.exists(mpacks_staging_location): + print_info_msg("Purging mpacks staging location: " + mpacks_staging_location) + sudo.rmtree(mpacks_staging_location) + sudo.makedir(mpacks_staging_location, 0755) + +def process_stack_definitions_artifact(artifact, artifact_source_dir, options): + """ + Process stack-definitions artifacts + :param artifact: Artifact metadata + :param artifact_source_dir: Location of artifact in the management pack + :param options: Command line options + """ + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + for file in os.listdir(artifact_source_dir): + if os.path.isfile(os.path.join(artifact_source_dir, file)): + # Example: /var/lib/ambari-server/resources/stacks/stack_advisor.py + create_symlink(artifact_source_dir, stack_location, file, options.force) + else: + src_stack_dir = os.path.join(artifact_source_dir, file) + dest_stack_dir = os.path.join(stack_location, file) + if not os.path.exists(dest_stack_dir): + sudo.makedir(dest_stack_dir, 0755) + for file in os.listdir(src_stack_dir): + if os.path.isfile(os.path.join(src_stack_dir, file)): + create_symlink(src_stack_dir, dest_stack_dir, file, options.force) + else: + src_stack_version_dir = os.path.join(src_stack_dir, file) + dest_stack_version_dir = os.path.join(dest_stack_dir, file) + if not os.path.exists(dest_stack_version_dir): + sudo.makedir(dest_stack_version_dir, 0755) + for file in os.listdir(src_stack_version_dir): + if file == "services": + src_stack_services_dir = os.path.join(src_stack_version_dir, file) + dest_stack_services_dir = os.path.join(dest_stack_version_dir, file) + if not os.path.exists(dest_stack_services_dir): + sudo.makedir(dest_stack_services_dir, 0755) + for file in os.listdir(src_stack_services_dir): + create_symlink(src_stack_services_dir, dest_stack_services_dir, file, options.force) + else: + create_symlink(src_stack_version_dir, dest_stack_version_dir, file, options.force) + +def process_stack_definition_artifact(artifact, artifact_source_dir, options): + """ + Process stack-definition artifact + :param artifact: Artifact metadata + :param artifact_source_dir: Location of artifact in the management pack + :param options: Command line options + """ + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + stack_name = None + if "stack_name" in artifact: + stack_name = artifact.stack_name + if not stack_name: + print_error_msg("Must provide stack name for stack-definition artifact!") + raise FatalException(-1, 'Must provide stack name for stack-definition artifact!') + stack_version = None + if "stack_version" in artifact: + stack_version = artifact.stack_version + if not stack_version: + print_error_msg("Must provide stack version for stack-definition artifact!") + raise FatalException(-1, 'Must provide stack version for stack-definition artifact!') + dest_link = os.path.join(stack_location, stack_name, stack_version) + if options.force and os.path.islink(dest_link): + sudo.unlink(dest_link) + sudo.symlink(artifact_source_dir, dest_link) + +def process_service_definitions_artifact(artifact, artifact_source_dir, options): + """ + Process service-definitions artifact + :param artifact: Artifact metadata + :param artifact_source_dir: Location of artifact in the management pack + :param options: Command line options + """ + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + for file in os.listdir(artifact_source_dir): + src_service_definitions_dir = os.path.join(artifact_source_dir, file) + dest_service_definitions_dir = os.path.join(service_definitions_location, file) + if not os.path.exists(dest_service_definitions_dir): + sudo.makedir(dest_service_definitions_dir, 0755) + for file in os.listdir(src_service_definitions_dir): + create_symlink(src_service_definitions_dir, dest_service_definitions_dir, file, options.force) + +def process_service_definition_artifact(artifact, artifact_source_dir, options): + """ + Process service-definition artifact + :param artifact: Artifact metadata + :param artifact_source_dir: Location of artifact in the management pack + :param options: Command line options + """ + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + service_name = None + if "service_name" in artifact: + service_name = artifact.service_name + if not service_name: + print_error_msg("Must provide service name for service-definition artifact!") + raise FatalException(-1, 'Must provide service name for service-definition artifact!') + service_version = None + if "service_version" in artifact: + service_version = artifact.service_version + if not service_version: + print_error_msg("Must provide service version for service-definition artifact!") + raise FatalException(-1, 'Must provide service version for service-definition artifact!') + dest_service_definition_dir = os.path.join(service_definitions_location, service_name) + if not os.path.exists(dest_service_definition_dir): + sudo.makedir(dest_service_definition_dir, 0755) + dest_link = os.path.join(dest_service_definition_dir, service_version) + if options.force and os.path.islink(dest_link): + sudo.unlink(dest_link) + sudo.symlink(artifact_source_dir, dest_link) + +def process_stack_extension_definitions_artifact(artifact, artifact_source_dir, options): + """ + Process stack-extension-definitions artifact + :param artifact: Artifact metadata + :param artifact_source_dir: Location of artifact in the management pack + :param options: Command line options + """ + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + service_versions_map = None + if "service_versions_map" in artifact: + service_versions_map = artifact.service_versions_map + if not service_versions_map: + print_error_msg("Must provide service versions map for stack-extension-definitions artifact!") + raise FatalException(-1, 'Must provide service versions map for stack-extension-definition artifact!') + for service_name in os.listdir(artifact_source_dir): + source_service_path = os.path.join(artifact_source_dir, service_name) + for service_version in os.listdir(source_service_path): + source_service_version_path = os.path.join(source_service_path, service_version) + for service_version_entry in service_versions_map: + if service_name == service_version_entry.service_name and service_version == service_version_entry.service_version: + applicable_stacks = service_version_entry.applicable_stacks + for applicable_stack in applicable_stacks: + stack_name = applicable_stack.stack_name + stack_version = applicable_stack.stack_version + dest_stack_path = os.path.join(stack_location, stack_name) + dest_stack_version_path = os.path.join(dest_stack_path, stack_version) + dest_stack_services_path = os.path.join(dest_stack_version_path, "services") + dest_link = os.path.join(dest_stack_services_path, service_name) + if os.path.exists(dest_stack_path) and os.path.exists(dest_stack_version_path): + if not os.path.exists(dest_stack_services_path): + sudo.makedir(dest_stack_services_path, 0755) + if options.force and os.path.islink(dest_link): + sudo.unlink(dest_link) + sudo.symlink(source_service_version_path, dest_link) + +def process_stack_extension_definition_artifact(artifact, artifact_source_dir, options): + """ + Process stack-extension-definition artifact + :param artifact: Artifact metadata + :param artifact_source_dir: Location of artifact in the management pack + :param options: Command line options + """ + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + service_name = None + if "service_name" in artifact: + service_name = artifact.service_name + if not service_name: + print_error_msg("Must provide service name for stack-extension-definition artifact!") + raise FatalException(-1, 'Must provide service name for stack-extension-definition artifact!') + applicable_stacks = None + if "applicable_stacks" in artifact: + applicable_stacks = artifact.applicable_stacks + if not applicable_stacks: + print_error_msg("Must provide applicable stacks for stack-extension-definition artifact!") + raise FatalException(-1, 'Must provide applicable stacks for stack-extension-definition artifact!') + for applicable_stack in applicable_stacks: + stack_name = applicable_stack.stack_name + stack_version = applicable_stack.stack_version + dest_stack_path = os.path.join(stack_location, stack_name) + dest_stack_version_path = os.path.join(dest_stack_path, stack_version) + dest_stack_services_path = os.path.join(dest_stack_version_path, "services") + dest_link = os.path.join(dest_stack_services_path, service_name) + if os.path.exists(dest_stack_path) and os.path.exists(dest_stack_version_path): + if not os.path.exists(dest_stack_services_path): + sudo.makedir(dest_stack_services_path, 0755) + if options.force and os.path.islink(dest_link): + sudo.unlink(dest_link) + sudo.symlink(artifact_source_dir, dest_link) + +def search_mpacks(mpack_name, max_mpack_version=None): + """ + Search for all "mpack_name" management packs installed. + If "max_mpack_version" is specified return only management packs < max_mpack_version + :param mpack_name: Management pack name + :param max_mpack_version: Max management pack version number + :return: List of management pack + """ + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + results = [] + if os.path.exists(mpacks_staging_location) and os.path.isdir(mpacks_staging_location): + staged_mpack_dirs = os.listdir(mpacks_staging_location) + for dir in staged_mpack_dirs: + staged_mpack_dir = os.path.join(mpacks_staging_location, dir) + if os.path.isdir(staged_mpack_dir): + staged_mpack_metadata = read_mpack_metadata(staged_mpack_dir) + if not staged_mpack_metadata: + print_error_msg("Skipping malformed management pack {0}-{1}. Metadata file missing!".format( + staged_mpack_name, staged_mpack_version)) + continue + staged_mpack_name = staged_mpack_metadata.name + staged_mpack_version = staged_mpack_metadata.version + if mpack_name == staged_mpack_name and \ + (not max_mpack_version or compare_versions(staged_mpack_version, max_mpack_version ) < 0): + results.append((staged_mpack_name, staged_mpack_version)) + return results + +def uninstall_mpacks(mpack_name, max_mpack_version=None): + """ + Uninstall all "mpack_name" management packs. + If "max_mpack_version" is specified uninstall only management packs < max_mpack_version + :param mpack_name: Management pack name + :param max_mpack_version: Max management pack version number + """ + results = search_mpacks(mpack_name, max_mpack_version) + for result in results: + uninstall_mpack(result[0], result[1]) + +def uninstall_mpack(mpack_name, mpack_version): + """ + Uninstall specific management pack + :param mpack_name: Management pack name + :param mpack_version: Management pack version + """ + print_info_msg("Uninstalling management pack {0}-{1}".format(mpack_name, mpack_version)) + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + found = False + if os.path.exists(mpacks_staging_location) and os.path.isdir(mpacks_staging_location): + staged_mpack_dirs = os.listdir(mpacks_staging_location) + for dir in staged_mpack_dirs: + staged_mpack_dir = os.path.join(mpacks_staging_location, dir) + if os.path.isdir(staged_mpack_dir): + staged_mpack_metadata = read_mpack_metadata(staged_mpack_dir) + if not staged_mpack_metadata: + print_error_msg("Skipping malformed management pack {0}-{1}. Metadata file missing!".format( + staged_mpack_name, staged_mpack_version)) + continue + staged_mpack_name = staged_mpack_metadata.name + staged_mpack_version = staged_mpack_metadata.version + if mpack_name == staged_mpack_name and compare_versions(staged_mpack_version, mpack_version ) == 0: + print_info_msg("Removing management pack staging location {0}".format(staged_mpack_dir)) + sudo.rmtree(staged_mpack_dir) + remove_symlinks(stack_location, service_definitions_location, staged_mpack_dir) + found = True + break + if not found: + print_error_msg("Management pack {0}-{1} is not installed!".format(mpack_name, mpack_version)) + else: + print_info_msg("Management pack {0}-{1} successfully uninstalled!".format(mpack_name, mpack_version)) + +def validate_mpack_prerequisites(mpack_metadata): + """ + Validate management pack prerequisites + :param mpack_name: Management pack metadata + """ + # Get ambari config properties + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + stack_location = get_stack_location(properties) + current_ambari_version = get_ambari_version(properties) + fail = False + + mpack_prerequisites = mpack_metadata.prerequisites + if "min_ambari_version" in mpack_prerequisites: + min_ambari_version = mpack_prerequisites.min_ambari_version + if(compare_versions(min_ambari_version, current_ambari_version, format=True) > 0): + print_error_msg("Prerequisite failure! Current Ambari Version = {0}, " + "Min Ambari Version = {1}".format(current_ambari_version, min_ambari_version)) + fail = True + if "max_ambari_version" in mpack_prerequisites: + max_ambari_version = mpack_prerequisites.max_ambari_version + if(compare_versions(max_ambari_version, current_ambari_version, format=True) < 0): + print_error_msg("Prerequisite failure! Current Ambari Version = {0}, " + "Max Ambari Version = {1}".format(current_ambari_version, max_ambari_version)) + if "min_stack_versions" in mpack_prerequisites: + min_stack_versions = mpack_prerequisites.min_stack_versions + stack_found = False + for min_stack_version in min_stack_versions: + stack_name = min_stack_version.stack_name + stack_version = min_stack_version.stack_version + stack_dir = os.path.join(stack_location, stack_name, stack_version) + if os.path.exists(stack_dir) and os.path.isdir(stack_dir): + stack_found = True + break + if not stack_found: + print_error_msg("Prerequisite failure! Min applicable stack not found") + fail = True + + if fail: + raise FatalException(-1, "Prerequisites for management pack {0}-{1} failed!".format( + mpack_metadata.name, mpack_metadata.version)) + +def install_mpack(options): + """ + Install management pack + :param options: Command line options + """ + + mpack_path = options.mpack_path + if not mpack_path: + print_error_msg("Management pack not specified!") + raise FatalException(-1, 'Management pack not specified!') + + print_info_msg("Installing management pack {0}".format(mpack_path)) + + # Download management pack to a temp location + tmp_archive_path = download_mpack(mpack_path) + + # Expand management pack in temp directory + tmp_root_dir = expand_mpack(tmp_archive_path) + + # Read mpack metadata + mpack_metadata = read_mpack_metadata(tmp_root_dir) + if not mpack_metadata: + raise FatalException(-1, 'Malformed management pack {0}. Metadata file missing!'.format(mpack_path)) + + # Validate management pack prerequisites + validate_mpack_prerequisites(mpack_metadata) + + # Purge previously installed stacks and management packs + if options.purge: + purge_stacks_and_mpacks() + + # Get ambari mpack properties + stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties() + # Create directories + if not os.path.exists(stack_location): + sudo.makedir(stack_location, 0755) + if not os.path.exists(service_definitions_location): + sudo.makedir(service_definitions_location, 0755) + if not os.path.exists(mpacks_staging_location): + sudo.makedir(mpacks_staging_location, 0755) + + # Stage management pack (Stage at /var/lib/ambari-server/resources/mpacks/mpack_name-mpack_version) + mpack_name = mpack_metadata.name + mpack_version = mpack_metadata.version + mpack_dirname = mpack_name + "-" + mpack_version + mpack_staging_dir = os.path.join(mpacks_staging_location, mpack_dirname) + + print_info_msg("Stage management pack {0}-{1} to staging location {2}".format( + mpack_name, mpack_version, mpack_staging_dir)) + if os.path.exists(mpack_staging_dir): + if options.force: + print_info_msg("Force removing previously installed management pack from {0}".format(mpack_staging_dir)) + sudo.rmtree(mpack_staging_dir) + else: + error_msg = "Management pack {0}-{1} already installed!".format(mpack_name, mpack_version) + print_error_msg(error_msg) + raise FatalException(-1, error_msg) + shutil.move(tmp_root_dir, mpack_staging_dir) + + # Process setup steps for all artifacts (stack-definitions, service-definitions, stack-extension-definitions) + # in the management pack + for artifact in mpack_metadata.artifacts: + # Artifact name (Friendly name) + artifact_name = artifact.name + # Artifact type (stack-definitions, service-definitions, stack-extension-definitions etc) + artifact_type = artifact.type + # Artifact directory with contents of the artifact + artifact_source_dir = os.path.join(mpack_staging_dir, artifact.source_dir) + + print_info_msg("Processing artifact {0} of type {1} in {2}".format( + artifact_name, artifact_type, artifact_source_dir)) + if artifact.type == "stack-definitions": + process_stack_definitions_artifact(artifact, artifact_source_dir, options) + elif artifact.type == "stack-definition": + process_stack_definition_artifact(artifact, artifact_source_dir, options) + elif artifact.type == "service-definitions": + process_service_definitions_artifact(artifact, artifact_source_dir, options) + elif artifact.type == "service-definition": + process_service_definition_artifact(artifact, artifact_source_dir, options) + elif artifact.type == "stack-extension-definitions": + process_stack_extension_definitions_artifact(artifact, artifact_source_dir, options) + elif artifact.type == "stack-extension-definition": + process_stack_extension_definition_artifact(artifact, artifact_source_dir, options) + else: + print_info_msg("Unknown artifact {0} of type {1}".format(artifact_name, artifact_type)) + + print_info_msg("Management pack {0}-{1} successfully installed!".format(mpack_name, mpack_version)) + return mpack_name, mpack_version, mpack_staging_dir + +def upgrade_mpack(options): + """ + Upgrade management pack + :param options: command line options + """ + mpack_path = options.mpack_path + if options.purge: + print_error_msg("Purge is not supported with upgrade_mpack action!") + raise FatalException(-1, "Purge is not supported with upgrade_mpack action!") + + if not mpack_path: + print_error_msg("Management pack not specified!") + raise FatalException(-1, 'Management pack not specified!') + + print_info_msg("Upgrading management pack {0}".format(mpack_path)) + + # Force install new management pack version + options.force = True + (mpack_name, mpack_version, mpack_staging_dir) = install_mpack(options) + + # Uninstall old management packs + uninstall_mpacks(mpack_name, mpack_version) + + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/9404dee9/ambari-server/src/test/python/TestAmbariServer.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index ac24f7d..1356dac 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -92,7 +92,7 @@ with patch.object(os_utils, "parse_log4j_file", return_value={'ambari.log.dir': get_pass_file_path, GET_FQDN_SERVICE_URL, JDBC_USE_INTEGRATED_AUTH_PROPERTY, SECURITY_KEY_ENV_VAR_NAME, \ JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, STACK_LOCATION_KEY, SERVER_VERSION_FILE_PATH, \ COMMON_SERVICES_PATH_PROPERTY, WEBAPP_DIR_PROPERTY, SHARED_RESOURCES_DIR, BOOTSTRAP_SCRIPT, \ - CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY + CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY, MPACKS_STAGING_PATH_PROPERTY from ambari_server.serverUtils import is_server_runing, refresh_stack_hash from ambari_server.serverSetup import check_selinux, check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED, SE_MODE_ENFORCING, configure_os_settings, \ download_and_install_jdk, prompt_db_properties, setup, \ @@ -4417,6 +4417,7 @@ class TestAmbariServer(TestCase): p.process_pair(BOOTSTRAP_SETUP_AGENT_SCRIPT, 'some_value') p.process_pair(STACKADVISOR_SCRIPT, 'some_value') p.process_pair(BOOTSTRAP_DIR_PROPERTY, 'some_value') + p.process_pair(MPACKS_STAGING_PATH_PROPERTY, 'some_value') get_ambari_properties_5_mock.return_value = get_ambari_properties_4_mock.return_value = \ get_ambari_properties_3_mock.return_value = get_ambari_properties_2_mock.return_value = \
