Repository: ambari Updated Branches: refs/heads/branch-2.4 77f90b9db -> 402b9d303
AMBARI-17233: Upgrade management packs should handle build numbers gracefully (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/402b9d30 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/402b9d30 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/402b9d30 Branch: refs/heads/branch-2.4 Commit: 402b9d3031fe3bfcca8126c13d24aaf93766abf0 Parents: 77f90b9 Author: Jayush Luniya <[email protected]> Authored: Tue Jun 14 21:26:00 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Tue Jun 14 21:27:07 2016 -0700 ---------------------------------------------------------------------- .../python/ambari_server/serverConfiguration.py | 13 +++++ .../main/python/ambari_server/setupMpacks.py | 52 +++++++++++++++++--- ambari-server/src/test/python/TestMpacks.py | 22 ++++++--- 3 files changed, 75 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/402b9d30/ambari-server/src/main/python/ambari_server/serverConfiguration.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py index 6e14b43..b14aeed 100644 --- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py +++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py @@ -193,6 +193,7 @@ CUSTOM_ACTION_DEFINITIONS = 'custom.action.definitions' BOOTSTRAP_SETUP_AGENT_SCRIPT = 'bootstrap.setup_agent.script' STACKADVISOR_SCRIPT = 'stackadvisor.script' PID_DIR_PROPERTY = 'pid.dir' +SERVER_TMP_DIR_PROPERTY = "server.tmp.dir" REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH, WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY, SECURITY_KEYS_DIR, JDBC_DATABASE_NAME_PROPERTY, NR_USER_PROPERTY, JAVA_HOME_PROPERTY, JDBC_PASSWORD_PROPERTY, SHARED_RESOURCES_DIR, @@ -375,6 +376,7 @@ class ServerConfigDefaults(object): self.STACK_LOCATION_DEFAULT = "" self.COMMON_SERVICES_LOCATION_DEFAULT = "" self.MPACKS_STAGING_LOCATION_DEFAULT = "" + self.SERVER_TMP_DIR_DEFAULT = "" self.DEFAULT_VIEWS_DIR = "" @@ -444,6 +446,7 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults): self.STACK_LOCATION_DEFAULT = "resources\\stacks" self.COMMON_SERVICES_LOCATION_DEFAULT = "resources\\common-services" self.MPACKS_STAGING_LOCATION_DEFAULT = "resources\\mpacks" + self.SERVER_TMP_DIR_DEFAULT = "data\\tmp" self.DEFAULT_VIEWS_DIR = "resources\\views" @@ -528,6 +531,7 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults): self.STACK_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/stacks") self.COMMON_SERVICES_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/common-services") self.MPACKS_STAGING_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/mpacks") + self.SERVER_TMP_DIR_DEFAULT = AmbariPath.get("/var/lib/ambari-server/data/tmp") self.DEFAULT_VIEWS_DIR = AmbariPath.get("/var/lib/ambari-server/resources/views") @@ -1385,6 +1389,15 @@ def get_mpacks_staging_location(properties): mpacks_staging_location = configDefaults.MPACKS_STAGING_LOCATION_DEFAULT return mpacks_staging_location +# +# Server temp location +# +def get_server_temp_location(properties): + server_tmp_dir = properties[SERVER_TMP_DIR_PROPERTY] + if not server_tmp_dir: + server_tmp_dir = configDefaults.SERVER_TMP_DIR_DEFAULT + return server_tmp_dir + def get_missing_properties(properties): missing_propertiers = [] for property in REQUIRED_PROPERTIES: http://git-wip-us.apache.org/repos/asf/ambari/blob/402b9d30/ambari-server/src/main/python/ambari_server/setupMpacks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/ambari_server/setupMpacks.py b/ambari-server/src/main/python/ambari_server/setupMpacks.py index 6994cbd..98811f5 100644 --- a/ambari-server/src/main/python/ambari_server/setupMpacks.py +++ b/ambari-server/src/main/python/ambari_server/setupMpacks.py @@ -20,7 +20,6 @@ limitations under the License. import os import shutil -import tempfile import json import ast @@ -29,7 +28,7 @@ from ambari_commons.inet_utils import download_file from ambari_commons.logging_utils import print_info_msg, print_error_msg from ambari_commons.os_utils import copy_file from ambari_server.serverConfiguration import get_ambari_properties, get_ambari_version, get_stack_location, \ - get_common_services_location, get_mpacks_staging_location + get_common_services_location, get_mpacks_staging_location, get_server_temp_location from resource_management.core import sudo from resource_management.libraries.functions.tar_archive import extract_archive, get_archive_root_dir @@ -69,6 +68,16 @@ class _named_dict(dict): else: dict.__getattr__(self, item) +def _get_temp_dir(): + properties = get_ambari_properties() + if properties == -1: + print_error_msg("Error getting ambari properties") + return -1 + tmpdir = get_server_temp_location(properties) + if not os.path.exists(tmpdir): + sudo.makedirs(tmpdir, 0755) + return tmpdir + def download_mpack(mpack_path): """ Download management pack @@ -76,7 +85,7 @@ def download_mpack(mpack_path): :return: Path where the management pack was downloaded """ # Download management pack to a temp location - tmpdir = tempfile.gettempdir() + tmpdir = _get_temp_dir() archive_filename = os.path.basename(mpack_path) tmp_archive_path = os.path.join(tmpdir, archive_filename) @@ -97,7 +106,7 @@ def expand_mpack(archive_path): :param archive_path: Local path to management pack :return: Path where the management pack was expanded """ - tmpdir = tempfile.gettempdir() + tmpdir = _get_temp_dir() archive_root_dir = get_archive_root_dir(archive_path) if not archive_root_dir: print_error_msg("Malformed management pack. Root directory missing!") @@ -146,6 +155,31 @@ def get_mpack_properties(): ambari_version = get_ambari_version(properties) return stack_location, service_definitions_location, mpacks_staging_location +def _get_mpack_name_version(mpack_path): + """ + Get mpack name and version + :param mpack_path: Path to mpack + """ + if not mpack_path: + print_error_msg("Management pack not specified!") + raise FatalException(-1, 'Management pack not specified!') + + # Download management pack to a temp location + tmp_archive_path = download_mpack(mpack_path) + if not (tmp_archive_path and os.path.exists(tmp_archive_path)): + print_error_msg("Management pack could not be downloaded!") + raise FatalException(-1, 'Management pack could not be downloaded!') + + # Expand management pack in temp directory + tmp_root_dir = expand_mpack(tmp_archive_path) + + # Read mpack metadata + mpack_metadata = read_mpack_metadata(tmp_root_dir) + if not mpack_metadata: + raise FatalException(-1, 'Malformed management pack {0}. Metadata file missing!'.format(mpack_path)) + + return (mpack_metadata.name, mpack_metadata.version) + def create_symlink(src_dir, dest_dir, file_name, force=False): """ Helper function to create symbolic link (dest_dir/file_name -> src_dir/file_name) @@ -420,7 +454,7 @@ def search_mpacks(mpack_name, max_mpack_version=None): staged_mpack_name = staged_mpack_metadata.name staged_mpack_version = staged_mpack_metadata.version if mpack_name == staged_mpack_name and \ - (not max_mpack_version or compare_versions(staged_mpack_version, max_mpack_version ) < 0): + (not max_mpack_version or compare_versions(staged_mpack_version, max_mpack_version, format=True) < 0): results.append((staged_mpack_name, staged_mpack_version)) return results @@ -459,7 +493,7 @@ def uninstall_mpack(mpack_name, mpack_version): continue staged_mpack_name = staged_mpack_metadata.name staged_mpack_version = staged_mpack_metadata.version - if mpack_name == staged_mpack_name and compare_versions(staged_mpack_version, mpack_version ) == 0: + if mpack_name == staged_mpack_name and compare_versions(staged_mpack_version, mpack_version, format=True) == 0: print_info_msg("Removing management pack staging location {0}".format(staged_mpack_dir)) sudo.rmtree(staged_mpack_dir) remove_symlinks(stack_location, service_definitions_location, staged_mpack_dir) @@ -665,6 +699,12 @@ def upgrade_mpack(options, replay_mode=False): print_error_msg("Management pack not specified!") raise FatalException(-1, 'Management pack not specified!') + (mpack_name, mpack_version) = _get_mpack_name_version(mpack_path) + results = search_mpacks(mpack_name, mpack_version) + if not results: + print_error_msg("No management packs found that can be upgraded!") + raise FatalException(-1, 'No management packs found that can be upgraded!') + print_info_msg("Upgrading management pack {0}".format(mpack_path)) # Force install new management pack version http://git-wip-us.apache.org/repos/asf/ambari/blob/402b9d30/ambari-server/src/test/python/TestMpacks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestMpacks.py b/ambari-server/src/test/python/TestMpacks.py index ef8e8b8..a7c6092 100644 --- a/ambari-server/src/test/python/TestMpacks.py +++ b/ambari-server/src/test/python/TestMpacks.py @@ -61,7 +61,8 @@ def get_configs(): configs = { serverConfiguration.STACK_LOCATION_KEY : "/var/lib/ambari-server/resources/stacks", serverConfiguration.COMMON_SERVICES_PATH_PROPERTY : "/var/lib/ambari-server/resources/common-services", - serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory + serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory, + serverConfiguration.SERVER_TMP_DIR_PROPERTY : "/tmp" } return configs @@ -136,10 +137,12 @@ class TestMpacks(TestCase): @patch("ambari_server.setupMpacks.extract_archive") @patch("ambari_server.setupMpacks.get_archive_root_dir") @patch("ambari_server.setupMpacks.download_mpack") - def test_install_mpack_with_malformed_mpack(self, download_mpack_mock, get_archive_root_dir_mock, extract_archive_mock, os_path_exists_mock): + @patch("ambari_server.setupMpacks.get_ambari_properties") + def test_install_mpack_with_malformed_mpack(self, get_ambari_properties_mock, download_mpack_mock, get_archive_root_dir_mock, extract_archive_mock, os_path_exists_mock): options = self._create_empty_options_mock() options.mpack_path = "/path/to/mpack.tar.gz" download_mpack_mock.return_value = "/tmp/mpack.tar.gz" + get_ambari_properties_mock.return_value = configs os_path_exists_mock.return_value = True get_archive_root_dir_mock.return_value = None @@ -152,7 +155,7 @@ class TestMpacks(TestCase): self.assertTrue(fail) get_archive_root_dir_mock.return_value = "mpack" - os_path_exists_mock.side_effect = [True, False, False] + os_path_exists_mock.side_effect = [True, True, False, False] extract_archive_mock.return_value = None fail = False try: @@ -163,7 +166,7 @@ class TestMpacks(TestCase): self.assertTrue(fail) get_archive_root_dir_mock.return_value = "mpack" - os_path_exists_mock.side_effect = [True, False, True, False] + os_path_exists_mock.side_effect = [True, True, False, True, False] extract_archive_mock.return_value = None fail = False try: @@ -367,12 +370,18 @@ class TestMpacks(TestCase): os_path_exists_mock): options = self._create_empty_options_mock() options.mpack_path = "/path/to/mystack-1.0.0.1.tar.gz" - download_mpack_mock.return_value = "/tmp/mystack-1.0.0.1.tar.gz" - expand_mpack_mock.return_value = "mpacks/mystack-ambari-mpack-1.0.0.1" + download_mpack_mock.side_effect = ["/tmp/mystack-1.0.0.1.tar.gz", "/tmp/mystack-1.0.0.1.tar.gz"] + expand_mpack_mock.side_effect = ["mpacks/mystack-ambari-mpack-1.0.0.1", "mpacks/mystack-ambari-mpack-1.0.0.1"] get_ambari_version_mock.return_value = "2.4.0.0" """ os_path_exists_calls = [call('/tmp/mystack-1.0.0.1.tar.gz'), call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'), + call(mpacks_directory), + call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'), + call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/mpack.json'), + call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/mpack.json'), + call('/tmp/mystack-1.0.0.1.tar.gz'), + call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'), call('/var/lib/ambari-server/resources/stacks'), call('/var/lib/ambari-server/resources/common-services'), call(mpacks_directory), @@ -396,6 +405,7 @@ class TestMpacks(TestCase): call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/mpack.json')] """ os_path_exists_mock.side_effect = [True, True, True, True, True, True, + True, True, True, True, True, True, False, True, True, False, True, True, True, True, True, True, True, False, False, True, True, True, True]
