Repository: ambari Updated Branches: refs/heads/branch-2.6 379a56a24 -> 4cc2ecca8
AMBARI-21931 - Use Correct Packages For Clients Where Stack Tools Support It (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4cc2ecca Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4cc2ecca Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4cc2ecca Branch: refs/heads/branch-2.6 Commit: 4cc2ecca8d56c3b778929ca524e3b4e8258c657b Parents: 379a56a Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Mon Sep 11 15:41:22 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Tue Sep 12 12:12:46 2017 -0400 ---------------------------------------------------------------------- .../libraries/functions/stack_select.py | 65 ++++++++++++++++++- .../ambari/server/topology/AmbariContext.java | 2 +- .../HDP/2.0.6/properties/stack_packages.json | 66 +++++++++++++------ .../src/test/python/TestStackSelect.py | 67 ++++++++++++++++++-- .../src/test/python/stacks/utils/RMFTestCase.py | 28 ++++---- 5 files changed, 191 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py index eac1bef..f5068e4 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py @@ -33,6 +33,7 @@ from resource_management.libraries.functions.get_stack_version import get_stack_ from resource_management.libraries.functions.format import format from resource_management.libraries.script.script import Script from resource_management.libraries.functions import stack_tools +from resource_management.core import shell from resource_management.core.shell import call from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root @@ -75,6 +76,11 @@ PACKAGE_SCOPE_INSTALL = "INSTALL" PACKAGE_SCOPE_STANDARD = "STANDARD" PACKAGE_SCOPE_PATCH = "PATCH" PACKAGE_SCOPE_STACK_SELECT = "STACK-SELECT-PACKAGE" + +# the legacy key is used when a package has changed from one version of the stack select tool to another. +_PACKAGE_SCOPE_LEGACY = "LEGACY" + +# the valid scopes which can be requested _PACKAGE_SCOPES = (PACKAGE_SCOPE_INSTALL, PACKAGE_SCOPE_STANDARD, PACKAGE_SCOPE_PATCH, PACKAGE_SCOPE_STACK_SELECT) # the orchestration types which equal to a partial (non-STANDARD) upgrade @@ -110,6 +116,37 @@ def get_package_name(default_package = None): raise +def is_package_supported(package, supported_packages = None): + """ + Gets whether the specified package is supported by the <stack_select> tool. + :param package: the package to check + :param supported_packages: the list of supported packages pre-fetched + :return: True if the package is support, False otherwise + """ + if supported_packages is None: + supported_packages = get_supported_packages() + + if package in supported_packages: + return True + + return False + + +def get_supported_packages(): + """ + Parses the output from <stack-select> packages and returns an array of the various packages. + :return: and array of packages support by <stack-select> + """ + stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) + command = (STACK_SELECT_PREFIX, stack_selector_path, "packages") + code, stdout = shell.call(command, sudo = True, quiet = True) + + if code != 0 or stdout is None: + raise Fail("Unable to query for supported packages using {0}".format(stack_selector_path)) + + # turn the output into lines, stripping each line + return [line.strip() for line in stdout.splitlines()] + def get_packages(scope, service_name = None, component_name = None): """ @@ -172,7 +209,33 @@ def get_packages(scope, service_name = None, component_name = None): Logger.info("Skipping stack-select on {0} because it does not exist in the stack-select package structure.".format(component_name)) return None - return data[component_name][scope] + # this one scope is not an array, so transform it into one for now so we can + # use the same code below + packages = data[component_name][scope] + if scope == PACKAGE_SCOPE_STACK_SELECT: + packages = [packages] + + # grab the package name from the JSON and validate it against the packages + # that the stack-select tool supports - if it doesn't support it, then try to find the legacy + # package name if it exists + supported_packages = get_supported_packages() + for index, package in enumerate(packages): + if not is_package_supported(package, supported_packages=supported_packages): + if _PACKAGE_SCOPE_LEGACY in data[component_name]: + legacy_package = data[component_name][_PACKAGE_SCOPE_LEGACY] + Logger.info( + "The package {0} is not supported by this version of the stack-select tool, defaulting to the legacy package of {1}".format(package, legacy_package)) + + # use the legacy package + packages[index] = legacy_package + else: + raise Fail("The package {0} is not supported by this version of the stack-select tool.".format(package)) + + # transform the array bcak to a single element + if scope == PACKAGE_SCOPE_STACK_SELECT: + packages = packages[0] + + return packages def select_all(version_to_select): http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 6a2d58d..134fffc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -206,7 +206,7 @@ public class AmbariContext { StackId stackId = new StackId(stack.getName(), stack.getVersion()); RepositoryVersionEntity repoVersion = null; - if (null == repoVersionString && null == repoVersionId) { + if (StringUtils.isEmpty(repoVersionString) && null == repoVersionId) { List<RepositoryVersionEntity> stackRepoVersions = repositoryVersionDAO.findByStack(stackId); if (stackRepoVersions.isEmpty()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json index 704fb54..ec35f2b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json @@ -298,12 +298,13 @@ ] }, "HDFS_CLIENT": { - "STACK-SELECT-PACKAGE": "hadoop-client", + "STACK-SELECT-PACKAGE": "hadoop-hdfs-client", + "LEGACY": "hadoop-client", "INSTALL": [ - "hadoop-client" + "hadoop-hdfs-client" ], "PATCH": [ - "INVALID" + "hadoop-hdfs-client" ], "STANDARD": [ "hadoop-client" @@ -420,12 +421,13 @@ ] }, "HIVE_CLIENT": { - "STACK-SELECT-PACKAGE": "hadoop-client", + "STACK-SELECT-PACKAGE": "hive-client", + "LEGACY": "hadoop-client", "INSTALL": [ - "hadoop-client" + "hive-client" ], "PATCH": [ - "INVALID" + "hive-client" ], "STANDARD": [ "hadoop-client" @@ -500,12 +502,13 @@ ] }, "MAPREDUCE2_CLIENT": { - "STACK-SELECT-PACKAGE": "hadoop-client", + "STACK-SELECT-PACKAGE": "hadoop-mapreduce-client", + "LEGACY": "hadoop-client", "INSTALL": [ - "hadoop-client" + "hadoop-mapreduce-client" ], "PATCH": [ - "hadoop-mapreduce-INVALID" + "hadoop-mapreduce-client" ], "STANDARD": [ "hadoop-client" @@ -542,12 +545,13 @@ }, "PIG": { "PIG": { - "STACK-SELECT-PACKAGE": "hadoop-client", + "STACK-SELECT-PACKAGE": "pig-client", + "LEGACY": "hadoop-client", "INSTALL": [ - "hadoop-client" + "pig-client" ], "PATCH": [ - "INVALID" + "pig-client" ], "STANDARD": [ "hadoop-client" @@ -636,6 +640,18 @@ } }, "SPARK": { + "LIVY_CLIENT": { + "STACK-SELECT-PACKAGE": "livy-client", + "INSTALL": [ + "livy-client" + ], + "PATCH": [ + "livy-client" + ], + "STANDARD": [ + "livy-client" + ] + }, "LIVY_SERVER": { "STACK-SELECT-PACKAGE": "livy-server", "INSTALL": [ @@ -686,6 +702,18 @@ } }, "SPARK2": { + "LIVY2_CLIENT": { + "STACK-SELECT-PACKAGE": "livy2-client", + "INSTALL": [ + "livy2-client" + ], + "PATCH": [ + "livy2-client" + ], + "STANDARD": [ + "livy2-client" + ] + }, "LIVY2_SERVER": { "STACK-SELECT-PACKAGE": "livy2-server", "INSTALL": [ @@ -819,12 +847,13 @@ }, "TEZ": { "TEZ_CLIENT": { - "STACK-SELECT-PACKAGE": "hadoop-client", + "STACK-SELECT-PACKAGE": "tez-client", + "LEGACY": "hadoop-client", "INSTALL": [ - "hadoop-client" + "tez-client" ], "PATCH": [ - "INVALID" + "tez-client" ], "STANDARD": [ "hadoop-client" @@ -895,12 +924,13 @@ ] }, "YARN_CLIENT": { - "STACK-SELECT-PACKAGE": "hadoop-client", + "STACK-SELECT-PACKAGE": "hadoop-yarn-client", + "LEGACY": "hadoop-client", "INSTALL": [ - "hadoop-client" + "hadoop-yarn-client" ], "PATCH": [ - "INVALID" + "hadoop-yarn-client" ], "STANDARD": [ "hadoop-client" http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/test/python/TestStackSelect.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestStackSelect.py b/ambari-server/src/test/python/TestStackSelect.py index 3d4e5b6..8cd8f2f 100644 --- a/ambari-server/src/test/python/TestStackSelect.py +++ b/ambari-server/src/test/python/TestStackSelect.py @@ -19,6 +19,7 @@ limitations under the License. ''' from mock.mock import patch +from mock.mock import MagicMock from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail @@ -43,13 +44,15 @@ class TestStackSelect(TestCase): self.assertRaises(Fail, stack_select.select_packages, version) - + @patch.object(stack_select, "get_supported_packages") @patch("resource_management.libraries.functions.stack_select.select") - def test_select_package_for_standard_orchestration(self, stack_select_select_mock): + def test_select_package_for_standard_orchestration(self, stack_select_select_mock, get_supported_packages_mock): """ Tests that missing the service & role throws an excpetion :return: """ + get_supported_packages_mock.return_value = TestStackSelect._get_supported_packages() + version = "2.5.9.9-9999" command_json = TestStackSelect._get_cluster_simple_upgrade_json() @@ -66,13 +69,15 @@ class TestStackSelect(TestCase): self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-master", version)) self.assertEqual(stack_select_select_mock.call_args_list[1][0], ("foo-client", version)) - + @patch.object(stack_select, "get_supported_packages") @patch("resource_management.libraries.functions.stack_select.select") - def test_select_package_for_patch_orchestration(self, stack_select_select_mock): + def test_select_package_for_patch_orchestration(self, stack_select_select_mock, get_supported_packages_mock): """ Tests that missing the service & role throws an excpetion :return: """ + get_supported_packages_mock.return_value = TestStackSelect._get_supported_packages() + version = "2.5.9.9-9999" command_json = TestStackSelect._get_cluster_simple_upgrade_json() @@ -98,6 +103,31 @@ class TestStackSelect(TestCase): self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-master", version)) + @patch.object(stack_select, "get_supported_packages") + @patch("resource_management.libraries.functions.stack_select.select") + def test_legacy_package_fallback(self, stack_select_select_mock, get_supported_packages_mock): + """ + Tests that if the package specified by the JSON isn't support by the stack-select tool, + the the fallback legacy value is used. + :return: + """ + get_supported_packages_mock.return_value = ["foo-legacy"] + + version = "2.5.9.9-9999" + + command_json = TestStackSelect._get_cluster_simple_upgrade_json() + + Script.config = dict() + Script.config.update(command_json) + Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } ) + Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages_with_legacy() + Script.config["hostLevelParams"] = { "stack_name" : "HDP" } + + stack_select.select_packages(version) + + self.assertEqual(len(stack_select_select_mock.call_args_list), 1) + self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-legacy", version)) + @staticmethod def _get_incomplete_cluster_simple_upgrade_json(): """ @@ -197,3 +227,32 @@ class TestStackSelect(TestCase): } } } ) + + @staticmethod + def _get_stack_packages_with_legacy(): + import json + return json.dumps( { + "HDP": { + "stack-select": { + "FOO_SERVICE": { + "FOO_MASTER": { + "LEGACY":"foo-legacy", + "STACK-SELECT-PACKAGE": "foo-master", + "INSTALL": [ + "foo-master" + ], + "PATCH": [ + "foo-master" + ], + "STANDARD": [ + "foo-master" + ] + } + } + } + } + } ) + + @staticmethod + def _get_supported_packages(): + return ["foo-master", "foo-client"] http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index 0341092..c10ff64 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -139,19 +139,21 @@ class RMFTestCase(TestCase): if 'status_params' in sys.modules: del(sys.modules["status_params"]) - with Environment(basedir, test_mode=True) as RMFTestCase.env: - with patch('resource_management.core.shell.checked_call', side_effect=checked_call_mocks) as mocks_dict['checked_call']: - with patch('resource_management.core.shell.call', side_effect=call_mocks) as mocks_dict['call']: - with patch.object(Script, 'get_config', return_value=self.config_dict) as mocks_dict['get_config']: # mocking configurations - with patch.object(Script, 'get_tmp_dir', return_value="/tmp") as mocks_dict['get_tmp_dir']: - with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local) as mocks_dict['get_kinit_path']: - with patch.object(platform, 'linux_distribution', return_value=os_type) as mocks_dict['linux_distribution']: - with patch.object(os, "environ", new=os_env) as mocks_dict['environ']: - if not try_install: - with patch.object(Script, 'install_packages') as install_mock_value: - method(RMFTestCase.env, *command_args) - else: - method(RMFTestCase.env, *command_args) + with Environment(basedir, test_mode=True) as RMFTestCase.env,\ + patch('resource_management.core.shell.checked_call', side_effect=checked_call_mocks) as mocks_dict['checked_call'],\ + patch('resource_management.core.shell.call', side_effect=call_mocks) as mocks_dict['call'],\ + patch.object(Script, 'get_config', return_value=self.config_dict) as mocks_dict['get_config'],\ + patch.object(Script, 'get_tmp_dir', return_value="/tmp") as mocks_dict['get_tmp_dir'],\ + patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local) as mocks_dict['get_kinit_path'],\ + patch.object(platform, 'linux_distribution', return_value=os_type) as mocks_dict['linux_distribution'],\ + patch('resource_management.libraries.functions.stack_select.is_package_supported', return_value=True),\ + patch('resource_management.libraries.functions.stack_select.get_supported_packages', return_value=MagicMock()),\ + patch.object(os, "environ", new=os_env) as mocks_dict['environ']: + if not try_install: + with patch.object(Script, 'install_packages') as install_mock_value: + method(RMFTestCase.env, *command_args) + else: + method(RMFTestCase.env, *command_args) sys.path.remove(scriptsdir)