Repository: ambari Updated Branches: refs/heads/branch-2.6 44ab10756 -> 22fe5ffae
AMBARI-21842 - PATCH and MAINT Orchestrations Should Only Invoke stack-select On Specific Packages (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/22fe5ffa Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/22fe5ffa Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/22fe5ffa Branch: refs/heads/branch-2.6 Commit: 22fe5ffaec5b4450262e83565031cdbbbf8b9a61 Parents: 44ab107 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Tue Aug 29 10:36:41 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Tue Aug 29 12:51:54 2017 -0400 ---------------------------------------------------------------------- .../libraries/functions/stack_select.py | 23 ++- .../services/HDFS/package/scripts/datanode.py | 3 +- .../HDFS/package/scripts/hdfs_client.py | 3 +- .../HDFS/package/scripts/journalnode.py | 3 +- .../services/HDFS/package/scripts/namenode.py | 8 +- .../services/HDFS/package/scripts/nfsgateway.py | 3 +- .../services/HDFS/package/scripts/snamenode.py | 3 +- .../OOZIE/package/scripts/oozie_client.py | 3 +- .../OOZIE/package/scripts/oozie_server.py | 15 +- .../src/test/python/TestStackSelect.py | 199 +++++++++++++++++++ 10 files changed, 230 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py index a83fd63..eac1bef 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py @@ -38,6 +38,7 @@ from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions import upgrade_summary STACK_SELECT_PREFIX = 'ambari-python-wrap' @@ -76,6 +77,8 @@ PACKAGE_SCOPE_PATCH = "PATCH" PACKAGE_SCOPE_STACK_SELECT = "STACK-SELECT-PACKAGE" _PACKAGE_SCOPES = (PACKAGE_SCOPE_INSTALL, PACKAGE_SCOPE_STANDARD, PACKAGE_SCOPE_PATCH, PACKAGE_SCOPE_STACK_SELECT) +# the orchestration types which equal to a partial (non-STANDARD) upgrade +_PARTIAL_ORCHESTRATION_SCOPES = ("PATCH", "MAINT") def get_package_name(default_package = None): """ @@ -197,13 +200,31 @@ def select_all(version_to_select): def select_packages(version): """ Uses the command's service and role to determine the stack-select packages which need to be invoked. + If in an upgrade, then the upgrade summary's orchestration is used to determine which packages + to install. :param version: the version to select :return: None """ - stack_select_packages = get_packages(PACKAGE_SCOPE_STANDARD) + package_scope = PACKAGE_SCOPE_STANDARD + orchestration = package_scope + summary = upgrade_summary.get_upgrade_summary() + + if summary is not None: + orchestration = summary.orchestration + if orchestration is None: + raise Fail("The upgrade summary for does not contain an orchestration type") + + # if the orchestration is patch or maint, use the "patch" key from the package JSON + if orchestration.upper() in _PARTIAL_ORCHESTRATION_SCOPES: + package_scope = PACKAGE_SCOPE_PATCH + + stack_select_packages = get_packages(package_scope) if stack_select_packages is None: return + Logger.info("The following packages will be stack-selected to version {0} using a {1} orchestration and {2} scope: {3}".format( + version, orchestration.upper(), package_scope, ", ".join(stack_select_packages))) + for stack_select_package_name in stack_select_packages: select(stack_select_package_name, version) http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/datanode.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/datanode.py index 79c32e4..370c8f1 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/datanode.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/datanode.py @@ -87,8 +87,7 @@ class DataNodeDefault(DataNode): import params env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "hadoop", params.version) - stack_select.select("hadoop-hdfs-datanode", params.version) + stack_select.select_packages(params.version) def post_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing DataNode Stack Upgrade post-restart") http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs_client.py index 95d1603..2febd33 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs_client.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs_client.py @@ -64,8 +64,7 @@ class HdfsClientDefault(HdfsClient): import params env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "hadoop", params.version) - stack_select.select("hadoop-client", params.version) + stack_select.select_packages(params.version) def security_status(self, env): import status_params http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/journalnode.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/journalnode.py index 9c5a124..969cd79 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/journalnode.py @@ -51,8 +51,7 @@ class JournalNodeDefault(JournalNode): env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "hadoop", params.version) - stack_select.select("hadoop-hdfs-journalnode", params.version) + stack_select.select_packages(params.version) def start(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode.py index 17d8107..d2e1f0c 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode.py @@ -196,13 +196,7 @@ class NameNodeDefault(NameNode): env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - # When downgrading an Express Upgrade, the first thing we do is to revert the symlinks. - # Therefore, we cannot call this code in that scenario. - call_if = [("rolling", "upgrade"), ("rolling", "downgrade"), ("nonrolling", "upgrade")] - for e in call_if: - if (upgrade_type, params.upgrade_direction) == e: - conf_select.select(params.stack_name, "hadoop", params.version) - stack_select.select("hadoop-hdfs-namenode", params.version) + stack_select.select_packages(params.version) def post_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Stack Upgrade post-restart") http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py index 770df59..782be20 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py @@ -47,8 +47,7 @@ class NFSGateway(Script): env.set_params(params) if params.stack_version_formatted and check_stack_feature(StackFeature.NFS, params.stack_version_formatted): - conf_select.select(params.stack_name, "hadoop", params.version) - stack_select.select("hadoop-hdfs-nfs3", params.version) + stack_select.select_packages(params.version) def start(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py index 30eee07..0cf32f1 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py @@ -73,8 +73,7 @@ class SNameNodeDefault(SNameNode): env.set_params(params) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "hadoop", params.version) - stack_select.select("hadoop-hdfs-secondarynamenode", params.version) + stack_select.select_packages(params.version) def security_status(self, env): import status_params http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py index 9bbca9f..1e89369 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_client.py @@ -58,8 +58,7 @@ class OozieClient(Script): return Logger.info("Executing Oozie Client Stack Upgrade pre-restart") - conf_select.select(params.stack_name, "oozie", params.version) - stack_select.select("oozie-client", params.version) + stack_select.select_packages(params.version) # We substitute some configs (oozie.authentication.kerberos.principal) before generation (see oozie.py and params.py). # This function returns changed configs (it's used for config generation before config download) http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py index b3a8643..da0c5e6 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie_server.py @@ -62,17 +62,7 @@ class OozieServer(Script): if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None: Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}")) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - # In order for the "<stack-root>/current/oozie-<client/server>" point to the new version of - # oozie, we need to create the symlinks both for server and client. - # This is required as both need to be pointing to new installed oozie version. - - # Sets the symlink : eg: <stack-root>/current/oozie-client -> <stack-root>/a.b.c.d-<version>/oozie - stack_select.select("oozie-client", params.version) - # Sets the symlink : eg: <stack-root>/current/oozie-server -> <stack-root>/a.b.c.d-<version>/oozie - stack_select.select("oozie-server", params.version) - - if params.version and check_stack_feature(StackFeature.CONFIG_VERSIONING, params.version): - conf_select.select(params.stack_name, "oozie", params.version) + stack_select.select_packages(params.version) env.set_params(params) oozie(is_server=True, upgrade_type=upgrade_type) @@ -126,8 +116,7 @@ class OozieServerDefault(OozieServer): Logger.info("Executing Oozie Server Stack Upgrade pre-restart") if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - conf_select.select(params.stack_name, "oozie", params.version) - stack_select.select("oozie-server", params.version) + stack_select.select_packages(params.version) OozieUpgrade.prepare_libext_directory(upgrade_type=upgrade_type) http://git-wip-us.apache.org/repos/asf/ambari/blob/22fe5ffa/ambari-server/src/test/python/TestStackSelect.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestStackSelect.py b/ambari-server/src/test/python/TestStackSelect.py new file mode 100644 index 0000000..3d4e5b6 --- /dev/null +++ b/ambari-server/src/test/python/TestStackSelect.py @@ -0,0 +1,199 @@ +# !/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +from mock.mock import patch + +from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail +from resource_management.libraries.functions import stack_select +from resource_management.libraries.script import Script + +from unittest import TestCase + +Logger.initialize_logger() + +class TestStackSelect(TestCase): + + def test_missing_role_information_throws_exception(self): + """ + Tests that missing the service & role throws an excpetion + :return: + """ + version = "2.5.9.9-9999" + + command_json = TestStackSelect._get_incomplete_cluster_simple_upgrade_json() + Script.config = command_json + + self.assertRaises(Fail, stack_select.select_packages, version) + + + @patch("resource_management.libraries.functions.stack_select.select") + def test_select_package_for_standard_orchestration(self, stack_select_select_mock): + """ + Tests that missing the service & role throws an excpetion + :return: + """ + version = "2.5.9.9-9999" + + command_json = TestStackSelect._get_cluster_simple_upgrade_json() + + Script.config = dict() + Script.config.update(command_json) + Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } ) + Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages() + Script.config["hostLevelParams"] = { "stack_name" : "HDP" } + + stack_select.select_packages(version) + + self.assertEqual(len(stack_select_select_mock.call_args_list), 2) + self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-master", version)) + self.assertEqual(stack_select_select_mock.call_args_list[1][0], ("foo-client", version)) + + + @patch("resource_management.libraries.functions.stack_select.select") + def test_select_package_for_patch_orchestration(self, stack_select_select_mock): + """ + Tests that missing the service & role throws an excpetion + :return: + """ + version = "2.5.9.9-9999" + + command_json = TestStackSelect._get_cluster_simple_upgrade_json() + command_json["upgradeSummary"]["orchestration"] = "PATCH" + + Script.config = dict() + Script.config.update(command_json) + Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } ) + Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages() + Script.config["hostLevelParams"] = { "stack_name" : "HDP" } + + stack_select.select_packages(version) + + self.assertEqual(len(stack_select_select_mock.call_args_list), 1) + self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-master", version)) + + stack_select_select_mock.reset_mock() + + command_json["upgradeSummary"]["orchestration"] = "MAINT" + stack_select.select_packages(version) + + self.assertEqual(len(stack_select_select_mock.call_args_list), 1) + self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-master", version)) + + + @staticmethod + def _get_incomplete_cluster_simple_upgrade_json(): + """ + A command missing the role and service name during an upgrade. + :return: + """ + return { + "roleCommand":"ACTIONEXECUTE", + "hostLevelParams": { + "stack_name": "HDP", + "stack_version": "2.4", + }, + "commandParams": { + "source_stack": "2.4", + "target_stack": "2.5", + "upgrade_direction": "upgrade", + "version": "2.5.9.9-9999" + }, + "upgradeSummary": { + "services":{ + "HDFS":{ + "sourceRepositoryId":1, + "sourceStackId":"HDP-2.4", + "sourceVersion":"2.4.0.0-1234", + "targetRepositoryId":2, + "targetStackId":"HDP-2.5", + "targetVersion":"2.5.9.9-9999" + } + }, + "direction":"UPGRADE", + "type":"rolling_upgrade", + "isRevert":False, + "orchestration":"STANDARD" + } + } + + @staticmethod + def _get_cluster_simple_upgrade_json(): + """ + A restart command during an upgrade. + :return: + """ + return { + "roleCommand":"ACTIONEXECUTE", + "serviceName": "FOO_SERVICE", + "role": "FOO_MASTER", + "hostLevelParams": { + "stack_name": "HDP", + "stack_version": "2.4", + }, + "commandParams": { + "source_stack": "2.4", + "target_stack": "2.5", + "upgrade_direction": "upgrade", + "version": "2.5.9.9-9999" + }, + "upgradeSummary": { + "services":{ + "HDFS":{ + "sourceRepositoryId":1, + "sourceStackId":"HDP-2.4", + "sourceVersion":"2.4.0.0-1234", + "targetRepositoryId":2, + "targetStackId":"HDP-2.5", + "targetVersion":"2.5.9.9-9999" + } + }, + "direction":"UPGRADE", + "type":"rolling_upgrade", + "isRevert":False, + "orchestration":"STANDARD" + } + } + + @staticmethod + def _get_stack_packages(): + import json + return json.dumps( { + "HDP": { + "stack-select": { + "FOO_SERVICE": { + "FOO_MASTER": { + "STACK-SELECT-PACKAGE": "foo-master", + "INSTALL": [ + "foo-master", + "foo-client" + ], + "PATCH": [ + "foo-master" + ], + "STANDARD": [ + "foo-master", + "foo-client" + ] + } + } + } + } + } )
