Repository: ambari Updated Branches: refs/heads/branch-2.2 16cda7576 -> 8eb62315f
AMBARI-15187. Introduce possibility to retry stack installation in case of network instability (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8eb62315 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8eb62315 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8eb62315 Branch: refs/heads/branch-2.2 Commit: 8eb62315f53e07ac922da07b8430e0fc1d0ea975 Parents: 16cda75 Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Mon Feb 29 20:20:02 2016 +0200 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Mon Feb 29 20:20:02 2016 +0200 ---------------------------------------------------------------------- .../resource_management/TestPackageResource.py | 6 + .../python/resource_management/TestScript.py | 12 +- .../core/providers/package/__init__.py | 69 +++++-- .../core/providers/package/apt.py | 11 +- .../core/providers/package/yumrpm.py | 6 +- .../core/providers/package/zypper.py | 11 +- .../core/resources/packaging.py | 10 +- .../libraries/script/script.py | 8 +- ambari-server/conf/unix/ambari.properties | 5 + .../ambari/server/agent/ExecutionCommand.java | 2 + .../ambari/server/agent/HeartbeatMonitor.java | 2 - .../server/configuration/Configuration.java | 16 ++ .../controller/AmbariActionExecutionHelper.java | 8 +- .../AmbariCustomCommandExecutionHelper.java | 4 + .../HBASE/0.96.0.2.0/package/scripts/hbase.py | 4 +- .../0.96.0.2.0/package/scripts/params_linux.py | 3 + .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 4 +- .../2.1.0.2.0/package/scripts/params_linux.py | 2 + .../0.12.0.2.0/package/scripts/params_linux.py | 2 + .../package/scripts/setup_atlas_hive.py | 2 +- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 4 +- .../4.0.0.2.0/package/scripts/params_linux.py | 2 + .../custom_actions/scripts/install_packages.py | 12 +- .../0.8/hooks/before-INSTALL/scripts/params.py | 3 + .../scripts/shared_initialization.py | 5 +- .../hooks/before-INSTALL/scripts/params.py | 3 + .../scripts/shared_initialization.py | 4 +- .../AmbariManagementControllerImplTest.java | 2 +- ...ClusterStackVersionResourceProviderTest.java | 3 + .../custom_actions/TestInstallPackages.py | 196 +++++++++++++++---- .../configs/install_packages_config.json | 2 + .../stacks/2.0.6/HBASE/test_hbase_master.py | 12 +- .../2.0.6/HBASE/test_hbase_regionserver.py | 2 +- .../python/stacks/2.0.6/configs/default.json | 5 +- .../2.0.6/configs/hbase-rs-2.2-phoenix.json | 2 + .../stacks/2.0.6/configs/hbase_no_phx.json | 2 + .../stacks/2.0.6/configs/hbase_with_phx.json | 2 + .../hooks/before-INSTALL/test_before_install.py | 4 +- .../src/test/python/stacks/2.3/PXF/test_pxf.py | 20 +- .../python/stacks/2.3/configs/pxf_default.json | 2 + 40 files changed, 373 insertions(+), 101 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-agent/src/test/python/resource_management/TestPackageResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestPackageResource.py b/ambari-agent/src/test/python/resource_management/TestPackageResource.py index 1d8ef01..1f2250d 100644 --- a/ambari-agent/src/test/python/resource_management/TestPackageResource.py +++ b/ambari-agent/src/test/python/resource_management/TestPackageResource.py @@ -93,6 +93,7 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'redhat') def test_action_install_rhel(self, shell_mock): + shell_mock.return_value = (0,'') sys.modules['rpm'] = MagicMock() sys.modules['rpm'].TransactionSet.return_value = MagicMock() sys.modules['rpm'].TransactionSet.return_value.dbMatch.return_value = [{'name':'some_packag'}] @@ -106,6 +107,7 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'redhat') def test_action_install_pattern_rhel(self, shell_mock): + shell_mock.return_value = (0,'') sys.modules['rpm'] = MagicMock() sys.modules['rpm'].TransactionSet.return_value = MagicMock() sys.modules['rpm'].TransactionSet.return_value.dbMatch.return_value = [{'name':'some_packag'}] @@ -118,12 +120,14 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'redhat') def test_action_install_pattern_installed_rhel(self, shell_mock): + shell_mock.return_value = (0,'') sys.modules['yum'] = MagicMock() sys.modules['yum'].YumBase.return_value = MagicMock() sys.modules['yum'].YumBase.return_value.rpmdb = MagicMock() sys.modules['yum'].YumBase.return_value.rpmdb.simplePkgList.return_value = [('some_package_1_2_3',)] with Environment('/') as env: Package("some_package*", + logoutput = False ) self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called") @@ -179,6 +183,7 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'redhat') def test_action_install_use_repos_rhel(self, shell_mock): + shell_mock.return_value = (0,'') with Environment('/') as env: Package("some_package", use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], logoutput = False @@ -233,6 +238,7 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'redhat') def test_action_install_version_attr(self, shell_mock): + shell_mock.return_value = (0,'') with Environment('/') as env: Package("some_package", version = "3.5.0", http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-agent/src/test/python/resource_management/TestScript.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestScript.py b/ambari-agent/src/test/python/resource_management/TestScript.py index fd31910..30e60d1 100644 --- a/ambari-agent/src/test/python/resource_management/TestScript.py +++ b/ambari-agent/src/test/python/resource_management/TestScript.py @@ -48,13 +48,17 @@ class TestScript(TestCase): def test_install_packages(self, package_provider_mock): no_packages_config = { 'hostLevelParams' : { - 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]" + 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", + 'agent_stack_retry_count': '5', + 'agent_stack_retry_on_unavailability': 'false' } } empty_config = { 'hostLevelParams' : { 'package_list' : '', - 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]" + 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", + 'agent_stack_retry_count': '5', + 'agent_stack_retry_on_unavailability': 'false' } } dummy_config = { @@ -62,7 +66,9 @@ class TestScript(TestCase): 'package_list' : "[{\"type\":\"rpm\",\"name\":\"hbase\"}," "{\"type\":\"rpm\",\"name\":\"yet-another-package\"}]", 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", - 'service_repo_info' : "[{\"mirrorsList\":\"abc\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]" + 'service_repo_info' : "[{\"mirrorsList\":\"abc\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]", + 'agent_stack_retry_count': '5', + 'agent_stack_retry_on_unavailability': 'false' } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py index 1fc4214..711c225 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py @@ -31,7 +31,9 @@ from resource_management.core.logger import Logger from resource_management.core.utils import suppress_stdout from resource_management.core import shell -PACKAGE_MANAGER_LOCK_ACQUIRED = "Package manager lock is acquired. Retrying after {0} seconds. Reason: {1}" +PACKAGE_MANAGER_LOCK_ACQUIRED_MSG = "Cannot obtain lock for Package manager. Retrying after {0} seconds. Reason: {1}" +PACKAGE_MANAGER_REPO_ERROR_MSG = "Cannot download the package due to repository unavailability. Retrying after {0} seconds. Reason: {1}" + class PackageProvider(Provider): def __init__(self, *args, **kwargs): @@ -39,8 +41,10 @@ class PackageProvider(Provider): def install_package(self, name, version): raise NotImplementedError() + def remove_package(self, name): raise NotImplementedError() + def upgrade_package(self, name, version): raise NotImplementedError() @@ -61,41 +65,64 @@ class PackageProvider(Provider): return self.resource.package_name + '-' + self.resource.version else: return self.resource.package_name + + def is_locked_output(self, out): + return False + + def is_repo_error_output(self, out): + return False def get_logoutput(self): return self.resource.logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or self.resource.logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG) - def call_until_not_locked(self, cmd, **kwargs): - return self.wait_until_not_locked(cmd, is_checked=False, **kwargs) + def call_with_retries(self, cmd, **kwargs): + return self._call_with_retries(cmd, is_checked=False, **kwargs) - def checked_call_until_not_locked(self, cmd, **kwargs): - return self.wait_until_not_locked(cmd, is_checked=True, **kwargs) - - def wait_until_not_locked(self, cmd, is_checked=True, **kwargs): + def checked_call_with_retries(self, cmd, **kwargs): + return self._call_with_retries(cmd, is_checked=True, **kwargs) + + def _call_with_retries(self, cmd, is_checked=True, **kwargs): func = shell.checked_call if is_checked else shell.call - - for i in range(self.resource.locked_tries): - is_last_time = (i == self.resource.locked_tries - 1) + retry_count = self.resource.retry_count + + for i in range(retry_count): + is_last_time = (i == retry_count - 1) try: code, out = func(cmd, **kwargs) except Fail as ex: # non-lock error - if not self.is_locked_output(str(ex)) or is_last_time: + if not self._is_handled_error(str(ex), is_last_time) or is_last_time: raise - - Logger.info(PACKAGE_MANAGER_LOCK_ACQUIRED.format(self.resource.locked_try_sleep, str(ex))) + + self._notify_about_handled_error(str(ex), is_last_time) else: # didn't fail or failed with non-lock error. - if not code or not self.is_locked_output(out): - break - - Logger.info(PACKAGE_MANAGER_LOCK_ACQUIRED.format(self.resource.locked_try_sleep, str(out))) - - time.sleep(self.resource.locked_try_sleep) + if not code or not self._is_handled_error(out, is_last_time): + break + + self._notify_about_handled_error(str(out), is_last_time) + + time.sleep(self.resource.retry_sleep) return code, out - - + + def _is_handled_error(self, output, is_last_time): + if self.resource.retry_on_locked and self.is_locked_output(output): + return True + elif self.resource.retry_on_repo_unavailability and self.is_repo_error_output(output): + return True + + return False + + def _notify_about_handled_error(self, output, is_last_time): + if is_last_time: + return + + if self.resource.retry_on_locked and self.is_locked_output(output): + Logger.info(PACKAGE_MANAGER_LOCK_ACQUIRED_MSG.format(self.resource.retry_sleep, str(output))) + elif self.resource.retry_on_repo_unavailability and self.is_repo_error_output(output): + Logger.info(PACKAGE_MANAGER_REPO_ERROR_MSG.format(self.resource.retry_sleep, str(output))) + def yum_check_package_available(self, name): """ Does the same as rpm_check_package_avaiable, but faster. http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-common/src/main/python/resource_management/core/providers/package/apt.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py index 70a553d..df70fb1 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py @@ -78,7 +78,7 @@ class AptProvider(PackageProvider): cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - code, out = self.call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) + code, out = self.call_with_retries(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if self.is_locked_output(out): err_msg = Logger.filter_text("Execution of '%s' returned %d. %s" % (cmd, code, out)) @@ -88,13 +88,13 @@ class AptProvider(PackageProvider): if code: Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out)) Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD))) - code, out = self.call_until_not_locked(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput()) + code, out = self.call_with_retries(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput()) if code: Logger.info("Execution of '%s' returned %d. %s" % (REPO_UPDATE_CMD, code, out)) Logger.info("Retrying to install package %s" % (name)) - self.checked_call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) + self.checked_call_with_retries(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if is_tmp_dir_created: for temporal_sources_file in copied_sources_files: @@ -108,6 +108,9 @@ class AptProvider(PackageProvider): def is_locked_output(self, out): return "Unable to lock the administration directory" in out + def is_repo_error_output(self, out): + return "Failure when receiving data from the peer" in out + @replace_underscores def upgrade_package(self, name, use_repos=[], skip_repos=[]): return self.install_package(name, use_repos, skip_repos) @@ -117,7 +120,7 @@ class AptProvider(PackageProvider): if self._check_existence(name): cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - self.checked_call_until_not_locked(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index 87938ad..41f980a 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -46,7 +46,7 @@ class YumProvider(PackageProvider): cmd = cmd + [disable_repo_option, enable_repo_option] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installation of existing package %s" % (name)) @@ -61,6 +61,10 @@ class YumProvider(PackageProvider): else: Logger.info("Skipping removal of non-existing package %s" % (name)) + def is_repo_error_output(self, out): + return "Failure when receiving data from the peer" in out or \ + "No more mirrors to try" in out + def _check_existence(self, name): """ For regexp names: http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py index 1fbc9bf..c3447e6 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py @@ -54,7 +54,7 @@ class ZypperProvider(PackageProvider): cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - self.checked_call_until_not_locked(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installation of existing package %s" % (name)) @@ -65,12 +65,12 @@ class ZypperProvider(PackageProvider): if self._check_existence(name): cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - self.checked_call_until_not_locked(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name)) def get_active_base_repos(self): - (code, output) = self.call_until_not_locked(LIST_ACTIVE_REPOS_CMD) + (code, output) = self.call_with_retries(LIST_ACTIVE_REPOS_CMD) enabled_repos = [] if not code: for line in output.split('\n')[2:]: @@ -81,9 +81,12 @@ class ZypperProvider(PackageProvider): return [line_list[1].strip()] return enabled_repos - def is_locked_output(self ,out): + def is_locked_output(self, out): return "System management is locked by the application" in out + def is_repo_error_output(self, out): + return "Failure when receiving data from the peer" in out + def _check_existence(self, name): """ For regexp names: http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-common/src/main/python/resource_management/core/resources/packaging.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/resources/packaging.py b/ambari-common/src/main/python/resource_management/core/resources/packaging.py index bb0aa56..07a137b 100644 --- a/ambari-common/src/main/python/resource_management/core/resources/packaging.py +++ b/ambari-common/src/main/python/resource_management/core/resources/packaging.py @@ -41,11 +41,13 @@ class Package(Resource): logoutput = ResourceArgument(default=None) """ - Retry if package manager is locked. (usually another process is running). - Note that this works only for apt-get and zypper, while yum manages lock retries itself. + Retry if package manager is locked or unavailable. + Note that retry_on_lock works only for apt-get and zypper, while yum manages lock retries itself. """ - locked_tries = ResourceArgument(default=8) - locked_try_sleep = ResourceArgument(default=30) # seconds + retry_count = ResourceArgument(default=4) + retry_sleep = ResourceArgument(default=30) + retry_on_repo_unavailability = ResourceArgument(default=False) + retry_on_locked = ResourceArgument(default=True) version = ResourceArgument() actions = ["install", "upgrade", "remove"] http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index e034f62..bd0e213 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -381,6 +381,7 @@ class Script(object): NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ? """ config = self.get_config() + if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host if config['hostLevelParams']['host_sys_prepped'] == True: @@ -389,6 +390,9 @@ class Script(object): pass try: package_list_str = config['hostLevelParams']['package_list'] + agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) + agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) + if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: @@ -401,7 +405,9 @@ class Script(object): if "ambari-metrics" in name: Package(name) else: - Package(name) + Package(name, + retry_on_repo_unavailability=agent_stack_retry_on_unavailability, + retry_count=agent_stack_retry_count) except KeyError: pass # No reason to worry http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index 5022712..bc8e463 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -77,6 +77,11 @@ agent.task.timeout=900 # Default timeout in seconds before package installation task is killed agent.package.install.task.timeout=1800 +# Enables package installation retry on repository unavailability error +agent.stack.retry.on_repo_unavailability=false +# Default count of tries +agent.stack.retry.tries=5 + # Default timeout in seconds before a server-side task is killed server.task.timeout=1200 http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 46151bd..2ae4a80 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -348,6 +348,8 @@ public class ExecutionCommand extends AgentCommand { String HOST_SYS_PREPPED = "host_sys_prepped"; String MAX_DURATION_OF_RETRIES = "max_duration_for_retries"; String COMMAND_RETRY_ENABLED = "command_retry_enabled"; + String AGENT_STACK_RETRY_ON_UNAVAILABILITY = "agent_stack_retry_on_unavailability"; + String AGENT_STACK_RETRY_COUNT = "agent_stack_retry_count"; /** * Comma separated list of config-types whose tags have be refreshed * at runtime before being executed. If all config-type tags have to be http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java index fa7a500..7915400 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java @@ -33,7 +33,6 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -import com.google.inject.Inject; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.actionmanager.ActionManager; @@ -342,7 +341,6 @@ public class HeartbeatMonitor implements Runnable { hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); - if (statusCmd.getPayloadLevel() == StatusCommand.StatusCommandPayload.EXECUTION_COMMAND) { ExecutionCommand ec = ambariManagementController.getExecutionCommand(cluster, sch, RoleCommand.START); statusCmd.setExecutionCommand(ec); http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 0d7b2f9..3fbcc6d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -552,6 +552,14 @@ public class Configuration { public static final String VIEWS_HTTP_X_XSS_PROTECTION_HEADER_VALUE_KEY = "views.http.x-xss-protection"; public static final String VIEWS_HTTP_X_XSS_PROTECTION_HEADER_VALUE_DEFAULT = "1; mode=block"; + /** + * For Agent Stack Install retry configuration + */ + public static final String AGENT_STACK_RETRY_ON_REPO_UNAVAILABILITY_KEY = "agent.stack.retry.on_repo_unavailability"; + public static final String AGENT_STACK_RETRY_ON_REPO_UNAVAILABILITY_DEFAULT = "false"; + public static final String AGENT_STACK_RETRY_COUNT_KEY = "agent.stack.retry.tries"; + public static final String AGENT_STACK_RETRY_COUNT_DEFAULT = "5"; + private static final Logger LOG = LoggerFactory.getLogger( Configuration.class); @@ -2535,4 +2543,12 @@ public class Configuration { } return attempts; } + + public String getAgentStackRetryOnInstallCount(){ + return properties.getProperty(AGENT_STACK_RETRY_COUNT_KEY, AGENT_STACK_RETRY_COUNT_DEFAULT); + } + + public String isAgentStackRetryOnInstallEnabled(){ + return properties.getProperty(AGENT_STACK_RETRY_ON_REPO_UNAVAILABILITY_KEY, AGENT_STACK_RETRY_ON_REPO_UNAVAILABILITY_DEFAULT); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 5e1bf9d..ba3163b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.controller; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; @@ -58,7 +60,6 @@ import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent; import org.apache.ambari.server.utils.SecretReference; -import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -414,7 +415,10 @@ public class AmbariActionExecutionHelper { execCmd.setComponentName(componentName == null || componentName.isEmpty() ? resourceFilter.getComponentName() : componentName); - addRepoInfoToHostLevelParams(cluster, execCmd.getHostLevelParams(), hostName); + Map<String, String> hostLevelParams = execCmd.getHostLevelParams(); + hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); + hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); + addRepoInfoToHostLevelParams(cluster, hostLevelParams, hostName); Map<String, String> roleParams = execCmd.getRoleParams(); if (roleParams == null) { http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 2335f74..d85f17e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.controller; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY; @@ -1121,6 +1123,8 @@ public class AmbariCustomCommandExecutionHelper { hostLevelParams.put(DB_DRIVER_FILENAME, configs.getMySQLJarName()); hostLevelParams.putAll(managementController.getRcaParameters()); hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); + hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); + hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); ClusterVersionEntity clusterVersionEntity = clusterVersionDAO.findByClusterAndStateCurrent(cluster.getClusterName()); if (clusterVersionEntity == null) { List<ClusterVersionEntity> clusterVersionEntityList = clusterVersionDAO http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py index d78c664..521dc93 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py @@ -192,7 +192,9 @@ def hbase(name=None): params.HdfsResource(None, action="execute") if params.phoenix_enabled: - Package(params.phoenix_package) + Package(params.phoenix_package, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) def hbase_TemplateConfig(name, tag=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 4052f83..b34c016 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -43,6 +43,9 @@ exec_tmp_dir = Script.get_tmp_dir() sudo = AMBARI_SUDO_BINARY stack_name = default("/hostLevelParams/stack_name", None) +agent_stack_retry_on_unavailability = config["hostLevelParams"]["agent_stack_retry_on_unavailability"] +agent_stack_retry_count = config["hostLevelParams"]["agent_stack_retry_count"] + version = default("/commandParams/version", None) component_directory = status_params.component_directory etc_prefix_dir = "/etc/hbase" http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py index 1a34a24..ad992c4 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py @@ -117,7 +117,9 @@ def hdfs(name=None): ) if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages) + Package(params.lzo_packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index bcfa4f3..c6a107d 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -47,6 +47,8 @@ stack_name = default("/hostLevelParams/stack_name", None) upgrade_direction = default("/commandParams/upgrade_direction", None) stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) +agent_stack_retry_on_unavailability = config["hostLevelParams"]["agent_stack_retry_on_unavailability"] +agent_stack_retry_count = config["hostLevelParams"]["agent_stack_retry_count"] # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade version = default("/commandParams/version", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index e25a9a0..bec5ebd 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -44,6 +44,8 @@ tmp_dir = Script.get_tmp_dir() sudo = AMBARI_SUDO_BINARY stack_name = default("/hostLevelParams/stack_name", None) +agent_stack_retry_on_unavailability = config["hostLevelParams"]["agent_stack_retry_on_unavailability"] +agent_stack_retry_count = config["hostLevelParams"]["agent_stack_retry_count"] # node hostname hostname = config["hostname"] http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py index c92e3db..1528f9c 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_atlas_hive.py @@ -32,7 +32,7 @@ def setup_atlas_hive(configuration_directory=None): if not params.host_sys_prepped: Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package, # FIXME HACK: install the package during RESTART/START when install_packages is not triggered. - ) + retry=params.agent_stack_reinstall, retry_count=params.agent_stack_reinstall_count) PropertiesFile(format('{configuration_directory}/client.properties'), properties = params.atlas_client_props, http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 1701bbd..c88ddde 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -299,7 +299,9 @@ def oozie_server_specific(): not_if = no_op_test) if params.lzo_enabled and len(params.all_lzo_packages) > 0: - Package(params.all_lzo_packages) + Package(params.all_lzo_packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), not_if = no_op_test, ) http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index b7ee0be..61816c2 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -46,6 +46,8 @@ hostname = config["hostname"] version = default("/commandParams/version", None) stack_name = default("/hostLevelParams/stack_name", None) upgrade_direction = default("/commandParams/upgrade_direction", None) +agent_stack_retry_on_unavailability = config["hostLevelParams"]["agent_stack_retry_on_unavailability"] +agent_stack_retry_count = config["hostLevelParams"]["agent_stack_retry_count"] stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 821df77..cc03338 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -359,10 +359,16 @@ class InstallPackages(Script): # Clear cache of package manager right before installation of the packages self._clear_package_manager_cache() + config = self.get_config() + agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) + agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) + # Install packages packages_were_checked = False try: - Package(self.get_base_packages_to_install()) + Package(self.get_base_packages_to_install(), + retry_on_repo_unavailability=agent_stack_retry_on_unavailability, + retry_count=agent_stack_retry_count) packages_installed_before = [] allInstalledPackages(packages_installed_before) @@ -373,7 +379,9 @@ class InstallPackages(Script): name = self.format_package_name(package['name'], self.repository_version) Package(name, use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories, - skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else []) + skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else [], + retry_on_repo_unavailability=agent_stack_retry_on_unavailability, + retry_count=agent_stack_retry_count) except Exception, err: ret_code = 1 Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err))) http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py index a687ea7..a6da746 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py @@ -28,6 +28,9 @@ tmp_dir = Script.get_tmp_dir() #RPM versioning support rpm_version = default("/configurations/cluster-env/rpm_version", None) +agent_stack_retry_on_unavailability = config["hostLevelParams"]["agent_stack_retry_on_unavailability"] +agent_stack_retry_count = config["hostLevelParams"]["agent_stack_retry_count"] + #users and groups hbase_user = config['configurations']['hbase-env']['hbase_user'] smoke_user = config['configurations']['cluster-env']['smokeuser'] http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py index 03afc44..54fb8a3 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py @@ -60,4 +60,7 @@ def install_packages(): packages = ['unzip', 'curl'] if params.rpm_version: packages.append('bigtop-select') - Package(packages) + Package(packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) + http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py index cf20df4..63b81fe 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py @@ -28,6 +28,9 @@ tmp_dir = Script.get_tmp_dir() sudo = AMBARI_SUDO_BINARY stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +agent_stack_retry_on_unavailability = config["hostLevelParams"]["agent_stack_retry_on_unavailability"] +agent_stack_retry_count = config["hostLevelParams"]["agent_stack_retry_count"] + hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) #users and groups http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py index f696543..274f29f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py @@ -29,4 +29,6 @@ def install_packages(): packages = ['unzip', 'curl'] if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0: packages.append('hdp-select') - Package(packages) + Package(packages, + retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, + retry_count=params.agent_stack_retry_count) http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java index 27f75f7..9c2a8bf 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java @@ -2017,7 +2017,7 @@ public class AmbariManagementControllerImplTest { Map<String, String> defaultHostParams = helper.createDefaultHostParams(cluster); - assertEquals(defaultHostParams.size(), 13); + assertEquals(defaultHostParams.size(), 15); assertEquals(defaultHostParams.get(DB_DRIVER_FILENAME), MYSQL_JAR); assertEquals(defaultHostParams.get(STACK_NAME), SOME_STACK_NAME); assertEquals(defaultHostParams.get(STACK_VERSION), SOME_STACK_VERSION); http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index 768c829..43d8b49 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -175,6 +175,7 @@ public class ClusterStackVersionResourceProviderTest { AmbariManagementController managementController = createMock(AmbariManagementController.class); Clusters clusters = createNiceMock(Clusters.class); Cluster cluster = createNiceMock(Cluster.class); + Map<String, String> hostLevelParams = new HashMap<>(); StackId stackId = new StackId("HDP", "2.0.1"); Map<String, Host> hostsForCluster = new HashMap<String, Host>(); @@ -245,6 +246,8 @@ public class ClusterStackVersionResourceProviderTest { expect(stage.getExecutionCommandWrapper(anyObject(String.class), anyObject(String.class))). andReturn(executionCommandWrapper).anyTimes(); + expect(executionCommand.getHostLevelParams()).andReturn(hostLevelParams).anyTimes(); + Map<Role, Float> successFactors = new HashMap<>(); expect(stage.getSuccessFactors()).andReturn(successFactors).atLeastOnce(); http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/custom_actions/TestInstallPackages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py index 423ae9e..a450167 100644 --- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py +++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py @@ -108,15 +108,43 @@ class TestInstallPackages(RMFTestCase): logoutput = False, sudo = True, ) - self.assertResourceCalled('Package', 'fuse') - self.assertResourceCalled('Package', 'fuse-libs') - self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'snappy', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'snappy-devel', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'lzo', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) + self.assertResourceCalled('Package', 'fuse', retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'fuse-libs', retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_*', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy-devel', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'lzo', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_*', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'ambari-log4j', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) self.assertNoMoreResources() @patch("ambari_commons.os_check.OSCheck.is_suse_family") @@ -172,15 +200,47 @@ class TestInstallPackages(RMFTestCase): logoutput = False, sudo = True, ) - self.assertResourceCalled('Package', 'fuse') - self.assertResourceCalled('Package', 'libfuse2') - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) + self.assertResourceCalled('Package', 'fuse', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'libfuse2', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy-devel', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'lzo', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885*', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*-libhdfs', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'ambari-log4j', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) self.assertNoMoreResources() @@ -239,15 +299,47 @@ class TestInstallPackages(RMFTestCase): logoutput = False, sudo = True, ) - self.assertResourceCalled('Package', 'fuse') - self.assertResourceCalled('Package', 'fuse-libs') - self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'snappy', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'snappy-devel', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'lzo', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) - self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=['HDP-*']) + self.assertResourceCalled('Package', 'fuse', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'fuse-libs', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_*', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy-devel', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'lzo', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_*', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'ambari-log4j', + use_repos=['HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=['HDP-*'], + retry_count=5, + retry_on_repo_unavailability=False) self.assertNoMoreResources() @@ -381,15 +473,47 @@ class TestInstallPackages(RMFTestCase): logoutput = False, sudo = True, ) - self.assertResourceCalled('Package', 'fuse') - self.assertResourceCalled('Package', 'libfuse2') - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) - self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], skip_repos=[]) + self.assertResourceCalled('Package', 'fuse', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'libfuse2', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy-devel', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'lzo', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885*', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*-libhdfs', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'ambari-log4j', + use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'], + skip_repos=[], + retry_count=5, + retry_on_repo_unavailability=False) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json index 50e327a..d4466c2 100644 --- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json +++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json @@ -5,6 +5,8 @@ "hostname": "0b3.vm", "passiveInfo": [], "hostLevelParams": { + "agent_stack_retry_count": "5", + "agent_stack_retry_on_unavailability": "false", "jdk_location": "http://0b3.vm:8080/resources/", "ambari_db_rca_password": "mapred", "java_home": "/usr/jdk64/jdk1.7.0_67", http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py index 8a6f2eb..8de7f96 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py @@ -37,7 +37,9 @@ class TestHBaseMaster(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES, try_install=True ) - self.assertResourceCalled('Package', 'hbase_2_3_*',) + self.assertResourceCalled('Package', 'hbase_2_3_*', + retry_count=5, + retry_on_repo_unavailability=False) self.assertNoMoreResources() @@ -50,8 +52,12 @@ class TestHBaseMaster(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES, try_install=True ) - self.assertResourceCalled('Package', 'hbase_2_3_*',) - self.assertResourceCalled('Package', 'phoenix_2_3_*',) + self.assertResourceCalled('Package', 'hbase_2_3_*', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'phoenix_2_3_*', + retry_count=5, + retry_on_repo_unavailability=False) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py index 6ea0bdc..dc471cf 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py @@ -549,7 +549,7 @@ class TestHbaseRegionServer(RMFTestCase): owner='hbase', content='log4jproperties\nline2') - self.assertResourceCalled('Package', 'phoenix_2_2_*') + self.assertResourceCalled('Package', 'phoenix_2_2_*', retry_count=5, retry_on_repo_unavailability=False) self.assertResourceCalled('Execute', '/usr/hdp/current/hbase-regionserver/bin/hbase-daemon.sh --config /usr/hdp/current/hbase-regionserver/conf start regionserver', not_if = 'ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hbase/hbase-hbase-regionserver.pid && ps -p `ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E cat /var/run/hbase/hbase-hbase-regionserver.pid` >/dev/null 2>&1', http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 6ae54f0..7626056 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -3,6 +3,8 @@ "clusterName": "c1", "hostname": "c6401.ambari.apache.org", "hostLevelParams": { + "agent_stack_retry_count": "5", + "agent_stack_retry_on_unavailability": "false", "agentCacheDir": "/var/lib/ambari-agent/cache", "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", "ambari_db_rca_password": "mapred", @@ -1132,9 +1134,6 @@ ], "metrics_collector_hosts": [ "c6401.ambari.apache.org" - ], - "metrics_grafana_hosts": [ - "c6401.ambari.apache.org" ] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-rs-2.2-phoenix.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-rs-2.2-phoenix.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-rs-2.2-phoenix.json index 13c7952..fea5e34 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-rs-2.2-phoenix.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-rs-2.2-phoenix.json @@ -46,6 +46,8 @@ "clusterName": "c1", "hostname": "c6403.ambari.apache.org", "hostLevelParams": { + "agent_stack_retry_count": "5", + "agent_stack_retry_on_unavailability": "false", "jdk_location": "http://10.0.0.15:8080/resources/", "ambari_db_rca_password": "mapred", "java_home": "/usr/jdk64/jdk1.7.0_45", http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json index e2e9965..c44f29c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_no_phx.json @@ -3,6 +3,8 @@ "clusterName": "c1", "hostname": "c6401.ambari.apache.org", "hostLevelParams": { + "agent_stack_retry_count": "5", + "agent_stack_retry_on_unavailability": "false", "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", "ambari_db_rca_password": "mapred", "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json index 1a3b217..e825dd3 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase_with_phx.json @@ -3,6 +3,8 @@ "clusterName": "c1", "hostname": "c6401.ambari.apache.org", "hostLevelParams": { + "agent_stack_retry_count": "5", + "agent_stack_retry_on_unavailability": "false", "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", "ambari_db_rca_password": "mapred", "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py index f9d69a8..aacd1f2 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py @@ -40,6 +40,6 @@ class TestHookBeforeInstall(RMFTestCase): repo_file_name='HDP', repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0' ) - self.assertResourceCalled('Package', 'unzip',) - self.assertResourceCalled('Package', 'curl',) + self.assertResourceCalled('Package', 'unzip', retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'curl', retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py b/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py index 13fec23..569b892 100644 --- a/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py +++ b/ambari-server/src/test/python/stacks/2.3/PXF/test_pxf.py @@ -67,11 +67,21 @@ class TestPxf(RMFTestCase): target=RMFTestCase.TARGET_COMMON_SERVICES, try_install=True) - self.assertResourceCalled('Package', 'pxf-service',) - self.assertResourceCalled('Package', 'apache-tomcat',) - self.assertResourceCalled('Package', 'pxf-hive',) - self.assertResourceCalled('Package', 'pxf-hdfs',) - self.assertResourceCalled('Package', 'pxf-hbase',) + self.assertResourceCalled('Package', 'pxf-service', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'apache-tomcat', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'pxf-hive', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'pxf-hdfs', + retry_count=5, + retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'pxf-hbase', + retry_count=5, + retry_on_repo_unavailability=False) self.assert_configure_default() http://git-wip-us.apache.org/repos/asf/ambari/blob/8eb62315/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json index a042dfd..e2853b7 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json @@ -3,6 +3,8 @@ "clusterName": "c1", "hostname": "c6401.ambari.apache.org", "hostLevelParams": { + "agent_stack_retry_count": "5", + "agent_stack_retry_on_unavailability": "false", "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", "ambari_db_rca_password": "mapred", "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",