AMBARI-14017. Service or component install fails when a non-ambari apt-get command is running (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2d8715a5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2d8715a5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2d8715a5 Branch: refs/heads/branch-2.1 Commit: 2d8715a5397e7fa8bd983885a19e9677e4c315b1 Parents: adf345b Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Mon Nov 23 17:32:42 2015 +0200 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Mon Nov 23 17:32:42 2015 +0200 ---------------------------------------------------------------------- .../resource_management/TestPackageResource.py | 10 ++++-- .../core/providers/package/__init__.py | 35 ++++++++++++++++++++ .../core/providers/package/apt.py | 18 +++++++--- .../core/providers/package/zypper.py | 9 +++-- .../core/resources/packaging.py | 7 ++++ 5 files changed, 68 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2d8715a5/ambari-agent/src/test/python/resource_management/TestPackageResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestPackageResource.py b/ambari-agent/src/test/python/resource_management/TestPackageResource.py index 18b2d00..5868108 100644 --- a/ambari-agent/src/test/python/resource_management/TestPackageResource.py +++ b/ambari-agent/src/test/python/resource_management/TestPackageResource.py @@ -34,7 +34,8 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'ubuntu') def test_action_install_ubuntu_update(self, shell_mock, call_mock): - call_mock.return_value= (1, None) + shell_mock.return_value= (0, '') + call_mock.return_value= (1, '') with Environment('/') as env: Package("some_package", ) @@ -43,13 +44,14 @@ class TestPackageResource(TestCase): call(['/usr/bin/apt-get', 'update', '-qq'], logoutput=False, sudo=True)]) shell_mock.assert_has_calls([call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', - '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True)]) + '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'})]) @patch.object(shell, "call") @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'ubuntu') def test_action_install_ubuntu(self, shell_mock, call_mock): - call_mock.side_effect = [(1, None), (0, None)] + call_mock.side_effect = [(1, ''), (0, '')] + shell_mock.return_value = (0, '') with Environment('/') as env: Package("some_package", ) @@ -124,6 +126,7 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'suse') def test_action_install_suse(self, shell_mock): + shell_mock.return_value = (0,'') sys.modules['rpm'] = MagicMock() sys.modules['rpm'].TransactionSet.return_value = MagicMock() sys.modules['rpm'].TransactionSet.return_value.dbMatch.return_value = [{'name':'some_packages'}] @@ -208,6 +211,7 @@ class TestPackageResource(TestCase): @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'suse') def test_action_remove_suse(self, shell_mock): + shell_mock.return_value = (0, '') sys.modules['rpm'] = MagicMock() sys.modules['rpm'].TransactionSet.return_value = MagicMock() sys.modules['rpm'].TransactionSet.return_value.dbMatch.return_value = [{'name':'some_package'}] http://git-wip-us.apache.org/repos/asf/ambari/blob/2d8715a5/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py index 7e532bc..1fc4214 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py @@ -20,6 +20,8 @@ Ambari Agent """ +import subprocess +import time import re import logging @@ -27,7 +29,9 @@ from resource_management.core.base import Fail from resource_management.core.providers import Provider from resource_management.core.logger import Logger from resource_management.core.utils import suppress_stdout +from resource_management.core import shell +PACKAGE_MANAGER_LOCK_ACQUIRED = "Package manager lock is acquired. Retrying after {0} seconds. Reason: {1}" class PackageProvider(Provider): def __init__(self, *args, **kwargs): @@ -61,6 +65,37 @@ class PackageProvider(Provider): def get_logoutput(self): return self.resource.logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or self.resource.logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG) + def call_until_not_locked(self, cmd, **kwargs): + return self.wait_until_not_locked(cmd, is_checked=False, **kwargs) + + def checked_call_until_not_locked(self, cmd, **kwargs): + return self.wait_until_not_locked(cmd, is_checked=True, **kwargs) + + def wait_until_not_locked(self, cmd, is_checked=True, **kwargs): + func = shell.checked_call if is_checked else shell.call + + for i in range(self.resource.locked_tries): + is_last_time = (i == self.resource.locked_tries - 1) + try: + code, out = func(cmd, **kwargs) + except Fail as ex: + # non-lock error + if not self.is_locked_output(str(ex)) or is_last_time: + raise + + Logger.info(PACKAGE_MANAGER_LOCK_ACQUIRED.format(self.resource.locked_try_sleep, str(ex))) + else: + # didn't fail or failed with non-lock error. + if not code or not self.is_locked_output(out): + break + + Logger.info(PACKAGE_MANAGER_LOCK_ACQUIRED.format(self.resource.locked_try_sleep, str(out))) + + time.sleep(self.resource.locked_try_sleep) + + return code, out + + def yum_check_package_available(self, name): """ Does the same as rpm_check_package_avaiable, but faster. http://git-wip-us.apache.org/repos/asf/ambari/blob/2d8715a5/ambari-common/src/main/python/resource_management/core/providers/package/apt.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py index ddd6952..70a553d 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py @@ -29,6 +29,7 @@ from resource_management.core import shell from resource_management.core import sudo from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail INSTALL_CMD_ENV = {'DEBIAN_FRONTEND':'noninteractive'} INSTALL_CMD = { @@ -77,19 +78,23 @@ class AptProvider(PackageProvider): cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - code, out = shell.call(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) + code, out = self.call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) - # apt-get update wasn't done too long + if self.is_locked_output(out): + err_msg = Logger.filter_text("Execution of '%s' returned %d. %s" % (cmd, code, out)) + raise Fail(err_msg) + + # apt-get update wasn't done too long maybe? if code: Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out)) Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD))) - code, out = shell.call(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput()) + code, out = self.call_until_not_locked(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput()) if code: Logger.info("Execution of '%s' returned %d. %s" % (REPO_UPDATE_CMD, code, out)) Logger.info("Retrying to install package %s" % (name)) - shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if is_tmp_dir_created: for temporal_sources_file in copied_sources_files: @@ -99,6 +104,9 @@ class AptProvider(PackageProvider): os.rmdir(apt_sources_list_tmp_dir) else: Logger.info("Skipping installation of existing package %s" % (name)) + + def is_locked_output(self, out): + return "Unable to lock the administration directory" in out @replace_underscores def upgrade_package(self, name, use_repos=[], skip_repos=[]): @@ -109,7 +117,7 @@ class AptProvider(PackageProvider): if self._check_existence(name): cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_until_not_locked(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/2d8715a5/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py index 3ff3dfd..b255254 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py @@ -38,7 +38,7 @@ REMOVE_CMD = { LIST_ACTIVE_REPOS_CMD = ['/usr/bin/zypper', 'repos'] def get_active_base_repos(): - (code, output) = shell.call(LIST_ACTIVE_REPOS_CMD) + (code, output) = self.call_until_not_locked(LIST_ACTIVE_REPOS_CMD) enabled_repos = [] if not code: for line in output.split('\n')[2:]: @@ -67,7 +67,7 @@ class ZypperProvider(PackageProvider): cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_until_not_locked(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installation of existing package %s" % (name)) @@ -78,9 +78,12 @@ class ZypperProvider(PackageProvider): if self._check_existence(name): cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) - shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) + self.checked_call_until_not_locked(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name)) + + def is_locked_output(self ,out): + return "System management is locked by the application" in out def _check_existence(self, name): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/2d8715a5/ambari-common/src/main/python/resource_management/core/resources/packaging.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/resources/packaging.py b/ambari-common/src/main/python/resource_management/core/resources/packaging.py index 1ca88af..bb0aa56 100644 --- a/ambari-common/src/main/python/resource_management/core/resources/packaging.py +++ b/ambari-common/src/main/python/resource_management/core/resources/packaging.py @@ -39,6 +39,13 @@ class Package(Resource): None (default) - log it in DEBUG mode """ logoutput = ResourceArgument(default=None) + + """ + Retry if package manager is locked. (usually another process is running). + Note that this works only for apt-get and zypper, while yum manages lock retries itself. + """ + locked_tries = ResourceArgument(default=8) + locked_try_sleep = ResourceArgument(default=30) # seconds version = ResourceArgument() actions = ["install", "upgrade", "remove"]
