---
Makefile.am | 4 +-
lib/utils/__init__.py | 157 +-----------------------------
lib/utils/retry.py | 184 +++++++++++++++++++++++++++++++++++
test/ganeti.utils.retry_unittest.py | 117 ++++++++++++++++++++++
test/ganeti.utils_unittest.py | 83 ----------------
5 files changed, 305 insertions(+), 240 deletions(-)
create mode 100644 lib/utils/retry.py
create mode 100755 test/ganeti.utils.retry_unittest.py
diff --git a/Makefile.am b/Makefile.am
index 76be174..73ebcfe 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -213,7 +213,8 @@ server_PYTHON = \
utils_PYTHON = \
lib/utils/__init__.py \
- lib/utils/algo.py
+ lib/utils/algo.py \
+ lib/utils/retry.py
docrst = \
doc/admin.rst \
@@ -480,6 +481,7 @@ python_tests = \
test/ganeti.ssh_unittest.py \
test/ganeti.uidpool_unittest.py \
test/ganeti.utils.algo_unittest.py \
+ test/ganeti.utils.retry_unittest.py \
test/ganeti.utils_mlockall_unittest.py \
test/ganeti.utils_unittest.py \
test/ganeti.workerpool_unittest.py \
diff --git a/lib/utils/__init__.py b/lib/utils/__init__.py
index 96db38e..1f2e0b7 100644
--- a/lib/utils/__init__.py
+++ b/lib/utils/__init__.py
@@ -63,6 +63,7 @@ from ganeti import constants
from ganeti import compat
from ganeti.utils.algo import * # pylint: disable-msg=W0401
+from ganeti.utils.retry import * # pylint: disable-msg=W0401
_locksheld = []
_re_shell_unquoted = re.compile('^[-.,=:/_...@a-za-z0-9]+$')
@@ -3353,162 +3354,6 @@ def ReadWatcherPauseFile(filename, now=None,
remove_after=3600):
return value
-class RetryTimeout(Exception):
- """Retry loop timed out.
-
- Any arguments which was passed by the retried function to RetryAgain will be
- preserved in RetryTimeout, if it is raised. If such argument was an exception
- the RaiseInner helper method will reraise it.
-
- """
- def RaiseInner(self):
- if self.args and isinstance(self.args[0], Exception):
- raise self.args[0]
- else:
- raise RetryTimeout(*self.args)
-
-
-class RetryAgain(Exception):
- """Retry again.
-
- Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
- arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
- of the RetryTimeout() method can be used to reraise it.
-
- """
-
-
-class _RetryDelayCalculator(object):
- """Calculator for increasing delays.
-
- """
- __slots__ = [
- "_factor",
- "_limit",
- "_next",
- "_start",
- ]
-
- def __init__(self, start, factor, limit):
- """Initializes this class.
-
- @type start: float
- @param start: Initial delay
- @type factor: float
- @param factor: Factor for delay increase
- @type limit: float or None
- @param limit: Upper limit for delay or None for no limit
-
- """
- assert start > 0.0
- assert factor >= 1.0
- assert limit is None or limit >= 0.0
-
- self._start = start
- self._factor = factor
- self._limit = limit
-
- self._next = start
-
- def __call__(self):
- """Returns current delay and calculates the next one.
-
- """
- current = self._next
-
- # Update for next run
- if self._limit is None or self._next < self._limit:
- self._next = min(self._limit, self._next * self._factor)
-
- return current
-
-
-#: Special delay to specify whole remaining timeout
-RETRY_REMAINING_TIME = object()
-
-
-def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
- _time_fn=time.time):
- """Call a function repeatedly until it succeeds.
-
- The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
- anymore. Between calls a delay, specified by C{delay}, is inserted. After a
- total of C{timeout} seconds, this function throws L{RetryTimeout}.
-
- C{delay} can be one of the following:
- - callable returning the delay length as a float
- - Tuple of (start, factor, limit)
- - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
- useful when overriding L{wait_fn} to wait for an external event)
- - A static delay as a number (int or float)
-
- @type fn: callable
- @param fn: Function to be called
- @param delay: Either a callable (returning the delay), a tuple of (start,
- factor, limit) (see L{_RetryDelayCalculator}),
- L{RETRY_REMAINING_TIME} or a number (int or float)
- @type timeout: float
- @param timeout: Total timeout
- @type wait_fn: callable
- @param wait_fn: Waiting function
- @return: Return value of function
-
- """
- assert callable(fn)
- assert callable(wait_fn)
- assert callable(_time_fn)
-
- if args is None:
- args = []
-
- end_time = _time_fn() + timeout
-
- if callable(delay):
- # External function to calculate delay
- calc_delay = delay
-
- elif isinstance(delay, (tuple, list)):
- # Increasing delay with optional upper boundary
- (start, factor, limit) = delay
- calc_delay = _RetryDelayCalculator(start, factor, limit)
-
- elif delay is RETRY_REMAINING_TIME:
- # Always use the remaining time
- calc_delay = None
-
- else:
- # Static delay
- calc_delay = lambda: delay
-
- assert calc_delay is None or callable(calc_delay)
-
- while True:
- retry_args = []
- try:
- # pylint: disable-msg=W0142
- return fn(*args)
- except RetryAgain, err:
- retry_args = err.args
- except RetryTimeout:
- raise errors.ProgrammerError("Nested retry loop detected that didn't"
- " handle RetryTimeout")
-
- remaining_time = end_time - _time_fn()
-
- if remaining_time < 0.0:
- # pylint: disable-msg=W0142
- raise RetryTimeout(*retry_args)
-
- assert remaining_time >= 0.0
-
- if calc_delay is None:
- wait_fn(remaining_time)
- else:
- current_delay = calc_delay()
- if current_delay > 0.0:
- wait_fn(current_delay)
-
-
def GetClosedTempfile(*args, **kwargs):
"""Creates a temporary file and returns its path.
diff --git a/lib/utils/retry.py b/lib/utils/retry.py
new file mode 100644
index 0000000..b5b7ff3
--- /dev/null
+++ b/lib/utils/retry.py
@@ -0,0 +1,184 @@
+#
+#
+
+# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+"""Utility functions for retrying function calls with a timeout.
+
+"""
+
+
+import time
+
+from ganeti import errors
+
+
+#: Special delay to specify whole remaining timeout
+RETRY_REMAINING_TIME = object()
+
+
+class RetryTimeout(Exception):
+ """Retry loop timed out.
+
+ Any arguments which was passed by the retried function to RetryAgain will be
+ preserved in RetryTimeout, if it is raised. If such argument was an exception
+ the RaiseInner helper method will reraise it.
+
+ """
+ def RaiseInner(self):
+ if self.args and isinstance(self.args[0], Exception):
+ raise self.args[0]
+ else:
+ raise RetryTimeout(*self.args)
+
+
+class RetryAgain(Exception):
+ """Retry again.
+
+ Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
+ arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
+ of the RetryTimeout() method can be used to reraise it.
+
+ """
+
+
+class _RetryDelayCalculator(object):
+ """Calculator for increasing delays.
+
+ """
+ __slots__ = [
+ "_factor",
+ "_limit",
+ "_next",
+ "_start",
+ ]
+
+ def __init__(self, start, factor, limit):
+ """Initializes this class.
+
+ @type start: float
+ @param start: Initial delay
+ @type factor: float
+ @param factor: Factor for delay increase
+ @type limit: float or None
+ @param limit: Upper limit for delay or None for no limit
+
+ """
+ assert start > 0.0
+ assert factor >= 1.0
+ assert limit is None or limit >= 0.0
+
+ self._start = start
+ self._factor = factor
+ self._limit = limit
+
+ self._next = start
+
+ def __call__(self):
+ """Returns current delay and calculates the next one.
+
+ """
+ current = self._next
+
+ # Update for next run
+ if self._limit is None or self._next < self._limit:
+ self._next = min(self._limit, self._next * self._factor)
+
+ return current
+
+
+def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
+ _time_fn=time.time):
+ """Call a function repeatedly until it succeeds.
+
+ The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
+ anymore. Between calls a delay, specified by C{delay}, is inserted. After a
+ total of C{timeout} seconds, this function throws L{RetryTimeout}.
+
+ C{delay} can be one of the following:
+ - callable returning the delay length as a float
+ - Tuple of (start, factor, limit)
+ - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
+ useful when overriding L{wait_fn} to wait for an external event)
+ - A static delay as a number (int or float)
+
+ @type fn: callable
+ @param fn: Function to be called
+ @param delay: Either a callable (returning the delay), a tuple of (start,
+ factor, limit) (see L{_RetryDelayCalculator}),
+ L{RETRY_REMAINING_TIME} or a number (int or float)
+ @type timeout: float
+ @param timeout: Total timeout
+ @type wait_fn: callable
+ @param wait_fn: Waiting function
+ @return: Return value of function
+
+ """
+ assert callable(fn)
+ assert callable(wait_fn)
+ assert callable(_time_fn)
+
+ if args is None:
+ args = []
+
+ end_time = _time_fn() + timeout
+
+ if callable(delay):
+ # External function to calculate delay
+ calc_delay = delay
+
+ elif isinstance(delay, (tuple, list)):
+ # Increasing delay with optional upper boundary
+ (start, factor, limit) = delay
+ calc_delay = _RetryDelayCalculator(start, factor, limit)
+
+ elif delay is RETRY_REMAINING_TIME:
+ # Always use the remaining time
+ calc_delay = None
+
+ else:
+ # Static delay
+ calc_delay = lambda: delay
+
+ assert calc_delay is None or callable(calc_delay)
+
+ while True:
+ retry_args = []
+ try:
+ # pylint: disable-msg=W0142
+ return fn(*args)
+ except RetryAgain, err:
+ retry_args = err.args
+ except RetryTimeout:
+ raise errors.ProgrammerError("Nested retry loop detected that didn't"
+ " handle RetryTimeout")
+
+ remaining_time = end_time - _time_fn()
+
+ if remaining_time < 0.0:
+ # pylint: disable-msg=W0142
+ raise RetryTimeout(*retry_args)
+
+ assert remaining_time >= 0.0
+
+ if calc_delay is None:
+ wait_fn(remaining_time)
+ else:
+ current_delay = calc_delay()
+ if current_delay > 0.0:
+ wait_fn(current_delay)
diff --git a/test/ganeti.utils.retry_unittest.py
b/test/ganeti.utils.retry_unittest.py
new file mode 100755
index 0000000..8173a30
--- /dev/null
+++ b/test/ganeti.utils.retry_unittest.py
@@ -0,0 +1,117 @@
+#!/usr/bin/python
+#
+
+# Copyright (C) 2011 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Script for testing ganeti.utils.retry"""
+
+import unittest
+
+from ganeti import constants
+from ganeti import errors
+from ganeti import utils
+
+import testutils
+
+
+class TestRetry(testutils.GanetiTestCase):
+ def setUp(self):
+ testutils.GanetiTestCase.setUp(self)
+ self.retries = 0
+
+ @staticmethod
+ def _RaiseRetryAgain():
+ raise utils.RetryAgain()
+
+ @staticmethod
+ def _RaiseRetryAgainWithArg(args):
+ raise utils.RetryAgain(*args)
+
+ def _WrongNestedLoop(self):
+ return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)
+
+ def _RetryAndSucceed(self, retries):
+ if self.retries < retries:
+ self.retries += 1
+ raise utils.RetryAgain()
+ else:
+ return True
+
+ def testRaiseTimeout(self):
+ self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
+ self._RaiseRetryAgain, 0.01, 0.02)
+ self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
+ self._RetryAndSucceed, 0.01, 0, args=[1])
+ self.failUnlessEqual(self.retries, 1)
+
+ def testComplete(self):
+ self.failUnlessEqual(utils.Retry(lambda: True, 0, 1), True)
+ self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2]),
+ True)
+ self.failUnlessEqual(self.retries, 2)
+
+ def testNestedLoop(self):
+ try:
+ self.failUnlessRaises(errors.ProgrammerError, utils.Retry,
+ self._WrongNestedLoop, 0, 1)
+ except utils.RetryTimeout:
+ self.fail("Didn't detect inner loop's exception")
+
+ def testTimeoutArgument(self):
+ retry_arg="my_important_debugging_message"
+ try:
+ utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]])
+ except utils.RetryTimeout, err:
+ self.failUnlessEqual(err.args, (retry_arg, ))
+ else:
+ self.fail("Expected timeout didn't happen")
+
+ def testRaiseInnerWithExc(self):
+ retry_arg="my_important_debugging_message"
+ try:
+ try:
+ utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
+ args=[[errors.GenericError(retry_arg, retry_arg)]])
+ except utils.RetryTimeout, err:
+ err.RaiseInner()
+ else:
+ self.fail("Expected timeout didn't happen")
+ except errors.GenericError, err:
+ self.failUnlessEqual(err.args, (retry_arg, retry_arg))
+ else:
+ self.fail("Expected GenericError didn't happen")
+
+ def testRaiseInnerWithMsg(self):
+ retry_arg="my_important_debugging_message"
+ try:
+ try:
+ utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
+ args=[[retry_arg, retry_arg]])
+ except utils.RetryTimeout, err:
+ err.RaiseInner()
+ else:
+ self.fail("Expected timeout didn't happen")
+ except utils.RetryTimeout, err:
+ self.failUnlessEqual(err.args, (retry_arg, retry_arg))
+ else:
+ self.fail("Expected RetryTimeout didn't happen")
+
+
+if __name__ == "__main__":
+ testutils.GanetiTestProgram()
diff --git a/test/ganeti.utils_unittest.py b/test/ganeti.utils_unittest.py
index 254c6d8..24deb07 100755
--- a/test/ganeti.utils_unittest.py
+++ b/test/ganeti.utils_unittest.py
@@ -1995,89 +1995,6 @@ class TestMakedirs(unittest.TestCase):
self.assert_(os.path.isdir(path))
-class TestRetry(testutils.GanetiTestCase):
- def setUp(self):
- testutils.GanetiTestCase.setUp(self)
- self.retries = 0
-
- @staticmethod
- def _RaiseRetryAgain():
- raise utils.RetryAgain()
-
- @staticmethod
- def _RaiseRetryAgainWithArg(args):
- raise utils.RetryAgain(*args)
-
- def _WrongNestedLoop(self):
- return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)
-
- def _RetryAndSucceed(self, retries):
- if self.retries < retries:
- self.retries += 1
- raise utils.RetryAgain()
- else:
- return True
-
- def testRaiseTimeout(self):
- self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
- self._RaiseRetryAgain, 0.01, 0.02)
- self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
- self._RetryAndSucceed, 0.01, 0, args=[1])
- self.failUnlessEqual(self.retries, 1)
-
- def testComplete(self):
- self.failUnlessEqual(utils.Retry(lambda: True, 0, 1), True)
- self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2]),
- True)
- self.failUnlessEqual(self.retries, 2)
-
- def testNestedLoop(self):
- try:
- self.failUnlessRaises(errors.ProgrammerError, utils.Retry,
- self._WrongNestedLoop, 0, 1)
- except utils.RetryTimeout:
- self.fail("Didn't detect inner loop's exception")
-
- def testTimeoutArgument(self):
- retry_arg="my_important_debugging_message"
- try:
- utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]])
- except utils.RetryTimeout, err:
- self.failUnlessEqual(err.args, (retry_arg, ))
- else:
- self.fail("Expected timeout didn't happen")
-
- def testRaiseInnerWithExc(self):
- retry_arg="my_important_debugging_message"
- try:
- try:
- utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
- args=[[errors.GenericError(retry_arg, retry_arg)]])
- except utils.RetryTimeout, err:
- err.RaiseInner()
- else:
- self.fail("Expected timeout didn't happen")
- except errors.GenericError, err:
- self.failUnlessEqual(err.args, (retry_arg, retry_arg))
- else:
- self.fail("Expected GenericError didn't happen")
-
- def testRaiseInnerWithMsg(self):
- retry_arg="my_important_debugging_message"
- try:
- try:
- utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
- args=[[retry_arg, retry_arg]])
- except utils.RetryTimeout, err:
- err.RaiseInner()
- else:
- self.fail("Expected timeout didn't happen")
- except utils.RetryTimeout, err:
- self.failUnlessEqual(err.args, (retry_arg, retry_arg))
- else:
- self.fail("Expected RetryTimeout didn't happen")
-
-
class TestLineSplitter(unittest.TestCase):
def test(self):
lines = []
--
1.7.3.1
