On Wed, Jun 08, 2011 at 11:00:32AM +0200, René Nussbaumer wrote:
> This includes an own simple cache implementation and an
> interface to a memcache instance.
> ---
> lib/cache.py | 308
> +++++++++++++++++++++++++++++++++++++++++
> test/ganeti.cache_unittest.py | 104 ++++++++++++++
> 2 files changed, 412 insertions(+), 0 deletions(-)
> create mode 100644 lib/cache.py
> create mode 100755 test/ganeti.cache_unittest.py
>
> diff --git a/lib/cache.py b/lib/cache.py
> new file mode 100644
> index 0000000..7789fb0
> --- /dev/null
> +++ b/lib/cache.py
> @@ -0,0 +1,308 @@
> +#
> +#
> +
> +# Copyright (C) 2011 Google Inc.
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful, but
> +# WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> +# General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> +# 02110-1301, USA.
> +
> +
> +"""This module implements caching."""
> +
> +
> +import threading
> +import time
> +
> +from ganeti import locking
> +from ganeti import serializer
> +
> +try:
> + _has_memcache = True
> + import memcache
> +except ImportError:
> + _has_memcache = False
> +
> +
> +_cache = None
> +_cache_lock = threading.Lock()
> +
> +
> +def Cache():
> + """Factory method to return appropriate cache.
> +
> + """
> + # We need to use the global keyword here
> + global _cache # pylint: disable-msg=W0603
> +
> + if not _cache:
> + _cache_lock.acquire()
> + try:
> + if not _cache:
> + if _has_memcache:
> + cache_obj = MemCache()
> + else:
> + cache_obj = SimpleCache()
> + # W0621: Redefine '_cache' from outer scope (used for singleton)
> + _cache = cache_obj # pylint: disable-msg=W0621
> + finally:
> + _cache_lock.release()
This will probably need to be enhanced later to select which kind of
cache we want, and (for memcache) what configuration we use.
> + # Make sure we have a fresh internal state and are ready to serve
> + _cache.ResetState()
> + return _cache
> +
> +
> +class CacheBase:
> + """This is the base class for all caches.
> +
> + """
> + def __init__(self):
> + """Base init method.
> +
> + """
> +
> + def Store(self, key, value, ttl=0):
> + """Stores key with value in the cache.
> +
> + @param key: The key to associate this cached value
> + @param value: The value to cache
> + @param ttl: TTL in seconds after when this entry is considered outdated
> + @returns True on success, False on failure
> +
> + """
> + raise NotImplementedError
> +
> + def Get(self, keys):
> + """Retrieve the value from the cache.
> +
> + @param keys: The keys to retrieve
> + @returns The list of values
> +
> + """
> + raise NotImplementedError
> +
> + def Invalidate(self, keys):
> + """Invalidate given keys.
> +
> + @param keys: The list of keys to invalidate
> + @returns True on success, False otherwise
> +
> + """
> + raise NotImplementedError
> +
> + def Flush(self):
> + """Invalidates all of the keys and flushes the cache.
> +
> + """
> + raise NotImplementedError
> +
> + def ResetState(self):
> + """Used to reset the state of the cache.
> +
> + This can be used to reinstantiate connection or any other state refresh
> +
> + """
> +
> + def Cleanup(self):
> + """Cleanup the cache from expired entries.
> +
> + """
> +
> +
> +class SimpleCache(CacheBase):
> + """Implements a very simple, dict base cache.
> +
> + """
> + CLEANUP_ROUND = 1800
> +
> + def __init__(self, _time_fn=time.time):
> + """Initialize this class.
> +
> + @param _time_fn: Function used to return time (unittest only)
> + """
Missing newline before """
> + CacheBase.__init__(self)
> +
> + self._time_fn = _time_fn
> +
> + self.cache = {}
> + self.lock = locking.SharedLock("SimpleCache-lock")
> + self.last_cleanup = self._time_fn()
> +
> + def _UnlockedCleanup(self):
> + """Does cleanup of the cache.
> +
> + """
> + check_time = self._time_fn()
> + if (self.last_cleanup + self.CLEANUP_ROUND) <= check_time:
> + keys = []
> + for key, value in self.cache.items():
> + if not value["ttl"]:
> + continue
> +
> + expired = value["timestamp"] + value["ttl"]
Both here and in the rest of the code, please get rid of hardcoded
strings and use module-level constants TIMESTAMP and TTL.
> + if expired < check_time:
> + keys.append(key)
> + self._UnlockedInvalidate(keys)
> + self.last_cleanup = check_time
> +
> + @locking.ssynchronized("lock")
> + def Cleanup(self):
> + """Cleanup our cache.
> +
> + """
> + self._UnlockedCleanup()
> +
> + @locking.ssynchronized("lock")
> + def Store(self, key, value, ttl=0):
> + """Stores a value at key in the cache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + assert ttl >= 0
> + self._UnlockedCleanup()
> + val = serializer.Dump(value)
> + cache_val = {
> + "timestamp": self._time_fn(),
> + "ttl": ttl,
> + "value": val
> + }
> + self.cache[key] = cache_val
> + return True
> +
> + @locking.ssynchronized("lock", shared=1)
> + def Get(self, keys):
> + """Retrieve the values of keys from cache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + return [self._ExtractValue(key) for key in keys]
> +
> + @locking.ssynchronized("lock")
> + def Invalidate(self, keys):
> + """Invalidates value for keys in cache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + self._UnlockedInvalidate(keys)
> + return True
This doesn't really make sense; move the return True in the
_UnlockedInvalidate and here just return self._UnlockedInvalidate.
> + @locking.ssynchronized("lock")
> + def Flush(self):
> + """Invalidates all keys and values in cache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + self.cache.clear()
> + self.last_cleanup = self._time_fn()
> +
> + def _UnlockedInvalidate(self, keys):
> + """Invalidate keys in cache.
> +
> + This is the unlocked version, see L{Invalidate} for parameter description
> +
> + """
> + for key in keys:
> + if key in self.cache:
> + del self.cache[key]
> +
> + def _ExtractValue(self, key):
> + """Extracts just the value for a key.
> +
> + This method is taking care if the value did not expire ans returns it
> +
> + @param key: The key to look for
> + @returns The value if key is not expired, None otherwise
> +
> + """
> + if key in self.cache:
> + cache_val = self.cache[key]
> + if cache_val["ttl"] == 0:
> + return serializer.Load(cache_val["value"])
> + else:
> + expired = cache_val["timestamp"] + cache_val["ttl"]
> +
> + if self._time_fn() <= expired:
> + return serializer.Load(cache_val["value"])
> + else:
> + return None
> + else:
> + return None
> +
> +
> +class MemCache(CacheBase):
> + """Implements the memcache cache.
> +
> + """
> + def __init__(self, servers=("localhost:11211",)):
> + """Initialize this class.
> +
> + @param servers: List of memcache servers in the format <host>:<port>
> +
> + """
> + CacheBase.__init__(self)
> +
> + self.mc = memcache.Client(servers)
> +
> + def Store(self, key, value, ttl=0):
> + """Stores key with value in memcache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + return (self.mc.add(key, value, time=ttl) or
> + self.mc.replace(key, value, time=ttl))
This does not make any mention of how/if the values are serialized. What
is happening here?
> + def Get(self, keys):
> + """Retrieve the list of keys from memcache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + result = self.mc.get_multi(keys)
> + return_result = []
> + for key in keys:
> + if key in result:
> + return_result.append(result[key])
> + else:
> + return_result.append(None)
> + return return_result
> +
> + def Invalidate(self, keys):
> + """Invalidates given keys in memcache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + return self.mc.delete_multi(self, keys) == 1
> +
> + def Flush(self):
> + """Invalidates all keys in memcache.
> +
> + See L{CacheBase} for parameter description
> +
> + """
> + self.mc.flush_all()
> +
> + def ResetState(self):
> + """Resets the memcache state.
> +
> + This basically means try to reconnect to possible memcache servers
> +
> + """
> + self.mc.forget_dead_hosts()
thanks,
iustin
