On Fri, Jun 10, 2011 at 10:46 AM, Iustin Pop <[email protected]> wrote:
> On Wed, Jun 08, 2011 at 11:00:32AM +0200, René Nussbaumer wrote:
>> This includes an own simple cache implementation and an
>> interface to a memcache instance.
>> ---
>> lib/cache.py | 308
>> +++++++++++++++++++++++++++++++++++++++++
>> test/ganeti.cache_unittest.py | 104 ++++++++++++++
>> 2 files changed, 412 insertions(+), 0 deletions(-)
>> create mode 100644 lib/cache.py
>> create mode 100755 test/ganeti.cache_unittest.py
>>
>> diff --git a/lib/cache.py b/lib/cache.py
>> new file mode 100644
>> index 0000000..7789fb0
>> --- /dev/null
>> +++ b/lib/cache.py
>> @@ -0,0 +1,308 @@
>> +#
>> +#
>> +
>> +# Copyright (C) 2011 Google Inc.
>> +#
>> +# This program is free software; you can redistribute it and/or modify
>> +# it under the terms of the GNU General Public License as published by
>> +# the Free Software Foundation; either version 2 of the License, or
>> +# (at your option) any later version.
>> +#
>> +# This program is distributed in the hope that it will be useful, but
>> +# WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> +# General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> +# 02110-1301, USA.
>> +
>> +
>> +"""This module implements caching."""
>> +
>> +
>> +import threading
>> +import time
>> +
>> +from ganeti import locking
>> +from ganeti import serializer
>> +
>> +try:
>> + _has_memcache = True
>> + import memcache
>> +except ImportError:
>> + _has_memcache = False
>> +
>> +
>> +_cache = None
>> +_cache_lock = threading.Lock()
>> +
>> +
>> +def Cache():
>> + """Factory method to return appropriate cache.
>> +
>> + """
>> + # We need to use the global keyword here
>> + global _cache # pylint: disable-msg=W0603
>> +
>> + if not _cache:
>> + _cache_lock.acquire()
>> + try:
>> + if not _cache:
>> + if _has_memcache:
>> + cache_obj = MemCache()
>> + else:
>> + cache_obj = SimpleCache()
>> + # W0621: Redefine '_cache' from outer scope (used for singleton)
>> + _cache = cache_obj # pylint: disable-msg=W0621
>> + finally:
>> + _cache_lock.release()
>
> This will probably need to be enhanced later to select which kind of
> cache we want, and (for memcache) what configuration we use.
Yes, this is very basic right now. But should be fairly easy to
enhance to our needs. The point is that we always get the same cache
for all invocation.
>> + # Make sure we have a fresh internal state and are ready to serve
>> + _cache.ResetState()
>> + return _cache
>> +
>> +
>> +class CacheBase:
>> + """This is the base class for all caches.
>> +
>> + """
>> + def __init__(self):
>> + """Base init method.
>> +
>> + """
>> +
>> + def Store(self, key, value, ttl=0):
>> + """Stores key with value in the cache.
>> +
>> + @param key: The key to associate this cached value
>> + @param value: The value to cache
>> + @param ttl: TTL in seconds after when this entry is considered outdated
>> + @returns True on success, False on failure
>> +
>> + """
>> + raise NotImplementedError
>> +
>> + def Get(self, keys):
>> + """Retrieve the value from the cache.
>> +
>> + @param keys: The keys to retrieve
>> + @returns The list of values
>> +
>> + """
>> + raise NotImplementedError
>> +
>> + def Invalidate(self, keys):
>> + """Invalidate given keys.
>> +
>> + @param keys: The list of keys to invalidate
>> + @returns True on success, False otherwise
>> +
>> + """
>> + raise NotImplementedError
>> +
>> + def Flush(self):
>> + """Invalidates all of the keys and flushes the cache.
>> +
>> + """
>> + raise NotImplementedError
>> +
>> + def ResetState(self):
>> + """Used to reset the state of the cache.
>> +
>> + This can be used to reinstantiate connection or any other state refresh
>> +
>> + """
>> +
>> + def Cleanup(self):
>> + """Cleanup the cache from expired entries.
>> +
>> + """
>> +
>> +
>> +class SimpleCache(CacheBase):
>> + """Implements a very simple, dict base cache.
>> +
>> + """
>> + CLEANUP_ROUND = 1800
>> +
>> + def __init__(self, _time_fn=time.time):
>> + """Initialize this class.
>> +
>> + @param _time_fn: Function used to return time (unittest only)
>> + """
>
> Missing newline before """
Thanks, must have missed that!
>> + CacheBase.__init__(self)
>> +
>> + self._time_fn = _time_fn
>> +
>> + self.cache = {}
>> + self.lock = locking.SharedLock("SimpleCache-lock")
>> + self.last_cleanup = self._time_fn()
>> +
>> + def _UnlockedCleanup(self):
>> + """Does cleanup of the cache.
>> +
>> + """
>> + check_time = self._time_fn()
>> + if (self.last_cleanup + self.CLEANUP_ROUND) <= check_time:
>> + keys = []
>> + for key, value in self.cache.items():
>> + if not value["ttl"]:
>> + continue
>> +
>> + expired = value["timestamp"] + value["ttl"]
>
> Both here and in the rest of the code, please get rid of hardcoded
> strings and use module-level constants TIMESTAMP and TTL.
Done.
>> + if expired < check_time:
>> + keys.append(key)
>> + self._UnlockedInvalidate(keys)
>> + self.last_cleanup = check_time
>> +
>> + @locking.ssynchronized("lock")
>> + def Cleanup(self):
>> + """Cleanup our cache.
>> +
>> + """
>> + self._UnlockedCleanup()
>> +
>> + @locking.ssynchronized("lock")
>> + def Store(self, key, value, ttl=0):
>> + """Stores a value at key in the cache.
>> +
>> + See L{CacheBase} for parameter description
>> +
>> + """
>> + assert ttl >= 0
>> + self._UnlockedCleanup()
>> + val = serializer.Dump(value)
>> + cache_val = {
>> + "timestamp": self._time_fn(),
>> + "ttl": ttl,
>> + "value": val
>> + }
>> + self.cache[key] = cache_val
>> + return True
>> +
>> + @locking.ssynchronized("lock", shared=1)
>> + def Get(self, keys):
>> + """Retrieve the values of keys from cache.
>> +
>> + See L{CacheBase} for parameter description
>> +
>> + """
>> + return [self._ExtractValue(key) for key in keys]
>> +
>> + @locking.ssynchronized("lock")
>> + def Invalidate(self, keys):
>> + """Invalidates value for keys in cache.
>> +
>> + See L{CacheBase} for parameter description
>> +
>> + """
>> + self._UnlockedInvalidate(keys)
>> + return True
>
> This doesn't really make sense; move the return True in the
> _UnlockedInvalidate and here just return self._UnlockedInvalidate.
Indeed, silly left over of some refactoring.
>> + @locking.ssynchronized("lock")
>> + def Flush(self):
>> + """Invalidates all keys and values in cache.
>> +
>> + See L{CacheBase} for parameter description
>> +
>> + """
>> + self.cache.clear()
>> + self.last_cleanup = self._time_fn()
>> +
>> + def _UnlockedInvalidate(self, keys):
>> + """Invalidate keys in cache.
>> +
>> + This is the unlocked version, see L{Invalidate} for parameter
>> description
>> +
>> + """
>> + for key in keys:
>> + if key in self.cache:
>> + del self.cache[key]
>> +
>> + def _ExtractValue(self, key):
>> + """Extracts just the value for a key.
>> +
>> + This method is taking care if the value did not expire ans returns it
>> +
>> + @param key: The key to look for
>> + @returns The value if key is not expired, None otherwise
>> +
>> + """
>> + if key in self.cache:
>> + cache_val = self.cache[key]
>> + if cache_val["ttl"] == 0:
>> + return serializer.Load(cache_val["value"])
>> + else:
>> + expired = cache_val["timestamp"] + cache_val["ttl"]
>> +
>> + if self._time_fn() <= expired:
>> + return serializer.Load(cache_val["value"])
>> + else:
>> + return None
>> + else:
>> + return None
>> +
>> +
>> +class MemCache(CacheBase):
>> + """Implements the memcache cache.
>> +
>> + """
>> + def __init__(self, servers=("localhost:11211",)):
>> + """Initialize this class.
>> +
>> + @param servers: List of memcache servers in the format <host>:<port>
>> +
>> + """
>> + CacheBase.__init__(self)
>> +
>> + self.mc = memcache.Client(servers)
>> +
>> + def Store(self, key, value, ttl=0):
>> + """Stores key with value in memcache.
>> +
>> + See L{CacheBase} for parameter description
>> +
>> + """
>> + return (self.mc.add(key, value, time=ttl) or
>> + self.mc.replace(key, value, time=ttl))
>
> This does not make any mention of how/if the values are serialized. What
> is happening here?
The module memcache uses Pickler to do serialization and
deserialization. So you want to have that documented in the docstring?
René
