Author: tv Date: Wed Nov 25 13:10:22 2015 New Revision: 1716376 URL: http://svn.apache.org/viewvc?rev=1716376&view=rev Log: JCS-153: Fix file size limitation for Block Disk Cache and Indexed Disk Cache
Added: commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheCountUnitTest.java (with props) commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheSizeUnitTest.java (with props) commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestAbstract.java (with props) commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheCountUnitTest.java (with props) commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheSizeUnitTest.java (with props) commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTestAbstract.java (with props) Removed: commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTest.java commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestCount.java commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestSize.java commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTest.java commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTestCount.java commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTestSize.java Modified: commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexedDiskCache.java commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/utils/struct/AbstractLRUMap.java commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/utils/struct/LRUMap.java commons/proper/jcs/trunk/src/changes/changes.xml Modified: commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java?rev=1716376&r1=1716375&r2=1716376&view=diff ============================================================================== --- commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java (original) +++ commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java Wed Nov 25 13:10:22 2015 @@ -125,11 +125,13 @@ public class BlockDiskCache<K, V> if ( this.blockDiskCacheAttributes.getBlockSizeBytes() > 0 ) { this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data" ), - this.blockDiskCacheAttributes.getBlockSizeBytes() ); + this.blockDiskCacheAttributes.getBlockSizeBytes(), + getElementSerializer() ); } else { - this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data" ), getElementSerializer() ); + this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data" ), + getElementSerializer() ); } keyStore = new BlockDiskKeyStore<K>( this.blockDiskCacheAttributes, this ); Modified: commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java?rev=1716376&r1=1716375&r2=1716376&view=diff ============================================================================== --- commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java (original) +++ commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java Wed Nov 25 13:10:22 2015 @@ -44,12 +44,13 @@ import org.apache.commons.logging.LogFac /** * This is responsible for storing the keys. * <p> + * * @author Aaron Smuts */ public class BlockDiskKeyStore<K> { /** The logger */ - private static final Log log = LogFactory.getLog( BlockDiskKeyStore.class ); + private static final Log log = LogFactory.getLog(BlockDiskKeyStore.class); /** Attributes governing the behavior of the block disk cache. */ private final BlockDiskCacheAttributes blockDiskCacheAttributes; @@ -79,35 +80,36 @@ public class BlockDiskKeyStore<K> /** * Set the configuration options. * <p> + * * @param cacheAttributes - * @param blockDiskCache used for freeing + * @param blockDiskCache + * used for freeing */ - public BlockDiskKeyStore( BlockDiskCacheAttributes cacheAttributes, - BlockDiskCache<K, ?> blockDiskCache) + public BlockDiskKeyStore(BlockDiskCacheAttributes cacheAttributes, BlockDiskCache<K, ?> blockDiskCache) { this.blockDiskCacheAttributes = cacheAttributes; this.logCacheName = "Region [" + this.blockDiskCacheAttributes.getCacheName() + "] "; this.fileName = this.blockDiskCacheAttributes.getCacheName(); this.maxKeySize = cacheAttributes.getMaxKeySize(); this.blockDiskCache = blockDiskCache; - this.diskLimitType = cacheAttributes.getDiskLimitType(); + this.diskLimitType = cacheAttributes.getDiskLimitType(); this.blockSize = cacheAttributes.getBlockSizeBytes(); File rootDirectory = cacheAttributes.getDiskPath(); - if ( log.isInfoEnabled() ) + if (log.isInfoEnabled()) { - log.info( logCacheName + "Cache file root directory [" + rootDirectory + "]" ); + log.info(logCacheName + "Cache file root directory [" + rootDirectory + "]"); } - this.keyFile = new File( rootDirectory, fileName + ".key" ); + this.keyFile = new File(rootDirectory, fileName + ".key"); - if ( log.isInfoEnabled() ) + if (log.isInfoEnabled()) { - log.info( logCacheName + "Key File [" + this.keyFile.getAbsolutePath() + "]" ); + log.info(logCacheName + "Key File [" + this.keyFile.getAbsolutePath() + "]"); } - if ( keyFile.length() > 0 ) + if (keyFile.length() > 0) { loadKeys(); // TODO verify somehow @@ -128,27 +130,26 @@ public class BlockDiskKeyStore<K> { ElapsedTimer timer = new ElapsedTimer(); int numKeys = keyHash.size(); - if ( log.isInfoEnabled() ) + if (log.isInfoEnabled()) { - log.info( logCacheName + "Saving keys to [" + this.keyFile.getAbsolutePath() + "], key count [" - + numKeys + "]" ); + log.info(logCacheName + "Saving keys to [" + this.keyFile.getAbsolutePath() + "], key count [" + numKeys + "]"); } synchronized (keyFile) { - FileOutputStream fos = new FileOutputStream( keyFile ); - BufferedOutputStream bos = new BufferedOutputStream( fos, 65536 ); - ObjectOutputStream oos = new ObjectOutputStream( bos ); + FileOutputStream fos = new FileOutputStream(keyFile); + BufferedOutputStream bos = new BufferedOutputStream(fos, 65536); + ObjectOutputStream oos = new ObjectOutputStream(bos); try { // don't need to synchronize, since the underlying collection makes a copy for (Map.Entry<K, int[]> entry : keyHash.entrySet()) { BlockDiskElementDescriptor<K> descriptor = new BlockDiskElementDescriptor<K>(); - descriptor.setKey( entry.getKey() ); - descriptor.setBlocks( entry.getValue() ); + descriptor.setKey(entry.getKey()); + descriptor.setBlocks(entry.getValue()); // stream these out in the loop. - oos.writeObject( descriptor ); + oos.writeUnshared(descriptor); } } finally @@ -158,15 +159,15 @@ public class BlockDiskKeyStore<K> } } - if ( log.isInfoEnabled() ) + if (log.isInfoEnabled()) { - log.info( logCacheName + "Finished saving keys. It took " + timer.getElapsedTimeString() + " to store " - + numKeys + " keys. Key file length [" + keyFile.length() + "]" ); + log.info(logCacheName + "Finished saving keys. It took " + timer.getElapsedTimeString() + " to store " + numKeys + + " keys. Key file length [" + keyFile.length() + "]"); } } - catch ( IOException e ) + catch (IOException e) { - log.error( logCacheName + "Problem storing keys.", e ); + log.error(logCacheName + "Problem storing keys.", e); } } @@ -180,7 +181,6 @@ public class BlockDiskKeyStore<K> clearMemoryMap(); saveKeys(); } - } /** @@ -197,16 +197,19 @@ public class BlockDiskKeyStore<K> private void initKeyMap() { keyHash = null; - if ( maxKeySize >= 0 ) + if (maxKeySize >= 0) { - if (this.diskLimitType.equals(DiskLimitType.SIZE)) { + if (this.diskLimitType == DiskLimitType.SIZE) + { keyHash = new LRUMapSizeLimited(maxKeySize); - } else { - keyHash = new LRUMapCountLimited( maxKeySize ); } - if ( log.isInfoEnabled() ) + else { - log.info( logCacheName + "Set maxKeySize to: '" + maxKeySize + "'" ); + keyHash = new LRUMapCountLimited(maxKeySize); + } + if (log.isInfoEnabled()) + { + log.info(logCacheName + "Set maxKeySize to: '" + maxKeySize + "'"); } } else @@ -214,9 +217,9 @@ public class BlockDiskKeyStore<K> // If no max size, use a plain map for memory and processing efficiency. keyHash = new HashMap<K, int[]>(); // keyHash = Collections.synchronizedMap( new HashMap() ); - if ( log.isInfoEnabled() ) + if (log.isInfoEnabled()) { - log.info( logCacheName + "Set maxKeySize to unlimited'" ); + log.info(logCacheName + "Set maxKeySize to unlimited'"); } } } @@ -227,9 +230,9 @@ public class BlockDiskKeyStore<K> */ protected void loadKeys() { - if ( log.isInfoEnabled() ) + if (log.isInfoEnabled()) { - log.info( logCacheName + "Loading keys for " + keyFile.toString() ); + log.info(logCacheName + "Loading keys for " + keyFile.toString()); } try @@ -241,22 +244,23 @@ public class BlockDiskKeyStore<K> synchronized (keyFile) { - FileInputStream fis = new FileInputStream( keyFile ); - BufferedInputStream bis = new BufferedInputStream( fis ); - ObjectInputStream ois = new ObjectInputStreamClassLoaderAware( bis , null); + FileInputStream fis = new FileInputStream(keyFile); + BufferedInputStream bis = new BufferedInputStream(fis, 65536); + ObjectInputStream ois = new ObjectInputStreamClassLoaderAware(bis, null); try { - while ( true ) + while (true) { - @SuppressWarnings("unchecked") // Need to cast from Object + @SuppressWarnings("unchecked") + // Need to cast from Object BlockDiskElementDescriptor<K> descriptor = (BlockDiskElementDescriptor<K>) ois.readObject(); - if ( descriptor != null ) + if (descriptor != null) { - keys.put( descriptor.getKey(), descriptor.getBlocks() ); + keys.put(descriptor.getKey(), descriptor.getBlocks()); } } } - catch ( EOFException eof ) + catch (EOFException eof) { // nothing } @@ -266,31 +270,32 @@ public class BlockDiskKeyStore<K> } } - if ( !keys.isEmpty() ) + if (!keys.isEmpty()) { - keyHash.putAll( keys ); + keyHash.putAll(keys); - if ( log.isDebugEnabled() ) + if (log.isDebugEnabled()) { - log.debug( logCacheName + "Found " + keys.size() + " in keys file." ); + log.debug(logCacheName + "Found " + keys.size() + " in keys file."); } - if ( log.isInfoEnabled() ) + if (log.isInfoEnabled()) { - log.info( logCacheName + "Loaded keys from [" + fileName + "], key count: " + keyHash.size() - + "; up to " + maxKeySize + " will be available." ); + log.info(logCacheName + "Loaded keys from [" + fileName + "], key count: " + keyHash.size() + "; up to " + + maxKeySize + " will be available."); } } } - catch ( Exception e ) + catch (Exception e) { - log.error( logCacheName + "Problem loading keys for file " + fileName, e ); + log.error(logCacheName + "Problem loading keys for file " + fileName, e); } } /** * Gets the entry set. * <p> + * * @return entry set. */ public Set<Map.Entry<K, int[]>> entrySet() @@ -301,6 +306,7 @@ public class BlockDiskKeyStore<K> /** * Gets the key set. * <p> + * * @return key set. */ public Set<K> keySet() @@ -311,6 +317,7 @@ public class BlockDiskKeyStore<K> /** * Gets the size of the key hash. * <p> + * * @return the number of keys. */ public int size() @@ -321,91 +328,123 @@ public class BlockDiskKeyStore<K> /** * gets the object for the key. * <p> + * * @param key * @return Object */ - public int[] get( K key ) + public int[] get(K key) { - return this.keyHash.get( key ); + return this.keyHash.get(key); } /** * Puts a int[] in the keyStore. * <p> + * * @param key * @param value */ - public void put( K key, int[] value ) + public void put(K key, int[] value) { - this.keyHash.put( key, value ); + this.keyHash.put(key, value); } /** * Remove by key. * <p> + * * @param key * @return BlockDiskElementDescriptor if it was present, else null */ - public int[] remove( K key ) + public int[] remove(K key) { - return this.keyHash.remove( key ); + return this.keyHash.remove(key); } - /** * Class for recycling and lru. This implements the LRU size overflow callback, so we can mark the * blocks as free. */ - - public class LRUMapSizeLimited - extends AbstractLRUMap<K, int[]> - + public class LRUMapSizeLimited extends AbstractLRUMap<K, int[]> { /** * <code>tag</code> tells us which map we are working on. */ - public String tag = "orig-lru-size"; + public final String tag = "orig-lru-size"; + // size of the content in kB - private AtomicInteger contentSize = new AtomicInteger(); - private int maxSize = -1; + private AtomicInteger contentSize; + private int maxSize; + /** * Default */ public LRUMapSizeLimited() { - super(); + this(-1); } /** - * @param maxKeySize + * @param maxSize maximum cache size in kB */ - public LRUMapSizeLimited( int maxKeySize ) + public LRUMapSizeLimited(int maxSize) { super(); - this.maxSize = maxKeySize; + this.maxSize = maxSize; + this.contentSize = new AtomicInteger(0); + } + + // keep the content size in kB, so 2^31 kB is reasonable value + private void subLengthFromCacheSize(int[] value) + { + contentSize.addAndGet(value.length * blockSize / -1024 - 1); + } + + // keep the content size in kB, so 2^31 kB is reasonable value + private void addLengthToCacheSize(int[] value) + { + contentSize.addAndGet(value.length * blockSize / 1024 + 1); } @Override - public int[] put(K key, int[] value) { - try { - return super.put(key, value); - } finally { - // keep the content size in kB, so 2^31 kB is reasonable value - contentSize.addAndGet((int) Math.ceil(value.length * blockSize / 1024.0)); + public int[] put(K key, int[] value) + { + int[] oldValue = null; + + try + { + oldValue = super.put(key, value); } + finally + { + if (value != null) + { + addLengthToCacheSize(value); + } + if (oldValue != null) + { + subLengthFromCacheSize(oldValue); + } + } + + return oldValue; } @Override - public int[] remove(Object key ) { + public int[] remove(Object key) + { int[] value = null; - try { + try + { value = super.remove(key); return value; - } finally { - if (value != null) { - // keep the content size in kB, so 2^31 kB is reasonable value - contentSize.addAndGet((int) ((Math.ceil(value.length * blockSize / 1024.0)) * -1)); + } + finally + { + if (value != null) + { + subLengthFromCacheSize(value); } } } @@ -414,57 +453,65 @@ public class BlockDiskKeyStore<K> * This is called when the may key size is reached. The least recently used item will be * passed here. We will store the position and size of the spot on disk in the recycle bin. * <p> + * * @param key * @param value */ @Override - protected void processRemovedLRU( K key, int[] value ) + protected void processRemovedLRU(K key, int[] value) { - blockDiskCache.freeBlocks( value ); - if ( log.isDebugEnabled() ) + blockDiskCache.freeBlocks(value); + if (log.isDebugEnabled()) { - log.debug( logCacheName + "Removing key: [" + key + "] from key store." ); - log.debug( logCacheName + "Key store size: [" + super.size() + "]." ); + log.debug(logCacheName + "Removing key: [" + key + "] from key store."); + log.debug(logCacheName + "Key store size: [" + super.size() + "]."); + } + + if (value != null) + { + subLengthFromCacheSize(value); } } + @Override - protected boolean shouldRemove() { - return maxSize > 0 && contentSize.intValue() > maxSize && this.size() > 1; + protected boolean shouldRemove() + { + return maxSize > 0 && contentSize.get() > maxSize && this.size() > 1; } } + /** * Class for recycling and lru. This implements the LRU overflow callback, so we can mark the * blocks as free. */ - public class LRUMapCountLimited - extends LRUMap<K, int[]> - // implements Serializable + public class LRUMapCountLimited extends LRUMap<K, int[]> { /** * <code>tag</code> tells us which map we are working on. */ - public String tag = "orig-lru-count"; + public final String tag = "orig-lru-count"; - public LRUMapCountLimited(int maxKeySize) { + public LRUMapCountLimited(int maxKeySize) + { super(maxKeySize); } - /** * This is called when the may key size is reached. The least recently used item will be * passed here. We will store the position and size of the spot on disk in the recycle bin. * <p> + * * @param key * @param value */ @Override - protected void processRemovedLRU( K key, int[] value ) + protected void processRemovedLRU(K key, int[] value) { - blockDiskCache.freeBlocks( value ); - if ( log.isDebugEnabled() ) + blockDiskCache.freeBlocks(value); + if (log.isDebugEnabled()) { - log.debug( logCacheName + "Removing key: [" + key + "] from key store." ); - log.debug( logCacheName + "Key store size: [" + super.size() + "]." ); + log.debug(logCacheName + "Removing key: [" + key + "] from key store."); + log.debug(logCacheName + "Key store size: [" + super.size() + "]."); } } }