Author: asmuts
Date: Sun Aug 27 19:29:54 2006
New Revision: 437511
URL: http://svn.apache.org/viewvc?rev=437511&view=rev
Log:
Added configurable, periodic key persistence to the block disk cache.
Added rough disk file verification to block disk cache.
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java
(original)
+++
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java
Sun Aug 27 19:29:54 2006
@@ -64,8 +64,13 @@
*/
public BlockDisk( File file )
throws FileNotFoundException
- {
+ {
this( file, DEFAULT_BLOCK_SIZE_BYTES );
+ if ( log.isInfoEnabled() )
+ {
+ log.info( "Used default block size [" + DEFAULT_BLOCK_SIZE_BYTES +
"]" );
+ }
+
}
/**
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
(original)
+++
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java
Sun Aug 27 19:29:54 2006
@@ -92,10 +92,9 @@
try
{
- if ( cacheAttributes.getBlockSizeBytes() > 0 )
+ if ( this.blockDiskCacheAttributes.getBlockSizeBytes() > 0 )
{
- this.dataFile = new BlockDisk( new File( rootDirectory,
fileName + ".data" ), cacheAttributes
- .getBlockSizeBytes() );
+ this.dataFile = new BlockDisk( new File( rootDirectory,
fileName + ".data" ), this.blockDiskCacheAttributes.getBlockSizeBytes() );
}
else
{
@@ -104,7 +103,9 @@
keyStore = new BlockDiskKeyStore( this.blockDiskCacheAttributes,
this );
- if ( keyStore.size() == 0 )
+ boolean alright = verifyDisk();
+
+ if ( keyStore.size() == 0 || !alright )
{
this.reset();
}
@@ -113,7 +114,7 @@
alive = true;
if ( log.isInfoEnabled() )
{
- log.info( logCacheName + "Indexed Disk Cache is alive." );
+ log.info( logCacheName + "Block Disk Cache is alive." );
}
}
catch ( Exception e )
@@ -126,6 +127,43 @@
}
/**
+ * We need to verify that the file on disk uses the same block size and
that the file is the
+ * proper size.
+ * <p>
+ * @return true if it looks ok
+ */
+ protected boolean verifyDisk()
+ {
+ boolean alright = false;
+ // simply try to read a few. If it works, then the file is probably ok.
+ // TODO add more.
+ try
+ {
+ int maxToTest = 100;
+ int count = 0;
+ Set keySet = this.keyStore.entrySet();
+ Iterator it = keySet.iterator();
+ while ( it.hasNext() && count < maxToTest )
+ {
+ count++;
+ Map.Entry entry = (Map.Entry) it.next();
+ Object data = this.dataFile.read( (int[]) entry.getValue() );
+ if ( data == null )
+ {
+ throw new Exception( "Couldn't find data for key [" +
entry.getKey() + "]" );
+ }
+ }
+ alright = true;
+ }
+ catch ( Exception e )
+ {
+ log.warn( "Problem verifying disk. Message [" + e.getMessage() +
"]" );
+ alright = false;
+ }
+ return alright;
+ }
+
+ /**
* This requires a full iteration through the keys.
* <p>
* (non-Javadoc)
@@ -459,7 +497,6 @@
storageLock.writeLock().acquire();
try
{
-
// Prevents any interaction with the cache while we're shutting
down.
alive = false;
@@ -472,7 +509,7 @@
log.debug( logCacheName + "Closing files, base filename: "
+ fileName );
}
dataFile.close();
- dataFile = null;
+ // dataFile = null;
// TOD make a close
// keyFile.close();
@@ -527,8 +564,15 @@
File dataFileTemp = new File( this.rootDirectory, fileName +
".data" );
dataFileTemp.delete();
- dataFile = new BlockDisk( new File( this.rootDirectory, fileName +
".data" ) );
-
+ if ( this.blockDiskCacheAttributes.getBlockSizeBytes() > 0 )
+ {
+ this.dataFile = new BlockDisk( new File( rootDirectory,
fileName + ".data" ), this.blockDiskCacheAttributes.getBlockSizeBytes() );
+ }
+ else
+ {
+ this.dataFile = new BlockDisk( new File( rootDirectory,
fileName + ".data" ) );
+ }
+
this.keyStore.reset();
}
catch ( Exception e )
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
(original)
+++
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheAttributes.java
Sun Aug 27 19:29:54 2006
@@ -19,6 +19,12 @@
/** -1 means no limit. */
private int maxKeySize = DEFAULT_MAX_KEY_SIZE;
+ private static final long DEFAULT_KEY_PERSISTENCE_INTERVAL_SECONDS = 5 *
60;
+
+ /** The keys will be persisted at this interval. -1 mean never. */
+ private long keyPersistenceIntervalSeconds =
DEFAULT_KEY_PERSISTENCE_INTERVAL_SECONDS;
+
+
/**
* The size of the blocks. All blocks are the same size.
* <p>
@@ -54,6 +60,22 @@
}
/**
+ * @param keyPersistenceIntervalSeconds The keyPersistenceIntervalSeconds
to set.
+ */
+ public void setKeyPersistenceIntervalSeconds( long
keyPersistenceIntervalSeconds )
+ {
+ this.keyPersistenceIntervalSeconds = keyPersistenceIntervalSeconds;
+ }
+
+ /**
+ * @return Returns the keyPersistenceIntervalSeconds.
+ */
+ public long getKeyPersistenceIntervalSeconds()
+ {
+ return keyPersistenceIntervalSeconds;
+ }
+
+ /**
* Write out the values for debugging purposes.
* <p>
* @return String
@@ -66,6 +88,7 @@
str.append( "\n MaxKeySize [" + this.getMaxKeySize() + "]" );
str.append( "\n MaxPurgatorySize [" + this.getMaxPurgatorySize() + "]"
);
str.append( "\n BlockSizeBytes [" + this.getBlockSizeBytes() + "]" );
+ str.append( "\n KeyPersistenceIntervalSeconds [" +
this.getKeyPersistenceIntervalSeconds() + "]" );
return str.toString();
}
}
Modified:
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
(original)
+++
jakarta/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
Sun Aug 27 19:29:54 2006
@@ -28,6 +28,9 @@
import org.apache.jcs.auxiliary.disk.LRUMapJCS;
import org.apache.jcs.utils.timing.ElapsedTimer;
+import EDU.oswego.cs.dl.util.concurrent.ClockDaemon;
+import EDU.oswego.cs.dl.util.concurrent.ThreadFactory;
+
/**
* This is responsible for storing the keys.
* <p>
@@ -35,7 +38,7 @@
*/
public class BlockDiskKeyStore
{
- private static final Log log = LogFactory.getLog( BlockDiskCache.class );
+ private static final Log log = LogFactory.getLog( BlockDiskKeyStore.class
);
private BlockDiskCacheAttributes blockDiskCacheAttributes;
@@ -55,6 +58,11 @@
private File rootDirectory;
/**
+ * The background key persister, one for all regions.
+ */
+ private static ClockDaemon persistenceDaemon;
+
+ /**
* Set the configuration options.
* <p>
* @param cacheAttributes
@@ -95,6 +103,26 @@
{
initKeyMap();
}
+
+ // add this region to the persistence thread.
+ // TODO we might need to stagger this a bit.
+ if ( this.blockDiskCacheAttributes.getKeyPersistenceIntervalSeconds()
> 0 )
+ {
+ if ( persistenceDaemon == null )
+ {
+ persistenceDaemon = new ClockDaemon();
+ persistenceDaemon.setThreadFactory( new MyThreadFactory() );
+ }
+ persistenceDaemon
+ .executePeriodically(
this.blockDiskCacheAttributes.getKeyPersistenceIntervalSeconds() * 1000,
+ new Runnable()
+ {
+ public void run()
+ {
+ saveKeys();
+ }
+ }, false );
+ }
}
/**
@@ -369,6 +397,27 @@
log.debug( logCacheName + "Removing key: [" + key + "] from
key store." );
log.debug( logCacheName + "Key store size: [" + this.size() +
"]." );
}
+ }
+ }
+
+ /**
+ * Allows us to set the daemon status on the clockdaemon
+ * @author aaronsm
+ */
+ class MyThreadFactory
+ implements ThreadFactory
+ {
+
+ /*
+ * (non-Javadoc)
+ * @see
EDU.oswego.cs.dl.util.concurrent.ThreadFactory#newThread(java.lang.Runnable)
+ */
+ public Thread newThread( Runnable runner )
+ {
+ Thread t = new Thread( runner );
+ t.setDaemon( true );
+ t.setPriority( Thread.MIN_PRIORITY );
+ return t;
}
}
}
Modified: jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
--- jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf (original)
+++ jakarta/jcs/trunk/src/test-conf/TestBlockDiskCacheHuge.ccf Sun Aug 27
19:29:54 2006
@@ -20,9 +20,9 @@
# Block Disk Cache
jcs.auxiliary.blockDiskCache=org.apache.jcs.auxiliary.disk.block.BlockDiskCacheFactory
jcs.auxiliary.blockDiskCache.attributes=org.apache.jcs.auxiliary.disk.block.BlockDiskCacheAttributes
-jcs.auxiliary.blockDiskCache.attributes.DiskPath=target/test-sandbox/block-disk-cache-conc
+jcs.auxiliary.blockDiskCache.attributes.DiskPath=target/test-sandbox/block-disk-cache-huge
jcs.auxiliary.blockDiskCache.attributes.MaxPurgatorySize=300000
-jcs.auxiliary.blockDiskCache.attributes.MaxKeySize=500000
+jcs.auxiliary.blockDiskCache.attributes.MaxKeySize=1000000
jcs.auxiliary.blockDiskCache.attributes.blockSizeBytes=500
jcs.auxiliary.blockDiskCache.attributes.EventQueueType=SINGLE
#jcs.auxiliary.blockDiskCache.attributes.EventQueuePoolName=disk_cache_event_queue
Modified:
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java
URL:
http://svn.apache.org/viewvc/jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java?rev=437511&r1=437510&r2=437511&view=diff
==============================================================================
---
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java
(original)
+++
jakarta/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/HugeQuantityBlockDiskCacheLoadTest.java
Sun Aug 27 19:29:54 2006
@@ -3,6 +3,8 @@
import junit.framework.TestCase;
import org.apache.jcs.JCS;
+import org.apache.jcs.utils.timing.ElapsedTimer;
+import org.apache.jcs.utils.timing.SleepUtil;
/**
* Put a few hundred thousand entries in the block disk cache.
@@ -38,15 +40,19 @@
int items = 300000;
String region = "testCache1";
- JCS jcs = JCS.getInstance( region );
+ System.out.println( "--------------------------" );
+ long initialMemory = measureMemoryUse();
+ System.out.println( "Before getting JCS: " + initialMemory );
+ JCS jcs = JCS.getInstance( region );
+ jcs.clear();
+
try
{
-
+ ElapsedTimer timer = new ElapsedTimer();
System.out.println( "Start: " + measureMemoryUse() );
// Add items to cache
-
for ( int i = 0; i <= items; i++ )
{
jcs.put( i + ":key", region + " data " + i );
@@ -61,24 +67,40 @@
System.out.println( jcs.getStats() );
System.out.println( "--------------------------" );
System.out.println( "After wait: " + measureMemoryUse() );
+
+ for ( int i = 0; i < 10; i++ )
+ {
+ SleepUtil.sleepAtLeast( 3000 );
+ System.out.println( "--------------------------" );
+ System.out.println( "After sleep. " +
timer.getElapsedTimeString() + " memory used = " + measureMemoryUse() );
+ System.out.println( jcs.getStats() );
+ }
// Test that all items are in cache
-
+ System.out.println( "--------------------------" );
+ System.out.println( "Retrieving all." );
for ( int i = 0; i <= items; i++ )
{
+ //System.out.print( "\033[s" );
String value = (String) jcs.get( i + ":key" );
-
- assertEquals( region + " data " + i, value );
+ if( i % 1000 == 0 )
+ {
+ //System.out.print( "\033[r" );
+ System.out.println( i + " ");
+ }
+ assertEquals( "Wrong value returned.", region + " data " + i,
value );
}
-
- System.out.println( "After get: " + measureMemoryUse() );
+ long aftetGet = measureMemoryUse();
+ System.out.println( "After get: " + aftetGet + " diff = " +
(aftetGet - initialMemory));
+
}
finally
{
// dump the stats to the report
System.out.println( jcs.getStats() );
System.out.println( "--------------------------" );
- System.out.println( "End: " + measureMemoryUse() );
+ long endMemory = measureMemoryUse();
+ System.out.println( "End: " + endMemory + " diff = " + (endMemory
- initialMemory) );
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
