taylor 2002/11/13 22:52:41 Modified: build build.xml src/java/org/apache/jetspeed/services/psmlmanager CastorPsmlManagerService.java webapp/WEB-INF/conf JetspeedResources.properties JetspeedSecurity.properties Added: src/java/org/apache/jetspeed/cache FileCache.java FileCacheEntry.java FileCacheEventListener.java TestFileCache.java test/testdata/psml/user/cachetest default.psml webapp/WEB-INF/conf HttpAgentLog4j.properties Log: Refactored the CastorPSMLManager service. After several days of (unplanned) JMeter testing, was seeing that: - the weakreferences *Never* worked. The cache was being completely bypassed - if the document failed to load for some reason, we had an infinite loop crash in the getDocument method - still not sure why the document was failing to load, but I can no longer repro with refactored impl The new implementation makes use of some new classes in the org.apache.jetspeed.cache package: * FileCache - a file cache, used by the CastorPSMLManager, to manager the caching of psml resources * FileCachEventListener - events are sent back to the CastorPSMLManager when a doc is refreshed or evicted * The JR.p now supports to more settings: - scanRate - in seconds for refreshing the cache - cacheSize - the max size of the cache before eviction kicks. Eviction is based least used * unit test for FileCache Revision Changes Path 1.167 +6 -1 jakarta-jetspeed/build/build.xml Index: build.xml =================================================================== RCS file: /home/cvs/jakarta-jetspeed/build/build.xml,v retrieving revision 1.166 retrieving revision 1.167 diff -u -r1.166 -r1.167 --- build.xml 7 Nov 2002 20:25:17 -0000 1.166 +++ build.xml 14 Nov 2002 06:52:36 -0000 1.167 @@ -261,7 +261,10 @@ <include name="org/apache/jetspeed/services/security/UserManagement.java"/> <include name="org/apache/jetspeed/util/MetaData.java"/> <!-- <include name="org/apache/jetspeed/services/security/ldap/UnixCrypt.java"/> --> - <include name="org.apache/jetspeed/services/security/ldap/JetspeedLDAPSecurityService"/> + <include name="org.apache/jetspeed/services/security/ldap/JetspeedLDAPSecurityService"/> + <include name="org.apache/jetspeed/cache/FileCache.java"/> + <include name="org.apache/jetspeed/cache/FileCacheEntry.java"/> + <include name="org.apache/jetspeed/cache/TestFileCache.java"/> </patternset> </fileset> </checkstyle> @@ -975,6 +978,7 @@ <test name="org.apache.jetspeed.util.rewriter.FrameRewriterTest"/> <test name="org.apache.jetspeed.services.profiler.TestProfilerService"/> <test name="org.apache.jetspeed.services.registry.TestRegistryCategories"/> + <test name="org.apache.jetspeed.cache.TestFileCache"/> <!-- <test name="org.apache.jetspeed.services.registry.TestRegistryPersistence"/> --> @@ -1212,6 +1216,7 @@ </ant> </target> + <!-- =================================================================== --> <!-- Include the usage target and the test targets for the different --> 1.1 jakarta-jetspeed/src/java/org/apache/jetspeed/cache/FileCache.java Index: FileCache.java =================================================================== /* ==================================================================== * The Apache Software License, Version 1.1 * * Copyright (c) 2000-2001 The Apache Software Foundation. All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The end-user documentation included with the redistribution, * if any, must include the following acknowledgment: * "This product includes software developed by the * Apache Software Foundation (http://www.apache.org/)." * Alternately, this acknowledgment may appear in the software itself, * if and wherever such third-party acknowledgments normally appear. * * 4. The names "Apache" and "Apache Software Foundation" and * "Apache Jetspeed" must not be used to endorse or promote products * derived from this software without prior written permission. For * written permission, please contact [EMAIL PROTECTED] * * 5. Products derived from this software may not be called "Apache" or * "Apache Jetspeed", nor may "Apache" appear in their name, without * prior written permission of the Apache Software Foundation. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation. For more * information on the Apache Software Foundation, please see * <http://www.apache.org/>. */ package org.apache.jetspeed.cache; import java.util.Collections; import java.util.Date; import java.util.Map; import java.util.HashMap; import java.util.List; import java.util.LinkedList; import java.util.Iterator; import java.io.File; import org.apache.turbine.util.Log; /** * FileCache keeps a cache of files up-to-date with a most simple eviction policy. * The eviction policy will keep n items in the cache, and then start evicting * the items ordered-by least used first. The cache runs a thread to check for * both evictions and refreshes. * * @author David S. Taylor <a href="mailto:taylor@;apache.org">David Sean Taylor</a> * @version $Id: FileCache.java,v 1.1 2002/11/14 06:52:36 taylor Exp $ */ public class FileCache implements java.util.Comparator { protected long scanRate = 300; // every 5 minutes protected int maxSize = 100; // maximum of 100 items protected List listeners = new LinkedList(); private FileCacheScanner scanner = null; private Map cache = null; /** * Default constructor. Use default values for scanReate and maxSize * */ public FileCache() { cache = new HashMap(); this.scanner = new FileCacheScanner(); this.scanner.setDaemon(true); } /** * Set scanRate and maxSize * * @param scanRate how often in seconds to refresh and evict from the cache * @param maxSize the maximum allowed size of the cache before eviction starts */ public FileCache(long scanRate, int maxSize) { cache = new HashMap(); this.scanRate = scanRate; this.maxSize = maxSize; this.scanner = new FileCacheScanner(); this.scanner.setDaemon(true); } /** * Set all parameters on the cache * * @param initialCapacity the initial size of the cache as passed to HashMap * @param loadFactor how full the hash table is allowed to get before increasing * @param scanRate how often in seconds to refresh and evict from the cache * @param maxSize the maximum allowed size of the cache before eviction starts */ public FileCache(int initialCapacity, int loadFactor, long scanRate, int maxSize) { cache = new HashMap(initialCapacity, loadFactor); this.scanRate = scanRate; this.maxSize = maxSize; this.scanner = new FileCacheScanner(); this.scanner.setDaemon(true); } /** * Set the new refresh scan rate on managed files. * * @param scanRate the new scan rate in seconds */ public void setScanRate(long scanRate) { this.scanRate= scanRate; } /** * Get the refresh scan rate * * @return the current refresh scan rate in seconds */ public long getScanRate() { return scanRate; } /** * Set the new maximum size of the cache * * @param maxSize the maximum size of the cache */ public void setMaxSize(int maxSize) { this.maxSize = maxSize; } /** * Get the maximum size of the cache * * @return the current maximum size of the cache */ public int getMaxSize() { return maxSize; } /** * Gets an entry from the cache given a key * * @param key the key to look up the entry by * @return the entry */ public FileCacheEntry get(String key) { return (FileCacheEntry) cache.get(key); } /** * Gets an entry from the cache given a key * * @param key the key to look up the entry by * @return the entry */ public Object getDocument(String key) { FileCacheEntry entry = (FileCacheEntry) cache.get(key); if (entry != null) { return entry.getDocument(); } return null; } /** * Puts a file entry in the file cache * * @param file The file to be put in the cache * @param document the cached document */ public void put(File file, Object document) throws java.io.IOException { FileCacheEntry entry = new FileCacheEntry(file, document); cache.put(file.getCanonicalPath(), entry); } /** * Puts a file entry in the file cache * * @param path the full path name of the file * @param document the cached document */ public void put(String key, Object document) throws java.io.IOException { File file = new File(key); FileCacheEntry entry = new FileCacheEntry(file, document); cache.put(file.getCanonicalPath(), entry); } /** * Removes a file entry from the file cache * * @param key the full path name of the file * @return the entry removed */ public Object remove(String key) { return cache.remove(key); } /** * Add a File Cache Event Listener * * @param listener the event listener */ public void addListener(FileCacheEventListener listener) { listeners.add(listener); } /** * Start the file Scanner running at the current scan rate. * */ public void startFileScanner() { try { this.scanner.start(); } catch (java.lang.IllegalThreadStateException e) { Log.error("Exception starting scanner", e); } } /** * Stop the file Scanner * */ public void stopFileScanner() { this.scanner.setStopping(true); } /** * Evicts entries based on last accessed time stamp * */ protected void evict() { synchronized (cache) { if (this.getMaxSize() >= cache.size()) { return; } List list = new LinkedList(cache.values()); Collections.sort(list, this); int count = 0; int limit = cache.size() - this.getMaxSize(); for (Iterator it = list.iterator(); it.hasNext(); ) { if (count >= limit) { break; } FileCacheEntry entry = (FileCacheEntry) it.next(); String key = null; try { key = entry.getFile().getCanonicalPath(); } catch (java.io.IOException e) { Log.error("Exception getting file path: ", e); } // notify that eviction will soon take place for (Iterator lit = this.listeners.iterator(); lit.hasNext(); ) { FileCacheEventListener listener = (FileCacheEventListener) lit.next(); listener.evict(entry); } cache.remove(key); count++; } } } /** * Comparator function for sorting by last accessed during eviction * */ public int compare(Object o1, Object o2) { FileCacheEntry e1 = (FileCacheEntry)o1; FileCacheEntry e2 = (FileCacheEntry)o2; if (e1.getLastAccessed() < e2.getLastAccessed()) { return -1; } else if (e1.getLastAccessed() == e2.getLastAccessed()) { return 0; } return 1; } /** * inner class that runs as a thread to scan the cache for updates or evictions * */ protected class FileCacheScanner extends Thread { private boolean stopping = false; public void setStopping(boolean flag) { this.stopping = flag; } /** * Run the file scanner thread * */ public void run() { boolean done = false; try { while(!done) { try { int count = 0; for (Iterator it = FileCache.this.cache.values().iterator(); it.hasNext(); ) { FileCacheEntry entry = (FileCacheEntry) it.next(); Date modified = new Date(entry.getFile().lastModified()); if (modified.after(entry.getLastModified())) { for (Iterator lit = FileCache.this.listeners.iterator(); lit.hasNext(); ) { FileCacheEventListener listener = (FileCacheEventListener) lit.next(); listener.refresh(entry); entry.setLastModified(modified); } } count++; } if (count > FileCache.this.getMaxSize()) { FileCache.this.evict(); } } catch (Exception e) { Log.error("FileCache Scanner: Error in iteration...", e); } sleep(FileCache.this.getScanRate() * 1000); if (this.stopping) { this.stopping = false; done = true; } } } catch (InterruptedException e) { Log.error("FileCacheScanner: recieved interruption, exiting.", e); } } } // end inner class: FileCacheScanner /** * get an iterator over the cache values * * @return iterator over the cache values */ public Iterator getIterator() { return cache.values().iterator(); } /** * get the size of the cache * * @return the size of the cache */ public int getSize() { return cache.size(); } } 1.1 jakarta-jetspeed/src/java/org/apache/jetspeed/cache/FileCacheEntry.java Index: FileCacheEntry.java =================================================================== /* ==================================================================== * The Apache Software License, Version 1.1 * * Copyright (c) 2000-2001 The Apache Software Foundation. All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The end-user documentation included with the redistribution, * if any, must include the following acknowledgment: * "This product includes software developed by the * Apache Software Foundation (http://www.apache.org/)." * Alternately, this acknowledgment may appear in the software itself, * if and wherever such third-party acknowledgments normally appear. * * 4. The names "Apache" and "Apache Software Foundation" and * "Apache Jetspeed" must not be used to endorse or promote products * derived from this software without prior written permission. For * written permission, please contact [EMAIL PROTECTED] * * 5. Products derived from this software may not be called "Apache" or * "Apache Jetspeed", nor may "Apache" appear in their name, without * prior written permission of the Apache Software Foundation. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation. For more * information on the Apache Software Foundation, please see * <http://www.apache.org/>. */ package org.apache.jetspeed.cache; import java.util.Date; import java.util.HashMap; import java.io.File; /** * FileCache entry keeps the cached content along with last access information. * * @author David S. Taylor <a href="mailto:taylor@;apache.org">David Sean Taylor</a> * @version $Id: FileCacheEntry.java,v 1.1 2002/11/14 06:52:36 taylor Exp $ */ public class FileCacheEntry { protected File file; protected Object document; protected long lastAccessed; protected Date lastModified; private FileCacheEntry() { } /** * Constructs a FileCacheEntry object * * @param document The user specific content being cached * @param lastModified The document's last modified stamp */ public FileCacheEntry(File file, Object document) { this.file = file; this.document = document; this.lastModified = new Date(file.lastModified()); this.lastAccessed = new Date().getTime(); } /** * Get the file descriptor * * @return the file descriptor */ public File getFile() { return this.file; } /** * Set the file descriptor * * @param file the new file descriptor */ public void setFile(File file) { this.file = file; } /** * Set the cache's last accessed stamp * * @param lastAccessed the cache's last access stamp */ public void setLastAccessed(long lastAccessed) { this.lastAccessed = lastAccessed; } /** * Get the cache's lastAccessed stamp * * @return the cache's last accessed stamp */ public long getLastAccessed() { return this.lastAccessed; } /** * Set the cache's last modified stamp * * @param lastModified the cache's last modified stamp */ public void setLastModified(Date lastModified) { this.lastModified = lastModified; } /** * Get the entry's lastModified stamp (which may be stale compared to file's stamp) * * @return the last modified stamp */ public Date getLastModified() { return this.lastModified; } /** * Set the Document in the cache * * @param document the document being cached */ public void setDocument(Object document) { this.document = document; } /** * Get the Document * * @return the document being cached */ public Object getDocument() { return this.document; } } 1.1 jakarta-jetspeed/src/java/org/apache/jetspeed/cache/FileCacheEventListener.java Index: FileCacheEventListener.java =================================================================== /* ==================================================================== * The Apache Software License, Version 1.1 * * Copyright (c) 2000-2001 The Apache Software Foundation. All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The end-user documentation included with the redistribution, * if any, must include the following acknowledgment: * "This product includes software developed by the * Apache Software Foundation (http://www.apache.org/)." * Alternately, this acknowledgment may appear in the software itself, * if and wherever such third-party acknowledgments normally appear. * * 4. The names "Apache" and "Apache Software Foundation" and * "Apache Jetspeed" must not be used to endorse or promote products * derived from this software without prior written permission. For * written permission, please contact [EMAIL PROTECTED] * * 5. Products derived from this software may not be called "Apache" or * "Apache Jetspeed", nor may "Apache" appear in their name, without * prior written permission of the Apache Software Foundation. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation. For more * information on the Apache Software Foundation, please see * <http://www.apache.org/>. */ package org.apache.jetspeed.cache; import java.util.Date; import java.util.HashMap; /** * FileCacheEventListener on notifications sent when FileCache events occur * * @author David S. Taylor <a href="mailto:taylor@;apache.org">David Sean Taylor</a> * @version $Id: FileCacheEventListener.java,v 1.1 2002/11/14 06:52:36 taylor Exp $ */ public interface FileCacheEventListener { /** * Refresh event, called when the entry is being refreshed from file system. * * @param entry the entry being refreshed. */ void refresh(FileCacheEntry entry); /** * Evict event, called when the entry is being evicted out of the cache * * @param entry the entry being refreshed. */ void evict(FileCacheEntry entry); } 1.1 jakarta-jetspeed/src/java/org/apache/jetspeed/cache/TestFileCache.java Index: TestFileCache.java =================================================================== /* ==================================================================== * The Apache Software License, Version 1.1 * * Copyright (c) 2000-2001 The Apache Software Foundation. All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The end-user documentation included with the redistribution, * if any, must include the following acknowledgment: * "This product includes software developed by the * Apache Software Foundation (http://www.apache.org/)." * Alternately, this acknowledgment may appear in the software itself, * if and wherever such third-party acknowledgments normally appear. * * 4. The names "Apache" and "Apache Software Foundation" and * "Apache Jetspeed" must not be used to endorse or promote products * derived from this software without prior written permission. For * written permission, please contact [EMAIL PROTECTED] * * 5. Products derived from this software may not be called "Apache" or * "Apache Jetspeed", nor may "Apache" appear in their name, without * prior written permission of the Apache Software Foundation. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * ==================================================================== * * This software consists of voluntary contributions made by many * individuals on behalf of the Apache Software Foundation. For more * information on the Apache Software Foundation, please see * <http://www.apache.org/>. */ package org.apache.jetspeed.cache; import java.util.Iterator; import java.io.File; import java.util.Date; import java.io.BufferedInputStream; import java.io.FileInputStream; // Junit imports import junit.framework.Test; import junit.framework.TestSuite; import junit.framework.TestCase; import org.apache.jetspeed.util.FileCopy; import org.apache.jetspeed.util.Streams; import org.apache.turbine.util.StringUtils; /** * Unit test for FileCache * * @author <a href="mailto:david@;bluesunrise.com">David Sean Taylor</a> * @version $Id: TestFileCache.java,v 1.1 2002/11/14 06:52:37 taylor Exp $ */ public class TestFileCache extends TestCase implements FileCacheEventListener { String refreshedEntry = null; /** * Defines the testcase name for JUnit. * * @param name the testcase's name. */ public TestFileCache( String name ) { super( name ); } /** * Start the tests. * * @param args the arguments. Not used */ public static void main(String args[]) { junit.awtui.TestRunner.main( new String[] { TestFileCache.class.getName() } ); } /** * Creates the test suite. * * @return a test suite (<code>TestSuite</code>) that includes all methods * starting with "test" */ public static Test suite() { // All methods starting with "test" will be executed in the test suite. return new TestSuite( TestFileCache.class ); } /** * Tests loading the cache * @throws Exception */ public void testLoadCache() throws Exception { String templateFile = "../test/testdata/psml/user/cachetest/default.psml"; try { File file = new File(templateFile); assertTrue(file.exists()); createTestFiles(templateFile); // create the Cache wake up after 10 seconds, cache size 20 FileCache cache = new FileCache(10, 20); // load the Cache File directory = new File("../test/testdata/psml/user/cachetest/"); File[] files = directory.listFiles(); for (int ix=0; ix < files.length; ix++) { String testData = readFile(files[ix]); cache.put(files[ix], testData); } assertTrue(cache.getSize() == 31); dumpCache(cache.getIterator()); cache.addListener(this); // start the cache's scanner cache.startFileScanner(); Thread.currentThread().sleep(2000); assertTrue(cache.getSize() == 20); dumpCache(cache.getIterator()); String stuff = (String) cache.getDocument(files[18].getCanonicalPath()); assertNotNull(stuff); files[18].setLastModified(new Date().getTime()); Thread.currentThread().sleep(9000); assertNotNull(refreshedEntry); System.out.println("refreshed entry = " + refreshedEntry); cache.stopFileScanner(); removeTestFiles(); } catch (Exception e) { fail(StringUtils.stackTrace(e)); } System.out.println("Completed loadCache Test OK "); } private void createTestFiles(String templateFile) throws java.io.IOException { for (int ix=1; ix < 31; ix++) { String testFile = "../test/testdata/psml/user/cachetest/testFile-" + ix + ".psml"; FileCopy.copy(templateFile, testFile); } } private void removeTestFiles() { for (int ix=1; ix < 31; ix++) { String testFile = "../test/testdata/psml/user/cachetest/testFile-" + ix + ".psml"; File file = new File(testFile); file.delete(); } } private String readFile(File file) throws java.io.IOException, java.io.FileNotFoundException { BufferedInputStream input; input = new BufferedInputStream(new FileInputStream(file)); String result = Streams.getAsString(input); input.close(); return result; } /** * Refresh event, called when the entry is being refreshed from file system. * * @param entry the entry being refreshed. */ public void refresh(FileCacheEntry entry) { System.out.println("entry is refreshing: " + entry.getFile().getName()); this.refreshedEntry = entry.getFile().getName(); } /** * Evict event, called when the entry is being evicted out of the cache * * @param entry the entry being refreshed. */ public void evict(FileCacheEntry entry) { System.out.println("entry is evicting: " + entry.getFile().getName()); } private void dumpCache(Iterator it) { for ( ; it.hasNext(); ) { FileCacheEntry entry = (FileCacheEntry) it.next(); System.out.println(entry.getFile().getName()); } } } 1.35 +105 -196 jakarta-jetspeed/src/java/org/apache/jetspeed/services/psmlmanager/CastorPsmlManagerService.java Index: CastorPsmlManagerService.java =================================================================== RCS file: /home/cvs/jakarta-jetspeed/src/java/org/apache/jetspeed/services/psmlmanager/CastorPsmlManagerService.java,v retrieving revision 1.34 retrieving revision 1.35 diff -u -r1.34 -r1.35 --- CastorPsmlManagerService.java 8 Nov 2002 10:05:18 -0000 1.34 +++ CastorPsmlManagerService.java 14 Nov 2002 06:52:37 -0000 1.35 @@ -115,9 +115,13 @@ import java.util.LinkedList; import java.util.Map; import java.util.HashMap; -import java.lang.ref.WeakReference; import javax.servlet.ServletConfig; +import org.apache.jetspeed.cache.FileCache; +import org.apache.jetspeed.cache.FileCacheEventListener; +import org.apache.jetspeed.cache.FileCacheEntry; + + /** * This service is responsible for loading and saving PSML documents. * @@ -126,8 +130,9 @@ * @author <a href="mailto:sgala@;apache.org">Santiago Gala</a> * @version $Id$ */ -public class CastorPsmlManagerService extends TurbineBaseService - implements PsmlManagerService +public class CastorPsmlManagerService extends TurbineBaseService + implements FileCacheEventListener, + PsmlManagerService { // resource path constants protected static final String PATH_GROUP = "group"; @@ -137,6 +142,8 @@ // configuration keys protected final static String CONFIG_ROOT = "root"; protected final static String CONFIG_EXT = "ext"; + protected final static String CONFIG_SCAN_RATE = "scanRate"; + protected final static String CONFIG_CACHE_SIZE = "cacheSize"; // default configuration values public final static String DEFAULT_ROOT = "/WEB-INF/psml"; @@ -153,16 +160,16 @@ protected String ext; /** The documents loaded by this manager */ - protected Map documents = new HashMap(); - - /** The watcher for the document locations */ - protected DocumentWatcher watcher = null; + protected FileCache documents = null; /** the output format for pretty printing when saving registries */ protected OutputFormat format = null; /** the base refresh rate for documents */ - protected long scanRate = 1000 * 60; + protected long scanRate = 1000 * 60; // every minute + + /** the default cache size */ + protected int cacheSize = 100; /** the import/export consumer service **/ protected PsmlManagerService consumer = null; @@ -233,8 +240,13 @@ mapFile = TurbineServlet.getRealPath( mapFile ); loadMapping(); - this.watcher = new DocumentWatcher(); - this.watcher.start(); + this.scanRate = serviceConf.getLong(CONFIG_SCAN_RATE, this.scanRate); + this.cacheSize= serviceConf.getInt(CONFIG_CACHE_SIZE, this.cacheSize); + + documents = new FileCache(this.scanRate, this.cacheSize); + documents.addListener(this); + documents.startFileScanner(); + //Mark that we are done setInit(true); @@ -269,7 +281,7 @@ */ public void shutdown() { - this.watcher.setDone(true); + documents.stopFileScanner(); } /** @@ -295,17 +307,8 @@ } PSMLDocument doc = null; - WeakReference ref = null; - - synchronized (documents) - { - ref = (WeakReference)documents.get(name); - } - if (ref != null) - { - doc = (PSMLDocument)ref.get(); - } + doc = (PSMLDocument)documents.getDocument(name); if (doc == null) { @@ -314,7 +317,14 @@ synchronized (documents) { // store the document in the hash and reference it to the watcher - documents.put(name, new WeakReference(doc)); + try + { + documents.put(name, doc); + } + catch (java.io.IOException e) + { + Log.error("Error puttin document: " + e); + } } } @@ -354,31 +364,9 @@ } PSMLDocument doc = null; - WeakReference ref = null; Profile profile = null; - synchronized (documents) - { - ref = (WeakReference)documents.get(name); - } - - if (ref != null) - { - /** - * Christophe Mestrallet ([EMAIL PROTECTED]) - * detected problems here. - * Since there is a fallback (reloading) I (SGP) will - * just catch the exception. The real problem is down, when - * we used a null profile as a key (profile --> newProfile) - */ - try - { - profile = (Profile)ref.get(); - } catch ( ClassCastException e ) - { - Log.error( e ); - } - } + profile = (Profile)documents.getDocument(name); if (profile == null) { @@ -397,11 +385,20 @@ // store the document in the hash and reference it to the watcher Profile newProfile = createProfile(locator); newProfile.setDocument(doc); - documents.put(name, new WeakReference(newProfile)); + try + { + documents.put(name, newProfile); + } + catch (IOException e) + { + Log.error("Error putting document: " + e); + } } } else + { doc = profile.getDocument(); + } return doc; } @@ -444,26 +441,26 @@ doc.setPortlets(portlets); - if (this.watcher != null) - { - this.watcher.addFile(fileOrUrl,f); - } } catch (IOException e) { Log.error("PSMLManager: Could not load the file "+f.getAbsolutePath(), e); + doc = null; } catch (MarshalException e) { Log.error("PSMLManager: Could not unmarshal the file "+f.getAbsolutePath(), e); + doc = null; } catch (MappingException e) { Log.error("PSMLManager: Could not unmarshal the file "+f.getAbsolutePath(), e); + doc = null; } catch (ValidationException e) { Log.error("PSMLManager: document "+f.getAbsolutePath()+" is not valid", e); + doc = null; } finally { @@ -474,57 +471,6 @@ return doc; } - /** Refresh a named document - * - * @param name the name of the document to refresh - */ - public void refresh(String name) throws IOException - { - PSMLDocument doc = null; - WeakReference ref = null; - - synchronized (documents) - { - ref = (WeakReference)documents.get(name); - } - - if (ref != null) - { - doc = (PSMLDocument)ref.get(); - - if (doc!=null) - { - File f = getFile(name); - FileReader reader = null; - try - { - reader = new FileReader(f); - doc.setPortlets(load(reader)); - } - catch (Exception e) - { - Log.error("PSMLManager: Error refrshing "+f.getAbsolutePath(),e); - } - finally - { - try { reader.close(); } catch (Exception e) {} - } - return; - } - else - { - // make sure the bogus key is removed - synchronized (documents) - { - documents.remove(name); - } - } - } - - // we don't know anything about this document, load it normally - getDocument(name); - } - /** Store the PSML document on disk, using its locator * * @param profile the profile locator description. @@ -554,7 +500,14 @@ // update it in cache synchronized (documents) { - documents.put(fullpath, new WeakReference(profile)); + try + { + documents.put(fullpath, profile); + } + catch (IOException e) + { + Log.error("Error storing document: " + e); + } } return ok; @@ -739,83 +692,6 @@ return null; } - protected class DocumentWatcher extends Thread - { - protected Map fileToName = new HashMap(); - protected Map fileToDate = new HashMap(); - protected boolean done = false; - - protected DocumentWatcher() - { - setDaemon(true); - setPriority(Thread.MIN_PRIORITY+1); - } - - protected void addFile(String name, File f) - { - synchronized (this) - { - fileToName.put(f, name); - fileToDate.put(f, new Date()); - } - } - - protected void removeFile(File f) - { - synchronized (this) - { - fileToName.remove(f); - fileToDate.remove(f); - } - } - - public void setDone(boolean done) - { - this.done = done; - } - - public void run() - { - try - { - while(!done) - { - try - { - Iterator i = fileToDate.keySet().iterator(); - - while(i.hasNext()) - { - File f = (File)i.next(); - Date modified = new Date(f.lastModified()); - - synchronized (this) - { - if ( modified.after((Date)fileToDate.get(f)) ) - { - CastorPsmlManagerService. - this.refresh((String)fileToName.get(f)); - fileToDate.put(f,modified); - } - } - } - } - catch (Exception e) - { - Log.error("DocumentWatcher: Error in iteration...", e); - } - - sleep(CastorPsmlManagerService.this.scanRate); - } - } - catch (InterruptedException e) - { - Log.info("DocumentWatcher: recieved interruption, aborting."); - } - } - - } - /** Create a new document. * * @param profile The description and default value for the new document. @@ -923,7 +799,6 @@ { documents.remove(name); } -// TODO: this.watcher.removeFile(file); file.delete(); @@ -965,16 +840,19 @@ synchronized (documents) { DirectoryUtils.rmdir(name); - Iterator it = documents.entrySet().iterator(); + Iterator it = documents.getIterator(); while (it.hasNext()) { - Map.Entry entry = (Map.Entry)it.next(); - WeakReference ref = (WeakReference)entry.getValue(); - if (null == ref) + FileCacheEntry entry = (FileCacheEntry)it.next(); + if (null == entry) + { continue; - Profile profile = (Profile)ref.get(); + } + Profile profile = (Profile)entry.getDocument(); if (null == profile) + { continue; + } JetspeedUser pUser = profile.getUser(); if (null != pUser && pUser.getUserName().equals(user.getUserName())) { @@ -1021,16 +899,19 @@ synchronized (documents) { DirectoryUtils.rmdir(name); - Iterator it = documents.entrySet().iterator(); + Iterator it = documents.getIterator(); while (it.hasNext()) { - Map.Entry entry = (Map.Entry)it.next(); - WeakReference ref = (WeakReference)entry.getValue(); - if (null == ref) + FileCacheEntry entry = (FileCacheEntry)it.next(); + if (null == entry) + { continue; - Profile profile = (Profile)ref.get(); + } + Profile profile = (Profile)entry.getDocument(); if (null == profile) + { continue; + } Role pRole = profile.getRole(); if (null != pRole && pRole.getName().equals(role.getName())) { @@ -1076,16 +957,19 @@ synchronized (documents) { DirectoryUtils.rmdir(name); - Iterator it = documents.entrySet().iterator(); + Iterator it = documents.getIterator(); while (it.hasNext()) { - Map.Entry entry = (Map.Entry)it.next(); - WeakReference ref = (WeakReference)entry.getValue(); - if (null == ref) + FileCacheEntry entry = (FileCacheEntry)it.next(); + if (null == entry) + { continue; - Profile profile = (Profile)ref.get(); + } + Profile profile = (Profile)entry.getDocument(); if (null == profile) + { continue; + } Group pGroup = profile.getGroup(); if (null != pGroup && pGroup.getName().equals(group.getName())) { @@ -1653,6 +1537,31 @@ } System.out.println("----------------------"); + } + + /** + * Refresh event, called when the entry is being refreshed from file system. + * + * @param entry the entry being refreshed. + */ + public void refresh(FileCacheEntry entry) + { + System.out.println("entry is refreshing: " + entry.getFile().getName()); + Profile profile = (Profile) entry.getDocument(); + if (profile != null) + { + profile.setDocument(loadDocument(entry.getFile().getName())); + } + } + + /** + * Evict event, called when the entry is being evicted out of the cache + * + * @param entry the entry being refreshed. + */ + public void evict(FileCacheEntry entry) + { + System.out.println("entry is evicting: " + entry.getFile().getName()); } } 1.1 jakarta-jetspeed/test/testdata/psml/user/cachetest/default.psml Index: default.psml =================================================================== <?xml version="1.0" encoding="iso-8859-1"?> <portlets id="100" xmlns="http://xml.apache.org/jetspeed/2000/psml"> <metainfo> <title>Default Jetspeed Page</title> </metainfo> <control name="TabControl"/> <controller name="CardPortletController"> <parameter name="parameter" value="pane"/> </controller> <skin name="orange-grey"/> <portlets id="101"> <controller name="RowController"> <parameter name="sizes" value="66%,34%"/> </controller> <metainfo> <title>Home Page</title> </metainfo> <portlets id="102"> <entry id="103" parent="JetspeedContent"/> </portlets> <portlets id="104"> <entry id="105" parent="Jetspeed"/> <entry id="106" parent="Welcome"/> </portlets> </portlets> <portlets id="107"> <metainfo> <title>RSS</title> </metainfo> <portlets id="108"> <control name="TabControl"/> <controller name="CardPortletController"> <parameter name="defaultcard" value="0"/> <parameter name="parameter" value="channel"/> <parameter name="ruler-size" value="0"/> </controller> <entry id="109" parent="http://jakarta.apache.org/jetspeed/channels/jetspeed.rss"> <control name="ClearPortletControl"/> </entry> <entry id="110" parent="http://jakarta.apache.org/jetspeed/channels/turbine.rss"> <control name="ClearPortletControl"/> </entry> <entry id="111" parent="http://www.mozilla.org/news.rdf"> <control name="ClearPortletControl"/> <parameter name="itemdisplayed" value="5"/> </entry> <entry id="112" parent="http://www.apacheweek.com/issues/apacheweek-headlines.xml"> <control name="ClearPortletControl"/> <parameter name="showtitle" value="false"/> <parameter name="showdescription" value="false"/> </entry> <entry id="113" parent="http://www.xmlhack.com/rsscat.php"> <control name="ClearPortletControl"/> </entry> <entry id="123" parent="BBCFrontPage"> <control name="ClearPortletControl"/> </entry> </portlets> </portlets> <portlets id="114"> <controller name="TwoColumns"/> <metainfo> <title>Dynamic</title> </metainfo> <portlets id="115"> <entry id="116" parent="JetspeedContent"/> <layout> <property name="row" value="0"/> <property name="column" value="0"/> </layout> </portlets> <portlets id="117"> <entry id="118" parent="HelloVelocity"/> <layout> <property name="row" value="0"/> <property name="column" value="1"/> </layout> </portlets> <portlets id="119"> <entry id="120" parent="HelloVelocityCached"/> <entry id="808" parent="InstanceExample"> <parameter name="country" value="Brazil"/> </entry> <entry id="809" parent="InstanceExample"> <parameter name="country" value="Germany"/> </entry> <layout> <property name="row" value="1"/> <property name="column" value="1"/> </layout> </portlets> <portlets id="121"> <entry id="122" parent="HelloJSP"/> <layout> <property name="row" value="1"/> <property name="column" value="0"/> </layout> </portlets> </portlets> </portlets> 1.92 +7 -1 jakarta-jetspeed/webapp/WEB-INF/conf/JetspeedResources.properties Index: JetspeedResources.properties =================================================================== RCS file: /home/cvs/jakarta-jetspeed/webapp/WEB-INF/conf/JetspeedResources.properties,v retrieving revision 1.91 retrieving revision 1.92 diff -u -r1.91 -r1.92 --- JetspeedResources.properties 30 Oct 2002 15:55:50 -0000 1.91 +++ JetspeedResources.properties 14 Nov 2002 06:52:41 -0000 1.92 @@ -357,6 +357,12 @@ #services.PsmlManager.media-types=html:wml #services.PsmlManager.admin=admin +## only used by Castor(File) PSML Manager +# scan rate in seconds (every 2 minutes) +services.PsmlManager.scanRate=120 +# cache size - number of PSML pages to cache +services.PsmlManager.cacheSize=100 + ######################################### # Portlet Factory # ######################################### 1.42 +0 -0 jakarta-jetspeed/webapp/WEB-INF/conf/JetspeedSecurity.properties Index: JetspeedSecurity.properties =================================================================== RCS file: /home/cvs/jakarta-jetspeed/webapp/WEB-INF/conf/JetspeedSecurity.properties,v retrieving revision 1.41 retrieving revision 1.42 diff -u -r1.41 -r1.42 1.1 jakarta-jetspeed/webapp/WEB-INF/conf/HttpAgentLog4j.properties Index: HttpAgentLog4j.properties =================================================================== # log4j.rootLogger=debug, stdout, AgentFile log4j.rootLogger=debug, AgentFile log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout # Pattern to output the caller's file name and line number. log4j.appender.stdout.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p [%t] (%C.%M:%L) - %m%n log4j.appender.AgentFile=org.apache.log4j.RollingFileAppender log4j.appender.AgentFile.File=httpAgent.log log4j.appender.AgentFile.MaxFileSize=100KB # Keep 10 backup files log4j.appender.AgentFile.MaxBackupIndex=10 log4j.appender.AgentFile.layout=org.apache.log4j.PatternLayout log4j.appender.AgentFile.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p [%t] (%C.%M:%L) - %m%n
-- To unsubscribe, e-mail: <mailto:jetspeed-dev-unsubscribe@;jakarta.apache.org> For additional commands, e-mail: <mailto:jetspeed-dev-help@;jakarta.apache.org>