cziegeler 01/10/23 04:55:19
Modified: src/org/apache/cocoon cocoon.roles
src/org/apache/cocoon/components/store MRUMemoryStore.java
webapp cocoon.xconf
Added: src/org/apache/cocoon/components/store FilesystemQueue.java
FilesystemQueueImpl.java FilesystemQueueObject.java
Log:
New Store component: FilesystemQueue
Submitted by: Gerhard Froehlich [[EMAIL PROTECTED]]
Revision Changes Path
1.24 +4 -0 xml-cocoon2/src/org/apache/cocoon/cocoon.roles
Index: cocoon.roles
===================================================================
RCS file: /home/cvs/xml-cocoon2/src/org/apache/cocoon/cocoon.roles,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -r1.23 -r1.24
--- cocoon.roles 2001/10/10 20:53:56 1.23
+++ cocoon.roles 2001/10/23 11:55:19 1.24
@@ -100,6 +100,10 @@
shorthand="store-janitor"
default-class="org.apache.cocoon.components.store.StoreJanitorImpl"/>
+ <role name="org.apache.cocoon.components.store.FilesystemQueue"
+ shorthand="filesystem-queue"
+ default-class="org.apache.cocoon.components.store.FilesystemQueueImpl"/>
+
<role name="org.apache.cocoon.components.hsqldb.Server"
shorthand="hsqldb-server"
default-class="org.apache.cocoon.components.hsqldb.ServerImpl"/>
1.22 +47 -105
xml-cocoon2/src/org/apache/cocoon/components/store/MRUMemoryStore.java
Index: MRUMemoryStore.java
===================================================================
RCS file:
/home/cvs/xml-cocoon2/src/org/apache/cocoon/components/store/MRUMemoryStore.java,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -r1.21 -r1.22
--- MRUMemoryStore.java 2001/10/10 09:23:08 1.21
+++ MRUMemoryStore.java 2001/10/23 11:55:19 1.22
@@ -7,6 +7,8 @@
*****************************************************************************/
package org.apache.cocoon.components.store;
+import org.apache.avalon.excalibur.collections.SynchronizedPriorityQueue;
+
import org.apache.avalon.framework.component.ComponentException;
import org.apache.avalon.framework.component.ComponentManager;
@@ -37,8 +39,6 @@
import org.apache.cocoon.util.IOUtils;
-
-
import java.io.File;
import java.io.IOException;
@@ -51,14 +51,10 @@
import java.util.LinkedList;
-import java.util.Stack;
-
/**
* This class provides a cache algorithm for the requested documents.
* It combines a HashMap and a LinkedList to create a so called MRU
* (Most Recently Used) cache.
- * The objects can also be stored onto the filesystem to hold them in a
- * persitent state over jvm restarts.
*
* The idea was taken from the "Writing Advanced Application Tutorial" from
* javasoft. Many thanx to the writers!
@@ -67,25 +63,24 @@
* @author <a href="mailto:[EMAIL PROTECTED]";>Davanum Srinivas</a>
*/
-public class MRUMemoryStore extends AbstractLoggable implements Store, Configurable,
- ThreadSafe,
Runnable,
- Composable,
Contextualizable {
+public class MRUMemoryStore extends AbstractLoggable implements Store,
+ Configurable,
+ ThreadSafe,
+ Composable,
+ Contextualizable {
- private int priority;
private int maxobjects;
private boolean filesystem;
private HashMap cache;
private LinkedList mrulist;
- private Runtime jvm;
private File cachefile;
private Store fsstore;
private StoreJanitor storejanitor;
- private Stack writerstack;
- private Thread writer;
private File cachedir;
private File workdir;
private String cachedirstr;
+ private FilesystemQueue filesystemQueue;
protected ComponentManager manager;
/**
@@ -100,6 +95,8 @@
this.fsstore = (Store)manager.lookup(Store.ROLE + "/Filesystem");
getLogger().debug("Looking up " + StoreJanitor.ROLE);
this.storejanitor = (StoreJanitor)manager.lookup(StoreJanitor.ROLE);
+ getLogger().debug("Looking up " + FilesystemQueue.ROLE);
+ this.filesystemQueue = (FilesystemQueue)manager.lookup(FilesystemQueue.ROLE);
} catch(ComponentException e) {
getLogger().error("Error in compose()!", e);
}
@@ -118,8 +115,13 @@
this.workdir = (File)context.get(Constants.CONTEXT_WORK_DIR);
this.cachedirstr = IOUtils.getContextFilePath(this.workdir.getPath(),
this.cachedir.getPath());
+
+
- getLogger().debug("Context path="
+ this.getLogger().debug("cachedir=" + this.cachedir);
+ this.getLogger().debug("workdir=" + this.workdir);
+ this.getLogger().debug("cachedirstr=" + this.cachedirstr);
+ this.getLogger().debug("Context path="
+
IOUtils.getContextFilePath(this.workdir.getPath(),this.cachedir.getPath()));
} catch (Exception e) {
@@ -132,7 +134,6 @@
* A few options can be used :
* <UL>
* <LI>maxobjects = how many objects will be stored in memory (Default: 10
objects)</LI>
- * <LI>threadpriority = priority of the threads (1-10). (Default: 10)</LI>
* <LI>filesystem = use filesystem storage to keep object persistent (Default:
false)</LI>
* </UL>
*
@@ -140,57 +141,17 @@
* @exception ConfigurationException
*/
public void configure(Configuration conf) throws ConfigurationException {
- this.jvm = Runtime.getRuntime();
- this.mrulist = new LinkedList();
- this.writerstack = new Stack();
-
- this.storejanitor.register(this);
-
Parameters params = Parameters.fromConfiguration(conf);
- this.maxobjects = params.getParameterAsInteger("maxobjects",100);
- this.cache = new HashMap((int)(this.maxobjects * 1.2));
- this.priority =
params.getParameterAsInteger("threadpriority",Thread.currentThread().getPriority());
- this.filesystem = params.getParameterAsBoolean("filesystem",false);
- if ((this.priority < 1) || (this.priority > 10)) {
- throw new ConfigurationException("MRUMemoryStore cleanup thread priority must
be between 1 and 10!");
- }
+ this.maxobjects = params.getParameterAsInteger("maxobjects",100);
+ this.filesystem = params.getParameterAsBoolean("filesystem",false);
if ((this.maxobjects < 1)) {
throw new ConfigurationException("MRUMemoryStore maxobjects must be at least
1 milli second!");
}
-
- if (this.filesystem) {
- getLogger().debug("Intializing writer thread");
- writer = new Thread(this);
- writer.setPriority(this.priority);
- writer.setDaemon(true);
- writer.setName("writer");
- writer.start();
- }
- }
-
- /**
- * The writer thread writes objects from the writer stack onto the filesystem.
- */
- public void run() {
- while (true) {
- getLogger().debug("Writerthread awake!");
- while(!writerstack.empty()) {
- try {
- TmpStackObject tmp = (TmpStackObject)this.writerstack.pop();
- this.fsstore.store(getFileName(tmp.getKey().toString()), tmp.getObject());
- } catch(IOException e) {
- getLogger().error("Error in writer thread",e);
- } catch(Exception ex) {
- getLogger().error("Error in writer thread",ex);
- }
- }
- synchronized (this.writer) {
- try {
- writer.wait();
- } catch (InterruptedException ignored) {}
- }
- }
+ this.cache = new HashMap((int)(this.maxobjects * 1.2));
+ this.mrulist = new LinkedList();
+
+ this.storejanitor.register(this);
}
/**
@@ -225,12 +186,13 @@
/** put the object on the filesystem */
if(this.filesystem) {
- if(this.checkSerializable(value)) {
- getLogger().debug("Storing object on fs");
- this.writerstack.push(new TmpStackObject(key,value));
- getLogger().debug("Stack size=" + writerstack.size());
- synchronized (this.writer) {
- this.writer.notify();
+ if(this.checkSerializable(value) &&
+ !this.fsstore.containsKey(getFileName(key.toString()))) {
+ this.getLogger().debug("Storing object on fs");
+ try {
+ this.filesystemQueue.insert(new FilesystemQueueObject(key,value));
+ } catch(Exception e) {
+ this.getLogger().error("Error storing Object on fs",e);
}
}
}
@@ -238,7 +200,7 @@
this.cache.put(key, value);
this.mrulist.remove(key);
this.mrulist.addFirst(key);
- getLogger().debug("Cache size=" + cache.size());
+ this.getLogger().debug("Cache size=" + cache.size());
}
/**
@@ -248,7 +210,7 @@
* @return the requested object
*/
public Object get(Object key) {
- getLogger().debug("Getting object from memory. Key: " + key);
+ this.getLogger().debug("Getting object from memory. Key: " + key);
Object tmpobject = this.cache.get(key);
if ( tmpobject != null ) {
@@ -258,24 +220,27 @@
return tmpobject;
}
- getLogger().debug("Object not found in memory");
+ this.getLogger().debug("Object not found in memory");
/** try to fetch from filesystem */
if(this.filesystem) {
tmpobject = this.fsstore.get(getFileName(key.toString()));
if (tmpobject == null) {
- getLogger().debug( "Object was NOT found on fs. Looked for: " +
URLEncoder.encode(key.toString()) );
+ this.getLogger().debug( "Object was NOT found on fs. Looked for: "
+ + getFileName(key.toString()));
return null;
} else {
- getLogger().debug("Object was found on fs");
+ this.getLogger().debug("Object was found on fs");
try {
tmpobject = IOUtils.deserializeObject((File)tmpobject);
- this.hold(key,tmpobject);
+ if(!this.cache.containsKey(key)) {
+ this.hold(key,tmpobject);
+ }
return tmpobject;
} catch (ClassNotFoundException ce) {
- getLogger().error("Error in get()!", ce);
+ this.getLogger().error("Error in get()!", ce);
return null;
} catch (IOException ioe) {
- getLogger().error("Error in get()!", ioe);
+ this.getLogger().error("Error in get()!", ioe);
return null;
}
}
@@ -289,7 +254,7 @@
* @param the key of to be removed object
*/
public void remove(Object key) {
- getLogger().debug("Removing object from store");
+ this.getLogger().debug("Removing object from store");
this.cache.remove(key);
this.mrulist.remove(key);
if(this.filesystem && key != null) {
@@ -305,7 +270,7 @@
*/
public boolean containsKey(Object key) {
synchronized(this.cache) {
- return this.cache.containsKey(key);
+ return this.cache.containsKey(key);
}
}
@@ -329,10 +294,10 @@
this.getLogger().debug("Freeing cache");
this.cache.remove(this.mrulist.getLast());
this.mrulist.removeLast();
- getLogger().debug("Cache size=" + cache.size());
+ this.getLogger().debug("Cache size=" + cache.size());
}
} catch (Exception e) {
- getLogger().error("Error in free()", e);
+ this.getLogger().error("Error in free()", e);
}
}
@@ -346,7 +311,7 @@
*/
private boolean checkSerializable(Object object) {
try {
- getLogger().debug("Object=" + object);
+ this.getLogger().debug("Object=" + object);
if((object.getClass().getName().equals("org.apache.cocoon.caching.CachedEventObject"))
||
(object.getClass().getName().equals("org.apache.cocoon.caching.CachedStreamObject"))
||
(ClassUtils.implementsInterface(object.getClass().getName(),"org.apache.cocoon.caching.CacheValidity")))
{
@@ -355,7 +320,7 @@
return false;
}
} catch (Exception e) {
- getLogger().error("Error in checkSerializable()!", e);
+ this.getLogger().error("Error in checkSerializable()!", e);
return false;
}
}
@@ -367,7 +332,7 @@
*
* @param the key of the object
* @return the filename of the key
- */
+ */
private String getFileName(String key)
{
return new StringBuffer()
@@ -375,28 +340,5 @@
.append(File.separator)
.append(URLEncoder.encode(key.toString()))
.toString();
- }
-
- /**
- * Temporary container object for the writerstack
- */
- static class TmpStackObject {
- private Object object;
- private Object key;
-
- public TmpStackObject (Object key, Object object) {
- this.object = object;
- this.key = key;
- }
-
- public TmpStackObject() {}
-
- public Object getKey() {
- return this.key;
- }
-
- public Object getObject() {
- return this.object;
- }
}
}
1.1
xml-cocoon2/src/org/apache/cocoon/components/store/FilesystemQueue.java
Index: FilesystemQueue.java
===================================================================
/*****************************************************************************
* Copyright (C) The Apache Software Foundation. All rights reserved. *
* ------------------------------------------------------------------------- *
* This software is published under the terms of the Apache Software License *
* version 1.1, a copy of which has been included with this distribution in *
* the LICENSE file. *
*****************************************************************************/
package org.apache.cocoon.components.store;
import org.apache.avalon.excalibur.collections.PriorityQueue;
import org.apache.avalon.framework.component.Component;
/**
* The interface for the filesystem queue
*
* @author <a href="mailto:[EMAIL PROTECTED]";>Gerhard Froehlich</a>
*/
public interface FilesystemQueue extends Component, PriorityQueue {
String ROLE = "org.apache.cocoon.components.store.FilesystemQueue";
}
1.1
xml-cocoon2/src/org/apache/cocoon/components/store/FilesystemQueueImpl.java
Index: FilesystemQueueImpl.java
===================================================================
/*****************************************************************************
* Copyright (C) The Apache Software Foundation. All rights reserved. *
* ------------------------------------------------------------------------- *
* This software is published under the terms of the Apache Software License *
* version 1.1, a copy of which has been included with this distribution in *
* the LICENSE file. *
*****************************************************************************/
package org.apache.cocoon.components.store;
import org.apache.avalon.framework.component.ComponentException;
import org.apache.avalon.framework.component.ComponentManager;
import org.apache.avalon.framework.component.Composable;
import org.apache.avalon.framework.configuration.Configurable;
import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.context.Context;
import org.apache.avalon.framework.context.ContextException;
import org.apache.avalon.framework.context.Contextualizable;
import org.apache.avalon.framework.logger.AbstractLoggable;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.avalon.framework.thread.ThreadSafe;
import org.apache.cocoon.Constants;
import org.apache.cocoon.util.IOUtils;
import java.io.File;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.NoSuchElementException;
/**
* This class provides a asynchron queue for storing objects on the filesystem.
* Every store component can uses this component to store object in the
* configured cache directory.
*
* @author <a href="mailto:[EMAIL PROTECTED]";>Gerhard Froehlich</a>
*/
public class FilesystemQueueImpl extends AbstractLoggable implements FilesystemQueue,
Configurable,
Contextualizable,
Runnable,
ThreadSafe,
Composable {
private int handlerinterval;
private int threadpriority;
private int maxobjects;
private LinkedList linkedList;
private File cachedir;
private File workdir;
private String cachedirstr;
private Store fsstore;
private Thread fsQueueHandler;
/**
* Get the context
*
* @param the Context of the application
*/
public void contextualize(Context context) throws ContextException {
this.cachedirstr = new String();
try {
this.cachedir = (File)context.get(Constants.CONTEXT_CACHE_DIR);
this.workdir = (File)context.get(Constants.CONTEXT_WORK_DIR);
this.cachedirstr = IOUtils.getContextFilePath(this.workdir.getPath(),
this.cachedir.getPath());
this.getLogger().debug("Context path="
+
IOUtils.getContextFilePath(this.workdir.getPath(),this.cachedir.getPath()));
} catch (Exception e) {
this.getLogger().error("Error in contextualize()",e);
}
}
/**
* Get components of the ComponentManager
*
* @param the ComponentManager
*/
public void compose(ComponentManager manager) throws ComponentException {
try {
getLogger().debug("Looking up " + Store.ROLE + "/Filesystem");
this.fsstore = (Store)manager.lookup(Store.ROLE + "/Filesystem");
} catch(ComponentException e) {
getLogger().error("Error in compose()!", e);
}
}
/**
* Configure the Filesystem Queue:
* <UL>
* <LI>handlerinterval = Interval of the Queue Handler Thread</LI>
* <LI>threadpriority = Priority of the Queue Handler Thread</LI>
* >LI>maxobjects = Defines the max. numbers of Objects in the queue</LI>
* </UL>
*/
public void configure(Configuration conf) throws ConfigurationException {
this.getLogger().debug("Configure Filesystem Queue");
Parameters params = Parameters.fromConfiguration(conf);
this.handlerinterval = params.getParameterAsInteger("handlerinterval",10);
this.threadpriority = params.getParameterAsInteger("threadpriority",5);
this.maxobjects = params.getParameterAsInteger("maxobjects",100);
this.getLogger().debug("handlerinterval=" + handlerinterval);
this.getLogger().debug("threadpriority=" + threadpriority);
this.linkedList = new LinkedList();
this.fsQueueHandler = new Thread(this);
this.fsQueueHandler.setDaemon(true);
this.fsQueueHandler.setPriority(this.threadpriority);
this.fsQueueHandler.setName("fsQueueHandler");
this.fsQueueHandler.start();
}
public void run() {
while(true) {
while(!this.isEmpty()) {
FilesystemQueueObject filesystemQueueObject =
(FilesystemQueueObject)this.pop();
try {
this.fsstore.store(this.getFileName(filesystemQueueObject.getKey().toString()),
filesystemQueueObject.getObject());
} catch(IOException e) {
this.getLogger().error("Error in fsQueueHandler",e);
}
}
try {
Thread.currentThread().sleep(this.handlerinterval * 1000);
} catch (InterruptedException ignore) {}
}
}
/**
* Clear all elements from queue.
*/
public void clear() {
this.getLogger().debug("Clearing the FilesystemQueue");
ListIterator listIterator = linkedList.listIterator(0);
while(listIterator.hasNext()) {
this.linkedList.remove(listIterator.nextIndex());
}
}
/**
* Test if queue is empty.
*
* @return true if queue is empty else false.
*/
public boolean isEmpty() {
if (this.linkedList.size() > 0) {
return false;
} else {
return true;
}
}
/**
* Insert an element into queue.
*
* @param element the element to be inserted
*/
public void insert(Comparable element) {
if(this.linkedList.size() < maxobjects) {
this.getLogger().debug("Insert Element in FilesystemQueue");
this.linkedList.addFirst(element);
this.reportSize();
} else {
this.getLogger().warn("Filesystem Queue full!");
}
}
/**
* Return element on top of heap but don't remove it.
*
* @return the element at top of heap
* @exception NoSuchElementException if isEmpty() == true
*/
public Comparable peek() throws NoSuchElementException {
try {
this.getLogger().debug("Peek Element in FilesystemQueue");
return (Comparable)linkedList.getLast();
} catch (NoSuchElementException e) {
this.getLogger().error("Error peeking element from the queue",e);
return null;
}
}
/**
* Return element on top of heap and remove it.
*
* @return the element at top of heap
* @exception NoSuchElementException if isEmpty() == true
*/
public Comparable pop() throws NoSuchElementException {
try {
this.getLogger().debug("Pop Element in FilesystemQueue");
return (Comparable)linkedList.removeLast();
} catch (NoSuchElementException e) {
this.getLogger().error("Error popping element from the queue",e);
return null;
}
}
/**
* This method puts together a filename for
* the object, which shall be stored on the
* filesystem.
*
* @param the key of the object
* @return the filename of the key
*/
private String getFileName(String key)
{
return new StringBuffer()
.append(this.cachedirstr)
.append(File.separator)
.append(URLEncoder.encode(key.toString()))
.toString();
}
public void reportSize() {
this.getLogger().debug("Size of FilesystemQueue=" + this.linkedList.size());
}
}
1.1
xml-cocoon2/src/org/apache/cocoon/components/store/FilesystemQueueObject.java
Index: FilesystemQueueObject.java
===================================================================
/*****************************************************************************
* Copyright (C) The Apache Software Foundation. All rights reserved. *
* ------------------------------------------------------------------------- *
* This software is published under the terms of the Apache Software License *
* version 1.1, a copy of which has been included with this distribution in *
* the LICENSE file. *
*****************************************************************************/
package org.apache.cocoon.components.store;
/**
* A "helper" container for the filesystem queue
*
* @author <a href="mailto:[EMAIL PROTECTED]";>Gerhard Froehlich</a>
*/
public class FilesystemQueueObject implements Comparable {
private Object object;
private Object key;
public FilesystemQueueObject (Object key, Object object) {
this.object = object;
this.key = key;
}
public FilesystemQueueObject() {}
public Object getKey() {
return this.key;
}
public Object getObject() {
return this.object;
}
public int compareTo(Object o) {
return 0;
}
}
1.38 +13 -15 xml-cocoon2/webapp/cocoon.xconf
Index: cocoon.xconf
===================================================================
RCS file: /home/cvs/xml-cocoon2/webapp/cocoon.xconf,v
retrieving revision 1.37
retrieving revision 1.38
diff -u -r1.37 -r1.38
--- cocoon.xconf 2001/10/10 20:53:57 1.37
+++ cocoon.xconf 2001/10/23 11:55:19 1.38
@@ -34,13 +34,10 @@
maxobjects: Indicates how many objects will be hold in the cache.
When the number of maxobjects has been reached. The
last object in the cache will be thrown out.
- threadpriority: Indicates the priority of the writer thread.
- (1 is the lowest priority and 10 is the highest).
filesystem: Turns the filesystem storage for objects on or off.
-->
<store class="org.apache.cocoon.components.store.MRUMemoryStore"
logger="root.store">
<parameter name="maxobjects" value="100"/>
- <parameter name="threadpriority" value="5"/>
<parameter name="filesystem" value="true"/>
</store>
@@ -69,6 +66,19 @@
<parameter name="threadpriority" value="5"/>
</store-janitor>
+ <!-- Filesystem Queue
+ The Filesystem Queue is the central queue for writing objects onto the
filesystem:
+ handlerinterval = How often (seconds) is the queue checked for new objects.
+ threadpriority = The priority of the Handler Thread
+ maxobjects = Defines the max. numbers of objects in queue. If the queue is
full no objects
+ can be inserted.
+ -->
+ <filesystem-queue class="org.apache.cocoon.components.store.FilesystemQueueImpl"
logger="root.store">
+ <parameter name="handlerinterval" value="10"/>
+ <parameter name="threadpriority" value="5"/>
+ <parameter name="maxobjects" value="100"/>
+ </filesystem-queue>
+
<!-- Entity resolution catalogs:
catalog:
The default catalog is distributed at /resources/entities/catalog
@@ -331,22 +341,13 @@
pool-max="32" pool-min="16" pool-grow="4"/>
<!-- Caching of stream pipeline:
- freememory: Indicates how much memory should be left free in the
- JVM for normal operation.
- heapsize: Indicates how big the heap size can grow to before the
- cleanup thread kicks in.
- cleanupthreadinterval: Indicates the interval of the cleanup thread in
seconds.
maxobjects: Indicates how many objects will be hold in the cache.
When the number of maxobjects has been reached. The
last object in the cache will be thrown out.
- usecleanupthread: Indicates whether we use a cleanup thread or not.
- threadpriority: Indicates the priority of the cleanup thread.
- (1 is the lowest priority and 10 is the highest).
filesystem: Turns the filesystem storage for objects on or off.
-->
<stream-cache class="org.apache.cocoon.components.store.MRUMemoryStore"
logger="root.store">
<parameter name="maxobjects" value="100"/>
- <parameter name="threadpriority" value="5"/>
<parameter name="filesystem" value="true"/>
</stream-cache>
@@ -365,13 +366,10 @@
maxobjects: Indicates how many objects will be hold in the cache.
When the number of maxobjects has been reached. The
last object in the cache will be thrown out.
- threadpriority: Indicates the priority of the cleanup thread.
- (1 is the lowest priority and 10 is the highest).
filesystem: Turns the filesystem storage for objects on or off.
-->
<event-cache class="org.apache.cocoon.components.store.MRUMemoryStore"
logger="root.store">
<parameter name="maxobjects" value="100"/>
- <parameter name="threadpriority" value="5"/>
<parameter name="filesystem" value="true"/>
</event-cache>
----------------------------------------------------------------------
In case of troubles, e-mail: [EMAIL PROTECTED]
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
