froehlich 01/12/12 16:04:34
Added: scratchpad/src/org/apache/cocoon/jispstore
MRUMemoryStore.java JispKey.java
JispFilesystemStore.java FilesystemQueueImpl.java
Log:
started to implement a Jisp based FilesystemStore, stand by for more.
Revision Changes Path
1.1
xml-cocoon2/scratchpad/src/org/apache/cocoon/jispstore/MRUMemoryStore.java
Index: MRUMemoryStore.java
===================================================================
/*****************************************************************************
* Copyright (C) The Apache Software Foundation. All rights reserved. *
* ------------------------------------------------------------------------- *
* This software is published under the terms of the Apache Software License *
* version 1.1, a copy of which has been included with this distribution in *
* the LICENSE file. *
*****************************************************************************/
package org.apache.cocoon.jispstore;
import org.apache.avalon.excalibur.collections.SynchronizedPriorityQueue;
import org.apache.avalon.framework.activity.Disposable;
import org.apache.avalon.excalibur.collections.SynchronizedPriorityQueue;
import org.apache.avalon.framework.component.ComponentException;
import org.apache.avalon.framework.component.ComponentManager;
import org.apache.avalon.framework.component.Composable;
import org.apache.avalon.framework.configuration.Configurable;
import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.context.Context;
import org.apache.avalon.framework.context.ContextException;
import org.apache.avalon.framework.context.Contextualizable;
import org.apache.avalon.framework.logger.AbstractLoggable;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.avalon.framework.thread.ThreadSafe;
import org.apache.cocoon.Constants;
import org.apache.cocoon.components.store.FilesystemQueue;
import org.apache.cocoon.components.store.FilesystemQueueObject;
import org.apache.cocoon.components.store.Store;
import org.apache.cocoon.components.store.StoreJanitor;
import org.apache.cocoon.util.ClassUtils;
import org.apache.cocoon.util.IOUtils;
import org.apache.regexp.RE;
import org.apache.regexp.RESyntaxException;
import java.io.File;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.LinkedList;
/**
* This class provides a cache algorithm for the requested documents.
* It combines a HashMap and a LinkedList to create a so called MRU
* (Most Recently Used) cache.
*
* The idea was taken from the "Writing Advanced Application Tutorial" from
* javasoft. Many thanx to the writers!
*
* @author <a href="mailto:[EMAIL PROTECTED]";>Gerhard Froehlich</a>
* @author <a href="mailto:[EMAIL PROTECTED]";>Davanum Srinivas</a>
*/
public final class MRUMemoryStore
extends AbstractLoggable
implements Store,
Configurable,
ThreadSafe,
Composable,
Disposable {
private int maxobjects;
private boolean filesystem;
private Hashtable cache;
private LinkedList mrulist;
private File cachefile;
private Store fsstore;
private StoreJanitor storejanitor;
private FilesystemQueue filesystemQueue;
private ComponentManager manager;
/**
* Get components of the ComponentManager
*
* @param the ComponentManager
*/
public void compose(ComponentManager manager) throws ComponentException {
this.manager = manager;
getLogger().debug("Looking up " + Store.ROLE + "/JispFilesystemStore");
this.fsstore = (Store)manager.lookup(Store.ROLE + "/JispFilesystemStore");
getLogger().debug("Looking up " + StoreJanitor.ROLE);
this.storejanitor = (StoreJanitor)manager.lookup(StoreJanitor.ROLE);
getLogger().debug("Looking up " + FilesystemQueue.ROLE);
this.filesystemQueue =
(FilesystemQueue)manager.lookup(FilesystemQueue.ROLE);
}
/**
* Initialize the MRUMemoryStore.
* A few options can be used :
* <UL>
* <LI>maxobjects = how many objects will be stored in memory (Default: 10
objects)</LI>
* <LI>filesystem = use filesystem storage to keep object persistent (Default:
false)</LI>
* </UL>
*
* @param the Configuration of the application
* @exception ConfigurationException
*/
public void configure(Configuration conf) throws ConfigurationException {
Parameters params = Parameters.fromConfiguration(conf);
this.maxobjects = params.getParameterAsInteger("maxobjects",100);
this.filesystem =
params.getParameterAsBoolean("filesystem",false);
if ((this.maxobjects < 1)) {
throw new ConfigurationException("MRUMemoryStore maxobjects must be at
least 1 milli second!");
}
this.cache = new Hashtable((int)(this.maxobjects * 1.2));
this.mrulist = new LinkedList();
this.storejanitor.register(this);
}
/**
* Dispose the component
*/
public void dispose() {
if (this.manager != null) {
this.manager.release(this.storejanitor);
this.storejanitor = null;
this.manager.release(this.filesystemQueue);
this.filesystemQueue = null;
this.manager.release(this.fsstore);
this.fsstore = null;
}
}
/**
* Store the given object in a persistent state. It is up to the
* caller to ensure that the key has a persistent state across
* different JVM executions.
*
* @param the key for the object to store
* @param the object to store
*/
public void store(Object key, Object value) {
this.hold(key,value);
}
/**
* This method holds the requested object in a HashMap combined
* with a LinkedList to create the MRU.
* It also stores objects onto the filesystem if configured.
*
* @param the key of the object to be stored
* @param the object to be stored
*/
public void hold(Object key, Object value) {
getLogger().debug("Holding object in memory. key: " + key);
getLogger().debug("Holding object in memory. value: " + value);
/** ...first test if the max. objects in cache is reached... */
while (this.mrulist.size() >= this.maxobjects) {
/** ...ok, heapsize is reached, remove the last element... */
this.free();
}
/** put the object on the filesystem */
if(this.filesystem) {
if(this.checkSerializable(value) &&
!this.fsstore.containsKey(key)) {
this.getLogger().debug("Storing object on fs");
try {
this.filesystemQueue.insert(new
FilesystemQueueObject(key,value));
} catch(Exception e) {
this.getLogger().error("Error storing Object on fs",e);
}
}
}
/** ..put the new object in the cache, on the top of course ... */
this.cache.put(key, value);
this.mrulist.remove(key);
this.mrulist.addFirst(key);
this.getLogger().debug("Cache size=" + cache.size());
}
/**
* Get the object associated to the given unique key.
*
* @param the key of the requested object
* @return the requested object
*/
public Object get(Object key) {
this.getLogger().debug("Getting object from memory. Key: " + key);
Object tmpobject = this.cache.get(key);
if ( tmpobject != null ) {
/** put the accessed key on top of the linked list */
this.mrulist.remove(key);
this.mrulist.addFirst(key);
return tmpobject;
}
this.getLogger().debug("Object not found in memory");
/** try to fetch from filesystem */
if(this.filesystem) {
tmpobject = this.fsstore.get(key);
if (tmpobject == null) {
this.getLogger().debug( "Object was NOT found on fs. Looked for: "
+ key);
return null;
} else {
this.getLogger().debug("Object was found on fs");
try {
tmpobject = IOUtils.deserializeObject((File)tmpobject);
if(!this.cache.containsKey(key)) {
this.hold(key,tmpobject);
}
return tmpobject;
} catch (ClassNotFoundException ce) {
this.getLogger().error("Error in get()!", ce);
return null;
} catch (IOException ioe) {
this.getLogger().error("Error in get()!", ioe);
return null;
}
}
}
return null;
}
/**
* Remove the object associated to the given key.
*
* @param the key of to be removed object
*/
public void remove(Object key) {
this.getLogger().debug("Removing object from store");
this.cache.remove(key);
this.mrulist.remove(key);
if(this.filesystem && key != null) {
this.fsstore.remove(key);
}
}
/**
* Indicates if the given key is associated to a contained object.
*
* @param the key of the object
* @return true if the key exists
*/
public boolean containsKey(Object key) {
if(filesystem) {
return (this.cache.containsKey(key) || this.fsstore.containsKey(key));
} else {
return this.cache.containsKey(key);
}
}
/**
* Returns the list of used keys as an Enumeration.
*
* @return the enumeration of the cache
*/
public Enumeration keys() {
return null;
}
/**
* Frees some of the fast memory used by this store.
* It removes the last element in the store.
*/
public void free() {
try {
if(this.cache.size() > 0) {
this.getLogger().debug("Freeing cache");
this.cache.remove(this.mrulist.getLast());
this.mrulist.removeLast();
this.getLogger().debug("Cache size=" + cache.size());
}
} catch (Exception e) {
this.getLogger().error("Error in free()", e);
}
}
/**
* This method checks if an object is seriazable.
* FIXME: In the moment only CachedEventObject or
* CachedStreamObject are stored.
*
* @param the object to be checked
* @return true if the object is storeable
*/
private boolean checkSerializable(Object object) {
try {
this.getLogger().debug("Object=" + object);
if((object.getClass().getName().equals("org.apache.cocoon.caching.CachedEventObject"))
||
(object.getClass().getName().equals("org.apache.cocoon.caching.CachedStreamObject"))
||
(ClassUtils.implementsInterface(object.getClass().getName(),"org.apache.cocoon.caching.CacheValidity")))
{
return true;
} else {
return false;
}
} catch (Exception e) {
this.getLogger().error("Error in checkSerializable()!", e);
return false;
}
}
}
1.1
xml-cocoon2/scratchpad/src/org/apache/cocoon/jispstore/JispKey.java
Index: JispKey.java
===================================================================
/*****************************************************************************
* Copyright (C) The Apache Software Foundation. All rights reserved. *
* ------------------------------------------------------------------------- *
* This software is published under the terms of the Apache Software License *
* version 1.1, a copy of which has been included with this distribution in *
* the LICENSE file. *
*****************************************************************************/
package org.apache.cocoon.jispstore;
import com.coyotegulch.jisp.KeyObject;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
final class JispKey extends KeyObject{
private String mKey;
public JispKey() {
mKey = null;
}
public JispKey(String keyValue) {
mKey = keyValue;
}
public int compareTo(KeyObject key) {
int code = -1;
if (key instanceof JispKey) {
code = System.identityHashCode(this) -
System.identityHashCode(key);
if(code < 0) {
code = KEY_LESS;
} else if(code > 0) {
code = KEY_MORE;
} else {
code = KEY_EQUAL;
}
}
return code;
}
public KeyObject makeNullKey() {
JispKey nullKey = new JispKey();
return nullKey;
}
public void writeExternal(ObjectOutput out) throws IOException {
out.writeObject(mKey);
}
public void readExternal(ObjectInput in)
throws IOException, ClassNotFoundException {
mKey = (String)in.readObject();
}
public boolean equals(Object obj)
{
if ((obj != null) && (obj instanceof JispKey))
return (mKey.equals(((JispKey)obj).mKey));
else
return false;
}
public String toString() {
return "Key: " + mKey;
}
}
1.1
xml-cocoon2/scratchpad/src/org/apache/cocoon/jispstore/JispFilesystemStore.java
Index: JispFilesystemStore.java
===================================================================
/*****************************************************************************
* Copyright (C) The Apache Software Foundation. All rights reserved. *
* ------------------------------------------------------------------------- *
* This software is published under the terms of the Apache Software License *
* version 1.1, a copy of which has been included with this distribution in *
* the LICENSE file. *
*****************************************************************************/
package org.apache.cocoon.jispstore;
import org.apache.avalon.framework.context.Context;
import org.apache.avalon.framework.context.ContextException;
import org.apache.avalon.framework.context.Contextualizable;
import org.apache.avalon.framework.activity.Initializable;
import org.apache.avalon.framework.logger.AbstractLoggable;
import org.apache.avalon.framework.thread.ThreadSafe;
import org.apache.cocoon.Constants;
import org.apache.cocoon.util.IOUtils;
import org.apache.cocoon.components.store.Store;
import com.coyotegulch.jisp.IndexedObjectDatabase;
import com.coyotegulch.jisp.BTreeIndex;
import com.coyotegulch.jisp.KeyObject;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.Enumeration;
public final class JispFilesystemStore
extends AbstractLoggable
implements Store,
Contextualizable,
ThreadSafe,
Initializable {
/** The directory repository */
protected File directoryFile;
protected volatile String directoryPath;
/** some statics for the moment */
private static String DATABASE_NAME = "cocoon.data";
private static String INDEX_NAME = "cocoon.idx";
private static int ORDER = 33;
/** The database */
private IndexedObjectDatabase database;
private BTreeIndex index;
/**
* Sets the repository's location
*/
public void setDirectory(final String directory)
throws IOException {
this.setDirectory(new File(directory));
}
public void contextualize(final Context context)
throws ContextException {
try {
setDirectory((File) context.get(Constants.CONTEXT_WORK_DIR));
} catch (Exception e) {
// ignore
}
}
public void initialize() {
/** determine datafile */
getLogger().debug("initialize() JispFilesystemStore");
try {
getLogger().debug("Path to data file=" + this.getDirectoryPath() +
DATABASE_NAME);
File myFile = new File(this.getDirectoryPath() + DATABASE_NAME);
getLogger().debug("File exists=" + myFile.exists());
if (myFile.exists()) {
this.getLogger().debug("Data file exists");
database = new IndexedObjectDatabase(getDirectoryPath() +
DATABASE_NAME,false);
index = new BTreeIndex(this.getDirectoryPath() + INDEX_NAME);
database.attachIndex(index);
index.dumpTree();
} else {
this.getLogger().debug("Data file not exists");
database = new IndexedObjectDatabase(getDirectoryPath() +
DATABASE_NAME,false);
index = new BTreeIndex(this.getDirectoryPath() + INDEX_NAME,
ORDER, new JispKey(),false);
database.attachIndex(index);
index.dumpTree();
}
} catch (Exception e) {
getLogger().error("initialize(..) Exception",e);
}
}
/**
* Sets the repository's location
*/
public void setDirectory(final File directory)
throws IOException {
this.directoryFile = directory;
/* Save directory path prefix */
this.directoryPath = IOUtils.getFullFilename(this.directoryFile);
this.directoryPath += File.separator;
/* Does directory exist? */
if (!this.directoryFile.exists()) {
/* Create it anew */
if (!this.directoryFile.mkdir()) {
throw new IOException(
"Error creating store directory '" + this.directoryPath + "': ");
}
}
/* Is given file actually a directory? */
if (!this.directoryFile.isDirectory()) {
throw new IOException("'" + this.directoryPath + "' is not a
directory");
}
/* Is directory readable and writable? */
if (!(this.directoryFile.canRead() && this.directoryFile.canWrite())) {
throw new IOException(
"Directory '" + this.directoryPath + "' is not readable/writable"
);
}
}
/**
* Returns the repository's full pathname
*/
public String getDirectoryPath() {
return this.directoryPath;
}
public Object get(Object key) {
this.getLogger().debug("get(): Get file with key: " + key.toString());
Object readObj = null;
try {
readObj = database.read(new JispKey(key.toString()),index);
if(readObj != null) {
this.getLogger().debug("get(): FOUND!!= " + readObj);
} else {
this.getLogger().debug("get(): NOT FOUND!!= " + readObj);
}
} catch (Exception e) {
getLogger().error("get(..): Exception",e);
}
return readObj;
}
public void store(Object key, Object value) throws IOException {
this.getLogger().debug("store(): Store file with key: " + key.toString());
//this.remove(key);
try {
KeyObject[] keyArray = new JispKey[3];
keyArray[0] = new JispKey(key.toString());
keyArray[1] = new JispKey(key.toString());
keyArray[2] = new JispKey(key.toString());
database.write(keyArray,(Serializable)value);
} catch (Exception e) {
this.getLogger().error("store(..): Exception",e);
}
}
public void hold(Object key, Object value) throws IOException {
this.store(key,value);
}
public void free() {
}
public void remove(Object key) {
this.getLogger().debug("remove(..) Remove item");
try {
KeyObject[] keyArray = new JispKey[3];
keyArray[0] = new JispKey(key.toString());
keyArray[1] = new JispKey(key.toString());
keyArray[2] = new JispKey(key.toString());
database.remove(keyArray);
} catch (Exception e) {
this.getLogger().error("remove(..): Exception",e);
}
}
public boolean containsKey(Object key) {
return false;
}
public Enumeration keys() {
return null;
}
}
1.1
xml-cocoon2/scratchpad/src/org/apache/cocoon/jispstore/FilesystemQueueImpl.java
Index: FilesystemQueueImpl.java
===================================================================
/*****************************************************************************
* Copyright (C) The Apache Software Foundation. All rights reserved. *
* ------------------------------------------------------------------------- *
* This software is published under the terms of the Apache Software License *
* version 1.1, a copy of which has been included with this distribution in *
* the LICENSE file. *
*****************************************************************************/
package org.apache.cocoon.jispstore;
import org.apache.avalon.framework.activity.Disposable;
import org.apache.avalon.framework.activity.Startable;
import org.apache.avalon.framework.component.ComponentException;
import org.apache.avalon.framework.component.ComponentManager;
import org.apache.avalon.framework.component.Composable;
import org.apache.avalon.framework.configuration.Configurable;
import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.logger.AbstractLoggable;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.avalon.framework.thread.ThreadSafe;
import org.apache.cocoon.Constants;
import org.apache.cocoon.components.store.FilesystemQueue;
import org.apache.cocoon.components.store.FilesystemQueueObject;
import org.apache.cocoon.components.store.Store;
import org.apache.cocoon.util.IOUtils;
import java.io.File;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.NoSuchElementException;
/**
* This class provides a asynchron queue for storing objects on the filesystem.
* Every store component can uses this component to store object in the
* configured cache directory.
*
* @author <a href="mailto:[EMAIL PROTECTED]";>Gerhard Froehlich</a>
*/
public final class FilesystemQueueImpl
extends AbstractLoggable
implements FilesystemQueue,
Configurable,
Runnable,
ThreadSafe,
Composable,
Disposable,
Startable {
private int handlerinterval;
private int threadpriority;
private int maxobjects;
private LinkedList linkedList;
private Store fsstore;
private Thread fsQueueHandler;
private static boolean doRun = false;
private ComponentManager manager;
/**
* Get components of the ComponentManager
*
* @param the ComponentManager
*/
public void compose(ComponentManager manager) throws ComponentException {
this.manager = manager;
if (this.getLogger().isDebugEnabled() == true) {
getLogger().debug("Looking up " + Store.ROLE + "/JispFilesystemStore");
}
this.fsstore = (Store)manager.lookup(Store.ROLE + "/JispFilesystemStore");
}
/**
* Dispose the component
*/
public void dispose() {
if (this.manager != null) {
this.manager.release(this.fsstore);
this.fsstore = null;
}
}
/**
* Configure the Filesystem Queue:
* <UL>
* <LI>handlerinterval = Interval of the Queue Handler Thread</LI>
* <LI>threadpriority = Priority of the Queue Handler Thread</LI>
* >LI>maxobjects = Defines the max. numbers of Objects in the queue</LI>
* </UL>
*/
public void configure(Configuration conf) throws ConfigurationException {
Parameters params = Parameters.fromConfiguration(conf);
this.handlerinterval = params.getParameterAsInteger("handlerinterval",10);
this.threadpriority = params.getParameterAsInteger("threadpriority",5);
this.maxobjects = params.getParameterAsInteger("maxobjects",100);
if (this.getLogger().isDebugEnabled() == true) {
this.getLogger().debug("Configure Filesystem Queue");
this.getLogger().debug("handlerinterval=" + handlerinterval);
this.getLogger().debug("threadpriority=" + threadpriority);
}
this.linkedList = new LinkedList();
}
public void start() {
doRun = true;
this.fsQueueHandler = new Thread(this);
this.fsQueueHandler.setDaemon(true);
this.fsQueueHandler.setPriority(this.threadpriority);
this.fsQueueHandler.setName("fsQueueHandler");
this.fsQueueHandler.start();
}
public void stop() {
doRun = false;
}
public void run() {
while (doRun) {
while(!this.isEmpty()) {
FilesystemQueueObject filesystemQueueObject =
(FilesystemQueueObject)this.pop();
try {
this.fsstore.store(filesystemQueueObject.getKey(),
filesystemQueueObject.getObject()
);
} catch(IOException e) {
this.getLogger().error("Error in fsQueueHandler",e);
}
}
try {
Thread.currentThread().sleep(this.handlerinterval * 1000);
} catch (InterruptedException ignore) {}
}
}
/**
* Clear all elements from queue.
*/
public void clear() {
if (this.getLogger().isDebugEnabled() == true) {
this.getLogger().debug("Clearing the FilesystemQueue");
}
ListIterator listIterator = linkedList.listIterator(0);
while(listIterator.hasNext()) {
this.linkedList.remove(listIterator.nextIndex());
}
}
/**
* Test if queue is empty.
*
* @return true if queue is empty else false.
*/
public boolean isEmpty() {
return (this.linkedList.size() == 0);
}
/**
* Insert an element into queue.
*
* @param element the element to be inserted
*/
public void insert(Object element) {
if(this.linkedList.size() < maxobjects) {
if (this.getLogger().isDebugEnabled() == true) {
this.getLogger().debug("Insert Element in FilesystemQueue");
}
this.linkedList.addFirst(element);
this.reportSize();
} else {
this.getLogger().warn("Filesystem Queue full!");
}
}
/**
* Return element on top of heap but don't remove it.
*
* @return the element at top of heap
* @exception NoSuchElementException if isEmpty() == true
*/
public Object peek() throws NoSuchElementException {
if (this.getLogger().isDebugEnabled() == true) {
this.getLogger().debug("Peek Element in FilesystemQueue");
}
return (Comparable)linkedList.getLast();
}
/**
* Return element on top of heap and remove it.
*
* @return the element at top of heap
* @exception NoSuchElementException if isEmpty() == true
*/
public Object pop() throws NoSuchElementException {
if (this.getLogger().isDebugEnabled() == true) {
this.getLogger().debug("Pop Element in FilesystemQueue");
}
return (Comparable)linkedList.removeLast();
}
public void reportSize() {
if (this.getLogger().isDebugEnabled() == true) {
this.getLogger().debug("Size of FilesystemQueue=" +
this.linkedList.size());
}
}
}
----------------------------------------------------------------------
In case of troubles, e-mail: [EMAIL PROTECTED]
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
