cutting 2004/10/04 12:44:11 Modified: src/java/org/apache/lucene/store MMapDirectory.java Log: Improved version by Paul Elschot that can handle files longer than 2^31. Revision Changes Path 1.3 +128 -12 jakarta-lucene/src/java/org/apache/lucene/store/MMapDirectory.java Index: MMapDirectory.java =================================================================== RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/store/MMapDirectory.java,v retrieving revision 1.2 retrieving revision 1.3 diff -u -r1.2 -r1.3 --- MMapDirectory.java 28 Sep 2004 21:55:59 -0000 1.2 +++ MMapDirectory.java 4 Oct 2004 19:44:11 -0000 1.3 @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + import java.io.IOException; import java.io.File; import java.io.RandomAccessFile; @@ -29,24 +29,17 @@ * org.apache.lucene.FSDirectory.class set to * org.apache.lucene.store.MMapDirectory. This will cause [EMAIL PROTECTED] * FSDirectory#getDirectory(File,boolean)} to return instances of this class. - * - * @author Doug Cutting */ public class MMapDirectory extends FSDirectory { - private class MMapIndexInput extends IndexInput { + private static class MMapIndexInput extends IndexInput { private ByteBuffer buffer; private final long length; - public MMapIndexInput(File file) throws IOException { - RandomAccessFile raf = new RandomAccessFile(file, "r"); - try { + private MMapIndexInput(RandomAccessFile raf) throws IOException { this.length = raf.length(); this.buffer = raf.getChannel().map(MapMode.READ_ONLY, 0, length); - } finally { - raf.close(); - } } public byte readByte() throws IOException { @@ -77,11 +70,134 @@ } public void close() throws IOException {} + } + /* Added class MultiMMapIndexInput, Paul Elschot. + * Slightly adapted constructor of MMapIndexInput. + * Licensed under the Apache License, Version 2.0. + */ + private static class MultiMMapIndexInput extends IndexInput { + + private ByteBuffer[] buffers; + private int[] bufSizes; // keep here, ByteBuffer.size() method is optional + + private final long length; + + private int curBufIndex; + private final int maxBufSize; + + private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex] + private int curAvail; // redundant for speed: (bufSizes[curBufIndex] - curBuf.position()) + + + public MultiMMapIndexInput(RandomAccessFile raf, int maxBufSize) + throws IOException { + this.length = raf.length(); + this.maxBufSize = maxBufSize; + + if (maxBufSize <= 0) + throw new IllegalArgumentException("Non positive maxBufSize: " + + maxBufSize); + + if ((length / maxBufSize) > Integer.MAX_VALUE) + throw new IllegalArgumentException + ("RandomAccessFile too big for maximum buffer size: " + + raf.toString()); + + int nrBuffers = (int) (length / maxBufSize); + if ((nrBuffers * maxBufSize) < length) nrBuffers++; + + this.buffers = new ByteBuffer[nrBuffers]; + this.bufSizes = new int[nrBuffers]; + + long bufferStart = 0; + FileChannel rafc = raf.getChannel(); + for (int bufNr = 0; bufNr < nrBuffers; bufNr++) { + int bufSize = (length > (bufferStart + maxBufSize)) + ? maxBufSize + : (int) (length - bufferStart); + //System.out.println("mapping from: "+bufferStart+", size: "+bufSize); + this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY,bufferStart,bufSize); + this.bufSizes[bufNr] = bufSize; + bufferStart += bufSize; + } + seek(0L); + } + + public byte readByte() throws IOException { + // Performance might be improved by reading ahead into an array of + // eg. 128 bytes and readByte() from there. + if (curAvail == 0) { + curBufIndex++; + curBuf = buffers[curBufIndex]; + curBuf.position(0); // index out of bounds when too many requested + curAvail = bufSizes[curBufIndex]; + } + curAvail--; + return curBuf.get(); + } + + public void readBytes(byte[] b, int offset, int len) throws IOException { + while (len > curAvail) { + curBuf.get(b, offset, curAvail); + len -= curAvail; + offset += curAvail; + curBufIndex++; + curBuf = buffers[curBufIndex]; + curBuf.position(0); // index out of bounds when too many requested + curAvail = bufSizes[curBufIndex]; + } + curBuf.get(b, offset, len); + curAvail -= len; + } + + public long getFilePointer() { + return (curBufIndex * (long) maxBufSize) + curBuf.position(); + } + + public void seek(long pos) throws IOException { + curBufIndex = (int) (pos / maxBufSize); + curBuf = buffers[curBufIndex]; + int bufOffset = (int) (pos - (curBufIndex * maxBufSize)); + curBuf.position(bufOffset); + curAvail = bufSizes[curBufIndex] - bufOffset; + } + + public long length() { + return length; + } + + public Object clone() { + MultiMMapIndexInput clone = (MultiMMapIndexInput)super.clone(); + clone.buffers = new ByteBuffer[buffers.length]; + // No need to clone bufSizes. + // Since most clones will use only one buffer, duplicate() could also be + // done lazy in clones, eg. when adapting curBuf. + for (int bufNr = 0; bufNr < buffers.length; bufNr++) { + clone.buffers[bufNr] = buffers[bufNr].duplicate(); + } + try { + clone.seek(getFilePointer()); + } catch(IOException ioe) { + throw new RuntimeException(ioe); + }; + return clone; + } + + public void close() throws IOException {} } + + private final int MAX_BBUF = Integer.MAX_VALUE; public IndexInput openInput(String name) throws IOException { - return new MMapIndexInput(new File(getFile(), name)); + File f = new File(getFile(), name); + RandomAccessFile raf = new RandomAccessFile(f, "r"); + try { + return (raf.length() <= MAX_BBUF) + ? (IndexInput) new MMapIndexInput(raf) + : (IndexInput) new MultiMMapIndexInput(raf, MAX_BBUF); + } finally { + raf.close(); + } } } -
--------------------------------------------------------------------- To unsubscribe, e-mail: [EMAIL PROTECTED] For additional commands, e-mail: [EMAIL PROTECTED]