Author: jukka Date: Wed Mar 6 14:13:15 2013 New Revision: 1453336 URL: http://svn.apache.org/r1453336 Log: OAK-593: Segment-based MK
Each new session starts a new SegmentWriter, so we need to keep the default memory use down. Use an automatically growing byte buffer to achieve that. Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1453336&r1=1453335&r2=1453336&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Wed Mar 6 14:13:15 2013 @@ -23,6 +23,7 @@ import static com.google.common.base.Pre import static com.google.common.base.Preconditions.checkPositionIndexes; import static com.google.common.base.Preconditions.checkState; import static org.apache.jackrabbit.oak.plugins.segment.MapRecord.BUCKETS_PER_LEVEL; +import static org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -55,6 +56,8 @@ import com.google.common.io.ByteStreams; public class SegmentWriter { + private static final byte[] EMPTY_BUFFER = new byte[0]; + static final int BLOCK_SIZE = 1 << 12; // 4kB private final SegmentStore store; @@ -74,12 +77,10 @@ public class SegmentWriter { /** * The segment write buffer, filled from the end to the beginning - * (see OAK-629). Note that we currently allocate the entire buffer - * right from the beginning. It might turn out that a better approach - * would be to start with a smaller buffer that grows automatically, - * or to use a pool of pre-allocated buffers. + * (see OAK-629). The buffer grows automatically up to + * {@link Segment#MAX_SEGMENT_SIZE}. */ - private final byte[] buffer = new byte[Segment.MAX_SEGMENT_SIZE]; + private byte[] buffer = EMPTY_BUFFER; /** * The number of bytes already written (or allocated). Counted from @@ -102,14 +103,15 @@ public class SegmentWriter { byte[] data = buffer; if (length < buffer.length) { data = new byte[length]; - int start = buffer.length - length; - System.arraycopy(buffer, start, data, 0, data.length); + System.arraycopy( + buffer, buffer.length - length, data, 0, data.length); } store.createSegment(new Segment( store, uuid, data, uuids.keySet(), strings, templates)); uuid = UUID.randomUUID(); + buffer = EMPTY_BUFFER; length = 0; uuids.clear(); strings.clear(); @@ -137,14 +139,26 @@ public class SegmentWriter { int alignment = Segment.RECORD_ALIGN_BYTES - 1; int alignedSize = (fullSize + alignment) & ~alignment; - if (length + alignedSize > buffer.length + if (length + alignedSize > MAX_SEGMENT_SIZE || uuids.size() + segmentIds.size() > 0x100) { flush(); } + if (length + alignedSize > buffer.length) { + int newBufferLength = Math.max(2 * buffer.length, 4096); + while (length + alignedSize > newBufferLength) { + newBufferLength *= 2; + } + byte[] newBuffer = new byte[newBufferLength]; + System.arraycopy( + buffer, buffer.length - length, + newBuffer, newBuffer.length - length, length); + buffer = newBuffer; + } length += alignedSize; + checkState(length <= MAX_SEGMENT_SIZE); position = buffer.length - length; - return new RecordId(uuid, position); + return new RecordId(uuid, MAX_SEGMENT_SIZE - length); } private synchronized void writeRecordId(RecordId id) { @@ -159,7 +173,7 @@ public class SegmentWriter { } int offset = id.getOffset(); - checkState(0 <= offset && offset < buffer.length); + checkState(0 <= offset && offset < MAX_SEGMENT_SIZE); checkState((offset & (Segment.RECORD_ALIGN_BYTES - 1)) == 0); buffer[position++] = segmentIndex.byteValue();