Author: frm Date: Wed Feb 28 11:58:42 2018 New Revision: 1825553 URL: http://svn.apache.org/viewvc?rev=1825553&view=rev Log: OAK-7280 - Remove superfluous methods from SegmentWriter
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java (with props) Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/server/DefaultStandbyReferenceReaderTest.java Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java Wed Feb 28 11:58:42 2018 @@ -139,12 +139,8 @@ public class DefaultSegmentWriter implem writeOperationHandler.flush(store); } - @Override @Nonnull - public RecordId writeMap(@Nullable final MapRecord base, - @Nonnull final Map<String, RecordId> changes - ) - throws IOException { + RecordId writeMap(@Nullable final MapRecord base, @Nonnull final Map<String, RecordId> changes) throws IOException { return writeOperationHandler.execute(new SegmentWriteOperation() { @Nonnull @@ -152,12 +148,12 @@ public class DefaultSegmentWriter implem public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException { return with(writer).writeMap(base, changes); } + }); } - @Override @Nonnull - public RecordId writeList(@Nonnull final List<RecordId> list) throws IOException { + RecordId writeList(@Nonnull final List<RecordId> list) throws IOException { return writeOperationHandler.execute(new SegmentWriteOperation() { @Nonnull @@ -165,12 +161,12 @@ public class DefaultSegmentWriter implem public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException { return with(writer).writeList(list); } + }); } - @Override @Nonnull - public RecordId writeString(@Nonnull final String string) throws IOException { + RecordId writeString(@Nonnull final String string) throws IOException { return writeOperationHandler.execute(new SegmentWriteOperation() { @Nonnull @@ -178,10 +174,10 @@ public class DefaultSegmentWriter implem public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException { return with(writer).writeString(string); } + }); } - @Override @Nonnull public RecordId writeBlob(@Nonnull final Blob blob) throws IOException { return writeOperationHandler.execute(new SegmentWriteOperation() { @@ -191,13 +187,12 @@ public class DefaultSegmentWriter implem public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException { return with(writer).writeBlob(blob); } + }); } - @Override @Nonnull - public RecordId writeBlock(@Nonnull final byte[] bytes, final int offset, final int length) - throws IOException { + RecordId writeBlock(@Nonnull final byte[] bytes, final int offset, final int length) throws IOException { return writeOperationHandler.execute(new SegmentWriteOperation() { @Nonnull @@ -205,6 +200,7 @@ public class DefaultSegmentWriter implem public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException { return with(writer).writeBlock(bytes, offset, length); } + }); } @@ -221,9 +217,8 @@ public class DefaultSegmentWriter implem }); } - @Override @Nonnull - public RecordId writeProperty(@Nonnull final PropertyState state) throws IOException { + RecordId writeProperty(@Nonnull final PropertyState state) throws IOException { return writeOperationHandler.execute(new SegmentWriteOperation() { @Nonnull @@ -236,16 +231,15 @@ public class DefaultSegmentWriter implem @Override @Nonnull - public RecordId writeNode( - @Nonnull final NodeState state, - @Nullable final ByteBuffer stableIdBytes) - throws IOException { + public RecordId writeNode(@Nonnull final NodeState state, @Nullable final ByteBuffer stableIdBytes) throws IOException { return writeOperationHandler.execute(new SegmentWriteOperation() { + @Nonnull @Override public RecordId execute(@Nonnull SegmentBufferWriter writer) throws IOException { return with(writer).writeNode(state, stableIdBytes); } + }); } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java Wed Feb 28 11:58:42 2018 @@ -20,14 +20,11 @@ package org.apache.jackrabbit.oak.segmen import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.NodeState; /** @@ -38,41 +35,6 @@ public interface SegmentWriter { void flush() throws IOException; /** - * Write a map record. - * - * @param base base map relative to which the {@code changes} are applied - * ot {@code null} for the empty map. - * @param changes the changed mapping to apply to the {@code base} map. - * @return the record id of the map written - * @throws IOException - */ - @Nonnull - // TODO frm this method is only used from test code, should it be removed? - RecordId writeMap(@Nullable MapRecord base, @Nonnull Map<String, RecordId> changes) throws IOException; - - /** - * Write a list record. - * - * @param list the list to write. - * @return the record id of the list written - * @throws IOException - */ - @Nonnull - // TODO frm this method is only used from test code, should it be removed? - RecordId writeList(@Nonnull List<RecordId> list) throws IOException; - - /** - * Write a string record. - * - * @param string the string to write. - * @return the record id of the string written. - * @throws IOException - */ - @Nonnull - // TODO frm this method is only used from test code, should it be removed? - RecordId writeString(@Nonnull String string) throws IOException; - - /** * Write a blob (as list of block records) * * @param blob blob to write @@ -83,18 +45,6 @@ public interface SegmentWriter { RecordId writeBlob(@Nonnull Blob blob) throws IOException; /** - * Writes a block record containing the given block of bytes. - * - * @param bytes source buffer - * @param offset offset within the source buffer - * @param length number of bytes to write - * @return the record id of the block written - */ - @Nonnull - // TODO frm this method is only used from test code, should it be removed? - RecordId writeBlock(@Nonnull byte[] bytes, int offset, int length) throws IOException; - - /** * Writes a stream value record. The given stream is consumed <em>and * closed</em> by this method. * @@ -107,17 +57,6 @@ public interface SegmentWriter { RecordId writeStream(@Nonnull InputStream stream) throws IOException; /** - * Write a property. - * - * @param state the property to write - * @return the record id of the property state written - * @throws IOException - */ - @Nonnull - // TODO frm this method is only used from test code, should it be removed? - RecordId writeProperty(@Nonnull PropertyState state) throws IOException; - - /** * Write a node state. If non null, the passed {@code stableId} will be assigned to * the persisted node. Otherwise the stable id will be inferred from {@code state}. * @@ -140,4 +79,5 @@ public interface SegmentWriter { default RecordId writeNode(@Nonnull NodeState state) throws IOException { return writeNode(state, null); } + } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java Wed Feb 28 11:58:42 2018 @@ -34,7 +34,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; -import java.util.Map; import java.util.Random; import javax.annotation.Nonnull; @@ -44,7 +43,6 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; -import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; @@ -200,50 +198,17 @@ public class CompactorTest { @Nonnull @Override - public RecordId writeMap( - @Nullable MapRecord base, @Nonnull Map<String, RecordId> changes) - throws IOException { - return delegate.writeMap(base, changes); - } - - @Nonnull - @Override - public RecordId writeList(@Nonnull List<RecordId> list) throws IOException { - return delegate.writeList(list); - } - - @Nonnull - @Override - public RecordId writeString(@Nonnull String string) throws IOException { - return delegate.writeString(string); - } - - @Nonnull - @Override public RecordId writeBlob(@Nonnull Blob blob) throws IOException { return delegate.writeBlob(blob); } @Nonnull @Override - public RecordId writeBlock(@Nonnull byte[] bytes, int offset, int length) - throws IOException { - return delegate.writeBlock(bytes, offset, length); - } - - @Nonnull - @Override public RecordId writeStream(@Nonnull InputStream stream) throws IOException { return delegate.writeStream(stream); } @Nonnull - @Override - public RecordId writeProperty(@Nonnull PropertyState state) throws IOException { - return delegate.writeProperty(state); - } - - @Nonnull @Override public RecordId writeNode(@Nonnull NodeState state, @Nullable ByteBuffer stableIdBytes) throws IOException { Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java?rev=1825553&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java Wed Feb 28 11:58:42 2018 @@ -0,0 +1,278 @@ +package org.apache.jackrabbit.oak.segment; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Maps.newHashMap; +import static org.apache.jackrabbit.oak.segment.DefaultSegmentWriterBuilder.defaultSegmentWriterBuilder; +import static org.apache.jackrabbit.oak.segment.ListRecord.LEVEL_SIZE; +import static org.apache.jackrabbit.oak.segment.ListRecord.MAX_ELEMENTS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import com.google.common.base.Charsets; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import org.apache.jackrabbit.oak.segment.test.TemporaryFileStore; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; + +public class DefaultSegmentWriterTest { + + private static final String HELLO_WORLD = "Hello, World!"; + + private final byte[] bytes = HELLO_WORLD.getBytes(Charsets.UTF_8); + + private TemporaryFolder folder = new TemporaryFolder(new File("target")); + + private TemporaryFileStore store = new TemporaryFileStore(folder, false); + + @Rule + public RuleChain rules = RuleChain.outerRule(folder).around(store); + + private DefaultSegmentWriter writer; + + @Before + public void setUp() throws Exception { + writer = defaultSegmentWriterBuilder("test").build(store.fileStore()); + } + + @Test + public void testBlockRecord() throws IOException { + RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); + BlockRecord block = new BlockRecord(blockId, bytes.length); + + // Check reading with all valid positions and lengths + for (int n = 1; n < bytes.length; n++) { + for (int i = 0; i + n <= bytes.length; i++) { + Arrays.fill(bytes, i, i + n, (byte) '.'); + assertEquals(n, block.read(i, bytes, i, n)); + assertEquals(HELLO_WORLD, new String(bytes, Charsets.UTF_8)); + } + } + + // Check reading with a too long length + byte[] large = new byte[bytes.length * 2]; + assertEquals(bytes.length, block.read(0, large, 0, large.length)); + assertEquals(HELLO_WORLD, new String(large, 0, bytes.length, Charsets.UTF_8)); + } + + @Test + public void testListRecord() throws IOException { + RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); + + ListRecord one = writeList(1, blockId); + ListRecord level1 = writeList(LEVEL_SIZE, blockId); + ListRecord level1p = writeList(LEVEL_SIZE + 1, blockId); + ListRecord level2 = writeList(LEVEL_SIZE * LEVEL_SIZE, blockId); + ListRecord level2p = writeList(LEVEL_SIZE * LEVEL_SIZE + 1, blockId); + + assertEquals(1, one.size()); + assertEquals(blockId, one.getEntry(0)); + assertEquals(LEVEL_SIZE, level1.size()); + assertEquals(blockId, level1.getEntry(0)); + assertEquals(blockId, level1.getEntry(LEVEL_SIZE - 1)); + assertEquals(LEVEL_SIZE + 1, level1p.size()); + assertEquals(blockId, level1p.getEntry(0)); + assertEquals(blockId, level1p.getEntry(LEVEL_SIZE)); + assertEquals(LEVEL_SIZE * LEVEL_SIZE, level2.size()); + assertEquals(blockId, level2.getEntry(0)); + assertEquals(blockId, level2.getEntry(LEVEL_SIZE * LEVEL_SIZE - 1)); + assertEquals(LEVEL_SIZE * LEVEL_SIZE + 1, level2p.size()); + assertEquals(blockId, level2p.getEntry(0)); + assertEquals(blockId, level2p.getEntry(LEVEL_SIZE * LEVEL_SIZE)); + + int count = 0; + for (RecordId entry : level2p.getEntries()) { + assertEquals(blockId, entry); + assertEquals(blockId, level2p.getEntry(count)); + count++; + } + assertEquals(LEVEL_SIZE * LEVEL_SIZE + 1, count); + } + + @Test + public void testLargeListRecord() throws IOException { + RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); + + ListRecord one = writeList(MAX_ELEMENTS, blockId); + one.getEntry(0); + } + + @Test(expected = IllegalArgumentException.class) + public void testTooLargeListRecord() throws IOException { + RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); + + ListRecord one = writeList(MAX_ELEMENTS + 1, blockId); + } + + private ListRecord writeList(int size, RecordId id) throws IOException { + List<RecordId> list = Collections.nCopies(size, id); + return new ListRecord(writer.writeList(list), size); + } + + @Test + public void testListWithLotsOfReferences() throws IOException { // OAK-1184 + List<RecordId> list = newArrayList(); + for (int i = 0; i < 1000; i++) { + list.add(new RecordId(store.fileStore().getSegmentIdProvider().newBulkSegmentId(), 0)); + } + writer.writeList(list); + } + + @Test + public void testStringRecord() throws IOException { + RecordId empty = writer.writeString(""); + RecordId space = writer.writeString(" "); + RecordId hello = writer.writeString("Hello, World!"); + + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < 2 * Segment.MAX_SEGMENT_SIZE + 1000; i++) { + builder.append((char) ('0' + i % 10)); + } + RecordId large = writer.writeString(builder.toString()); + + Segment segment = large.getSegmentId().getSegment(); + + assertEquals("", store.fileStore().getReader().readString(empty)); + assertEquals(" ", store.fileStore().getReader().readString(space)); + assertEquals("Hello, World!", store.fileStore().getReader().readString(hello)); + assertEquals(builder.toString(), store.fileStore().getReader().readString(large)); + } + + @Test + public void testMapRecord() throws IOException { + RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); + + MapRecord zero = new MapRecord(store.fileStore().getReader(), writer.writeMap(null, ImmutableMap.<String, RecordId>of())); + MapRecord one = new MapRecord(store.fileStore().getReader(), writer.writeMap(null, ImmutableMap.of("one", blockId))); + MapRecord two = new MapRecord(store.fileStore().getReader(), writer.writeMap(null, ImmutableMap.of("one", blockId, "two", blockId))); + Map<String, RecordId> map = newHashMap(); + for (int i = 0; i < 1000; i++) { + map.put("key" + i, blockId); + } + MapRecord many = new MapRecord(store.fileStore().getReader(), writer.writeMap(null, map)); + + Iterator<MapEntry> iterator; + + assertEquals(0, zero.size()); + assertNull(zero.getEntry("one")); + iterator = zero.getEntries().iterator(); + assertFalse(iterator.hasNext()); + + assertEquals(1, one.size()); + assertEquals(blockId, one.getEntry("one").getValue()); + assertNull(one.getEntry("two")); + iterator = one.getEntries().iterator(); + assertTrue(iterator.hasNext()); + assertEquals("one", iterator.next().getName()); + assertFalse(iterator.hasNext()); + + assertEquals(2, two.size()); + assertEquals(blockId, two.getEntry("one").getValue()); + assertEquals(blockId, two.getEntry("two").getValue()); + assertNull(two.getEntry("three")); + iterator = two.getEntries().iterator(); + assertTrue(iterator.hasNext()); + iterator.next(); + assertTrue(iterator.hasNext()); + iterator.next(); + assertFalse(iterator.hasNext()); + + assertEquals(1000, many.size()); + iterator = many.getEntries().iterator(); + for (int i = 0; i < 1000; i++) { + assertTrue(iterator.hasNext()); + assertEquals(blockId, iterator.next().getValue()); + assertEquals(blockId, many.getEntry("key" + i).getValue()); + } + assertFalse(iterator.hasNext()); + assertNull(many.getEntry("foo")); + + Map<String, RecordId> changes = newHashMap(); + changes.put("key0", null); + changes.put("key1000", blockId); + MapRecord modified = new MapRecord(store.fileStore().getReader(), writer.writeMap(many, changes)); + assertEquals(1000, modified.size()); + iterator = modified.getEntries().iterator(); + for (int i = 1; i <= 1000; i++) { + assertTrue(iterator.hasNext()); + assertEquals(blockId, iterator.next().getValue()); + assertEquals(blockId, modified.getEntry("key" + i).getValue()); + } + assertFalse(iterator.hasNext()); + assertNull(many.getEntry("foo")); + } + + @Test + public void testMapRemoveNonExisting() throws IOException { + RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); + + Map<String, RecordId> changes = newHashMap(); + changes.put("one", null); + MapRecord zero = new MapRecord(store.fileStore().getReader(), writer.writeMap(null, changes)); + assertEquals(0, zero.size()); + } + + @Test + public void testWorstCaseMap() throws IOException { + RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); + Map<String, RecordId> map = newHashMap(); + char[] key = new char[2]; + for (int i = 0; i <= MapRecord.BUCKETS_PER_LEVEL; i++) { + key[0] = (char) ('A' + i); + key[1] = (char) ('\u1000' - key[0] * 31); + map.put(new String(key), blockId); + } + + MapRecord bad = new MapRecord(store.fileStore().getReader(), writer.writeMap(null, map)); + + assertEquals(map.size(), bad.size()); + Iterator<MapEntry> iterator = bad.getEntries().iterator(); + for (int i = 0; i < map.size(); i++) { + assertTrue(iterator.hasNext()); + assertEquals('\u1000', iterator.next().getName().hashCode()); + } + assertFalse(iterator.hasNext()); + } + + @Test // See OAK-2049 + public void segmentOverflow() throws Exception { + for (int n = 1; n < 255; n++) { // 255 = ListRecord.LEVEL_SIZE + DefaultSegmentWriter writer = defaultSegmentWriterBuilder("test").build(store.fileStore()); + // writer.length == 32 (from the root node) + + // adding 15 strings with 16516 bytes each + for (int k = 0; k < 15; k++) { + // 16516 = (Segment.MEDIUM_LIMIT - 1 + 2 + 3) + // 1 byte per char, 2 byte to store the length and 3 bytes for the + // alignment to the integer boundary + writer.writeString(Strings.repeat("abcdefghijklmno".substring(k, k + 1), + SegmentTestConstants.MEDIUM_LIMIT - 1)); + } + + // adding 14280 bytes. 1 byte per char, and 2 bytes to store the length + RecordId x = writer.writeString(Strings.repeat("x", 14278)); + // writer.length == 262052 + + // Adding 765 bytes (255 recordIds) + // This should cause the current segment to flush + List<RecordId> list = Collections.nCopies(n, x); + writer.writeList(list); + + writer.flush(); + } + } + +} Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordTest.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/RecordTest.java Wed Feb 28 11:58:42 2018 @@ -19,7 +19,6 @@ package org.apache.jackrabbit.oak.segment; import static com.google.common.collect.Lists.newArrayList; -import static com.google.common.collect.Maps.newHashMap; import static java.lang.Math.min; import static java.util.Collections.singletonList; import static junit.framework.Assert.assertNotNull; @@ -29,27 +28,16 @@ import static org.apache.jackrabbit.oak. import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.api.Type.STRINGS; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.apache.jackrabbit.oak.segment.ListRecord.LEVEL_SIZE; -import static org.apache.jackrabbit.oak.segment.ListRecord.MAX_ELEMENTS; import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Random; -import com.google.common.base.Charsets; -import com.google.common.collect.ImmutableMap; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; @@ -69,10 +57,6 @@ public class RecordTest { @Rule public TemporaryFolder folder = new TemporaryFolder(new File("target")); - private static final String HELLO_WORLD = "Hello, World!"; - - private final byte[] bytes = HELLO_WORLD.getBytes(Charsets.UTF_8); - private FileStore store; private SegmentWriter writer; @@ -91,89 +75,6 @@ public class RecordTest { } @Test - public void testBlockRecord() throws IOException { - RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - BlockRecord block = new BlockRecord(blockId, bytes.length); - - // Check reading with all valid positions and lengths - for (int n = 1; n < bytes.length; n++) { - for (int i = 0; i + n <= bytes.length; i++) { - Arrays.fill(bytes, i, i + n, (byte) '.'); - assertEquals(n, block.read(i, bytes, i, n)); - assertEquals(HELLO_WORLD, new String(bytes, Charsets.UTF_8)); - } - } - - // Check reading with a too long length - byte[] large = new byte[bytes.length * 2]; - assertEquals(bytes.length, block.read(0, large, 0, large.length)); - assertEquals(HELLO_WORLD, new String(large, 0, bytes.length, Charsets.UTF_8)); - } - - @Test - public void testListRecord() throws IOException { - RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - - ListRecord one = writeList(1, blockId); - ListRecord level1 = writeList(LEVEL_SIZE, blockId); - ListRecord level1p = writeList(LEVEL_SIZE + 1, blockId); - ListRecord level2 = writeList(LEVEL_SIZE * LEVEL_SIZE, blockId); - ListRecord level2p = writeList(LEVEL_SIZE * LEVEL_SIZE + 1, blockId); - - assertEquals(1, one.size()); - assertEquals(blockId, one.getEntry(0)); - assertEquals(LEVEL_SIZE, level1.size()); - assertEquals(blockId, level1.getEntry(0)); - assertEquals(blockId, level1.getEntry(LEVEL_SIZE - 1)); - assertEquals(LEVEL_SIZE + 1, level1p.size()); - assertEquals(blockId, level1p.getEntry(0)); - assertEquals(blockId, level1p.getEntry(LEVEL_SIZE)); - assertEquals(LEVEL_SIZE * LEVEL_SIZE, level2.size()); - assertEquals(blockId, level2.getEntry(0)); - assertEquals(blockId, level2.getEntry(LEVEL_SIZE * LEVEL_SIZE - 1)); - assertEquals(LEVEL_SIZE * LEVEL_SIZE + 1, level2p.size()); - assertEquals(blockId, level2p.getEntry(0)); - assertEquals(blockId, level2p.getEntry(LEVEL_SIZE * LEVEL_SIZE)); - - int count = 0; - for (RecordId entry : level2p.getEntries()) { - assertEquals(blockId, entry); - assertEquals(blockId, level2p.getEntry(count)); - count++; - } - assertEquals(LEVEL_SIZE * LEVEL_SIZE + 1, count); - } - - @Test - public void testLargeListRecord() throws IOException { - RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - - ListRecord one = writeList(MAX_ELEMENTS, blockId); - one.getEntry(0); - } - - @Test(expected = IllegalArgumentException.class) - public void testTooLargeListRecord() throws IOException { - RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - - ListRecord one = writeList(MAX_ELEMENTS + 1, blockId); - } - - private ListRecord writeList(int size, RecordId id) throws IOException { - List<RecordId> list = Collections.nCopies(size, id); - return new ListRecord(writer.writeList(list), size); - } - - @Test - public void testListWithLotsOfReferences() throws IOException { // OAK-1184 - List<RecordId> list = newArrayList(); - for (int i = 0; i < 1000; i++) { - list.add(new RecordId(store.getSegmentIdProvider().newBulkSegmentId(), 0)); - } - writer.writeList(list); - } - - @Test public void testStreamRecord() throws IOException { checkRandomStreamRecord(0); checkRandomStreamRecord(1); @@ -219,122 +120,6 @@ public class RecordTest { } @Test - public void testStringRecord() throws IOException { - RecordId empty = writer.writeString(""); - RecordId space = writer.writeString(" "); - RecordId hello = writer.writeString("Hello, World!"); - - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < 2 * Segment.MAX_SEGMENT_SIZE + 1000; i++) { - builder.append((char) ('0' + i % 10)); - } - RecordId large = writer.writeString(builder.toString()); - - Segment segment = large.getSegmentId().getSegment(); - - assertEquals("", store.getReader().readString(empty)); - assertEquals(" ", store.getReader().readString(space)); - assertEquals("Hello, World!", store.getReader().readString(hello)); - assertEquals(builder.toString(), store.getReader().readString(large)); - } - - @Test - public void testMapRecord() throws IOException { - RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - - MapRecord zero = new MapRecord(store.getReader(), writer.writeMap(null, ImmutableMap.<String, RecordId>of())); - MapRecord one = new MapRecord(store.getReader(), writer.writeMap(null, ImmutableMap.of("one", blockId))); - MapRecord two = new MapRecord(store.getReader(), writer.writeMap(null, ImmutableMap.of("one", blockId, "two", blockId))); - Map<String, RecordId> map = newHashMap(); - for (int i = 0; i < 1000; i++) { - map.put("key" + i, blockId); - } - MapRecord many = new MapRecord(store.getReader(), writer.writeMap(null, map)); - - Iterator<MapEntry> iterator; - - assertEquals(0, zero.size()); - assertNull(zero.getEntry("one")); - iterator = zero.getEntries().iterator(); - assertFalse(iterator.hasNext()); - - assertEquals(1, one.size()); - assertEquals(blockId, one.getEntry("one").getValue()); - assertNull(one.getEntry("two")); - iterator = one.getEntries().iterator(); - assertTrue(iterator.hasNext()); - assertEquals("one", iterator.next().getName()); - assertFalse(iterator.hasNext()); - - assertEquals(2, two.size()); - assertEquals(blockId, two.getEntry("one").getValue()); - assertEquals(blockId, two.getEntry("two").getValue()); - assertNull(two.getEntry("three")); - iterator = two.getEntries().iterator(); - assertTrue(iterator.hasNext()); - iterator.next(); - assertTrue(iterator.hasNext()); - iterator.next(); - assertFalse(iterator.hasNext()); - - assertEquals(1000, many.size()); - iterator = many.getEntries().iterator(); - for (int i = 0; i < 1000; i++) { - assertTrue(iterator.hasNext()); - assertEquals(blockId, iterator.next().getValue()); - assertEquals(blockId, many.getEntry("key" + i).getValue()); - } - assertFalse(iterator.hasNext()); - assertNull(many.getEntry("foo")); - - Map<String, RecordId> changes = newHashMap(); - changes.put("key0", null); - changes.put("key1000", blockId); - MapRecord modified = new MapRecord(store.getReader(), writer.writeMap(many, changes)); - assertEquals(1000, modified.size()); - iterator = modified.getEntries().iterator(); - for (int i = 1; i <= 1000; i++) { - assertTrue(iterator.hasNext()); - assertEquals(blockId, iterator.next().getValue()); - assertEquals(blockId, modified.getEntry("key" + i).getValue()); - } - assertFalse(iterator.hasNext()); - assertNull(many.getEntry("foo")); - } - - @Test - public void testMapRemoveNonExisting() throws IOException { - RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - - Map<String, RecordId> changes = newHashMap(); - changes.put("one", null); - MapRecord zero = new MapRecord(store.getReader(), writer.writeMap(null, changes)); - assertEquals(0, zero.size()); - } - - @Test - public void testWorstCaseMap() throws IOException { - RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - Map<String, RecordId> map = newHashMap(); - char[] key = new char[2]; - for (int i = 0; i <= MapRecord.BUCKETS_PER_LEVEL; i++) { - key[0] = (char) ('A' + i); - key[1] = (char) ('\u1000' - key[0] * 31); - map.put(new String(key), blockId); - } - - MapRecord bad = new MapRecord(store.getReader(), writer.writeMap(null, map)); - - assertEquals(map.size(), bad.size()); - Iterator<MapEntry> iterator = bad.getEntries().iterator(); - for (int i = 0; i < map.size(); i++) { - assertTrue(iterator.hasNext()); - assertEquals('\u1000', iterator.next().getName().hashCode()); - } - assertFalse(iterator.hasNext()); - } - - @Test public void testEmptyNode() throws IOException { NodeState before = EMPTY_NODE; NodeState after = new SegmentNodeState(store.getReader(), writer, store.getBlobStore(), writer.writeNode(before)); Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriterTest.java Wed Feb 28 11:58:42 2018 @@ -26,6 +26,7 @@ import static org.junit.Assert.assertNot import java.io.File; import java.util.List; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.junit.Rule; @@ -82,7 +83,7 @@ public class SegmentBufferWriterTest { try (FileStore store = openFileStore()) { SegmentWriter writer = defaultSegmentWriterBuilder("t").build(store); - writer.writeString("test"); + writer.writeNode(EmptyNodeState.EMPTY_NODE); writer.flush(); } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentParserTest.java Wed Feb 28 11:58:42 2018 @@ -61,7 +61,7 @@ public class SegmentParserTest { private MemoryStore store; - private SegmentWriter writer; + private DefaultSegmentWriter writer; private static class TestParser extends SegmentParser { private final String name; Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentReferencesTest.java Wed Feb 28 11:58:42 2018 @@ -22,10 +22,11 @@ import static org.junit.Assert.assertEqu import static org.junit.Assert.assertNotEquals; import java.io.File; -import java.util.Arrays; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -46,18 +47,20 @@ public class SegmentReferencesTest { // Write two records, one referencing the other. SegmentWriter writer = defaultSegmentWriterBuilder("test").build(store); - RecordId stringId = writer.writeString("test"); - RecordId listId = writer.writeList(Arrays.asList(stringId, stringId)); + RecordId a = writer.writeNode(EmptyNodeState.EMPTY_NODE); + NodeBuilder builder = EmptyNodeState.EMPTY_NODE.builder(); + builder.setChildNode("referred", store.getReader().readNode(a)); + RecordId b = writer.writeNode(builder.getNodeState()); writer.flush(); // The two records should be living in the same segment. - assertEquals(listId.getSegmentId(), stringId.getSegmentId()); + assertEquals(b.getSegmentId(), a.getSegmentId()); // This inter-segment reference shouldn't generate a reference from // this segment to itself. - assertEquals(0, listId.getSegment().getReferencedSegmentIdCount()); + assertEquals(0, b.getSegment().getReferencedSegmentIdCount()); } } @@ -69,21 +72,23 @@ public class SegmentReferencesTest { SegmentWriter writer = defaultSegmentWriterBuilder("test").build(store); - RecordId stringId = writer.writeString("test"); + RecordId a = writer.writeNode(EmptyNodeState.EMPTY_NODE); writer.flush(); - RecordId listId = writer.writeList(Arrays.asList(stringId, stringId)); + NodeBuilder builder = EmptyNodeState.EMPTY_NODE.builder(); + builder.setChildNode("referred", store.getReader().readNode(a)); + RecordId b = writer.writeNode(builder.getNodeState()); writer.flush(); // The two records should be living in two different segments. - assertNotEquals(listId.getSegmentId(), stringId.getSegmentId()); + assertNotEquals(a.getSegmentId(), b.getSegmentId()); // This intra-segment reference should generate a reference from the // segment containing the list to the segment containing the string. - assertEquals(1, listId.getSegment().getReferencedSegmentIdCount()); - assertEquals(stringId.getSegmentId().asUUID(), listId.getSegment().getReferencedSegmentId(0)); + assertEquals(1, b.getSegment().getReferencedSegmentIdCount()); + assertEquals(a.getSegmentId().asUUID(), b.getSegment().getReferencedSegmentId(0)); } } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreIT.java Wed Feb 28 11:58:42 2018 @@ -25,18 +25,14 @@ import static org.junit.Assert.assertFal import java.io.ByteArrayInputStream; import java.io.File; import java.io.RandomAccessFile; -import java.util.Collections; -import java.util.List; import java.util.Random; -import com.google.common.base.Strings; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder; import org.apache.jackrabbit.oak.segment.SegmentNodeState; import org.apache.jackrabbit.oak.segment.SegmentTestConstants; -import org.apache.jackrabbit.oak.segment.SegmentWriter; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -132,39 +128,6 @@ public class FileStoreIT { store.close(); } - @Test // See OAK-2049 - public void segmentOverflow() throws Exception { - for (int n = 1; n < 255; n++) { // 255 = ListRecord.LEVEL_SIZE - FileStore store = fileStoreBuilder(getFileStoreFolder()).withMaxFileSize(1).withMemoryMapping(false).build(); - SegmentWriter writer = store.getWriter(); - // writer.length == 32 (from the root node) - - // adding 15 strings with 16516 bytes each - for (int k = 0; k < 15; k++) { - // 16516 = (Segment.MEDIUM_LIMIT - 1 + 2 + 3) - // 1 byte per char, 2 byte to store the length and 3 bytes for the - // alignment to the integer boundary - writer.writeString(Strings.repeat("abcdefghijklmno".substring(k, k + 1), - SegmentTestConstants.MEDIUM_LIMIT - 1)); - } - - // adding 14280 bytes. 1 byte per char, and 2 bytes to store the length - RecordId x = writer.writeString(Strings.repeat("x", 14278)); - // writer.length == 262052 - - // Adding 765 bytes (255 recordIds) - // This should cause the current segment to flush - List<RecordId> list = Collections.nCopies(n, x); - writer.writeList(list); - - writer.flush(); - - // Don't close the store in a finally clause as if a failure happens - // this will also fail an cover up the earlier exception - store.close(); - } - } - @Test public void nonBlockingROStore() throws Exception { FileStore store = fileStoreBuilder(getFileStoreFolder()).withMaxFileSize(1).withMemoryMapping(false).build(); Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/server/DefaultStandbyReferenceReaderTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/server/DefaultStandbyReferenceReaderTest.java?rev=1825553&r1=1825552&r2=1825553&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/server/DefaultStandbyReferenceReaderTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/server/DefaultStandbyReferenceReaderTest.java Wed Feb 28 11:58:42 2018 @@ -17,7 +17,6 @@ package org.apache.jackrabbit.oak.segment.standby.server; -import static java.util.Arrays.asList; import static java.util.UUID.randomUUID; import static org.apache.jackrabbit.oak.segment.DefaultSegmentWriterBuilder.defaultSegmentWriterBuilder; import static org.junit.Assert.assertEquals; @@ -28,10 +27,12 @@ import static org.junit.Assert.assertTru import java.io.File; import java.util.Iterator; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.SegmentWriter; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -59,7 +60,7 @@ public class DefaultStandbyReferenceRead try (FileStore store = newFileStore()) { SegmentWriter writer = defaultSegmentWriterBuilder("test").build(store); - RecordId id = writer.writeString("test"); + RecordId id = writer.writeNode(EmptyNodeState.EMPTY_NODE); writer.flush(); DefaultStandbyReferencesReader reader = new DefaultStandbyReferencesReader(store); @@ -73,10 +74,12 @@ public class DefaultStandbyReferenceRead try (FileStore store = newFileStore()) { SegmentWriter writer = defaultSegmentWriterBuilder("test").build(store); - RecordId a = writer.writeString("test"); + RecordId a = writer.writeNode(EmptyNodeState.EMPTY_NODE); writer.flush(); - RecordId b = writer.writeList(asList(a, a)); + NodeBuilder builder = EmptyNodeState.EMPTY_NODE.builder(); + builder.setChildNode("reference", store.getReader().readNode(a)); + RecordId b = writer.writeNode(builder.getNodeState()); writer.flush(); DefaultStandbyReferencesReader reader = new DefaultStandbyReferencesReader(store);