Author: mduerig
Date: Fri Feb 20 15:58:21 2015
New Revision: 1661146
URL: http://svn.apache.org/r1661146
Log:
OAK-2294: Corrupt repository after concurrent version operations
Committing v8 patch with minor modifications. With this patch we introduce
segment version 11 (previous 10), which stores properties in lists instead as
lists
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
Fri Feb 20 15:58:21 2015
@@ -32,6 +32,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.BLOCK_SIZE;
import static
org.apache.jackrabbit.oak.plugins.segment.Template.MANY_CHILD_NODES;
import static
org.apache.jackrabbit.oak.plugins.segment.Template.ZERO_CHILD_NODES;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import java.util.Formatter;
import java.util.Map;
@@ -227,10 +228,25 @@ public class RecordUsageAnalyser {
int ids = template.getChildName() == ZERO_CHILD_NODES ? 1 : 2;
nodeSize += ids * RECORD_ID_BYTES;
PropertyTemplate[] propertyTemplates =
template.getPropertyTemplates();
- for (PropertyTemplate propertyTemplate : propertyTemplates) {
- nodeSize += RECORD_ID_BYTES;
- RecordId propertyId = segment.readRecordId(offset + ids++ *
RECORD_ID_BYTES);
- analyseProperty(propertyId, propertyTemplate);
+ if (segment.getSegmentVersion().onOrAfter(V_11)) {
+ if (propertyTemplates.length > 0) {
+ nodeSize += RECORD_ID_BYTES;
+ RecordId id = segment.readRecordId(offset + ids *
RECORD_ID_BYTES);
+ ListRecord pIds = new ListRecord(id,
+ propertyTemplates.length);
+ for (int i = 0; i < propertyTemplates.length; i++) {
+ RecordId propertyId = pIds.getEntry(i);
+ analyseProperty(propertyId, propertyTemplates[i]);
+ }
+ analyseList(id, propertyTemplates.length);
+ }
+ } else {
+ for (PropertyTemplate propertyTemplate : propertyTemplates) {
+ nodeSize += RECORD_ID_BYTES;
+ RecordId propertyId = segment.readRecordId(offset + ids++
+ * RECORD_ID_BYTES);
+ analyseProperty(propertyId, propertyTemplate);
+ }
}
}
}
@@ -238,6 +254,7 @@ public class RecordUsageAnalyser {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
+ @SuppressWarnings("resource")
Formatter formatter = new Formatter(sb);
formatter.format(
"%s in maps (%s leaf and branch records)%n",
@@ -295,11 +312,26 @@ public class RecordUsageAnalyser {
size += Segment.RECORD_ID_BYTES;
}
- for (int i = 0; i < propertyCount; i++) {
- RecordId propertyNameId = segment.readRecordId(offset + size);
- size += Segment.RECORD_ID_BYTES;
- size++; // type
- analyseString(propertyNameId);
+ if (segment.getSegmentVersion().onOrAfter(V_11)) {
+ if (propertyCount > 0) {
+ RecordId listId = segment.readRecordId(offset + size);
+ ListRecord propertyNames = new ListRecord(listId,
+ propertyCount);
+ for (int i = 0; i < propertyCount; i++) {
+ RecordId propertyNameId = propertyNames.getEntry(i);
+ size += Segment.RECORD_ID_BYTES;
+ size++; // type
+ analyseString(propertyNameId);
+ }
+ analyseList(listId, propertyCount);
+ }
+ } else {
+ for (int i = 0; i < propertyCount; i++) {
+ RecordId propertyNameId = segment.readRecordId(offset +
size);
+ size += Segment.RECORD_ID_BYTES;
+ size++; // type
+ analyseString(propertyNameId);
+ }
}
templateSize += size;
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
Fri Feb 20 15:58:21 2015
@@ -22,6 +22,7 @@ import static com.google.common.base.Pre
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static com.google.common.collect.Maps.newConcurrentMap;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import static
org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.BLOCK_SIZE;
import java.io.IOException;
@@ -52,14 +53,6 @@ import org.apache.jackrabbit.oak.plugins
public class Segment {
/**
- * Version of the segment storage format.
- * <ul>
- * <li>10 = all Oak versions released so far</li>
- * </ul>
- */
- public static final byte STORAGE_FORMAT_VERSION = 10;
-
- /**
* Number of bytes used for storing a record identifier. One byte
* is used for identifying the segment and two for the record offset
* within that segment.
@@ -117,6 +110,11 @@ public class Segment {
private final ByteBuffer data;
/**
+ * Version of the segment storage format.
+ */
+ private final SegmentVersion version;
+
+ /**
* Referenced segment identifiers. Entries are initialized lazily in
* {@link #getRefId(int)}. Set to {@code null} for bulk segments.
*/
@@ -137,19 +135,25 @@ public class Segment {
private volatile long accessed = 0;
public Segment(SegmentTracker tracker, SegmentId id, ByteBuffer data) {
+ this(tracker, id, data, V_11);
+ }
+
+ public Segment(SegmentTracker tracker, SegmentId id, ByteBuffer data,
SegmentVersion version) {
this.tracker = checkNotNull(tracker);
this.id = checkNotNull(id);
this.data = checkNotNull(data);
-
if (id.isDataSegmentId()) {
+ byte segmentVersion = data.get(3);
checkState(data.get(0) == '0'
&& data.get(1) == 'a'
&& data.get(2) == 'K'
- && data.get(3) == STORAGE_FORMAT_VERSION);
+ && SegmentVersion.isValid(segmentVersion));
this.refids = new SegmentId[getRefCount()];
- refids[0] = id;
+ this.refids[0] = id;
+ this.version = SegmentVersion.fromByte(segmentVersion);
} else {
this.refids = null;
+ this.version = version;
}
}
@@ -157,9 +161,9 @@ public class Segment {
this.tracker = checkNotNull(tracker);
this.id = tracker.newDataSegmentId();
this.data = ByteBuffer.wrap(checkNotNull(buffer));
-
this.refids = new SegmentId[SEGMENT_REFERENCE_LIMIT + 1];
- refids[0] = id;
+ this.refids[0] = id;
+ this.version = SegmentVersion.fromByte(buffer[3]);
}
void access() {
@@ -171,6 +175,10 @@ public class Segment {
return accessed != 0;
}
+ SegmentVersion getSegmentVersion() {
+ return version;
+ }
+
/**
* Maps the given record offset to the respective position within the
* internal {@link #data} array. The validity of a record with the given
@@ -436,19 +444,41 @@ public class Segment {
offset += Segment.RECORD_ID_BYTES;
}
- PropertyTemplate[] properties =
- new PropertyTemplate[propertyCount];
- for (int i = 0; i < properties.length; i++) {
+ PropertyTemplate[] properties;
+ if (version.onOrAfter(V_11)) {
+ properties = readPropsV11(propertyCount, offset);
+ } else {
+ properties = readPropsV10(propertyCount, offset);
+ }
+ return new Template(primaryType, mixinTypes, properties, childName);
+ }
+
+ private PropertyTemplate[] readPropsV10(int propertyCount, int offset) {
+ PropertyTemplate[] properties = new PropertyTemplate[propertyCount];
+ for (int i = 0; i < propertyCount; i++) {
RecordId propertyNameId = readRecordId(offset);
offset += Segment.RECORD_ID_BYTES;
byte type = readByte(offset++);
- properties[i] = new PropertyTemplate(
- i, readString(propertyNameId),
+ properties[i] = new PropertyTemplate(i, readString(propertyNameId),
Type.fromTag(Math.abs(type), type < 0));
}
+ return properties;
+ }
- return new Template(
- primaryType, mixinTypes, properties, childName);
+ private PropertyTemplate[] readPropsV11(int propertyCount, int offset) {
+ PropertyTemplate[] properties = new PropertyTemplate[propertyCount];
+ if (propertyCount > 0) {
+ RecordId id = readRecordId(offset);
+ ListRecord propertyNames = new ListRecord(id, properties.length);
+ offset += Segment.RECORD_ID_BYTES;
+ for (int i = 0; i < propertyCount; i++) {
+ byte type = readByte(offset++);
+ properties[i] = new PropertyTemplate(i,
+ readString(propertyNames.getEntry(i)), Type.fromTag(
+ Math.abs(type), type < 0));
+ }
+ }
+ return properties;
}
long readLength(RecordId id) {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeState.java
Fri Feb 20 15:58:21 2015
@@ -16,21 +16,6 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
-import java.util.Collections;
-import java.util.List;
-
-import javax.annotation.CheckForNull;
-import javax.annotation.Nonnull;
-
-import org.apache.jackrabbit.oak.api.PropertyState;
-import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
-import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry;
-import org.apache.jackrabbit.oak.spi.state.AbstractNodeState;
-import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.oak.spi.state.NodeStateDiff;
-
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
@@ -46,8 +31,24 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.api.Type.STRINGS;
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import static
org.apache.jackrabbit.oak.spi.state.AbstractNodeState.checkValidName;
+import java.util.Collections;
+import java.util.List;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.AbstractNodeState;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateDiff;
+
/**
* A record of type "NODE". This class can read a node record from a segment.
It
* currently doesn't cache data (but the template is fully loaded).
@@ -134,17 +135,39 @@ public class SegmentNodeState extends Re
template.getPropertyTemplate(name);
if (propertyTemplate != null) {
Segment segment = getSegment();
- int ids = 1 + propertyTemplate.getIndex();
- if (template.getChildName() != Template.ZERO_CHILD_NODES) {
- ids++;
+ RecordId id;
+ if (getSegment().getSegmentVersion().onOrAfter(V_11)) {
+ id = getRecordIdV11(segment, template, propertyTemplate);
+ } else {
+ id = getRecordIdV10(segment, template, propertyTemplate);
}
- return new SegmentPropertyState(
- segment.readRecordId(getOffset(0, ids)), propertyTemplate);
+ return new SegmentPropertyState(id, propertyTemplate);
} else {
return null;
}
}
+ private RecordId getRecordIdV10(Segment segment, Template template,
+ PropertyTemplate propertyTemplate) {
+ int ids = 1 + propertyTemplate.getIndex();
+ if (template.getChildName() != Template.ZERO_CHILD_NODES) {
+ ids++;
+ }
+ return segment.readRecordId(getOffset(0, ids));
+ }
+
+ private RecordId getRecordIdV11(Segment segment, Template template,
+ PropertyTemplate propertyTemplate) {
+ int ids = 1;
+ if (template.getChildName() != Template.ZERO_CHILD_NODES) {
+ ids++;
+ }
+ RecordId rid = segment.readRecordId(getOffset(0, ids));
+ ListRecord pIds = new ListRecord(rid,
+ template.getPropertyTemplates().length);
+ return pIds.getEntry(propertyTemplate.getIndex());
+ }
+
@Override @Nonnull
public Iterable<PropertyState> getProperties() {
Template template = getTemplate();
@@ -167,10 +190,24 @@ public class SegmentNodeState extends Re
if (template.getChildName() != Template.ZERO_CHILD_NODES) {
ids++;
}
- for (int i = 0; i < propertyTemplates.length; i++) {
- RecordId propertyId = segment.readRecordId(getOffset(0, ids++));
- list.add(new SegmentPropertyState(
- propertyId, propertyTemplates[i]));
+
+ if (segment.getSegmentVersion().onOrAfter(V_11)) {
+ if (propertyTemplates.length > 0) {
+ ListRecord pIds = new ListRecord(
+ segment.readRecordId(getOffset(0, ids)),
+ propertyTemplates.length);
+ for (int i = 0; i < propertyTemplates.length; i++) {
+ RecordId propertyId = pIds.getEntry(i);
+ list.add(new SegmentPropertyState(propertyId,
+ propertyTemplates[i]));
+ }
+ }
+ } else {
+ for (int i = 0; i < propertyTemplates.length; i++) {
+ RecordId propertyId = segment.readRecordId(getOffset(0,
ids++));
+ list.add(new SegmentPropertyState(propertyId,
+ propertyTemplates[i]));
+ }
}
return list;
@@ -247,11 +284,13 @@ public class SegmentNodeState extends Re
}
Segment segment = getSegment();
- int ids = 1 + propertyTemplate.getIndex();
- if (template.getChildName() != Template.ZERO_CHILD_NODES) {
- ids++;
+ RecordId id;
+ if (getSegment().getSegmentVersion().onOrAfter(V_11)) {
+ id = getRecordIdV11(segment, template, propertyTemplate);
+ } else {
+ id = getRecordIdV10(segment, template, propertyTemplate);
}
- return segment.readString(segment.readRecordId(getOffset(0, ids)));
+ return segment.readString(id);
}
/**
@@ -288,12 +327,12 @@ public class SegmentNodeState extends Re
}
Segment segment = getSegment();
- int ids = 1 + propertyTemplate.getIndex();
- if (template.getChildName() != Template.ZERO_CHILD_NODES) {
- ids++;
+ RecordId id;
+ if (getSegment().getSegmentVersion().onOrAfter(V_11)) {
+ id = getRecordIdV11(segment, template, propertyTemplate);
+ } else {
+ id = getRecordIdV10(segment, template, propertyTemplate);
}
-
- RecordId id = segment.readRecordId(getOffset(0, ids));
segment = id.getSegment();
int size = segment.readInt(id.getOffset());
if (size == 0) {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
Fri Feb 20 15:58:21 2015
@@ -93,20 +93,25 @@ public class SegmentTracker {
private long currentSize = 0;
- public SegmentTracker(SegmentStore store, int cacheSizeMB) {
+ public SegmentTracker(SegmentStore store, int cacheSizeMB,
+ SegmentVersion version) {
for (int i = 0; i < tables.length; i++) {
tables[i] = new SegmentIdTable(this);
}
this.store = store;
- this.writer = new SegmentWriter(store, this);
+ this.writer = new SegmentWriter(store, this, version);
this.cacheSize = cacheSizeMB * MB;
this.compactionMap = new AtomicReference<CompactionMap>(
new CompactionMap(1, this));
}
+ public SegmentTracker(SegmentStore store, SegmentVersion version) {
+ this(store, DEFAULT_MEMORY_CACHE_SIZE, version);
+ }
+
public SegmentTracker(SegmentStore store) {
- this(store, DEFAULT_MEMORY_CACHE_SIZE);
+ this(store, DEFAULT_MEMORY_CACHE_SIZE, SegmentVersion.V_11);
}
public SegmentWriter getWriter() {
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java?rev=1661146&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersion.java
Fri Feb 20 15:58:21 2015
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+/**
+ * Version of the segment storage format.
+ * <ul>
+ * <li>10 = all Oak versions previous to 11</li>
+ * <li>11 = all Oak versions starting from 1.0.12, 1.1.7 and 1.2</li>
+ * </ul>
+ */
+public enum SegmentVersion {
+
+ /**
+ * @deprecated Use latest version V11
+ */
+ @Deprecated
+ V_10((byte) 10),
+
+ V_11((byte) 11);
+
+ private final byte version;
+
+ SegmentVersion(byte version) {
+ this.version = version;
+ }
+
+ public boolean onOrAfter(SegmentVersion other) {
+ return compareTo(other) >= 0;
+ }
+
+ public static byte asByte(SegmentVersion v) {
+ return v.version;
+ }
+
+ public static SegmentVersion fromByte(byte v) {
+ if (v == V_11.version) {
+ return V_11;
+ } else if (v == V_10.version) {
+ return V_10;
+ } else {
+ throw new IllegalArgumentException("Unknown version " + v);
+ }
+ }
+
+ public static boolean isValid(byte v) {
+ return v == V_10.version || v == V_11.version;
+ }
+
+}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
Fri Feb 20 15:58:21 2015
@@ -38,7 +38,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE;
import static
org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ID_BYTES;
import static
org.apache.jackrabbit.oak.plugins.segment.Segment.SEGMENT_REFERENCE_LIMIT;
-import static
org.apache.jackrabbit.oak.plugins.segment.Segment.STORAGE_FORMAT_VERSION;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -57,6 +57,11 @@ import java.util.Set;
import javax.jcr.PropertyType;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Closeables;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
@@ -65,13 +70,6 @@ import org.apache.jackrabbit.oak.spi.blo
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.DefaultNodeStateDiff;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.ByteStreams;
-import com.google.common.io.Closeables;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,12 +88,12 @@ public class SegmentWriter {
static final int BLOCK_SIZE = 1 << 12; // 4kB
- static byte[] createNewBuffer() {
+ static byte[] createNewBuffer(SegmentVersion v) {
byte[] buffer = new byte[Segment.MAX_SEGMENT_SIZE];
buffer[0] = '0';
buffer[1] = 'a';
buffer[2] = 'K';
- buffer[3] = STORAGE_FORMAT_VERSION;
+ buffer[3] = SegmentVersion.asByte(v);
buffer[4] = 0; // reserved
buffer[5] = 0; // refcount
return buffer;
@@ -119,6 +117,7 @@ public class SegmentWriter {
* avoid storing duplicates of frequently occurring data.
* Should only be accessed from synchronized blocks to prevent corruption.
*/
+ @SuppressWarnings("serial")
private final Map<Object, RecordId> records =
new LinkedHashMap<Object, RecordId>(15000, 0.75f, true) {
@Override
@@ -142,7 +141,7 @@ public class SegmentWriter {
* The segment write buffer, filled from the end to the beginning
* (see OAK-629).
*/
- private byte[] buffer = createNewBuffer();
+ private byte[] buffer;
/**
* The number of bytes already written (or allocated). Counted from
@@ -158,9 +157,16 @@ public class SegmentWriter {
private Segment segment;
- public SegmentWriter(SegmentStore store, SegmentTracker tracker) {
+ /**
+ * Version of the segment storage format.
+ */
+ private final SegmentVersion version;
+
+ public SegmentWriter(SegmentStore store, SegmentTracker tracker,
SegmentVersion version) {
this.store = store;
this.tracker = tracker;
+ this.version = version;
+ this.buffer = createNewBuffer(version);
this.segment = new Segment(tracker, buffer);
segment.getSegmentId().setSegment(segment);
}
@@ -240,7 +246,7 @@ public class SegmentWriter {
}
tracker.setSegment(id, new Segment(tracker, id, data));
- buffer = createNewBuffer();
+ buffer = createNewBuffer(version);
roots.clear();
blobrefs.clear();
length = 0;
@@ -444,7 +450,6 @@ public class SegmentWriter {
}
}
-
private synchronized RecordId writeListBucket(List<RecordId> bucket) {
checkArgument(bucket.size() > 1);
RecordId bucketId = prepare(RecordType.BUCKET, 0, bucket);
@@ -968,7 +973,17 @@ public class SegmentWriter {
propertyTypes[i] = (byte) type.tag();
}
}
- ids.addAll(Arrays.asList(propertyNames));
+
+ RecordId propNamesId = null;
+ if (segment.getSegmentVersion().onOrAfter(V_11)) {
+ if (propertyNames.length > 0) {
+ propNamesId = writeList(Arrays.asList(propertyNames));
+ ids.add(propNamesId);
+ }
+ } else {
+ ids.addAll(Arrays.asList(propertyNames));
+ }
+
checkState(propertyNames.length < (1 << 18));
head |= propertyNames.length;
@@ -985,8 +1000,16 @@ public class SegmentWriter {
if (childNameId != null) {
writeRecordId(childNameId);
}
+ if (segment.getSegmentVersion().onOrAfter(V_11)) {
+ if (propNamesId != null) {
+ writeRecordId(propNamesId);
+ }
+ }
for (int i = 0; i < propertyNames.length; i++) {
- writeRecordId(propertyNames[i]);
+ if (!segment.getSegmentVersion().onOrAfter(V_11)) {
+ // V10 only
+ writeRecordId(propertyNames[i]);
+ }
buffer[position++] = propertyTypes[i];
}
@@ -1087,36 +1110,45 @@ public class SegmentWriter {
ids.add(writeNode(state.getChildNode(template.getChildName())).getRecordId());
}
+ List<RecordId> pIds = Lists.newArrayList();
for (PropertyTemplate pt : template.getPropertyTemplates()) {
String name = pt.getName();
PropertyState property = state.getProperty(name);
if (property instanceof SegmentPropertyState
&& store.containsSegment(((SegmentPropertyState)
property).getRecordId().getSegmentId())) {
- ids.add(((SegmentPropertyState) property).getRecordId());
+ pIds.add(((SegmentPropertyState) property).getRecordId());
} else if (before == null
||
!store.containsSegment(before.getRecordId().getSegmentId())) {
- ids.add(writeProperty(property));
+ pIds.add(writeProperty(property));
} else {
// reuse previously stored property, if possible
PropertyTemplate bt = beforeTemplate.getPropertyTemplate(name);
if (bt == null) {
- ids.add(writeProperty(property)); // new property
+ pIds.add(writeProperty(property)); // new property
} else {
SegmentPropertyState bp = beforeTemplate.getProperty(
before.getRecordId(), bt.getIndex());
if (property.equals(bp)) {
- ids.add(bp.getRecordId()); // no changes
+ pIds.add(bp.getRecordId()); // no changes
} else if (bp.isArray() && bp.getType() != BINARIES) {
// reuse entries from the previous list
- ids.add(writeProperty(property, bp.getValueRecords()));
+ pIds.add(writeProperty(property,
bp.getValueRecords()));
} else {
- ids.add(writeProperty(property));
+ pIds.add(writeProperty(property));
}
}
}
}
+ if (!pIds.isEmpty()) {
+ if (segment.getSegmentVersion().onOrAfter(V_11)) {
+ ids.add(writeList(pIds));
+ } else {
+ ids.addAll(pIds);
+ }
+ }
+
synchronized (this) {
RecordId recordId = prepare(RecordType.NODE, 0, ids);
for (RecordId id : ids) {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java
Fri Feb 20 15:58:21 2015
@@ -22,6 +22,7 @@ import static com.google.common.base.Pre
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE;
import static org.apache.jackrabbit.oak.plugins.segment.Record.fastEquals;
import static
org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ID_BYTES;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import java.util.Arrays;
import java.util.Collections;
@@ -173,9 +174,16 @@ public class Template {
if (childName != ZERO_CHILD_NODES) {
offset += RECORD_ID_BYTES;
}
- offset += index * RECORD_ID_BYTES;
- return new SegmentPropertyState(
- segment.readRecordId(offset), properties[index]);
+ RecordId rid = null;
+ if (segment.getSegmentVersion().onOrAfter(V_11)) {
+ RecordId lid = segment.readRecordId(offset);
+ ListRecord props = new ListRecord(lid, properties.length);
+ rid = props.getEntry(index);
+ } else {
+ offset += index * RECORD_ID_BYTES;
+ rid = segment.readRecordId(offset);
+ }
+ return new SegmentPropertyState(rid, properties[index]);
}
MapRecord getChildNodeMap(RecordId recordId) {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Fri Feb 20 15:58:21 2015
@@ -52,6 +52,7 @@ import java.util.regex.Pattern;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Maps;
+
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.segment.CompactionMap;
@@ -64,6 +65,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.SegmentNotFoundException;
import org.apache.jackrabbit.oak.plugins.segment.SegmentStore;
import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentVersion;
import org.apache.jackrabbit.oak.plugins.segment.SegmentWriter;
import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
@@ -151,6 +153,11 @@ public class FileStore implements Segmen
*/
private final LinkedList<File> toBeRemoved = newLinkedList();
+ /**
+ * Version of the segment storage format.
+ */
+ private final SegmentVersion version = SegmentVersion.V_11;
+
public FileStore(BlobStore blobStore, File directory, int maxFileSizeMB,
boolean memoryMapping)
throws IOException {
this(blobStore, directory, EMPTY_NODE, maxFileSizeMB, 0,
memoryMapping);
@@ -188,11 +195,11 @@ public class FileStore implements Segmen
throws IOException {
checkNotNull(directory).mkdirs();
if (cacheSizeMB < 0) {
- this.tracker = new SegmentTracker(this, 0);
+ this.tracker = new SegmentTracker(this, 0, getVersion());
} else if (cacheSizeMB > 0) {
- this.tracker = new SegmentTracker(this, cacheSizeMB);
+ this.tracker = new SegmentTracker(this, cacheSizeMB, getVersion());
} else {
- this.tracker = new SegmentTracker(this);
+ this.tracker = new SegmentTracker(this, getVersion());
}
this.blobStore = blobStore;
this.directory = directory;
@@ -537,7 +544,7 @@ public class FileStore implements Segmen
log.info("TarMK compaction running, strategy={}", compactionStrategy);
long start = System.currentTimeMillis();
- SegmentWriter writer = new SegmentWriter(this, tracker);
+ SegmentWriter writer = new SegmentWriter(this, tracker, getVersion());
final Compactor compactor = new Compactor(writer,
compactionStrategy.cloneBinaries());
SegmentNodeState before = getHead();
long existing = before.getChildNode(SegmentNodeStore.CHECKPOINTS)
@@ -613,6 +620,7 @@ public class FileStore implements Segmen
flush();
writer.close();
+ tracker.getWriter().dropCache();
List<TarReader> list = readers;
readers = newArrayList();
@@ -885,4 +893,8 @@ public class FileStore implements Segmen
}
}
}
+
+ public SegmentVersion getVersion() {
+ return version;
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
Fri Feb 20 15:58:21 2015
@@ -29,6 +29,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.segment.ListRecord.LEVEL_SIZE;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.MEDIUM_LIMIT;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.SMALL_LIMIT;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -52,7 +53,7 @@ public class RecordUsageAnalyserTest {
store = mock(SegmentStore.class);
SegmentTracker tracker = new SegmentTracker(store);
when(store.getTracker()).thenReturn(tracker);
- writer = new SegmentWriter(store, store.getTracker());
+ writer = new SegmentWriter(store, store.getTracker(), V_11);
analyser = new RecordUsageAnalyser();
}
@@ -270,7 +271,7 @@ public class RecordUsageAnalyserTest {
SegmentNodeState node = writer.writeNode(builder.getNodeState());
analyser.analyseNode(node.getRecordId());
- assertCounts(analyser, 1, 3, 6, 1, 1, 1, 0, 10, 1, 1, 2, 3);
+ assertCounts(analyser, 1, 5, 6, 1, 1, 1, 0, 10, 1, 1, 2, 3);
}
private static Blob createRandomBlob(int size) {
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java
Fri Feb 20 15:58:21 2015
@@ -90,7 +90,7 @@ public class SegmentIdFactoryTest {
@Test(expected = IllegalStateException.class)
public void dataAIOOBE() {
SegmentId id = factory.newDataSegmentId();
- byte[] buffer = SegmentWriter.createNewBuffer();
+ byte[] buffer = SegmentWriter.createNewBuffer(SegmentVersion.V_11);
ByteBuffer data = ByteBuffer.allocate(Segment.MAX_SEGMENT_SIZE);
data.put(buffer);
data.rewind();
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
Fri Feb 20 15:58:21 2015
@@ -51,8 +51,8 @@ public class SegmentSizeTest {
builder = EMPTY_NODE.builder();
builder.setProperty("foo", "bar");
builder.setProperty("baz", 123);
- assertEquals(64, getSize(builder));
- assertEquals(12, getAmortizedSize(builder));
+ assertEquals(80, getSize(builder));
+ assertEquals(16, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.child("foo");
@@ -127,15 +127,15 @@ public class SegmentSizeTest {
deny.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
assertEquals(176, getSize(builder));
- assertEquals(28, getAmortizedSize(builder));
+ assertEquals(32, getAmortizedSize(builder));
NodeBuilder allow = builder.child("allow");
allow.setProperty("jcr:primaryType", "rep:GrantACE");
allow.setProperty("rep:principalName", "administrators");
allow.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(288, getSize(builder));
- assertEquals(72, getAmortizedSize(builder));
+ assertEquals(320, getSize(builder));
+ assertEquals(84, getAmortizedSize(builder));
NodeBuilder deny0 = builder.child("deny0");
deny0.setProperty("jcr:primaryType", "rep:DenyACE", Type.NAME);
@@ -143,16 +143,16 @@ public class SegmentSizeTest {
deny0.setProperty("rep:glob", "*/activities/*");
builder.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
- assertEquals(384, getSize(builder));
- assertEquals(108, getAmortizedSize(builder));
+ assertEquals(416, getSize(builder));
+ assertEquals(124, getAmortizedSize(builder));
NodeBuilder allow0 = builder.child("allow0");
allow0.setProperty("jcr:primaryType", "rep:GrantACE");
allow0.setProperty("rep:principalName", "user-administrators");
allow0.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(432, getSize(builder));
- assertEquals(136, getAmortizedSize(builder));
+ assertEquals(480, getSize(builder));
+ assertEquals(160, getAmortizedSize(builder));
}
@Test
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java?rev=1661146&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentVersionTest.java
Fri Feb 20 15:58:21 2015
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.apache.jackrabbit.oak.api.Type.LONG;
+import static org.apache.jackrabbit.oak.api.Type.LONGS;
+import static org.apache.jackrabbit.oak.api.Type.STRING;
+import static org.apache.jackrabbit.oak.api.Type.STRINGS;
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.json.JsopDiff;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SegmentVersionTest {
+
+ private static final Logger log = LoggerFactory
+ .getLogger(SegmentVersionTest.class);
+
+ private File directory;
+
+ @Before
+ public void setUp() throws IOException {
+ directory = File.createTempFile("VersionTest", "dir",
+ new File("target"));
+ directory.delete();
+ directory.mkdir();
+ }
+
+ @After
+ public void cleanDir() {
+ try {
+ deleteDirectory(directory);
+ } catch (IOException e) {
+ log.error("Error cleaning directory", e);
+ }
+ }
+
+ @Test
+ public void compareOldRevision() throws Exception {
+ FileStore fileStoreV10 = new FileStore(directory, 1) {
+ @SuppressWarnings("deprecation")
+ @Override
+ public SegmentVersion getVersion() {
+ return SegmentVersion.V_10;
+ }
+ };
+ try {
+ NodeState content =
addTestContent(fileStoreV10).getChildNode("content");
+ NodeBuilder builder = content.builder();
+ builder.setChildNode("foo");
+ JsopDiff diff = new JsopDiff();
+ content.compareAgainstBaseState(builder.getNodeState(), diff);
+ assertEquals("-\"/foo\"", diff.toString());
+ } finally {
+ fileStoreV10.close();
+ }
+ }
+
+ @Test
+ public void readOldVersions() throws Exception {
+ FileStore fileStoreV10 = new FileStore(directory, 1) {
+ @SuppressWarnings("deprecation")
+ @Override
+ public SegmentVersion getVersion() {
+ return SegmentVersion.V_10;
+ }
+ };
+ try {
+ addTestContent(fileStoreV10);
+ } finally {
+ fileStoreV10.close();
+ }
+
+ FileStore fileStoreV11 = new FileStore(directory, 1);
+ try {
+ verifyContent(fileStoreV11);
+ } finally {
+ fileStoreV11.close();
+ }
+
+ }
+
+ @SuppressWarnings("deprecation")
+ private static NodeState addTestContent(FileStore fs)
+ throws CommitFailedException {
+ NodeStore store = new SegmentNodeStore(fs);
+ NodeBuilder builder = store.getRoot().builder();
+
+ NodeBuilder content = builder.child("content");
+ content.setProperty("a", 1);
+ content.setProperty("aM", ImmutableList.of(1L, 2L, 3L, 4L), LONGS);
+
+ content.setProperty("b", "azerty");
+ content.setProperty("bM",
+ ImmutableList.of("a", "z", "e", "r", "t", "y"), STRINGS);
+
+ // add blobs?
+
+ NodeState root = store.merge(builder, EmptyHook.INSTANCE,
CommitInfo.EMPTY);
+ assertEquals(SegmentVersion.V_10, fs.getHead().getSegment()
+ .getSegmentVersion());
+ return root;
+ }
+
+ private static void verifyContent(FileStore fs)
+ throws CommitFailedException {
+ NodeStore store = new SegmentNodeStore(fs);
+ SegmentNodeState content = (SegmentNodeState) store.getRoot()
+ .getChildNode("content");
+
+ assertEquals(new Long(1), content.getProperty("a").getValue(LONG));
+ assertEquals(ImmutableList.of(1L, 2L, 3L, 4L),
+ Lists.newArrayList(content.getProperty("aM").getValue(LONGS)));
+
+ assertEquals("azerty", content.getProperty("b").getValue(STRING));
+ assertEquals("azerty", content.getString("b"));
+
+ assertEquals(ImmutableList.of("a", "z", "e", "r", "t", "y"),
+
Lists.newArrayList(content.getProperty("bM").getValue(STRINGS)));
+ assertEquals(ImmutableList.of("a", "z", "e", "r", "t", "y"),
+ Lists.newArrayList(content.getStrings("bM")));
+ }
+}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
Fri Feb 20 15:58:21 2015
@@ -21,6 +21,7 @@ import static com.google.common.collect.
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -139,7 +140,7 @@ public class FileStoreTest {
store = new FileStore(directory, 1, false);
head = store.getHead();
assertTrue(store.size() > largeBinarySize);
- writer = new SegmentWriter(store, store.getTracker());
+ writer = new SegmentWriter(store, store.getTracker(), V_11);
compactor = new Compactor(writer);
compacted = compactor.compact(EmptyNodeState.EMPTY_NODE, head);
builder = head.builder();
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java?rev=1661146&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java
Fri Feb 20 15:58:21 2015
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LargeNumberOfPropertiesTestIT {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SegmentReferenceLimitTestIT.class);
+
+ private File directory;
+
+ @Before
+ public void setUp() throws IOException {
+ directory = File.createTempFile("LargeNumberOfPropertiesTestIT", "dir",
+ new File("target"));
+ directory.delete();
+ directory.mkdir();
+ }
+
+ @After
+ public void cleanDir() {
+ try {
+ FileUtils.deleteDirectory(directory);
+ } catch (IOException e) {
+ LOG.error("Error cleaning directory", e);
+ }
+ }
+
+ /**
+ * OAK-2481 IllegalStateException in TarMk with large number of properties
+ *
+ * TODO Test is currently ignored because of how memory intensive it is
+ *
+ * @see <a
+ * href="https://issues.apache.org/jira/browse/OAK-2481";>OAK-2481</a>
+ */
+ @Test
+ @Ignore
+ public void corruption() throws Exception {
+ FileStore fileStore = new FileStore(directory, 5, 0, false);
+ SegmentNodeStore nodeStore = new SegmentNodeStore(fileStore);
+
+ NodeBuilder root = nodeStore.getRoot().builder();
+
+ try {
+ NodeBuilder c = root.child("c" + System.currentTimeMillis());
+ // i=26 hits the hard limit for the number of properties a node can
+ // have (262144)
+ for (int i = 0; i < 25; i++) {
+ System.out.println(i);
+ for (int j = 0; j < 10000; j++) {
+ c.setProperty("int-" + i + "-" + j, i);
+ }
+ }
+ nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ } finally {
+ fileStore.close();
+ }
+ }
+
+}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java?rev=1661146&r1=1661145&r2=1661146&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java
Fri Feb 20 15:58:21 2015
@@ -19,11 +19,10 @@
package org.apache.jackrabbit.oak.plugins.segment.file;
-import static java.lang.String.valueOf;
-import static org.junit.Assert.assertTrue;
-
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
@@ -31,7 +30,6 @@ import java.util.concurrent.FutureTask;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentOverflowException;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -43,13 +41,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SegmentReferenceLimitTestIT {
+
private static final Logger LOG =
LoggerFactory.getLogger(SegmentReferenceLimitTestIT.class);
private File directory;
@Before
public void setUp() throws IOException {
- directory = File.createTempFile("FileStoreTest", "dir", new
File("target"));
+ directory = File.createTempFile("SegmentReferenceLimitTestIT", "dir",
new File("target"));
directory.delete();
directory.mkdir();
}
@@ -63,6 +62,12 @@ public class SegmentReferenceLimitTestIT
}
}
+ /**
+ * OAK-2294 Corrupt repository after concurrent version operations
+ *
+ * @see <a
+ * href="https://issues.apache.org/jira/browse/OAK-2294";>OAK-2294</a>
+ */
@Test
public void corruption() throws IOException, CommitFailedException,
ExecutionException, InterruptedException {
FileStore fileStore = new FileStore(directory, 1, 0, false);
@@ -72,19 +77,15 @@ public class SegmentReferenceLimitTestIT
root.setChildNode("test");
nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- FutureTask<Void> w1 = run(new Worker(nodeStore, "w1"));
- FutureTask<Void> w2 = run(new Worker(nodeStore, "w2"));
- FutureTask<Void> w3 = run(new Worker(nodeStore, "w3"));
- FutureTask<Void> w4 = run(new Worker(nodeStore, "w4"));
+ List<FutureTask<Void>> l = new ArrayList<FutureTask<Void>>();
+ for (int i = 0; i < 10; i++) {
+ l.add(run(new Worker(nodeStore, "w" + i)));
+ }
try {
- w1.get();
- w2.get();
- w3.get();
- w4.get();
- } catch (ExecutionException e) {
- assertTrue(valueOf(e.getCause()), e.getCause() instanceof
CommitFailedException);
- assertTrue(valueOf(e.getCause()), e.getCause().getCause()
instanceof SegmentOverflowException);
+ for (FutureTask<Void> w : l) {
+ w.get();
+ }
} finally {
fileStore.close();
}
@@ -107,11 +108,12 @@ public class SegmentReferenceLimitTestIT
@Override
public Void call() throws Exception {
- for (int k = 0; ; k++) {
+ for (int k = 0; k < 400; k++) {
NodeBuilder root = nodeStore.getRoot().builder();
root.getChildNode("test").setProperty(name + ' ' + k, name + "
value " + k);
nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
+ return null;
}
}
