Author: baedke
Date: Tue Sep 27 12:40:27 2016
New Revision: 1762470
URL: http://svn.apache.org/viewvc?rev=1762470&view=rev
Log:
OAK-4553: Retain journal entries for valid checkpoints
Added:
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java
- copied, changed from r1749424,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeStateDiffer.java
- copied unchanged from r1749424,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeStateDiffer.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java
- copied unchanged from r1752292,
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 12:40:27 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886
,1751410,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1759433,1760340,1760373,1760387,1760661-1760662,1761412,1761444,1761571,1761762,1761787
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626
,1750809,1750886,1751410,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1759433,1760340,1760373,1760387,1760661-1760662,1761412,1761444,1761571,1761762,1761787
/jackrabbit/trunk:1345480
Copied:
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java
(from r1749424,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java?p2=jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java&r1=1749424&r2=1762470&rev=1762470&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java
Tue Sep 27 12:40:27 2016
@@ -36,11 +36,6 @@ public abstract class AbstractDocumentNo
private static final PerfLogger perfLogger = new PerfLogger(
LoggerFactory.getLogger(AbstractDocumentNodeState.class.getName()
+ ".perf"));
- protected final NodeStateDiffer differ;
-
- protected AbstractDocumentNodeState(NodeStateDiffer differ) {
- this.differ = differ;
- }
public abstract String getPath();
@@ -70,6 +65,8 @@ public abstract class AbstractDocumentNo
public abstract boolean hasNoChildren();
+ protected abstract NodeStateDiffer getNodeStateDiffer();
+
//--------------------------< NodeState
>-----------------------------------
@Override
@@ -116,7 +113,7 @@ public abstract class AbstractDocumentNo
// use DocumentNodeStore compare
final long start = perfLogger.start();
try {
- return differ.compare(this, mBase, diff);
+ return getNodeStateDiffer().compare(this, mBase, diff);
} finally {
if (start > 0) {
perfLogger
Modified:
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java?rev=1762470&r1=1762469&r2=1762470&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java
Tue Sep 27 12:40:27 2016
@@ -40,14 +40,10 @@ import org.apache.jackrabbit.oak.json.Js
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder;
-import org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState;
import org.apache.jackrabbit.oak.spi.state.AbstractChildNodeEntry;
-import org.apache.jackrabbit.oak.spi.state.AbstractNodeState;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
-import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.oak.spi.state.NodeStateDiff;
import org.apache.jackrabbit.oak.util.PerfLogger;
import org.slf4j.LoggerFactory;
@@ -57,12 +53,11 @@ import com.google.common.collect.Iterato
import static com.google.common.base.Preconditions.checkNotNull;
import static
org.apache.jackrabbit.oak.commons.StringUtils.estimateMemoryUsage;
-import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
/**
* A {@link NodeState} implementation for the {@link DocumentNodeStore}.
*/
-public class DocumentNodeState extends AbstractNodeState implements CacheValue
{
+public class DocumentNodeState extends AbstractDocumentNodeState implements
CacheValue {
private static final PerfLogger perfLogger = new PerfLogger(
LoggerFactory.getLogger(DocumentNodeState.class.getName()
@@ -133,7 +128,8 @@ public class DocumentNodeState extends A
* @return a copy of this node state with the given root revision and
* external change flag.
*/
- private DocumentNodeState withRootRevision(@Nonnull RevisionVector root,
+ @Override
+ public DocumentNodeState withRootRevision(@Nonnull RevisionVector root,
boolean externalChange) {
if (rootRevision.equals(root) && fromExternalChange == externalChange)
{
return this;
@@ -157,12 +153,16 @@ public class DocumentNodeState extends A
* @return {@code true} if this node state was created as a result of an
* external change; {@code false} otherwise.
*/
- boolean isFromExternalChange() {
+ @Override
+ public boolean isFromExternalChange() {
return fromExternalChange;
}
+ //--------------------------< AbstractDocumentNodeState
>-----------------------------------
+
+ @Override
@Nonnull
- RevisionVector getRevision() {
+ public RevisionVector getRevision() {
return readRevision;
}
@@ -178,39 +178,23 @@ public class DocumentNodeState extends A
* same value as returned by {@link #getRevision()}.
*/
@Nonnull
- RevisionVector getRootRevision() {
+ public RevisionVector getRootRevision() {
return rootRevision;
}
- //--------------------------< NodeState
>-----------------------------------
+ @Override
+ public String getPath() {
+ return path;
+ }
@Override
- public boolean equals(Object that) {
- if (this == that) {
- return true;
- } else if (that instanceof DocumentNodeState) {
- DocumentNodeState other = (DocumentNodeState) that;
- if (!getPath().equals(other.getPath())) {
- // path does not match: not equals
- // (even if the properties are equal)
- return false;
- }
- if (revisionEquals(other)) {
- return true;
- }
- // revision does not match: might still be equals
- } else if (that instanceof ModifiedNodeState) {
- ModifiedNodeState modified = (ModifiedNodeState) that;
- if (modified.getBaseState() == this) {
- return EqualsDiff.equals(this, modified);
- }
- }
- if (that instanceof NodeState) {
- return AbstractNodeState.equals(this, (NodeState) that);
- }
- return false;
+ public RevisionVector getLastRevision() {
+ return lastRevision;
}
+ //--------------------------< NodeState
>-----------------------------------
+
+
@Override
public boolean exists() {
return true;
@@ -323,42 +307,6 @@ public class DocumentNodeState extends A
}
}
- @Override
- public boolean compareAgainstBaseState(NodeState base, NodeStateDiff diff)
{
- if (this == base) {
- return true;
- } else if (base == EMPTY_NODE || !base.exists()) {
- // special case
- return EmptyNodeState.compareAgainstEmptyState(this, diff);
- } else if (base instanceof DocumentNodeState) {
- DocumentNodeState mBase = (DocumentNodeState) base;
- if (store == mBase.store) {
- if (getPath().equals(mBase.getPath())) {
- if (revisionEquals(mBase)) {
- // no differences
- return true;
- } else {
- // use DocumentNodeStore compare
- final long start = perfLogger.start();
- try {
- return store.compare(this, mBase, diff);
- } finally {
- perfLogger
- .end(start,
- 1,
- "compareAgainstBaseState, path={},
readRevision={}, lastRevision={}, base.path={}, base.readRevision={},
base.lastRevision={}",
- path, readRevision, lastRevision,
- mBase.path, mBase.readRevision,
- mBase.lastRevision);
- }
- }
- }
- }
- }
- // fall back to the generic node state diff algorithm
- return super.compareAgainstBaseState(base, diff);
- }
-
void setProperty(String propertyName, String value) {
if (value == null) {
properties.remove(propertyName);
@@ -392,11 +340,17 @@ public class DocumentNodeState extends A
newNode.properties.putAll(properties);
}
- boolean hasNoChildren() {
+ @Override
+ public boolean hasNoChildren() {
return !hasChildren;
}
@Override
+ protected NodeStateDiffer getNodeStateDiffer() {
+ return store;
+ }
+
+ @Override
public String toString() {
StringBuilder buff = new StringBuilder();
buff.append("{ path: '").append(path).append("', ");
@@ -426,10 +380,6 @@ public class DocumentNodeState extends A
return op;
}
- String getPath() {
- return path;
- }
-
String getId() {
return path + "@" + lastRevision;
}
@@ -447,10 +397,6 @@ public class DocumentNodeState extends A
this.lastRevision = lastRevision;
}
- RevisionVector getLastRevision() {
- return lastRevision;
- }
-
@Override
public int getMemory() {
int size = 40 // shallow
@@ -512,10 +458,10 @@ public class DocumentNodeState extends A
@Nonnull
private Iterable<ChildNodeEntry> getChildNodeEntries(@Nullable String name,
int limit) {
- Iterable<DocumentNodeState> children = store.getChildNodes(this, name,
limit);
- return Iterables.transform(children, new Function<DocumentNodeState,
ChildNodeEntry>() {
+ Iterable<? extends AbstractDocumentNodeState> children =
store.getChildNodes(this, name, limit);
+ return Iterables.transform(children, new
Function<AbstractDocumentNodeState, ChildNodeEntry>() {
@Override
- public ChildNodeEntry apply(final DocumentNodeState input) {
+ public ChildNodeEntry apply(final AbstractDocumentNodeState input)
{
return new AbstractChildNodeEntry() {
@Nonnull
@Override
Modified:
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1762470&r1=1762469&r2=1762470&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
Tue Sep 27 12:40:27 2016
@@ -121,7 +121,7 @@ import org.slf4j.LoggerFactory;
* Implementation of a NodeStore on {@link DocumentStore}.
*/
public final class DocumentNodeStore
- implements NodeStore, RevisionContext, Observable, Clusterable {
+ implements NodeStore, RevisionContext, Observable, Clusterable,
NodeStateDiffer {
private static final Logger LOG =
LoggerFactory.getLogger(DocumentNodeStore.class);
@@ -845,7 +845,7 @@ public final class DocumentNodeStore
}
}
- DocumentNodeState.Children getChildren(@Nonnull final DocumentNodeState
parent,
+ DocumentNodeState.Children getChildren(@Nonnull final
AbstractDocumentNodeState parent,
@Nullable final String name,
final int limit)
throws DocumentStoreException {
@@ -894,7 +894,7 @@ public final class DocumentNodeStore
* @param limit the maximum number of child nodes to return.
* @return the children of {@code parent}.
*/
- DocumentNodeState.Children readChildren(DocumentNodeState parent,
+ DocumentNodeState.Children readChildren(AbstractDocumentNodeState parent,
String name, int limit) {
String queriedName = name;
String path = parent.getPath();
@@ -1447,8 +1447,9 @@ public final class DocumentNodeStore
* {@code false} if it was aborted as requested by the handler
* (see the {@link NodeStateDiff} contract for more details)
*/
- boolean compare(@Nonnull final DocumentNodeState node,
- @Nonnull final DocumentNodeState base,
+ @Override
+ public boolean compare(@Nonnull final AbstractDocumentNodeState node,
+ @Nonnull final AbstractDocumentNodeState base,
@Nonnull NodeStateDiff diff) {
if (!AbstractNodeState.comparePropertiesAgainstBaseState(node, base,
diff)) {
return false;
@@ -2144,8 +2145,8 @@ public final class DocumentNodeStore
}
private boolean dispatch(@Nonnull final String jsonDiff,
- @Nonnull final DocumentNodeState node,
- @Nonnull final DocumentNodeState base,
+ @Nonnull final AbstractDocumentNodeState node,
+ @Nonnull final AbstractDocumentNodeState base,
@Nonnull final NodeStateDiff diff) {
return DiffCache.parseJsopDiff(jsonDiff, new DiffCache.Diff() {
@Override
@@ -2224,7 +2225,7 @@ public final class DocumentNodeStore
return false;
}
- private String diffImpl(DocumentNodeState from, DocumentNodeState to)
+ private String diffImpl(AbstractDocumentNodeState from,
AbstractDocumentNodeState to)
throws DocumentStoreException {
JsopWriter w = new JsopStream();
// TODO this does not work well for large child node lists
Modified:
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java?rev=1762470&r1=1762469&r2=1762470&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
Tue Sep 27 12:40:27 2016
@@ -38,12 +38,12 @@ import com.google.common.base.Stopwatch;
*/
public class JournalGarbageCollector {
- private final DocumentStore ds;
+ private final DocumentNodeStore ns;
private static final Logger log =
LoggerFactory.getLogger(JournalGarbageCollector.class);
public JournalGarbageCollector(DocumentNodeStore nodeStore) {
- this.ds = nodeStore.getDocumentStore();
+ this.ns = nodeStore;
}
/**
@@ -55,7 +55,16 @@ public class JournalGarbageCollector {
* @return the number of entries that have been removed
*/
public int gc(long maxRevisionAge, int batchSize, TimeUnit unit) {
+ DocumentStore ds = ns.getDocumentStore();
+ Revision keep = ns.getCheckpoints().getOldestRevisionToKeep();
long maxRevisionAgeInMillis = unit.toMillis(maxRevisionAge);
+ long now = ns.getClock().getTime();
+ long gcOlderThan = now - maxRevisionAgeInMillis;
+ if (keep != null && keep.getTimestamp() < gcOlderThan) {
+ gcOlderThan = keep.getTimestamp();
+ log.debug("gc: Checkpoint {} is older than maxRevisionAge: {} min",
+ keep, unit.toMinutes(maxRevisionAge));
+ }
if (log.isDebugEnabled()) {
log.debug("gc: Journal garbage collection starts with maxAge: {}
min., batch size: {}.",
TimeUnit.MILLISECONDS.toMinutes(maxRevisionAgeInMillis),
batchSize);
@@ -100,7 +109,7 @@ public class JournalGarbageCollector {
long startPointer = 0;
while (true) {
String fromKey = JournalEntry.asId(new Revision(startPointer,
0, clusterNodeId, branch));
- String toKey = JournalEntry.asId(new
Revision(System.currentTimeMillis() - maxRevisionAgeInMillis,
Integer.MAX_VALUE, clusterNodeId, branch));
+ String toKey = JournalEntry.asId(new Revision(gcOlderThan, 0,
clusterNodeId, branch));
List<JournalEntry> deletionBatch =
ds.query(Collection.JOURNAL, fromKey, toKey, batchSize);
if (deletionBatch.size() > 0) {
ds.remove(Collection.JOURNAL, asKeys(deletionBatch));
@@ -123,7 +132,10 @@ public class JournalGarbageCollector {
sw.stop();
- log.info("gc: Journal garbage collection took {}, deleted {} entries
that were older than {} min.", sw, numDeleted,
TimeUnit.MILLISECONDS.toMinutes(maxRevisionAgeInMillis));
+ if (numDeleted > 0) {
+ log.info("gc: Journal garbage collection took {}, deleted {}
entries that were older than {} min.",
+ sw, numDeleted, TimeUnit.MILLISECONDS.toMinutes(now -
gcOlderThan));
+ }
return numDeleted;
}
