Author: baedke Date: Tue Sep 27 12:40:27 2016 New Revision: 1762470 URL: http://svn.apache.org/viewvc?rev=1762470&view=rev Log: OAK-4553: Retain journal entries for valid checkpoints
Added: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java - copied, changed from r1749424, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeStateDiffer.java - copied unchanged from r1749424, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeStateDiffer.java jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java - copied unchanged from r1752292, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java Modified: jackrabbit/oak/branches/1.4/ (props changed) jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java Propchange: jackrabbit/oak/branches/1.4/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Tue Sep 27 12:40:27 2016 @@ -1,3 +1,3 @@ /jackrabbit/oak/branches/1.0:1665962 -/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886 ,1751410,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1759433,1760340,1760373,1760387,1760661-1760662,1761412,1761444,1761571,1761762,1761787 +/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626 ,1750809,1750886,1751410,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1759433,1760340,1760373,1760387,1760661-1760662,1761412,1761444,1761571,1761762,1761787 /jackrabbit/trunk:1345480 Copied: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java (from r1749424, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java) URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java?p2=jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java&r1=1749424&r2=1762470&rev=1762470&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentNodeState.java Tue Sep 27 12:40:27 2016 @@ -36,11 +36,6 @@ public abstract class AbstractDocumentNo private static final PerfLogger perfLogger = new PerfLogger( LoggerFactory.getLogger(AbstractDocumentNodeState.class.getName() + ".perf")); - protected final NodeStateDiffer differ; - - protected AbstractDocumentNodeState(NodeStateDiffer differ) { - this.differ = differ; - } public abstract String getPath(); @@ -70,6 +65,8 @@ public abstract class AbstractDocumentNo public abstract boolean hasNoChildren(); + protected abstract NodeStateDiffer getNodeStateDiffer(); + //--------------------------< NodeState >----------------------------------- @Override @@ -116,7 +113,7 @@ public abstract class AbstractDocumentNo // use DocumentNodeStore compare final long start = perfLogger.start(); try { - return differ.compare(this, mBase, diff); + return getNodeStateDiffer().compare(this, mBase, diff); } finally { if (start > 0) { perfLogger Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java?rev=1762470&r1=1762469&r2=1762470&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java Tue Sep 27 12:40:27 2016 @@ -40,14 +40,10 @@ import org.apache.jackrabbit.oak.json.Js import org.apache.jackrabbit.oak.plugins.document.util.Utils; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder; -import org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState; import org.apache.jackrabbit.oak.spi.state.AbstractChildNodeEntry; -import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; -import org.apache.jackrabbit.oak.spi.state.EqualsDiff; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; import org.apache.jackrabbit.oak.util.PerfLogger; import org.slf4j.LoggerFactory; @@ -57,12 +53,11 @@ import com.google.common.collect.Iterato import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.oak.commons.StringUtils.estimateMemoryUsage; -import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; /** * A {@link NodeState} implementation for the {@link DocumentNodeStore}. */ -public class DocumentNodeState extends AbstractNodeState implements CacheValue { +public class DocumentNodeState extends AbstractDocumentNodeState implements CacheValue { private static final PerfLogger perfLogger = new PerfLogger( LoggerFactory.getLogger(DocumentNodeState.class.getName() @@ -133,7 +128,8 @@ public class DocumentNodeState extends A * @return a copy of this node state with the given root revision and * external change flag. */ - private DocumentNodeState withRootRevision(@Nonnull RevisionVector root, + @Override + public DocumentNodeState withRootRevision(@Nonnull RevisionVector root, boolean externalChange) { if (rootRevision.equals(root) && fromExternalChange == externalChange) { return this; @@ -157,12 +153,16 @@ public class DocumentNodeState extends A * @return {@code true} if this node state was created as a result of an * external change; {@code false} otherwise. */ - boolean isFromExternalChange() { + @Override + public boolean isFromExternalChange() { return fromExternalChange; } + //--------------------------< AbstractDocumentNodeState >----------------------------------- + + @Override @Nonnull - RevisionVector getRevision() { + public RevisionVector getRevision() { return readRevision; } @@ -178,39 +178,23 @@ public class DocumentNodeState extends A * same value as returned by {@link #getRevision()}. */ @Nonnull - RevisionVector getRootRevision() { + public RevisionVector getRootRevision() { return rootRevision; } - //--------------------------< NodeState >----------------------------------- + @Override + public String getPath() { + return path; + } @Override - public boolean equals(Object that) { - if (this == that) { - return true; - } else if (that instanceof DocumentNodeState) { - DocumentNodeState other = (DocumentNodeState) that; - if (!getPath().equals(other.getPath())) { - // path does not match: not equals - // (even if the properties are equal) - return false; - } - if (revisionEquals(other)) { - return true; - } - // revision does not match: might still be equals - } else if (that instanceof ModifiedNodeState) { - ModifiedNodeState modified = (ModifiedNodeState) that; - if (modified.getBaseState() == this) { - return EqualsDiff.equals(this, modified); - } - } - if (that instanceof NodeState) { - return AbstractNodeState.equals(this, (NodeState) that); - } - return false; + public RevisionVector getLastRevision() { + return lastRevision; } + //--------------------------< NodeState >----------------------------------- + + @Override public boolean exists() { return true; @@ -323,42 +307,6 @@ public class DocumentNodeState extends A } } - @Override - public boolean compareAgainstBaseState(NodeState base, NodeStateDiff diff) { - if (this == base) { - return true; - } else if (base == EMPTY_NODE || !base.exists()) { - // special case - return EmptyNodeState.compareAgainstEmptyState(this, diff); - } else if (base instanceof DocumentNodeState) { - DocumentNodeState mBase = (DocumentNodeState) base; - if (store == mBase.store) { - if (getPath().equals(mBase.getPath())) { - if (revisionEquals(mBase)) { - // no differences - return true; - } else { - // use DocumentNodeStore compare - final long start = perfLogger.start(); - try { - return store.compare(this, mBase, diff); - } finally { - perfLogger - .end(start, - 1, - "compareAgainstBaseState, path={}, readRevision={}, lastRevision={}, base.path={}, base.readRevision={}, base.lastRevision={}", - path, readRevision, lastRevision, - mBase.path, mBase.readRevision, - mBase.lastRevision); - } - } - } - } - } - // fall back to the generic node state diff algorithm - return super.compareAgainstBaseState(base, diff); - } - void setProperty(String propertyName, String value) { if (value == null) { properties.remove(propertyName); @@ -392,11 +340,17 @@ public class DocumentNodeState extends A newNode.properties.putAll(properties); } - boolean hasNoChildren() { + @Override + public boolean hasNoChildren() { return !hasChildren; } @Override + protected NodeStateDiffer getNodeStateDiffer() { + return store; + } + + @Override public String toString() { StringBuilder buff = new StringBuilder(); buff.append("{ path: '").append(path).append("', "); @@ -426,10 +380,6 @@ public class DocumentNodeState extends A return op; } - String getPath() { - return path; - } - String getId() { return path + "@" + lastRevision; } @@ -447,10 +397,6 @@ public class DocumentNodeState extends A this.lastRevision = lastRevision; } - RevisionVector getLastRevision() { - return lastRevision; - } - @Override public int getMemory() { int size = 40 // shallow @@ -512,10 +458,10 @@ public class DocumentNodeState extends A @Nonnull private Iterable<ChildNodeEntry> getChildNodeEntries(@Nullable String name, int limit) { - Iterable<DocumentNodeState> children = store.getChildNodes(this, name, limit); - return Iterables.transform(children, new Function<DocumentNodeState, ChildNodeEntry>() { + Iterable<? extends AbstractDocumentNodeState> children = store.getChildNodes(this, name, limit); + return Iterables.transform(children, new Function<AbstractDocumentNodeState, ChildNodeEntry>() { @Override - public ChildNodeEntry apply(final DocumentNodeState input) { + public ChildNodeEntry apply(final AbstractDocumentNodeState input) { return new AbstractChildNodeEntry() { @Nonnull @Override Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1762470&r1=1762469&r2=1762470&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Tue Sep 27 12:40:27 2016 @@ -121,7 +121,7 @@ import org.slf4j.LoggerFactory; * Implementation of a NodeStore on {@link DocumentStore}. */ public final class DocumentNodeStore - implements NodeStore, RevisionContext, Observable, Clusterable { + implements NodeStore, RevisionContext, Observable, Clusterable, NodeStateDiffer { private static final Logger LOG = LoggerFactory.getLogger(DocumentNodeStore.class); @@ -845,7 +845,7 @@ public final class DocumentNodeStore } } - DocumentNodeState.Children getChildren(@Nonnull final DocumentNodeState parent, + DocumentNodeState.Children getChildren(@Nonnull final AbstractDocumentNodeState parent, @Nullable final String name, final int limit) throws DocumentStoreException { @@ -894,7 +894,7 @@ public final class DocumentNodeStore * @param limit the maximum number of child nodes to return. * @return the children of {@code parent}. */ - DocumentNodeState.Children readChildren(DocumentNodeState parent, + DocumentNodeState.Children readChildren(AbstractDocumentNodeState parent, String name, int limit) { String queriedName = name; String path = parent.getPath(); @@ -1447,8 +1447,9 @@ public final class DocumentNodeStore * {@code false} if it was aborted as requested by the handler * (see the {@link NodeStateDiff} contract for more details) */ - boolean compare(@Nonnull final DocumentNodeState node, - @Nonnull final DocumentNodeState base, + @Override + public boolean compare(@Nonnull final AbstractDocumentNodeState node, + @Nonnull final AbstractDocumentNodeState base, @Nonnull NodeStateDiff diff) { if (!AbstractNodeState.comparePropertiesAgainstBaseState(node, base, diff)) { return false; @@ -2144,8 +2145,8 @@ public final class DocumentNodeStore } private boolean dispatch(@Nonnull final String jsonDiff, - @Nonnull final DocumentNodeState node, - @Nonnull final DocumentNodeState base, + @Nonnull final AbstractDocumentNodeState node, + @Nonnull final AbstractDocumentNodeState base, @Nonnull final NodeStateDiff diff) { return DiffCache.parseJsopDiff(jsonDiff, new DiffCache.Diff() { @Override @@ -2224,7 +2225,7 @@ public final class DocumentNodeStore return false; } - private String diffImpl(DocumentNodeState from, DocumentNodeState to) + private String diffImpl(AbstractDocumentNodeState from, AbstractDocumentNodeState to) throws DocumentStoreException { JsopWriter w = new JsopStream(); // TODO this does not work well for large child node lists Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java?rev=1762470&r1=1762469&r2=1762470&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java Tue Sep 27 12:40:27 2016 @@ -38,12 +38,12 @@ import com.google.common.base.Stopwatch; */ public class JournalGarbageCollector { - private final DocumentStore ds; + private final DocumentNodeStore ns; private static final Logger log = LoggerFactory.getLogger(JournalGarbageCollector.class); public JournalGarbageCollector(DocumentNodeStore nodeStore) { - this.ds = nodeStore.getDocumentStore(); + this.ns = nodeStore; } /** @@ -55,7 +55,16 @@ public class JournalGarbageCollector { * @return the number of entries that have been removed */ public int gc(long maxRevisionAge, int batchSize, TimeUnit unit) { + DocumentStore ds = ns.getDocumentStore(); + Revision keep = ns.getCheckpoints().getOldestRevisionToKeep(); long maxRevisionAgeInMillis = unit.toMillis(maxRevisionAge); + long now = ns.getClock().getTime(); + long gcOlderThan = now - maxRevisionAgeInMillis; + if (keep != null && keep.getTimestamp() < gcOlderThan) { + gcOlderThan = keep.getTimestamp(); + log.debug("gc: Checkpoint {} is older than maxRevisionAge: {} min", + keep, unit.toMinutes(maxRevisionAge)); + } if (log.isDebugEnabled()) { log.debug("gc: Journal garbage collection starts with maxAge: {} min., batch size: {}.", TimeUnit.MILLISECONDS.toMinutes(maxRevisionAgeInMillis), batchSize); @@ -100,7 +109,7 @@ public class JournalGarbageCollector { long startPointer = 0; while (true) { String fromKey = JournalEntry.asId(new Revision(startPointer, 0, clusterNodeId, branch)); - String toKey = JournalEntry.asId(new Revision(System.currentTimeMillis() - maxRevisionAgeInMillis, Integer.MAX_VALUE, clusterNodeId, branch)); + String toKey = JournalEntry.asId(new Revision(gcOlderThan, 0, clusterNodeId, branch)); List<JournalEntry> deletionBatch = ds.query(Collection.JOURNAL, fromKey, toKey, batchSize); if (deletionBatch.size() > 0) { ds.remove(Collection.JOURNAL, asKeys(deletionBatch)); @@ -123,7 +132,10 @@ public class JournalGarbageCollector { sw.stop(); - log.info("gc: Journal garbage collection took {}, deleted {} entries that were older than {} min.", sw, numDeleted, TimeUnit.MILLISECONDS.toMinutes(maxRevisionAgeInMillis)); + if (numDeleted > 0) { + log.info("gc: Journal garbage collection took {}, deleted {} entries that were older than {} min.", + sw, numDeleted, TimeUnit.MILLISECONDS.toMinutes(now - gcOlderThan)); + } return numDeleted; }