Author: mreutegg Date: Tue Sep 20 12:52:19 2016 New Revision: 1761569 URL: http://svn.apache.org/viewvc?rev=1761569&view=rev Log: OAK-4819: Improve revision GC resilience
Merged revision 1761444 from trunk Modified: jackrabbit/oak/branches/1.4/ (props changed) jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java Propchange: jackrabbit/oak/branches/1.4/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Tue Sep 20 12:52:19 2016 @@ -1,3 +1,3 @@ /jackrabbit/oak/branches/1.0:1665962 -/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886 ,1751410,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1760340,1760373,1760387,1760661-1760662,1761412 +/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886 ,1751410,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1760340,1760373,1760387,1760661-1760662,1761412,1761444 /jackrabbit/trunk:1345480 Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java?rev=1761569&r1=1761568&r2=1761569&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java Tue Sep 20 12:52:19 2016 @@ -219,10 +219,17 @@ public class VersionGarbageCollector { */ void possiblyDeleted(NodeDocument doc) throws IOException { + // construct an id that also contains + // the _modified time of the document + String id = doc.getId() + "/" + doc.getModified(); + // check if id is valid + try { + Utils.getDepthFromId(id); + } catch (IllegalArgumentException e) { + log.warn("Invalid GC id {} for document {}", id, doc); + return; + } if (doc.getNodeAtRevision(nodeStore, headRevision, null) == null) { - // construct an id that also contains - // the _modified time of the document - String id = doc.getId() + "/" + doc.getModified(); addDocument(id); // Collect id of all previous docs also Iterator<NodeDocument> it = doc.getAllPreviousDocs(); Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java?rev=1761569&r1=1761568&r2=1761569&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java Tue Sep 20 12:52:19 2016 @@ -315,10 +315,16 @@ public class Utils { return id.substring(index + 1); } - public static int getDepthFromId(String id) { - int index = id.indexOf(':'); - assert index > 0 : "Invalid id " + id; - return Integer.parseInt(id.substring(0, index)); + public static int getDepthFromId(String id) throws IllegalArgumentException { + try { + int index = id.indexOf(':'); + if (index >= 0) { + return Integer.parseInt(id.substring(0, index)); + } + } catch (NumberFormatException e) { + // ignore and throw IllegalArgumentException + } + throw new IllegalArgumentException("Invalid id: " + id); } public static String getPreviousPathFor(String path, Revision r, int height) { Modified: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java?rev=1761569&r1=1761568&r2=1761569&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java Tue Sep 20 12:52:19 2016 @@ -39,6 +39,7 @@ import static org.apache.jackrabbit.oak. import static org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.MEMORY_NS; import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures; import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; +import static org.apache.jackrabbit.oak.plugins.document.Document.ID; import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.NUM_REVS_THRESHOLD; import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.PREV_SPLIT_FACTOR; import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType; @@ -92,10 +93,6 @@ public class VersionGarbageCollectorIT { @Parameterized.Parameters(name="{0}") public static Collection<Object[]> fixtures() throws IOException { List<Object[]> fixtures = Lists.newArrayList(); - if (getFixtures().contains(MEMORY_NS)) { - fixtures.add(new Object[] { new DocumentStoreFixture.MemoryFixture() }); - } - DocumentStoreFixture mongo = new DocumentStoreFixture.MongoFixture(); if (getFixtures().contains(DOCUMENT_NS) && mongo.isAvailable()) { fixtures.add(new Object[] { mongo }); @@ -105,6 +102,10 @@ public class VersionGarbageCollectorIT { if (getFixtures().contains(DOCUMENT_RDB) && rdb.isAvailable()) { fixtures.add(new Object[] { rdb }); } + if (fixtures.isEmpty() || getFixtures().contains(MEMORY_NS)) { + fixtures.add(new Object[] { new DocumentStoreFixture.MemoryFixture() }); + } + return fixtures; } @@ -499,6 +500,37 @@ public class VersionGarbageCollectorIT { assertEquals(2, stats.splitDocGCCount); } + // OAK-4819 + @Test + public void malformedId() throws Exception { + long maxAge = 1; //hrs + long delta = TimeUnit.MINUTES.toMillis(10); + + NodeBuilder builder = store.getRoot().builder(); + builder.child("foo"); + store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + // remove again + builder = store.getRoot().builder(); + builder.child("foo").remove(); + store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + + store.runBackgroundOperations(); + + // add a document with a malformed id + String id = "42"; + UpdateOp op = new UpdateOp(id, true); + op.set(ID, id); + NodeDocument.setDeletedOnce(op); + NodeDocument.setModified(op, store.newRevision()); + store.getDocumentStore().create(NODES, Lists.newArrayList(op)); + + clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge) + delta); + + // gc must not fail + VersionGCStats stats = gc.gc(maxAge, HOURS); + assertEquals(1, stats.deletedDocGCCount); + } + private void createTestNode(String name) throws CommitFailedException { DocumentStore ds = store.getDocumentStore(); NodeBuilder builder = store.getRoot().builder(); Modified: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java?rev=1761569&r1=1761568&r2=1761569&view=diff ============================================================================== --- jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java (original) +++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/util/UtilsTest.java Tue Sep 20 12:52:19 2016 @@ -237,4 +237,14 @@ public class UtilsTest { assertEquals(4, Utils.getMinTimestampForDiff(to, from, minRevs)); } + + @Test(expected = IllegalArgumentException.class) + public void getDepthFromIdIllegalArgumentException1() { + Utils.getDepthFromId("a:/foo"); + } + + @Test(expected = IllegalArgumentException.class) + public void getDepthFromIdIllegalArgumentException2() { + Utils.getDepthFromId("42"); + } }