This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch OAK-10632
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit c4e9e7131953657bad443482a4e9089b738804dc
Author: Rishabh Kumar <d...@adobe.com>
AuthorDate: Fri Jan 26 01:59:26 2024 +0530
OAK-10632 : make embedded verification configurable in dryRun mode
---
.../plugins/document/DocumentNodeStoreHelper.java | 7 +++---
.../jackrabbit/oak/run/RevisionsCommand.java | 13 +++++++++--
.../oak/plugins/document/RevisionsCommandTest.java | 27 ++++++++++++++++++++++
.../oak/plugins/document/DocumentNodeStore.java | 2 +-
.../plugins/document/VersionGarbageCollector.java | 8 +++----
.../oak/plugins/document/VersionGCQueryTest.java | 4 ++--
.../oak/plugins/document/VersionGCTest.java | 2 +-
.../document/VersionGarbageCollectorIT.java | 13 +++++------
8 files changed, 56 insertions(+), 20 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
index f11232b24f..1cede25f26 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
@@ -71,9 +71,10 @@ public class DocumentNodeStoreHelper {
System.out.println("Collected in " + sw.stop());
}
- public static VersionGarbageCollector createVersionGC(
- DocumentNodeStore nodeStore, VersionGCSupport gcSupport, boolean
detailedGCEnabled, boolean isDetailedGCDryRun) {
- return new VersionGarbageCollector(nodeStore, gcSupport,
detailedGCEnabled, isDetailedGCDryRun);
+ public static VersionGarbageCollector createVersionGC(DocumentNodeStore
nodeStore, VersionGCSupport gcSupport,
+ boolean
detailedGCEnabled, boolean isDetailedGCDryRun,
+ boolean
embeddedVerification) {
+ return new VersionGarbageCollector(nodeStore, gcSupport,
detailedGCEnabled, isDetailedGCDryRun, embeddedVerification);
}
private static Iterable<BlobReferences> scan(DocumentNodeStore store,
diff --git
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
index 317659fd2b..b01c0512a0 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -56,7 +56,6 @@ import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
import static java.util.concurrent.TimeUnit.SECONDS;
import static
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper.createVersionGC;
@@ -108,6 +107,7 @@ public class RevisionsCommand implements Command {
final OptionSpec<?> detailedGCOnly;
final OptionSpec<?> verbose;
final OptionSpec<Boolean> dryRun;
+ final OptionSpec<Boolean> embeddedVerification;
RevisionsOptions(String usage) {
super(usage);
@@ -126,6 +126,10 @@ public class RevisionsCommand implements Command {
.ofType(Long.class).defaultsTo(-1L);
dryRun = parser.accepts("dryRun", "dryRun of detailedGC i.e. only
print what would be deleted")
.withRequiredArg().ofType(Boolean.class).defaultsTo(TRUE);
+ embeddedVerification = parser.accepts("verify", "enable embedded
verification of detailedGC " +
+ "during dryRun mode i.e. will verify the effect of
detailedGC operation on each document after " +
+ "applying the changes in memory and will raise
flag if it can cause issues")
+ .withRequiredArg().ofType(Boolean.class).defaultsTo(TRUE);
continuous = parser
.accepts("continuous", "run continuously (collect only)");
detailedGC = parser
@@ -161,6 +165,10 @@ public class RevisionsCommand implements Command {
return dryRun.value(options);
}
+ boolean isEmbeddedVerificationEnabled() {
+ return embeddedVerification.value(options);
+ }
+
long getOlderThan() {
return olderThan.value(options);
}
@@ -261,8 +269,9 @@ public class RevisionsCommand implements Command {
// create a version GC that operates on a read-only DocumentNodeStore
// and a GC support with a writable DocumentStore
System.out.println("DryRun is enabled : " + options.isDryRun());
+ System.out.println("EmbeddedVerification is enabled : " +
options.isEmbeddedVerificationEnabled());
VersionGarbageCollector gc = createVersionGC(builder.build(),
gcSupport, isDetailedGCEnabled(builder),
- options.isDryRun());
+ options.isDryRun(), options.isEmbeddedVerificationEnabled());
VersionGCOptions gcOptions = gc.getOptions();
gcOptions = gcOptions.withDelayFactor(options.getDelay());
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
index 919052106d..40938861c1 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/document/RevisionsCommandTest.java
@@ -184,6 +184,33 @@ public class RevisionsCommandTest {
assertTrue(output.contains("starting gc collect"));
}
+ @Test
+ public void embeddedVerification() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("detailedGC"));
+ assertTrue(output.contains("EmbeddedVerification is enabled : true"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void detailedGCWithoutEmbeddedVerification() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("detailedGC",
"--verify", "false"));
+ assertTrue(output.contains("EmbeddedVerification is enabled : false"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
+ @Test
+ public void detailedGCWithEmbeddedVerification() {
+ ns.dispose();
+
+ String output = captureSystemOut(new RevisionsCmd("detailedGC",
"--verify", "true"));
+ assertTrue(output.contains("EmbeddedVerification is enabled : true"));
+ assertTrue(output.contains("starting gc collect"));
+ }
+
@Test
public void sweep() throws Exception {
int clusterId = ns.getClusterId();
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
index 4cb808aa0a..e4e04fb759 100644
---
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
@@ -642,7 +642,7 @@ public final class DocumentNodeStore
this.branches = new UnmergedBranches();
this.asyncDelay = builder.getAsyncDelay();
this.versionGarbageCollector = new VersionGarbageCollector(
- this, builder.createVersionGCSupport(),
isDetailedGCEnabled(builder), false);
+ this, builder.createVersionGCSupport(),
isDetailedGCEnabled(builder), false, false);
this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
this.journalGarbageCollector = new JournalGarbageCollector(
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index 08b05aea91..aca0b2a746 100644
---
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -153,13 +153,14 @@ public class VersionGarbageCollector {
VersionGarbageCollector(DocumentNodeStore nodeStore,
VersionGCSupport gcSupport,
final boolean detailedGCEnabled,
- final boolean isDetailedGCDryRun) {
+ final boolean isDetailedGCDryRun,
+ final boolean embeddedVerification) {
this.nodeStore = nodeStore;
this.versionStore = gcSupport;
this.ds = gcSupport.getDocumentStore();
this.detailedGCEnabled = detailedGCEnabled;
this.isDetailedGCDryRun = isDetailedGCDryRun;
- this.embeddedVerification = true; //TODO: make this "configurable"
+ this.embeddedVerification = embeddedVerification; //TODO: make this
"configurable" for actual detailedGC
this.options = new VersionGCOptions();
}
@@ -1294,8 +1295,7 @@ public class VersionGarbageCollector {
if (oldNS == null && newNS == null) {
// both don't exist - fine, that's considered equal
return true;
- } else if ((oldNS == null && newNS != null)
- || (oldNS != null && newNS == null)) {
+ } else if (oldNS == null || newNS == null) {
// failure : one is deleted/missing, the other not
log.error("removeGarbage.verify : failure in DetailedGC"
+ " with id : {}, oldNS exists : {}, newNS exists: {},
update: {}",
diff --git
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
index ab3cfd30f3..0b8453011e 100644
---
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
+++
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
@@ -107,7 +107,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store), false, false);
+ ns, new VersionGCSupport(store), false, false, false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(11, stats.deletedDocGCCount);
@@ -140,7 +140,7 @@ public class VersionGCQueryTest {
clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
VersionGarbageCollector gc = new VersionGarbageCollector(
- ns, new VersionGCSupport(store), false, false);
+ ns, new VersionGCSupport(store), false, false, false);
prevDocIds.clear();
VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
assertEquals(1, stats.deletedDocGCCount);
diff --git
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
index 985aea7078..2489637fc7 100644
---
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
+++
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java
@@ -458,7 +458,7 @@ public class VersionGCTest {
deletedOnceCountCalls.incrementAndGet();
return Iterables.size(Utils.getSelectedDocuments(store,
NodeDocument.DELETED_ONCE, 1));
}
- }, false, false);
+ }, false, false, false);
// run first RGC
gc.gc(1, TimeUnit.HOURS);
diff --git
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index a8dfc849f7..92ce1a888d 100644
---
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -99,7 +99,6 @@ import org.apache.jackrabbit.oak.api.Type;
import
org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.RDBFixture;
import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import
org.apache.jackrabbit.oak.plugins.document.bundlor.BundlingConfigInitializer;
-import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoTestUtils;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBOptions;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
@@ -1022,7 +1021,7 @@ public class VersionGarbageCollectorIT {
}
};
- VersionGarbageCollector gc = new VersionGarbageCollector(store1,
gcSupport, true, false);
+ VersionGarbageCollector gc = new VersionGarbageCollector(store1,
gcSupport, true, false, false);
stats = gc.gc(maxAge*2, HOURS);
assertEquals(0, stats.updatedDetailedGCDocsCount);
assertEquals(0, stats.deletedPropsCount);
@@ -1088,7 +1087,7 @@ public class VersionGarbageCollectorIT {
}
};
- gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false));
+ gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false,
false));
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
@@ -1812,7 +1811,7 @@ public class VersionGarbageCollectorIT {
});
}
};
- final VersionGarbageCollector gc = new VersionGarbageCollector(store1,
gcSupport, false, false);
+ final VersionGarbageCollector gc = new VersionGarbageCollector(store1,
gcSupport, false, false, false);
// start GC -> will try to remove /foo and /bar
Future<VersionGCStats> f = execService.submit(new
Callable<VersionGCStats>() {
@Override
@@ -1968,7 +1967,7 @@ public class VersionGarbageCollectorIT {
return super.getPossiblyDeletedDocs(fromModified, toModified);
}
};
- gcRef.set(new VersionGarbageCollector(store1, gcSupport, false,
false));
+ gcRef.set(new VersionGarbageCollector(store1, gcSupport, false, false,
false));
VersionGCStats stats = gcRef.get().gc(30, TimeUnit.MINUTES);
assertTrue(stats.canceled);
assertEquals(0, stats.deletedDocGCCount);
@@ -2020,7 +2019,7 @@ public class VersionGarbageCollectorIT {
return super.getPossiblyDeletedDocs(prevLastModifiedTime,
lastModifiedTime).iterator();
}
};
- gcRef.set(new VersionGarbageCollector(store1, gcSupport, false,
false));
+ gcRef.set(new VersionGarbageCollector(store1, gcSupport, false, false,
false));
VersionGCStats stats = gcRef.get().gc(30, TimeUnit.MINUTES);
assertTrue(stats.canceled);
assertEquals(0, stats.deletedDocGCCount);
@@ -2049,7 +2048,7 @@ public class VersionGarbageCollectorIT {
});
}
};
- final VersionGarbageCollector gc = new VersionGarbageCollector(store1,
nonReportingGcSupport, false, false);
+ final VersionGarbageCollector gc = new VersionGarbageCollector(store1,
nonReportingGcSupport, false, false, false);
final long maxAgeHours = 1;
final long clockDelta = HOURS.toMillis(maxAgeHours) +
MINUTES.toMillis(5);
