This is an automated email from the ASF dual-hosted git repository. adulceanu pushed a commit to branch issues/OAK-9455 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 60eda2c96661ac0a16dff53af531199247ace7db Author: Andrei Dulceanu <dulce...@adobe.com> AuthorDate: Tue Jun 4 13:00:02 2024 +0200 OAK-9455 - Improve oak-run check to allow fast remote consistency checks --- .../site/markdown/nodestore/segment/overview.md | 7 + .../apache/jackrabbit/oak/run/CheckCommand.java | 91 +++-- .../oak/segment/azure/tool/AzureCheck.java | 408 +++++++++++++++++++++ .../apache/jackrabbit/oak/segment/tool/Check.java | 284 ++------------ .../tool/{Check.java => check/CheckHelper.java} | 370 +++++-------------- 5 files changed, 611 insertions(+), 549 deletions(-) diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md b/oak-doc/src/site/markdown/nodestore/segment/overview.md index 7e07b02ca2..b350f8e470 100644 --- a/oak-doc/src/site/markdown/nodestore/segment/overview.md +++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md @@ -803,6 +803,8 @@ The `--bin` option has no effect on binary properties stored in an external Blob The optional `--last [Integer]` argument can be used to control the maximum number of revisions to be verified (default is `1`). +The optional `--fail-fast` argument can be used to stop the check as soon as an inconsistency is found. If not specified, the tool will continue to check the entire journal. + If the `--head` option is specified, the tool will scan **only** the head state, ignoring any available checkpoints. If the `--checkpoints` option is specified, the tool will scan **only** the specified checkpoints, ignoring the head state. At least one argument is expected with this option; multiple arguments need to be comma-separated. @@ -820,6 +822,11 @@ If the option is not specified, the full traversal of the repository (rooted at If the `--io-stats` option is specified, the tool will print some statistics about the I/O operations performed during the execution of the check command. This option is optional and is disabled by default. +The optional `--persistent-cache-path PERSISTENT_CACHE_PATH` argument allows to specify the path for the persistent disk cache. `PERSISTENT_CACHE_PATH` must be a valid path. + +The optional `--persistent-cache-size-gb <PERSISTENT_CACHE_SIZE_GB>` argument allows to limit the maximum size of the persistent disk cache to `<PERSISTENT_CACHE_SIZE_GB>`. If not specified, the default size will be limited to `50` GB. + + ### <a name="compact"/> Compact ``` diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java index 7a0815679e..601e23cebf 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java @@ -27,6 +27,7 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.apache.jackrabbit.oak.run.commons.Command; +import org.apache.jackrabbit.oak.segment.azure.tool.AzureCheck; import org.apache.jackrabbit.oak.segment.tool.Check; class CheckCommand implements Command { @@ -38,9 +39,9 @@ class CheckCommand implements Command { .withOptionalArg() .ofType(Boolean.class) .defaultsTo(true); - OptionSpec<File> journal = parser.accepts("journal", "journal file") + OptionSpec<String> journal = parser.accepts("journal", "journal file") .withRequiredArg() - .ofType(File.class); + .ofType(String.class); OptionSpec<Long> notify = parser.accepts("notify", "number of seconds between progress notifications") .withRequiredArg() .ofType(Long.class) @@ -61,9 +62,22 @@ class CheckCommand implements Command { .withValuesSeparatedBy(',') .defaultsTo("all"); OptionSpec<?> ioStatistics = parser.accepts("io-stats", "Print I/O statistics (only for oak-segment-tar)"); - OptionSpec<File> dir = parser.nonOptions() - .describedAs("path") - .ofType(File.class); + OptionSpec<String> dir = parser.nonOptions() + .describedAs("Path/URI to TAR/remote segment store (required)") + .ofType(String.class); + OptionSpec<Boolean> failFast = parser.accepts("fail-fast", "eagerly fail if first path/revision checked is inconsistent (default: false)") + .withOptionalArg() + .ofType(Boolean.class) + .defaultsTo(false); + OptionSpec<String> persistentCachePath = parser.accepts("persistent-cache-path", "Path/URI to persistent cache where " + + "resulting segments will be written") + .withRequiredArg() + .ofType(String.class); + OptionSpec<Integer> persistentCacheSizeGb = parser.accepts("persistent-cache-size-gb", "Size in GB (defaults to 50 GB) for " + + "the persistent disk cache") + .withRequiredArg() + .ofType(Integer.class) + .defaultsTo(50); OptionSet options = parser.parse(args); if (options.valuesOf(dir).isEmpty()) { @@ -74,27 +88,56 @@ class CheckCommand implements Command { printUsageAndExit(parser, "Too many Segment Store paths specified"); } - Check.Builder builder = Check.builder() - .withPath(options.valueOf(dir)) - .withMmap(mmapArg.value(options)) - .withDebugInterval(notify.value(options)) - .withCheckBinaries(options.has(bin)) - .withCheckHead(shouldCheckHead(options, head, cp)) - .withCheckpoints(toCheckpointsSet(options, head, cp)) - .withFilterPaths(toSet(options, filter)) - .withIOStatistics(options.has(ioStatistics)) - .withOutWriter(new PrintWriter(System.out, true)) - .withErrWriter(new PrintWriter(System.err, true)); - - if (options.has(journal)) { - builder.withJournal(journal.value(options)); - } - - if (options.has(last)) { - builder.withRevisionsCount(options.valueOf(last) != null ? last.value(options) : 1); + int code; + if (options.valueOf(dir).startsWith("az:")) { + AzureCheck.Builder builder = AzureCheck.builder() + .withPath(options.valueOf(dir)) + .withDebugInterval(notify.value(options)) + .withCheckBinaries(options.has(bin)) + .withCheckHead(shouldCheckHead(options, head, cp)) + .withCheckpoints(toCheckpointsSet(options, head, cp)) + .withFilterPaths(toSet(options, filter)) + .withIOStatistics(options.has(ioStatistics)) + .withOutWriter(new PrintWriter(System.out, true)) + .withErrWriter(new PrintWriter(System.err, true)) + .withFailFast(failFast.value(options)); + + if (options.has(last)) { + builder.withRevisionsCount(options.valueOf(last) != null ? last.value(options) : 1); + } + + if (options.has(persistentCachePath)) { + builder.withPersistentCachePath(persistentCachePath.value(options)); + builder.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options)); + } + + code = builder.build().run(); + } else { + Check.Builder builder = Check.builder() + .withPath(new File(options.valueOf(dir))) + .withMmap(mmapArg.value(options)) + .withDebugInterval(notify.value(options)) + .withCheckBinaries(options.has(bin)) + .withCheckHead(shouldCheckHead(options, head, cp)) + .withCheckpoints(toCheckpointsSet(options, head, cp)) + .withFilterPaths(toSet(options, filter)) + .withIOStatistics(options.has(ioStatistics)) + .withOutWriter(new PrintWriter(System.out, true)) + .withErrWriter(new PrintWriter(System.err, true)) + .withFailFast(failFast.value(options)); + + if (options.has(journal)) { + builder.withJournal(new File(journal.value(options))); + } + + if (options.has(last)) { + builder.withRevisionsCount(options.valueOf(last) != null ? last.value(options) : 1); + } + + code = builder.build().run(); } - System.exit(builder.build().run()); + System.exit(code); } private void printUsageAndExit(OptionParser parser, String... messages) throws IOException { diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java new file mode 100644 index 0000000000..2452e8b12b --- /dev/null +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java @@ -0,0 +1,408 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.segment.azure.tool; + +import com.google.common.io.Files; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.segment.file.JournalReader; +import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; +import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; +import org.apache.jackrabbit.oak.segment.tool.Check; +import org.apache.jackrabbit.oak.segment.tool.check.CheckHelper; + +import java.io.File; +import java.io.PrintWriter; +import java.text.MessageFormat; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument; +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; +import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; + +public class AzureCheck { + /** + * Create a builder for the {@link Check} command. + * + * @return an instance of {@link Check.Builder}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Collect options for the {@link Check} command. + */ + public static class Builder { + + private String path; + + private String journal; + + private long debugInterval = Long.MAX_VALUE; + + private boolean checkBinaries; + + private boolean checkHead; + + private Integer revisionsCount; + + private Set<String> checkpoints; + + private Set<String> filterPaths; + + private boolean ioStatistics; + + private Check.RepositoryStatistics repoStatistics; + + private PrintWriter outWriter; + + private PrintWriter errWriter; + + private boolean failFast; + + private String persistentCachePath; + + private Integer persistentCacheSizeGb; + + private Builder() { + // Prevent external instantiation. + } + + /** + * The path to an existing segment store. This parameter is required. + * + * @param path the path to an existing segment store. + * @return this builder. + */ + public Builder withPath(String path) { + this.path = checkNotNull(path); + return this; + } + + /** + * The path to the journal of the segment store. This parameter is + * optional. If not provided, the journal in the default location is + * used. + * + * @param journal the path to the journal of the segment store. + * @return this builder. + */ + public Builder withJournal(String journal) { + this.journal = checkNotNull(journal); + return this; + } + + /** + * Number of seconds between successive debug print statements. This + * parameter is not required and defaults to an arbitrary large number. + * + * @param debugInterval number of seconds between successive debug print + * statements. It must be positive. + * @return this builder. + */ + public Builder withDebugInterval(long debugInterval) { + checkArgument(debugInterval >= 0); + this.debugInterval = debugInterval; + return this; + } + + /** + * Instruct the command to scan the full content of binary properties. + * This parameter is not required and defaults to {@code false}. + * + * @param checkBinaries {@code true} if binary properties should be + * scanned, {@code false} otherwise. + * @return this builder. + */ + public Builder withCheckBinaries(boolean checkBinaries) { + this.checkBinaries = checkBinaries; + return this; + } + + /** + * Instruct the command to check head state. + * This parameter is not required and defaults to {@code true}. + * @param checkHead if {@code true}, will check the head state. + * @return this builder. + */ + public Builder withCheckHead(boolean checkHead) { + this.checkHead = checkHead; + return this; + } + + /** + * Instruct the command to check only the last {@code revisionsCount} revisions. + * This parameter is not required and defaults to {@code 1}. + * @param revisionsCount number of revisions to check. + * @return this builder. + */ + public Builder withRevisionsCount(Integer revisionsCount){ + this.revisionsCount = revisionsCount; + return this; + } + + /** + * Instruct the command to check specified checkpoints. + * This parameter is not required and defaults to "/checkpoints", + * i.e. will check all checkpoints when not explicitly overridden. + * + * @param checkpoints checkpoints to be checked + * @return this builder. + */ + public Builder withCheckpoints(Set<String> checkpoints) { + this.checkpoints = checkpoints; + return this; + } + + /** + * Content paths to be checked. This parameter is not required and + * defaults to "/". + * + * @param filterPaths + * paths to be checked + * @return this builder. + */ + public Builder withFilterPaths(Set<String> filterPaths) { + this.filterPaths = filterPaths; + return this; + } + + /** + * Instruct the command to print statistics about I/O operations + * performed during the check. This parameter is not required and + * defaults to {@code false}. + * + * @param ioStatistics {@code true} if I/O statistics should be + * provided, {@code false} otherwise. + * @return this builder. + */ + public Builder withIOStatistics(boolean ioStatistics) { + this.ioStatistics = ioStatistics; + return this; + } + + /** + * Attach a repository statistics instance to collect info on nodes + * and properties checked on head. + * + * @param repoStatistics instance to collect statistics + * @return this builder. + */ + public Builder withRepositoryStatistics(Check.RepositoryStatistics repoStatistics) { + this.repoStatistics = repoStatistics; + return this; + } + + /** + * The text output stream writer used to print normal output. + * @param outWriter the output writer. + * @return this builder. + */ + public Builder withOutWriter(PrintWriter outWriter) { + this.outWriter = outWriter; + + return this; + } + + /** + * The text error stream writer used to print erroneous output. + * @param errWriter the error writer. + * @return this builder. + */ + public Builder withErrWriter(PrintWriter errWriter) { + this.errWriter = errWriter; + + return this; + } + + /** + * Instruct the command to fail fast if the first path/revision checked + * is inconsistent. This parameter is not required and defaults to + * {@code false}. + * + * @param failFast {@code true} if the command should fail fast, + * {@code false} otherwise. + * @return this builder. + */ + public Builder withFailFast(boolean failFast) { + this.failFast = failFast; + return this; + } + + /** + * The path where segments in the persistent cache will be stored. + * + * @param persistentCachePath + * the path to the persistent cache. + * @return this builder + */ + public Builder withPersistentCachePath(String persistentCachePath) { + this.persistentCachePath = checkNotNull(persistentCachePath); + return this; + } + + /** + * The maximum size in GB of the persistent disk cache. + * + * @param persistentCacheSizeGb + * the maximum size of the persistent cache. + * @return this builder + */ + public Builder withPersistentCacheSizeGb(Integer persistentCacheSizeGb) { + this.persistentCacheSizeGb = checkNotNull(persistentCacheSizeGb); + return this; + } + + /** + * Create an executable version of the {@link Check} command. + * + * @return an instance of {@link Runnable}. + */ + public AzureCheck build() { + checkNotNull(path); + return new AzureCheck(this); + } + + } + + private static class StatisticsIOMonitor extends IOMonitorAdapter { + + AtomicLong ops = new AtomicLong(0); + + AtomicLong bytes = new AtomicLong(0); + + AtomicLong time = new AtomicLong(0); + + @Override + public void afterSegmentRead(File file, long msb, long lsb, int length, long elapsed) { + ops.incrementAndGet(); + bytes.addAndGet(length); + time.addAndGet(elapsed); + } + + } + + private final String path; + + private final String journal; + + private final long debugInterval; + + private final boolean checkBinaries; + + private final boolean checkHead; + + private final Integer revisionsCount; + + private final Set<String> requestedCheckpoints; + + private final Set<String> filterPaths; + + private final boolean ioStatistics; + + private Check.RepositoryStatistics repoStatistics; + + private final PrintWriter out; + + private final PrintWriter err; + + private final boolean failFast; + + private final String persistentCachePath; + + private final Integer persistentCacheSizeGb; + + private AzureCheck(Builder builder) { + this.path = builder.path; + this.debugInterval = builder.debugInterval; + this.checkHead = builder.checkHead; + this.checkBinaries = builder.checkBinaries; + this.requestedCheckpoints = builder.checkpoints; + this.filterPaths = builder.filterPaths; + this.ioStatistics = builder.ioStatistics; + this.repoStatistics = builder.repoStatistics; + this.out = builder.outWriter; + this.err = builder.errWriter; + this.journal = builder.journal; + this.revisionsCount = revisionsToCheckCount(builder.revisionsCount); + this.failFast = builder.failFast; + this.persistentCachePath = builder.persistentCachePath; + this.persistentCacheSizeGb = builder.persistentCacheSizeGb; + } + + private static Integer revisionsToCheckCount(Integer revisionsCount) { + return revisionsCount != null ? revisionsCount : Integer.MAX_VALUE; + } + + public int run() { + StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor(); + SegmentNodeStorePersistence persistence; + if (persistentCachePath != null) { + persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, persistentCachePath, persistentCacheSizeGb); + } else { + persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); + } + + FileStoreBuilder builder = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(persistence); + + if (ioStatistics) { + builder.withIOMonitor(ioMonitor); + } + + CheckHelper checkHelper = CheckHelper.builder() + .withCheckBinaries(checkBinaries) + .withCheckpoints(requestedCheckpoints) + .withCheckHead(checkHead) + .withDebugInterval(debugInterval) + .withFailFast(failFast) + .withFilterPaths(filterPaths) + .withRevisionsCount(revisionsCount) + .withErrWriter(err) + .withOutWriter(out) + .build(); + + try ( + ReadOnlyFileStore store = builder.buildReadOnly(); + JournalReader journal = new JournalReader(persistence.getJournalFile()) + ) { + int result = checkHelper.run(store, journal); + + if (ioStatistics) { + print("[I/O] Segment read: Number of operations: {0}", ioMonitor.ops.get()); + print("[I/O] Segment read: Total size: {0} ({1} bytes)", humanReadableByteCount(ioMonitor.bytes.get()), ioMonitor.bytes.get()); + print("[I/O] Segment read: Total time: {0} ns", ioMonitor.time.get()); + } + + if (repoStatistics != null) { + repoStatistics.setHeadNodeCount(checkHelper.getHeadNodeCount()); + repoStatistics.setHeadPropertyCount(checkHelper.getHeadPropertyCount()); + } + + return result; + } catch (Exception e) { + e.printStackTrace(err); + return 1; + } + } + + private void print(String format, Object... arguments) { + out.println(MessageFormat.format(format, arguments)); + } +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java index cafb684aa1..88f2889425 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java @@ -19,33 +19,22 @@ package org.apache.jackrabbit.oak.segment.tool; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; -import static java.text.DateFormat.getDateTimeInstance; import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; import java.io.File; import java.io.PrintWriter; import java.text.MessageFormat; -import java.util.Date; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; -import org.apache.jackrabbit.guava.common.base.Strings; -import org.apache.jackrabbit.guava.common.collect.Sets; -import org.apache.jackrabbit.oak.api.PropertyState; -import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile; import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; -import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker; -import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.ConsistencyCheckResult; -import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.Revision; import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter; +import org.apache.jackrabbit.oak.segment.tool.check.CheckHelper; /** * Perform a consistency check on an existing segment store. @@ -259,6 +248,15 @@ public class Check { return this; } + /** + * Instruct the command to fail fast if the first path/revision checked + * is inconsistent. This parameter is not required and defaults to + * {@code false}. + * + * @param failFast {@code true} if the command should fail fast, + * {@code false} otherwise. + * @return this builder. + */ public Builder withFailFast(boolean failFast) { this.failFast = failFast; return this; @@ -297,6 +295,14 @@ public class Check { int headNodeCount; int headPropertyCount; + public void setHeadPropertyCount(int headPropertyCount) { + this.headPropertyCount = headPropertyCount; + } + + public void setHeadNodeCount(int headNodeCount) { + this.headNodeCount = headNodeCount; + } + public int getHeadNodeCount() { return headNodeCount; } @@ -334,16 +340,6 @@ public class Check { private final boolean failFast; - private int currentNodeCount; - - private int currentPropertyCount; - - private int headNodeCount; - - private int headPropertyCount; - - private long lastDebugEvent; - private Check(Builder builder) { this.path = builder.path; this.mmap = builder.mmap; @@ -383,11 +379,23 @@ public class Check { builder.withIOMonitor(ioMonitor); } + CheckHelper checkHelper = CheckHelper.builder() + .withCheckBinaries(checkBinaries) + .withCheckpoints(requestedCheckpoints) + .withCheckHead(checkHead) + .withDebugInterval(debugInterval) + .withFailFast(failFast) + .withFilterPaths(filterPaths) + .withRevisionsCount(revisionsCount) + .withErrWriter(err) + .withOutWriter(out) + .build(); + try ( ReadOnlyFileStore store = builder.buildReadOnly(); JournalReader journal = new JournalReader(new LocalJournalFile(this.journal)) ) { - int result = run(store, journal); + int result = checkHelper.run(store, journal); if (ioStatistics) { print("[I/O] Segment read: Number of operations: {0}", ioMonitor.ops.get()); @@ -396,8 +404,8 @@ public class Check { } if (repoStatistics != null) { - repoStatistics.headNodeCount = headNodeCount; - repoStatistics.headPropertyCount = headPropertyCount; + repoStatistics.headNodeCount = checkHelper.getHeadNodeCount(); + repoStatistics.headPropertyCount = checkHelper.getHeadPropertyCount(); } return result; @@ -407,233 +415,7 @@ public class Check { } } - private int run(ReadOnlyFileStore store, JournalReader journal) { - Set<String> checkpoints = requestedCheckpoints; - - if (requestedCheckpoints.contains("all")) { - checkpoints = Sets.newLinkedHashSet(SegmentNodeStoreBuilders.builder(store).build().checkpoints()); - } - - ConsistencyCheckResult result = newConsistencyChecker().checkConsistency( - store, - journal, - checkHead, - checkpoints, - filterPaths, - checkBinaries, - revisionsCount, - failFast - ); - - print("\nSearched through {0} revisions and {1} checkpoints", result.getCheckedRevisionsCount(), checkpoints.size()); - - if (isGoodRevisionFound(result)) { - if (checkHead) { - print("\nHead"); - for (Entry<String, Revision> e : result.getHeadRevisions().entrySet()) { - printRevision(0, e.getKey(), e.getValue()); - } - } - if (checkpoints.size() > 0) { - print("\nCheckpoints"); - for (String checkpoint : result.getCheckpointRevisions().keySet()) { - print("- {0}", checkpoint); - for (Entry<String, Revision> e : result.getCheckpointRevisions().get(checkpoint).entrySet()) { - printRevision(2, e.getKey(), e.getValue()); - } - - } - } - print("\nOverall"); - printOverallRevision(result.getOverallRevision()); - return 0; - } else { - print("No good revision found"); - return 1; - } - } - - private boolean isGoodRevisionFound(ConsistencyCheckResult result) { - return failFast ? hasAllRevision(result) : hasAnyRevision(result); - } - - private ConsistencyChecker newConsistencyChecker() { - return new ConsistencyChecker() { - - @Override - protected void onCheckRevision(String revision) { - print("\nChecking revision {0}", revision); - } - - @Override - protected void onCheckHead() { - headNodeCount = 0; - headPropertyCount = 0; - print("\nChecking head\n"); - } - - @Override - protected void onCheckChekpoints() { - print("\nChecking checkpoints"); - } - - @Override - protected void onCheckCheckpoint(String checkpoint) { - print("\nChecking checkpoint {0}", checkpoint); - } - - @Override - protected void onCheckpointNotFoundInRevision(String checkpoint) { - printError("Checkpoint {0} not found in this revision!", checkpoint); - } - - @Override - protected void onCheckRevisionError(String revision, Exception e) { - printError("Skipping invalid record id {0}: {1}", revision, e); - } - - @Override - protected void onConsistentPath(String path) { - print("Path {0} is consistent", path); - } - - @Override - protected void onPathNotFound(String path) { - printError("Path {0} not found", path); - } - - @Override - protected void onCheckTree(String path, boolean head) { - currentNodeCount = 0; - currentPropertyCount = 0; - print("Checking {0}", path); - } - - @Override - protected void onCheckTreeEnd(boolean head) { - if (head) { - headNodeCount += currentNodeCount; - headPropertyCount += currentPropertyCount; - } - - print("Checked {0} nodes and {1} properties", currentNodeCount, currentPropertyCount); - } - - @Override - protected void onCheckNode(String path) { - debug("Traversing {0}", path); - currentNodeCount++; - } - - @Override - protected void onCheckProperty() { - currentPropertyCount++; - } - - @Override - protected void onCheckPropertyEnd(String path, PropertyState property) { - debug("Checked {0}/{1}", path, property); - } - - @Override - protected void onCheckNodeError(String path, Exception e) { - printError("Error while traversing {0}: {1}", path, e); - } - - @Override - protected void onCheckTreeError(String path, Exception e) { - printError("Error while traversing {0}: {1}", path, e.getMessage()); - } - - }; - } - private void print(String format, Object... arguments) { out.println(MessageFormat.format(format, arguments)); } - - private void printError(String format, Object... args) { - err.println(MessageFormat.format(format, args)); - } - - private void debug(String format, Object... arg) { - if (debug()) { - print(format, arg); - } - } - - private boolean debug() { - // Avoid calling System.currentTimeMillis(), which is slow on some systems. - if (debugInterval == Long.MAX_VALUE) { - return false; - } - - if (debugInterval == 0) { - return true; - } - - long t = System.currentTimeMillis(); - if ((t - this.lastDebugEvent) / 1000 > debugInterval) { - this.lastDebugEvent = t; - return true; - } else { - return false; - } - } - - private static boolean hasAnyRevision(ConsistencyCheckResult result) { - return hasAnyHeadRevision(result) || hasAnyCheckpointRevision(result); - } - - private static boolean hasAllRevision(ConsistencyCheckResult result) { - return hasAnyHeadRevision(result) && hasAllCheckpointRevision(result); - } - - private static boolean hasAnyHeadRevision(ConsistencyCheckResult result) { - return result.getHeadRevisions() - .values() - .stream() - .anyMatch(Objects::nonNull); - } - - private static boolean hasAnyCheckpointRevision(ConsistencyCheckResult result) { - return result.getCheckpointRevisions() - .values() - .stream() - .flatMap(m -> m.values().stream()) - .anyMatch(Objects::nonNull); - } - - private static boolean hasAllCheckpointRevision(ConsistencyCheckResult result) { - return result.getCheckpointRevisions() - .values() - .stream() - .flatMap(m -> m.values().stream()) - .allMatch(Objects::nonNull); - } - - private void printRevision(int indent, String path, Revision revision) { - Optional<Revision> r = Optional.ofNullable(revision); - print( - "{0}Latest good revision for path {1} is {2} from {3}", - Strings.repeat(" ", indent), - path, - r.map(Revision::getRevision).orElse("none"), - r.map(Revision::getTimestamp).map(Check::timestampToString).orElse("unknown time") - ); - } - - private void printOverallRevision(Revision revision) { - Optional<Revision> r = Optional.ofNullable(revision); - print( - "Latest good revision for paths and checkpoints checked is {0} from {1}", - r.map(Revision::getRevision).orElse("none"), - r.map(Revision::getTimestamp).map(Check::timestampToString).orElse("unknown time") - ); - } - - private static String timestampToString(long timestamp) { - return getDateTimeInstance().format(new Date(timestamp)); - } - } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/check/CheckHelper.java similarity index 59% copy from oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java copy to oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/check/CheckHelper.java index cafb684aa1..0bbb1b1c60 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/check/CheckHelper.java @@ -14,46 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.jackrabbit.oak.segment.tool; - -import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument; -import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; -import static java.text.DateFormat.getDateTimeInstance; -import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; -import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; - -import java.io.File; -import java.io.PrintWriter; -import java.text.MessageFormat; -import java.util.Date; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; +package org.apache.jackrabbit.oak.segment.tool.check; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; -import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; -import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile; -import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker; -import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.ConsistencyCheckResult; -import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.Revision; -import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter; +import org.apache.jackrabbit.oak.segment.tool.Check; -/** - * Perform a consistency check on an existing segment store. - */ -public class Check { +import java.io.PrintWriter; +import java.text.MessageFormat; +import java.util.*; +import static java.text.DateFormat.getDateTimeInstance; +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument; + +public class CheckHelper { /** - * Create a builder for the {@link Check} command. + * Create a builder for the {@link CheckHelper}. * * @return an instance of {@link Builder}. */ @@ -66,12 +47,6 @@ public class Check { */ public static class Builder { - private File path; - - private boolean mmap; - - private File journal; - private long debugInterval = Long.MAX_VALUE; private boolean checkBinaries; @@ -84,10 +59,6 @@ public class Check { private Set<String> filterPaths; - private boolean ioStatistics; - - private RepositoryStatistics repoStatistics; - private PrintWriter outWriter; private PrintWriter errWriter; @@ -98,44 +69,6 @@ public class Check { // Prevent external instantiation. } - /** - * The path to an existing segment store. This parameter is required. - * - * @param path the path to an existing segment store. - * @return this builder. - */ - public Builder withPath(File path) { - this.path = checkNotNull(path); - return this; - } - - /** - * Whether to use memory mapped access or file access. - * - * @param mmap {@code true} for memory mapped access, {@code false} for - * file access {@code null} to determine the access mode - * from the system architecture: memory mapped on 64 bit - * systems, file access on 32 bit systems. - * @return this builder. - */ - public Builder withMmap(boolean mmap) { - this.mmap = mmap; - return this; - } - - /** - * The path to the journal of the segment store. This parameter is - * optional. If not provided, the journal in the default location is - * used. - * - * @param journal the path to the journal of the segment store. - * @return this builder. - */ - public Builder withJournal(File journal) { - this.journal = checkNotNull(journal); - return this; - } - /** * Number of seconds between successive debug print statements. This * parameter is not required and defaults to an arbitrary large number. @@ -211,32 +144,6 @@ public class Check { return this; } - /** - * Instruct the command to print statistics about I/O operations - * performed during the check. This parameter is not required and - * defaults to {@code false}. - * - * @param ioStatistics {@code true} if I/O statistics should be - * provided, {@code false} otherwise. - * @return this builder. - */ - public Builder withIOStatistics(boolean ioStatistics) { - this.ioStatistics = ioStatistics; - return this; - } - - /** - * Attach a repository statistics instance to collect info on nodes - * and properties checked on head. - * - * @param repoStatistics instance to collect statistics - * @return this builder. - */ - public Builder withRepositoryStatistics(RepositoryStatistics repoStatistics) { - this.repoStatistics = repoStatistics; - return this; - } - /** * The text output stream writer used to print normal output. * @param outWriter the output writer. @@ -265,55 +172,16 @@ public class Check { } /** - * Create an executable version of the {@link Check} command. + * Create an executable version of the {@link CheckHelper} command. * - * @return an instance of {@link Runnable}. + * @return an instance of {@link CheckHelper}. */ - public Check build() { - checkNotNull(path); - return new Check(this); - } - - } - - private static class StatisticsIOMonitor extends IOMonitorAdapter { - - AtomicLong ops = new AtomicLong(0); - - AtomicLong bytes = new AtomicLong(0); - - AtomicLong time = new AtomicLong(0); - - @Override - public void afterSegmentRead(File file, long msb, long lsb, int length, long elapsed) { - ops.incrementAndGet(); - bytes.addAndGet(length); - time.addAndGet(elapsed); - } - - } - - public static class RepositoryStatistics { - int headNodeCount; - int headPropertyCount; - - public int getHeadNodeCount() { - return headNodeCount; + public CheckHelper build() { + return new CheckHelper(this); } - public int getHeadPropertyCount() { - return headPropertyCount; - } } - private final File path; - - private final boolean mmap; - - private final File journal; - - private final long debugInterval; - private final boolean checkBinaries; private final boolean checkHead; @@ -324,16 +192,14 @@ public class Check { private final Set<String> filterPaths; - private final boolean ioStatistics; + private final boolean failFast; - private RepositoryStatistics repoStatistics; + private final long debugInterval; private final PrintWriter out; private final PrintWriter err; - private final boolean failFast; - private int currentNodeCount; private int currentPropertyCount; @@ -344,85 +210,34 @@ public class Check { private long lastDebugEvent; - private Check(Builder builder) { - this.path = builder.path; - this.mmap = builder.mmap; + private CheckHelper(Builder builder) { this.debugInterval = builder.debugInterval; this.checkHead = builder.checkHead; this.checkBinaries = builder.checkBinaries; this.requestedCheckpoints = builder.checkpoints; this.filterPaths = builder.filterPaths; - this.ioStatistics = builder.ioStatistics; - this.repoStatistics = builder.repoStatistics; this.out = builder.outWriter; this.err = builder.errWriter; - this.journal = journalPath(builder.path, builder.journal); - this.revisionsCount = revisionsToCheckCount(builder.revisionsCount); this.failFast = builder.failFast; + this.revisionsCount = builder.revisionsCount; } - private static File journalPath(File segmentStore, File journal) { - if (journal == null) { - return new File(segmentStore, "journal.log"); - } - return journal; - } - - private static Integer revisionsToCheckCount(Integer revisionsCount) { - return revisionsCount != null ? revisionsCount : Integer.MAX_VALUE; - } - - public int run() { - StatisticsIOMonitor ioMonitor = new StatisticsIOMonitor(); - - FileStoreBuilder builder = fileStoreBuilder(path) - .withMemoryMapping(mmap) - .withCustomPersistence(new TarPersistence(this.path, this.journal)); - - if (ioStatistics) { - builder.withIOMonitor(ioMonitor); - } - - try ( - ReadOnlyFileStore store = builder.buildReadOnly(); - JournalReader journal = new JournalReader(new LocalJournalFile(this.journal)) - ) { - int result = run(store, journal); - - if (ioStatistics) { - print("[I/O] Segment read: Number of operations: {0}", ioMonitor.ops.get()); - print("[I/O] Segment read: Total size: {0} ({1} bytes)", humanReadableByteCount(ioMonitor.bytes.get()), ioMonitor.bytes.get()); - print("[I/O] Segment read: Total time: {0} ns", ioMonitor.time.get()); - } - - if (repoStatistics != null) { - repoStatistics.headNodeCount = headNodeCount; - repoStatistics.headPropertyCount = headPropertyCount; - } - - return result; - } catch (Exception e) { - e.printStackTrace(err); - return 1; - } - } - - private int run(ReadOnlyFileStore store, JournalReader journal) { + public int run(ReadOnlyFileStore store, JournalReader journal) { Set<String> checkpoints = requestedCheckpoints; if (requestedCheckpoints.contains("all")) { checkpoints = Sets.newLinkedHashSet(SegmentNodeStoreBuilders.builder(store).build().checkpoints()); } - ConsistencyCheckResult result = newConsistencyChecker().checkConsistency( - store, - journal, - checkHead, - checkpoints, - filterPaths, - checkBinaries, - revisionsCount, - failFast + ConsistencyChecker.ConsistencyCheckResult result = newConsistencyChecker().checkConsistency( + store, + journal, + checkHead, + checkpoints, + filterPaths, + checkBinaries, + revisionsCount, + failFast ); print("\nSearched through {0} revisions and {1} checkpoints", result.getCheckedRevisionsCount(), checkpoints.size()); @@ -430,15 +245,15 @@ public class Check { if (isGoodRevisionFound(result)) { if (checkHead) { print("\nHead"); - for (Entry<String, Revision> e : result.getHeadRevisions().entrySet()) { + for (Map.Entry<String, ConsistencyChecker.Revision> e : result.getHeadRevisions().entrySet()) { printRevision(0, e.getKey(), e.getValue()); } } - if (checkpoints.size() > 0) { + if (!checkpoints.isEmpty()) { print("\nCheckpoints"); for (String checkpoint : result.getCheckpointRevisions().keySet()) { print("- {0}", checkpoint); - for (Entry<String, Revision> e : result.getCheckpointRevisions().get(checkpoint).entrySet()) { + for (Map.Entry<String, ConsistencyChecker.Revision> e : result.getCheckpointRevisions().get(checkpoint).entrySet()) { printRevision(2, e.getKey(), e.getValue()); } @@ -453,7 +268,15 @@ public class Check { } } - private boolean isGoodRevisionFound(ConsistencyCheckResult result) { + public int getHeadNodeCount() { + return headNodeCount; + } + + public int getHeadPropertyCount() { + return headPropertyCount; + } + + private boolean isGoodRevisionFound(ConsistencyChecker.ConsistencyCheckResult result) { return failFast ? hasAllRevision(result) : hasAnyRevision(result); } @@ -548,63 +371,30 @@ public class Check { }; } - private void print(String format, Object... arguments) { - out.println(MessageFormat.format(format, arguments)); - } - - private void printError(String format, Object... args) { - err.println(MessageFormat.format(format, args)); - } - - private void debug(String format, Object... arg) { - if (debug()) { - print(format, arg); - } - } - - private boolean debug() { - // Avoid calling System.currentTimeMillis(), which is slow on some systems. - if (debugInterval == Long.MAX_VALUE) { - return false; - } - - if (debugInterval == 0) { - return true; - } - - long t = System.currentTimeMillis(); - if ((t - this.lastDebugEvent) / 1000 > debugInterval) { - this.lastDebugEvent = t; - return true; - } else { - return false; - } - } - - private static boolean hasAnyRevision(ConsistencyCheckResult result) { + private static boolean hasAnyRevision(ConsistencyChecker.ConsistencyCheckResult result) { return hasAnyHeadRevision(result) || hasAnyCheckpointRevision(result); } - private static boolean hasAllRevision(ConsistencyCheckResult result) { + private static boolean hasAllRevision(ConsistencyChecker.ConsistencyCheckResult result) { return hasAnyHeadRevision(result) && hasAllCheckpointRevision(result); } - private static boolean hasAnyHeadRevision(ConsistencyCheckResult result) { + private static boolean hasAnyHeadRevision(ConsistencyChecker.ConsistencyCheckResult result) { return result.getHeadRevisions() - .values() - .stream() - .anyMatch(Objects::nonNull); + .values() + .stream() + .anyMatch(Objects::nonNull); } - private static boolean hasAnyCheckpointRevision(ConsistencyCheckResult result) { + private static boolean hasAnyCheckpointRevision(ConsistencyChecker.ConsistencyCheckResult result) { return result.getCheckpointRevisions() - .values() - .stream() - .flatMap(m -> m.values().stream()) - .anyMatch(Objects::nonNull); + .values() + .stream() + .flatMap(m -> m.values().stream()) + .anyMatch(Objects::nonNull); } - private static boolean hasAllCheckpointRevision(ConsistencyCheckResult result) { + private static boolean hasAllCheckpointRevision(ConsistencyChecker.ConsistencyCheckResult result) { return result.getCheckpointRevisions() .values() .stream() @@ -612,23 +402,23 @@ public class Check { .allMatch(Objects::nonNull); } - private void printRevision(int indent, String path, Revision revision) { - Optional<Revision> r = Optional.ofNullable(revision); + private void printRevision(int indent, String path, ConsistencyChecker.Revision revision) { + Optional<ConsistencyChecker.Revision> r = Optional.ofNullable(revision); print( - "{0}Latest good revision for path {1} is {2} from {3}", - Strings.repeat(" ", indent), - path, - r.map(Revision::getRevision).orElse("none"), - r.map(Revision::getTimestamp).map(Check::timestampToString).orElse("unknown time") + "{0}Latest good revision for path {1} is {2} from {3}", + Strings.repeat(" ", indent), + path, + r.map(ConsistencyChecker.Revision::getRevision).orElse("none"), + r.map(ConsistencyChecker.Revision::getTimestamp).map(CheckHelper::timestampToString).orElse("unknown time") ); } - private void printOverallRevision(Revision revision) { - Optional<Revision> r = Optional.ofNullable(revision); + private void printOverallRevision(ConsistencyChecker.Revision revision) { + Optional<ConsistencyChecker.Revision> r = Optional.ofNullable(revision); print( - "Latest good revision for paths and checkpoints checked is {0} from {1}", - r.map(Revision::getRevision).orElse("none"), - r.map(Revision::getTimestamp).map(Check::timestampToString).orElse("unknown time") + "Latest good revision for paths and checkpoints checked is {0} from {1}", + r.map(ConsistencyChecker.Revision::getRevision).orElse("none"), + r.map(ConsistencyChecker.Revision::getTimestamp).map(CheckHelper::timestampToString).orElse("unknown time") ); } @@ -636,4 +426,36 @@ public class Check { return getDateTimeInstance().format(new Date(timestamp)); } + private void printError(String format, Object... args) { + err.println(MessageFormat.format(format, args)); + } + + private void print(String format, Object... arguments) { + out.println(MessageFormat.format(format, arguments)); + } + + private void debug(String format, Object... arg) { + if (debug()) { + print(format, arg); + } + } + + private boolean debug() { + // Avoid calling System.currentTimeMillis(), which is slow on some systems. + if (debugInterval == Long.MAX_VALUE) { + return false; + } + + if (debugInterval == 0) { + return true; + } + + long t = System.currentTimeMillis(); + if ((t - this.lastDebugEvent) / 1000 > debugInterval) { + this.lastDebugEvent = t; + return true; + } else { + return false; + } + } }