Author: frm Date: Tue Apr 24 15:19:15 2018 New Revision: 1830010 URL: http://svn.apache.org/viewvc?rev=1830010&view=rev Log: OAK-7445 - Introduce a cleanup-first compaction strategy
Introduce a new garbage collection strategy that runs a pre-cleanup phase before the compaction phase. The new strategy doesn't replace the default garbage collection strategy, but lives alongside it hidden by a feature flag. Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java (with props) jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java (with props) jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java (with props) jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java (with props) Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java?rev=1830010&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018 @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.file; + +import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.ESTIMATION; +import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.IDLE; + +import java.io.IOException; +import java.util.List; + +import com.google.common.base.Predicate; +import org.apache.jackrabbit.oak.segment.Revisions; +import org.apache.jackrabbit.oak.segment.SegmentCache; +import org.apache.jackrabbit.oak.segment.SegmentReader; +import org.apache.jackrabbit.oak.segment.SegmentTracker; +import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; +import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; +import org.apache.jackrabbit.oak.segment.file.tar.TarFiles; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; + +abstract class AbstractGarbageCollectionStrategy implements GarbageCollectionStrategy { + + abstract EstimationStrategy getFullEstimationStrategy(); + + abstract EstimationStrategy getTailEstimationStrategy(); + + abstract CompactionStrategy getFullCompactionStrategy(); + + abstract CompactionStrategy getTailCompactionStrategy(); + + abstract CleanupStrategy getCleanupStrategy(); + + @Override + public void collectGarbage(Context context) throws IOException { + switch (context.getGCOptions().getGCType()) { + case FULL: + collectFullGarbage(context); + break; + case TAIL: + collectTailGarbage(context); + break; + default: + throw new IllegalStateException("Invalid GC type"); + } + } + + @Override + public void collectFullGarbage(Context context) throws IOException { + run(context, getFullEstimationStrategy(), getFullCompactionStrategy()); + } + + @Override + public void collectTailGarbage(Context context) throws IOException { + run(context, getTailEstimationStrategy(), getTailCompactionStrategy()); + } + + @Override + public CompactionResult compactFull(Context context) throws IOException { + return getFullCompactionStrategy().compact(newCompactionStrategyContext(context)); + } + + @Override + public CompactionResult compactTail(Context context) throws IOException { + return getTailCompactionStrategy().compact(newCompactionStrategyContext(context)); + } + + @Override + public List<String> cleanup(Context context) throws IOException { + return cleanup(context, CompactionResult.skipped( + context.getLastCompactionType(), + getGcGeneration(context), + context.getGCOptions(), + context.getRevisions().getHead(), + context.getGCCount() + )); + } + + void run(Context context, EstimationStrategy estimationStrategy, CompactionStrategy compactionStrategy) throws IOException { + try { + context.getGCListener().info("started"); + + long dt = System.currentTimeMillis() - context.getLastSuccessfulGC(); + + if (dt < context.getGCBackOff()) { + context.getGCListener().skipped("skipping garbage collection as it already ran less than {} hours ago ({} s).", context.getGCBackOff() / 3600000, dt / 1000); + return; + } + + boolean sufficientEstimatedGain = true; + if (context.getGCOptions().isEstimationDisabled()) { + context.getGCListener().info("estimation skipped because it was explicitly disabled"); + } else if (context.getGCOptions().isPaused()) { + context.getGCListener().info("estimation skipped because compaction is paused"); + } else { + context.getGCListener().info("estimation started"); + context.getGCListener().updateStatus(ESTIMATION.message()); + + PrintableStopwatch watch = PrintableStopwatch.createStarted(); + EstimationResult estimation = estimationStrategy.estimate(newEstimationStrategyContext(context)); + sufficientEstimatedGain = estimation.isGcNeeded(); + String gcLog = estimation.getGcLog(); + if (sufficientEstimatedGain) { + context.getGCListener().info("estimation completed in {}. {}", watch, gcLog); + } else { + context.getGCListener().skipped("estimation completed in {}. {}", watch, gcLog); + } + } + + if (sufficientEstimatedGain) { + try (GCMemoryBarrier ignored = new GCMemoryBarrier(context.getSufficientMemory(), context.getGCListener(), context.getGCOptions())) { + if (context.getGCOptions().isPaused()) { + context.getGCListener().skipped("compaction paused"); + } else if (!context.getSufficientMemory().get()) { + context.getGCListener().skipped("compaction skipped. Not enough memory"); + } else { + CompactionResult compactionResult = compactionStrategy.compact(newCompactionStrategyContext(context)); + if (compactionResult.isSuccess()) { + context.getSuccessfulGarbageCollectionListener().onSuccessfulGarbageCollection(); + } else { + context.getGCListener().info("cleaning up after failed compaction"); + } + context.getFileReaper().add(cleanup(context, compactionResult)); + } + } + } + } finally { + context.getCompactionMonitor().finished(); + context.getGCListener().updateStatus(IDLE.message()); + } + } + + private GCGeneration getGcGeneration(Context context) { + return context.getRevisions().getHead().getSegmentId().getGcGeneration(); + } + + private List<String> cleanup(Context context, CompactionResult compactionResult) throws IOException { + return getCleanupStrategy().cleanup(newCleanupStrategyContext(context, compactionResult)); + } + + private EstimationStrategy.Context newEstimationStrategyContext(Context context) { + return new EstimationStrategy.Context() { + + @Override + public long getSizeDelta() { + return context.getGCOptions().getGcSizeDeltaEstimation(); + } + + @Override + public long getCurrentSize() { + return context.getTarFiles().size(); + } + + @Override + public GCJournal getGCJournal() { + return context.getGCJournal(); + } + + }; + } + + private static CompactionStrategy.Context newCompactionStrategyContext(Context context) { + return new CompactionStrategy.Context() { + + @Override + public SegmentTracker getSegmentTracker() { + return context.getSegmentTracker(); + } + + @Override + public GCListener getGCListener() { + return context.getGCListener(); + } + + @Override + public GCJournal getGCJournal() { + return context.getGCJournal(); + } + + @Override + public SegmentGCOptions getGCOptions() { + return context.getGCOptions(); + } + + @Override + public GCNodeWriteMonitor getCompactionMonitor() { + return context.getCompactionMonitor(); + } + + @Override + public SegmentReader getSegmentReader() { + return context.getSegmentReader(); + } + + @Override + public SegmentWriterFactory getSegmentWriterFactory() { + return context.getSegmentWriterFactory(); + } + + @Override + public Revisions getRevisions() { + return context.getRevisions(); + } + + @Override + public TarFiles getTarFiles() { + return context.getTarFiles(); + } + + @Override + public BlobStore getBlobStore() { + return context.getBlobStore(); + } + + @Override + public CancelCompactionSupplier getCanceller() { + return context.getCanceller(); + } + + @Override + public int getGCCount() { + return context.getGCCount(); + } + + @Override + public SuccessfulCompactionListener getSuccessfulCompactionListener() { + return context.getSuccessfulCompactionListener(); + } + + @Override + public Flusher getFlusher() { + return context.getFlusher(); + } + + }; + } + + private CleanupStrategy.Context newCleanupStrategyContext(Context context, CompactionResult compactionResult) { + return new CleanupStrategy.Context() { + + @Override + public GCListener getGCListener() { + return context.getGCListener(); + } + + @Override + public SegmentCache getSegmentCache() { + return context.getSegmentCache(); + } + + @Override + public SegmentTracker getSegmentTracker() { + return context.getSegmentTracker(); + } + + @Override + public FileStoreStats getFileStoreStats() { + return context.getFileStoreStats(); + } + + @Override + public GCNodeWriteMonitor getCompactionMonitor() { + return context.getCompactionMonitor(); + } + + @Override + public GCJournal getGCJournal() { + return context.getGCJournal(); + } + + @Override + public Predicate<GCGeneration> getReclaimer() { + return compactionResult.reclaimer(); + } + + @Override + public TarFiles getTarFiles() { + return context.getTarFiles(); + } + + @Override + public Revisions getRevisions() { + return context.getRevisions(); + } + + @Override + public String getCompactedRootId() { + return compactionResult.getCompactedRootId().toString10(); + } + + @Override + public String getSegmentEvictionReason() { + return compactionResult.gcInfo(); + } + + }; + } + +} Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java?rev=1830010&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java Tue Apr 24 15:19:15 2018 @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.file; + +import static com.google.common.collect.Sets.newHashSet; +import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId; +import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.CLEANUP; +import static org.apache.jackrabbit.oak.segment.file.PrintableBytes.newPrintableBytes; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import javax.annotation.Nonnull; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import org.apache.jackrabbit.oak.segment.SegmentId; +import org.apache.jackrabbit.oak.segment.file.tar.CleanupContext; +import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; +import org.apache.jackrabbit.oak.segment.file.tar.TarFiles; + +class CleanupFirstCompactionStrategy implements CompactionStrategy { + + private final GarbageCollectionStrategy.Context parentContext; + + private final CompactionStrategy strategy; + + CleanupFirstCompactionStrategy(GarbageCollectionStrategy.Context parentContext, CompactionStrategy strategy) { + this.parentContext = parentContext; + this.strategy = strategy; + } + + @Override + public CompactionResult compact(Context context) throws IOException { + + // This is a slightly modified version of the default cleanup phase when + // invoked with a successful compaction result. There are some important + // differences deriving from the fact that we are assuming that the compaction + // for `newGeneration` is going to succeed. + + // First, we don't have a RecordId for the compacted state, because it didn't + // happen yet. This shouldn't matter, because we are not going to advance the + // `gcJournal` to the `RecordId` of the compacted state. + + // Second, we are using a custom reclaimer that is similar to the one returned + // by `newOldReclaimer`, but that also takes in consideration that + // `newGeneration` has not been committed yet, and the most recent transient + // state shouldn't be removed. + + // Third, we don't clear the segment cache. There might be transient segments + // in there, and we don't want those segments to be removed. + + // Fourth, the following code assumes the number of retained generations fixed + // to two, which is also the default value for the Segment Store. A complete + // solution should be flexible enough to accommodate other values for the + // number of retained generations. + + PrintableStopwatch watch = PrintableStopwatch.createStarted(); + + Predicate<GCGeneration> reclaimer; + + GCGeneration currentGeneration = context.getRevisions().getHead().getSegmentId().getGcGeneration(); + + switch (context.getGCOptions().getGCType()) { + case FULL: + reclaimer = generation -> { + if (generation == null) { + return false; + } + if (generation.getFullGeneration() < currentGeneration.getFullGeneration()) { + return true; + } + if (generation.getFullGeneration() > currentGeneration.getFullGeneration()) { + return true; + } + return generation.getGeneration() < currentGeneration.getGeneration() && !generation.isCompacted(); + }; + break; + case TAIL: + reclaimer = generation -> { + if (generation == null) { + return false; + } + if (generation.getFullGeneration() < currentGeneration.getFullGeneration() - 1) { + return true; + } + if (generation.getFullGeneration() == currentGeneration.getFullGeneration() - 1) { + return !generation.isCompacted(); + } + if (generation.getFullGeneration() > currentGeneration.getFullGeneration()) { + return true; + } + return generation.getGeneration() < currentGeneration.getGeneration() && !generation.isCompacted(); + }; + break; + default: + throw new IllegalArgumentException("invalid garbage collection type"); + } + + context.getGCListener().info("pre-compaction cleanup started"); + context.getGCListener().updateStatus(CLEANUP.message()); + + // Suggest to the JVM that now would be a good time to clear stale weak + // references in the SegmentTracker + + System.gc(); + + TarFiles.CleanupResult cleanupResult = context.getTarFiles().cleanup(newCleanupContext(context, reclaimer)); + + if (cleanupResult.isInterrupted()) { + context.getGCListener().info("cleanup interrupted"); + } + + context.getSegmentTracker().clearSegmentIdTables(cleanupResult.getReclaimedSegmentIds(), "[pre-compaction cleanup]"); + context.getGCListener().info("cleanup marking files for deletion: {}", toFileNames(cleanupResult.getRemovableFiles())); + + long finalSize = context.getTarFiles().size(); + long reclaimedSize = cleanupResult.getReclaimedSize(); + parentContext.getFileStoreStats().reclaimed(reclaimedSize); + context.getGCListener().cleaned(reclaimedSize, finalSize); + context.getGCListener().info( + "pre-compaction cleanup completed in {}. Post cleanup size is {} and space reclaimed {}.", + watch, + newPrintableBytes(finalSize), + newPrintableBytes(reclaimedSize)); + parentContext.getFileReaper().add(cleanupResult.getRemovableFiles()); + + // Invoke the wrapped compaction phase + + return strategy.compact(context); + } + + private static CleanupContext newCleanupContext(Context context, Predicate<GCGeneration> old) { + return new CleanupContext() { + + private boolean isUnreferencedBulkSegment(UUID id, boolean referenced) { + return !isDataSegmentId(id.getLeastSignificantBits()) && !referenced; + } + + private boolean isOldDataSegment(UUID id, GCGeneration generation) { + return isDataSegmentId(id.getLeastSignificantBits()) && old.apply(generation); + } + + @Override + public Collection<UUID> initialReferences() { + Set<UUID> references = newHashSet(); + for (SegmentId id : context.getSegmentTracker().getReferencedSegmentIds()) { + if (id.isBulkSegmentId()) { + references.add(id.asUUID()); + } + } + return references; + } + + @Override + public boolean shouldReclaim(UUID id, GCGeneration generation, boolean referenced) { + return isUnreferencedBulkSegment(id, referenced) || isOldDataSegment(id, generation); + } + + @Override + public boolean shouldFollow(UUID from, UUID to) { + return !isDataSegmentId(to.getLeastSignificantBits()); + } + + }; + } + + private static String toFileNames(@Nonnull List<String> files) { + if (files.isEmpty()) { + return "none"; + } else { + return Joiner.on(",").join(files); + } + } + + private static GCGeneration getGcGeneration(Context context) { + return context.getRevisions().getHead().getSegmentId().getGcGeneration(); + } + + private static long size(Context context) { + return context.getTarFiles().size(); + } + +} Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java?rev=1830010&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018 @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.file; + +import java.io.IOException; + +class CleanupFirstGarbageCollectionStrategy extends AbstractGarbageCollectionStrategy { + + @Override + EstimationStrategy getFullEstimationStrategy() { + return new FullSizeDeltaEstimationStrategy(); + } + + @Override + EstimationStrategy getTailEstimationStrategy() { + return new TailSizeDeltaEstimationStrategy(); + } + + @Override + CompactionStrategy getFullCompactionStrategy() { + return new FullCompactionStrategy(); + } + + @Override + CompactionStrategy getTailCompactionStrategy() { + return new FallbackCompactionStrategy(new TailCompactionStrategy(), new FullCompactionStrategy()); + } + + @Override + CleanupStrategy getCleanupStrategy() { + return new DefaultCleanupStrategy(); + } + + @Override + void run(Context context, EstimationStrategy estimationStrategy, CompactionStrategy compactionStrategy) throws IOException { + super.run(context, estimationStrategy, new CleanupFirstCompactionStrategy(context, compactionStrategy)); + } + +} Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java?rev=1830010&r1=1830009&r2=1830010&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java Tue Apr 24 15:19:15 2018 @@ -19,6 +19,8 @@ package org.apache.jackrabbit.oak.segment.file; +import java.io.IOException; + import org.apache.jackrabbit.oak.segment.Revisions; import org.apache.jackrabbit.oak.segment.SegmentReader; import org.apache.jackrabbit.oak.segment.SegmentTracker; @@ -60,6 +62,6 @@ interface CompactionStrategy { } - CompactionResult compact(Context context); + CompactionResult compact(Context context) throws IOException; } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java?rev=1830010&r1=1830009&r2=1830010&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018 @@ -19,311 +19,31 @@ package org.apache.jackrabbit.oak.segment.file; -import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.ESTIMATION; -import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.IDLE; - -import java.io.IOException; -import java.util.List; - -import com.google.common.base.Predicate; -import org.apache.jackrabbit.oak.segment.Revisions; -import org.apache.jackrabbit.oak.segment.SegmentCache; -import org.apache.jackrabbit.oak.segment.SegmentReader; -import org.apache.jackrabbit.oak.segment.SegmentTracker; -import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; -import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; -import org.apache.jackrabbit.oak.segment.file.tar.TarFiles; -import org.apache.jackrabbit.oak.spi.blob.BlobStore; - -class DefaultGarbageCollectionStrategy implements GarbageCollectionStrategy { - - private final EstimationStrategy fullEstimationStrategy = new FullSizeDeltaEstimationStrategy(); - - private final EstimationStrategy tailEstimationStrategy = new TailSizeDeltaEstimationStrategy(); - - private final CompactionStrategy fullCompactionStrategy = new FullCompactionStrategy(); - - private final CompactionStrategy tailCompactionStrategy = new FallbackCompactionStrategy(new TailCompactionStrategy(), fullCompactionStrategy); - - private final CleanupStrategy cleanupStrategy = new DefaultCleanupStrategy(); - - private GCGeneration getGcGeneration(Context context) { - return context.getRevisions().getHead().getSegmentId().getGcGeneration(); - } +class DefaultGarbageCollectionStrategy extends AbstractGarbageCollectionStrategy { @Override - public synchronized void collectGarbage(Context context) throws IOException { - switch (context.getGCOptions().getGCType()) { - case FULL: - collectFullGarbage(context); - break; - case TAIL: - collectTailGarbage(context); - break; - default: - throw new IllegalStateException("Invalid GC type"); - } + EstimationStrategy getFullEstimationStrategy() { + return new FullSizeDeltaEstimationStrategy(); } @Override - public synchronized void collectFullGarbage(Context context) throws IOException { - run(context, true, this::compactFull); + EstimationStrategy getTailEstimationStrategy() { + return new TailSizeDeltaEstimationStrategy(); } @Override - public synchronized void collectTailGarbage(Context context) throws IOException { - run(context, false, this::compactTail); - } - - private interface Compactor { - - CompactionResult compact(Context contex) throws IOException; - - } - - private void run(Context context, boolean full, Compactor compact) throws IOException { - try { - context.getGCListener().info("started"); - - long dt = System.currentTimeMillis() - context.getLastSuccessfulGC(); - - if (dt < context.getGCBackOff()) { - context.getGCListener().skipped("skipping garbage collection as it already ran less than {} hours ago ({} s).", context.getGCBackOff() / 3600000, dt / 1000); - return; - } - - boolean sufficientEstimatedGain = true; - if (context.getGCOptions().isEstimationDisabled()) { - context.getGCListener().info("estimation skipped because it was explicitly disabled"); - } else if (context.getGCOptions().isPaused()) { - context.getGCListener().info("estimation skipped because compaction is paused"); - } else { - context.getGCListener().info("estimation started"); - context.getGCListener().updateStatus(ESTIMATION.message()); - - PrintableStopwatch watch = PrintableStopwatch.createStarted(); - EstimationResult estimation = estimateCompactionGain(context, full); - sufficientEstimatedGain = estimation.isGcNeeded(); - String gcLog = estimation.getGcLog(); - if (sufficientEstimatedGain) { - context.getGCListener().info("estimation completed in {}. {}", watch, gcLog); - } else { - context.getGCListener().skipped("estimation completed in {}. {}", watch, gcLog); - } - } - - if (sufficientEstimatedGain) { - try (GCMemoryBarrier ignored = new GCMemoryBarrier(context.getSufficientMemory(), context.getGCListener(), context.getGCOptions())) { - if (context.getGCOptions().isPaused()) { - context.getGCListener().skipped("compaction paused"); - } else if (!context.getSufficientMemory().get()) { - context.getGCListener().skipped("compaction skipped. Not enough memory"); - } else { - CompactionResult compactionResult = compact.compact(context); - if (compactionResult.isSuccess()) { - context.getSuccessfulGarbageCollectionListener().onSuccessfulGarbageCollection(); - } else { - context.getGCListener().info("cleaning up after failed compaction"); - } - context.getFileReaper().add(cleanup(context, compactionResult)); - } - } - } - } finally { - context.getCompactionMonitor().finished(); - context.getGCListener().updateStatus(IDLE.message()); - } - } - - private static CompactionStrategy.Context newCompactionStrategyContext(Context context) { - return new CompactionStrategy.Context() { - - @Override - public SegmentTracker getSegmentTracker() { - return context.getSegmentTracker(); - } - - @Override - public GCListener getGCListener() { - return context.getGCListener(); - } - - @Override - public GCJournal getGCJournal() { - return context.getGCJournal(); - } - - @Override - public SegmentGCOptions getGCOptions() { - return context.getGCOptions(); - } - - @Override - public GCNodeWriteMonitor getCompactionMonitor() { - return context.getCompactionMonitor(); - } - - @Override - public SegmentReader getSegmentReader() { - return context.getSegmentReader(); - } - - @Override - public SegmentWriterFactory getSegmentWriterFactory() { - return context.getSegmentWriterFactory(); - } - - @Override - public Revisions getRevisions() { - return context.getRevisions(); - } - - @Override - public TarFiles getTarFiles() { - return context.getTarFiles(); - } - - @Override - public BlobStore getBlobStore() { - return context.getBlobStore(); - } - - @Override - public CancelCompactionSupplier getCanceller() { - return context.getCanceller(); - } - - @Override - public int getGCCount() { - return context.getGCCount(); - } - - @Override - public SuccessfulCompactionListener getSuccessfulCompactionListener() { - return context.getSuccessfulCompactionListener(); - } - - @Override - public Flusher getFlusher() { - return context.getFlusher(); - } - - }; - } - - @Override - public synchronized CompactionResult compactFull(Context context) { - return fullCompactionStrategy.compact(newCompactionStrategyContext(context)); + CompactionStrategy getFullCompactionStrategy() { + return new FullCompactionStrategy(); } @Override - public synchronized CompactionResult compactTail(Context context) { - return tailCompactionStrategy.compact(newCompactionStrategyContext(context)); - } - - private EstimationResult estimateCompactionGain(Context context, boolean full) { - EstimationStrategy strategy; - - if (full) { - strategy = fullEstimationStrategy; - } else { - strategy = tailEstimationStrategy; - } - - return estimateCompactionGain(context, strategy); - } - - private EstimationResult estimateCompactionGain(Context context, EstimationStrategy strategy) { - return strategy.estimate(new EstimationStrategy.Context() { - - @Override - public long getSizeDelta() { - return context.getGCOptions().getGcSizeDeltaEstimation(); - } - - @Override - public long getCurrentSize() { - return context.getTarFiles().size(); - } - - @Override - public GCJournal getGCJournal() { - return context.getGCJournal(); - } - - }); + CompactionStrategy getTailCompactionStrategy() { + return new FallbackCompactionStrategy(new TailCompactionStrategy(), new FullCompactionStrategy()); } @Override - public synchronized List<String> cleanup(Context context) throws IOException { - return cleanup(context, CompactionResult.skipped( - context.getLastCompactionType(), - getGcGeneration(context), - context.getGCOptions(), - context.getRevisions().getHead(), - context.getGCCount() - )); - } - - private List<String> cleanup(Context context, CompactionResult compactionResult) throws IOException { - return cleanupStrategy.cleanup(new CleanupStrategy.Context() { - - @Override - public GCListener getGCListener() { - return context.getGCListener(); - } - - @Override - public SegmentCache getSegmentCache() { - return context.getSegmentCache(); - } - - @Override - public SegmentTracker getSegmentTracker() { - return context.getSegmentTracker(); - } - - @Override - public FileStoreStats getFileStoreStats() { - return context.getFileStoreStats(); - } - - @Override - public GCNodeWriteMonitor getCompactionMonitor() { - return context.getCompactionMonitor(); - } - - @Override - public GCJournal getGCJournal() { - return context.getGCJournal(); - } - - @Override - public Predicate<GCGeneration> getReclaimer() { - return compactionResult.reclaimer(); - } - - @Override - public TarFiles getTarFiles() { - return context.getTarFiles(); - } - - @Override - public Revisions getRevisions() { - return context.getRevisions(); - } - - @Override - public String getCompactedRootId() { - return compactionResult.getCompactedRootId().toString10(); - } - - @Override - public String getSegmentEvictionReason() { - return compactionResult.gcInfo(); - } - - }); + CleanupStrategy getCleanupStrategy() { + return new DefaultCleanupStrategy(); } } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java?rev=1830010&r1=1830009&r2=1830010&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java Tue Apr 24 15:19:15 2018 @@ -19,6 +19,8 @@ package org.apache.jackrabbit.oak.segment.file; +import java.io.IOException; + class FallbackCompactionStrategy implements CompactionStrategy { private final CompactionStrategy primary; @@ -31,7 +33,7 @@ class FallbackCompactionStrategy impleme } @Override - public CompactionResult compact(Context context) { + public CompactionResult compact(Context context) throws IOException { CompactionResult result = primary.compact(context); if (result.isNotApplicable()) { Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1830010&r1=1830009&r2=1830010&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Apr 24 15:19:15 2018 @@ -108,7 +108,15 @@ public class FileStore extends AbstractF @Nonnull private final SegmentNotFoundExceptionListener snfeListener; - private final GarbageCollectionStrategy garbageCollectionStrategy = new DefaultGarbageCollectionStrategy(); + private final GarbageCollectionStrategy garbageCollectionStrategy; + + { + if (Boolean.getBoolean("gc.cleanup.first")) { + garbageCollectionStrategy = new SynchronizedGarbageCollectionStrategy(new CleanupFirstGarbageCollectionStrategy()); + } else { + garbageCollectionStrategy = new SynchronizedGarbageCollectionStrategy(new DefaultGarbageCollectionStrategy()); + } + } FileStore(final FileStoreBuilder builder) throws InvalidFileStoreVersionException, IOException { super(builder); Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java?rev=1830010&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018 @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.file; + +import java.io.IOException; +import java.util.List; + +class SynchronizedGarbageCollectionStrategy implements GarbageCollectionStrategy { + + private final Object lock = new Object(); + + private final GarbageCollectionStrategy strategy; + + SynchronizedGarbageCollectionStrategy(GarbageCollectionStrategy strategy) { + this.strategy = strategy; + } + + @Override + public void collectGarbage(Context context) throws IOException { + synchronized (lock) { + strategy.collectGarbage(context); + } + } + + @Override + public void collectFullGarbage(Context context) throws IOException { + synchronized (lock) { + strategy.collectFullGarbage(context); + } + } + + @Override + public void collectTailGarbage(Context context) throws IOException { + synchronized (lock) { + strategy.collectTailGarbage(context); + } + } + + @Override + public CompactionResult compactFull(Context context) throws IOException { + synchronized (lock) { + return strategy.compactFull(context); + } + } + + @Override + public CompactionResult compactTail(Context context) throws IOException { + synchronized (lock) { + return strategy.compactTail(context); + } + } + + @Override + public List<String> cleanup(Context context) throws IOException { + synchronized (lock) { + return strategy.cleanup(context); + } + } + +} Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java ------------------------------------------------------------------------------ svn:eol-style = native