Author: tomekr Date: Tue Feb 13 11:17:42 2018 New Revision: 1824115 URL: http://svn.apache.org/viewvc?rev=1824115&view=rev Log: OAK-6921: Support pluggable segment storage
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarManager.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarReader.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarWriter.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarPersistence.java Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/History.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/JournalEntryTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/JournalReaderTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ManifestCheckerTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ManifestTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java Tue Feb 13 11:17:42 2018 @@ -47,11 +47,13 @@ import org.apache.jackrabbit.oak.segment import org.apache.jackrabbit.oak.segment.SegmentId; import org.apache.jackrabbit.oak.segment.SegmentNodeState; import org.apache.jackrabbit.oak.segment.SegmentNodeStateHelper; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.JournalFile; import org.apache.jackrabbit.oak.segment.SegmentPropertyState; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.file.JournalEntry; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; +import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile; import org.apache.jackrabbit.oak.spi.state.NodeState; class SegmentTarExplorerBackend implements ExplorerBackend { @@ -85,7 +87,7 @@ class SegmentTarExplorerBackend implemen @Override public List<String> readRevisions() { - File journal = new File(path, "journal.log"); + JournalFile journal = new LocalJournalFile(path, "journal.log"); if (!journal.exists()) { return newArrayList(); Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java?rev=1824115&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java Tue Feb 13 11:17:42 2018 @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment; + +import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; +import org.apache.jackrabbit.oak.segment.file.tar.TarEntry; +import org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndex; +import org.apache.jackrabbit.oak.segment.file.tar.binaries.InvalidBinaryReferencesIndexException; +import org.apache.jackrabbit.oak.segment.file.tar.index.Index; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * SegmentArchiveManager provides a low-level access to the segment files (eg. + * stored in the .tar). It allows to perform a few FS-like operations (delete, + * rename, copy, etc.) and also opens the segment archives either for reading + * or reading and writing. + */ +public interface SegmentArchiveManager { + + /** + * List names of the available archives. + * + * @return archive list + */ + @Nonnull + List<String> listArchives() throws IOException; + + /** + * Opens a given archive for reading. + * + * @param archiveName + * @return the archive reader or null if the archive doesn't exist + */ + @Nullable + SegmentArchiveReader open(@Nonnull String archiveName) throws IOException; + + /** + * Creates a new archive. + * + * @param archiveName + * @return the archive writer + */ + @Nonnull + SegmentArchiveWriter create(@Nonnull String archiveName) throws IOException; + + /** + * Deletes the archive if exists. + * + * @param archiveName + * @return true if the archive was removed, false otherwise + */ + boolean delete(@Nonnull String archiveName); + + /** + * Renames the archive. + * + * @param from the existing archive + * @param to new name + * @return true if the archive was renamed, false otherwise + */ + boolean renameTo(@Nonnull String from, @Nonnull String to); + + /** + * Copies the archive with all the segments. + * + * @param from the existing archive + * @param to new name + */ + void copyFile(@Nonnull String from, @Nonnull String to) throws IOException; + + /** + * Check if archive exists. + * + * @param archiveName archive to check + * @return true if archive exists, false otherwise + */ + boolean exists(@Nonnull String archiveName); + + /** + * Finds all the segments included in the archive. + * + * @param archiveName archive to recover + * @param entries results will be put there, in the order of presence in the + * archive + */ + void recoverEntries(@Nonnull String archiveName, @Nonnull LinkedHashMap<UUID, byte[]> entries) throws IOException; + + /** + * Allows to write in the new archive. + */ + interface SegmentArchiveWriter { + + /** + * Write the new segment to the archive. + * + * @param msb + * @param lsb + * @param data + * @param offset + * @param size + * @param generation + * @return the entry representing the new segment. Can be later used for the {@link #readSegment(TarEntry)} method. + */ + @Nonnull + TarEntry writeSegment(long msb, long lsb, @Nonnull byte[] data, int offset, int size, @Nonnull GCGeneration generation) throws IOException; + + /** + * Read the segment. + * + * @param tarEntry + * @return byte buffer containing the segment data or null if segment doesn't exist + */ + @Nullable + ByteBuffer readSegment(@Nonnull TarEntry tarEntry) throws IOException; + + /** + * Write the index data. + * + * @param data + */ + void writeIndex(@Nonnull byte[] data) throws IOException; + + /** + * Write the graph data. + * + * @param data + */ + void writeGraph(@Nonnull byte[] data) throws IOException; + + /** + * Write the binary references data. + * + * @param data + */ + void writeBinaryReferences(@Nonnull byte[] data) throws IOException; + + /** + * Get the current length of the archive. + * + * @return length of the archive, in bytes + */ + long getLength(); + + /** + * Close the archive. + */ + void close() throws IOException; + + /** + * Check if the archive has been created (eg. something has been written). + * + * @return true if the archive has been created, false otherwise + */ + boolean isCreated(); + + /** + * Flush all the data to the storage. + */ + void flush() throws IOException; + + /** + * Get the name of the archive. + * + * @return archive name + */ + @Nonnull + String getName(); + } + + interface SegmentArchiveReader { + + /** + * Read the segment. + * + * @param msb + * @param lsb + * @return byte buffer containing the segment data or null if segment doesn't exist + */ + @Nullable + ByteBuffer readSegment(long msb, long lsb) throws IOException; + + /** + * Returns the index. + * + * @return segment index + */ + @Nonnull + Index getIndex(); + + /** + * Loads and returns the graph. + * + * @return the segment graph or null if the persisted graph doesn't exist. + */ + @Nullable + Map<UUID, List<UUID>> getGraph() throws IOException; + + /** + * Check if the persisted graph exists. + * + * @return true if the graph exists, false otherwise + */ + boolean hasGraph(); + + /** + * Loads and returns the binary references. + * + * @return binary references + */ + @Nonnull + BinaryReferencesIndex getBinaryReferences() throws IOException, InvalidBinaryReferencesIndexException; + + /** + * Get the current length of the archive. + * + * @return length of the archive, in bytes + */ + long length(); + + /** + * Get the name of the archive. + * + * @return archive name + */ + @Nonnull + String getName(); + + /** + * Close the archive. + */ + void close() throws IOException; + + /** + * Returns the size of the entry + * @param size + * @return + */ + int getEntrySize(int size); + } + +} Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java Tue Feb 13 11:17:42 2018 @@ -75,6 +75,12 @@ public class SegmentNodeStoreFactory { public static final String CUSTOM_BLOB_STORE = "customBlobStore"; @Property(boolValue = false, + label = "Custom segment store", + description = "Boolean value indicating that a custom (non-tar) segment store is used" + ) + public static final String CUSTOM_SEGMENT_STORE = "customSegmentStore"; + + @Property(boolValue = false, label = "Register JCR descriptors as OSGi services", description="Should only be done for one factory instance") public static final String REGISTER_DESCRIPTORS = "registerDescriptors"; @@ -89,6 +95,13 @@ public class SegmentNodeStoreFactory { ) private volatile BlobStore blobStore; + @Reference( + cardinality = ReferenceCardinality.OPTIONAL_UNARY, + policy = ReferencePolicy.STATIC, + policyOption = ReferencePolicyOption.GREEDY + ) + private volatile SegmentNodeStorePersistence segmentStore; + @Reference private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP; @@ -100,6 +113,7 @@ public class SegmentNodeStoreFactory { // In secondaryNodeStore mode customBlobStore is always enabled boolean isSecondaryStoreMode = "secondary".equals(role); boolean customBlobStore = Boolean.parseBoolean(property(CUSTOM_BLOB_STORE, context)) || isSecondaryStoreMode; + boolean customSegmentStore = Boolean.parseBoolean(property(CUSTOM_SEGMENT_STORE, context)); boolean registerRepositoryDescriptors = Boolean.parseBoolean(property(REGISTER_DESCRIPTORS, context)); log.info("activate: SegmentNodeStore '" + role + "' starting."); @@ -108,10 +122,15 @@ public class SegmentNodeStoreFactory { return; } + if (segmentStore == null && customSegmentStore) { + log.info("customSegmentStore enabled. SegmentNodeStore will be initialized once the custom segment store becomes available"); + return; + } + if (role != null) { registrations = Closer.create(); OsgiWhiteboard whiteboard = new OsgiWhiteboard(context.getBundleContext()); - final SegmentNodeStore store = SegmentNodeStoreService.registerSegmentStore(context, blobStore, + final SegmentNodeStore store = SegmentNodeStoreService.registerSegmentStore(context, blobStore, segmentStore, statisticsProvider, registrations, whiteboard, role, registerRepositoryDescriptors); if (store != null) { Map<String, Object> props = new HashMap<String, Object>(); Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java?rev=1824115&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java Tue Feb 13 11:17:42 2018 @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment; + +import org.apache.jackrabbit.oak.segment.file.tar.FileStoreMonitor; +import org.apache.jackrabbit.oak.segment.file.tar.IOMonitor; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +public interface SegmentNodeStorePersistence { + + SegmentArchiveManager createArchiveManager(boolean memoryMapping, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) throws IOException; + + boolean segmentFilesExist(); + + JournalFile getJournalFile(); + + GCJournalFile getGCJournalFile() throws IOException; + + ManifestFile getManifestFile() throws IOException; + + RepositoryLock lockRepository() throws IOException; + + interface JournalFile { + + JournalFileReader openJournalReader() throws IOException; + + JournalFileWriter openJournalWriter() throws IOException; + + String getName(); + + boolean exists(); + } + + interface JournalFileReader extends Closeable { + + String readLine() throws IOException; + + } + + interface JournalFileWriter extends Closeable { + + void truncate() throws IOException; + + void writeLine(String line) throws IOException; + + } + + interface GCJournalFile { + + void writeLine(String line) throws IOException; + + List<String> readLines() throws IOException; + + } + + interface ManifestFile { + + boolean exists(); + + Properties load() throws IOException; + + void save(Properties properties) throws IOException; + + } + + interface RepositoryLock { + + void unlock() throws IOException; + + } +} Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java Tue Feb 13 11:17:42 2018 @@ -35,6 +35,7 @@ import static org.apache.jackrabbit.oak. import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.COMPACTION_RETRY_COUNT; import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.COMPACTION_SIZE_DELTA_ESTIMATION; import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.CUSTOM_BLOB_STORE; +import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.CUSTOM_SEGMENT_STORE; import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DEFAULT_BLOB_GC_MAX_AGE; import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DEFAULT_BLOB_SNAPSHOT_INTERVAL; import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.GC_PROGRESS_LOG; @@ -316,6 +317,12 @@ public class SegmentNodeStoreService { ) public static final String CUSTOM_BLOB_STORE = "customBlobStore"; + @Property(boolValue = false, + label = "Custom segment store", + description = "Boolean value indicating that a custom (non-tar) segment store is used" + ) + public static final String CUSTOM_SEGMENT_STORE = "customSegmentStore"; + @Property( label = "Backup directory", description = "Directory (relative to current working directory) for storing repository backups. " + @@ -331,6 +338,13 @@ public class SegmentNodeStoreService { ) private volatile BlobStore blobStore; + @Reference( + cardinality = ReferenceCardinality.OPTIONAL_UNARY, + policy = ReferencePolicy.STATIC, + policyOption = ReferencePolicyOption.GREEDY + ) + private volatile SegmentNodeStorePersistence segmentStore; + @Reference private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP; @@ -374,9 +388,14 @@ public class SegmentNodeStoreService { "store becomes available"); return; } + if (segmentStore == null && configuration.hasCustomSegmentStore()) { + log.info("customSegmentStore enabled. SegmentNodeStore will be initialized once the custom segment " + + "store becomes available"); + return; + } closer = Closer.create(); OsgiWhiteboard whiteboard = new OsgiWhiteboard(context.getBundleContext()); - registerSegmentStore(context, blobStore, statisticsProvider, closer, whiteboard, null, true); + registerSegmentStore(context, blobStore, segmentStore, statisticsProvider, closer, whiteboard, null, true); } /** @@ -388,6 +407,8 @@ public class SegmentNodeStoreService { * @param context An instance of {@link ComponentContext}. * @param blobStore An instance of {@link BlobStore}. It can be * {@code null}. + * @param segmentStore An instance of {@link SegmentNodeStorePersistence}. It can be + * {@code null}. * @param statisticsProvider An instance of {@link StatisticsProvider}. * @param closer An instance of {@link Closer}. It will be used * to track every registered service or @@ -406,6 +427,7 @@ public class SegmentNodeStoreService { static SegmentNodeStore registerSegmentStore( @Nonnull ComponentContext context, @Nullable BlobStore blobStore, + @Nullable SegmentNodeStorePersistence segmentStore, @Nonnull StatisticsProvider statisticsProvider, @Nonnull Closer closer, @Nonnull Whiteboard whiteboard, @@ -469,6 +491,11 @@ public class SegmentNodeStoreService { builder.withBlobStore(blobStore); } + if (configuration.hasCustomSegmentStore() && segmentStore != null) { + log.info("Initializing SegmentNodeStore with custom persistence [{}]", segmentStore); + builder.withCustomPersistence(segmentStore); + } + if (configuration.isStandbyInstance()) { builder.withSnfeListener(IGNORE_SNFE); } @@ -934,6 +961,10 @@ class Configuration { return toBoolean(property(CUSTOM_BLOB_STORE), false); } + boolean hasCustomSegmentStore() { + return toBoolean(property(CUSTOM_SEGMENT_STORE), false); + } + long getBlobGcMaxAge() { return toLong(property(PROP_BLOB_GC_MAX_AGE), DEFAULT_BLOB_GC_MAX_AGE); } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java Tue Feb 13 11:17:42 2018 @@ -24,7 +24,6 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collection; import java.util.HashSet; import java.util.Set; import java.util.UUID; @@ -32,7 +31,6 @@ import java.util.UUID; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; -import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean; import org.apache.jackrabbit.oak.segment.CachingSegmentReader; import org.apache.jackrabbit.oak.segment.RecordType; @@ -45,6 +43,7 @@ import org.apache.jackrabbit.oak.segment import org.apache.jackrabbit.oak.segment.SegmentIdFactory; import org.apache.jackrabbit.oak.segment.SegmentIdProvider; import org.apache.jackrabbit.oak.segment.SegmentNodeState; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.SegmentNotFoundException; import org.apache.jackrabbit.oak.segment.SegmentReader; import org.apache.jackrabbit.oak.segment.SegmentStore; @@ -66,8 +65,6 @@ public abstract class AbstractFileStore private static final Logger log = LoggerFactory.getLogger(AbstractFileStore.class); - private static final String MANIFEST_FILE_NAME = "manifest"; - /** * The minimum supported store version. It is possible for an implementation * to support in a transparent and backwards-compatible way older versions @@ -88,20 +85,15 @@ public abstract class AbstractFileStore */ private static final int MAX_STORE_VERSION = 2; - static ManifestChecker newManifestChecker(File directory, boolean strictVersionCheck) { + static ManifestChecker newManifestChecker(SegmentNodeStorePersistence persistence, boolean strictVersionCheck) throws IOException { return ManifestChecker.newManifestChecker( - new File(directory, MANIFEST_FILE_NAME), - notEmptyDirectory(directory), + persistence.getManifestFile(), + persistence.segmentFilesExist(), strictVersionCheck ? MAX_STORE_VERSION : MIN_STORE_VERSION, MAX_STORE_VERSION ); } - private static boolean notEmptyDirectory(File path) { - Collection<File> entries = FileUtils.listFiles(path, new String[] {"tar"}, false); - return !entries.isEmpty(); - } - @Nonnull final SegmentTracker tracker; Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java Tue Feb 13 11:17:42 2018 @@ -19,37 +19,42 @@ package org.apache.jackrabbit.oak.segmen import static com.google.common.collect.Lists.newArrayList; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.HashSet; import java.util.List; import java.util.Set; import com.google.common.base.Joiner; +import org.apache.jackrabbit.oak.segment.SegmentArchiveManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Thread-safe class tracking files to be removed. */ -class FileReaper { +public class FileReaper { private static final Logger logger = LoggerFactory.getLogger(FileReaper.class); - private final Set<File> files = new HashSet<>(); + private final Set<String> files = new HashSet<>(); private final Object lock = new Object(); + private final SegmentArchiveManager archiveManager; + + public FileReaper(SegmentArchiveManager archiveManager) { + this.archiveManager = archiveManager; + } + /** * Add files to be removed. The same file can be added more than once. * Duplicates are ignored. * * @param files group of files to be removed. */ - void add(Iterable<File> files) { + void add(Iterable<String> files) { synchronized (lock) { - for (File file : files) { + for (String file : files) { this.files.add(file); } } @@ -59,21 +64,20 @@ class FileReaper { * Reap previously added files. */ void reap() { - Set<File> reap; + Set<String> reap; synchronized (lock) { reap = new HashSet<>(files); files.clear(); } - Set<File> redo = new HashSet<>(); - List<File> removed = newArrayList(); - for (File file : reap) { - try { - Files.deleteIfExists(file.toPath()); + Set<String> redo = new HashSet<>(); + List<String> removed = newArrayList(); + for (String file : reap) { + if (archiveManager.delete(file)) { removed.add(file); - } catch (IOException e) { - logger.warn(String.format("Unable to remove file %s", file), e); + } else { + logger.warn("Unable to remove file {}", file); redo.add(file); } } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Feb 13 11:17:42 2018 @@ -43,12 +43,8 @@ import static org.apache.jackrabbit.oak. import static org.apache.jackrabbit.oak.segment.file.TarRevisions.EXPEDITE_OPTION; import static org.apache.jackrabbit.oak.segment.file.TarRevisions.timeout; -import java.io.File; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; -import java.nio.channels.FileLock; -import java.nio.channels.OverlappingFileLockException; import java.util.Collection; import java.util.List; import java.util.Set; @@ -74,6 +70,7 @@ import org.apache.jackrabbit.oak.segment import org.apache.jackrabbit.oak.segment.Segment; import org.apache.jackrabbit.oak.segment.SegmentId; import org.apache.jackrabbit.oak.segment.SegmentNodeState; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.SegmentNotFoundException; import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener; import org.apache.jackrabbit.oak.segment.SegmentWriter; @@ -85,7 +82,6 @@ import org.apache.jackrabbit.oak.segment import org.apache.jackrabbit.oak.segment.file.tar.CleanupContext; import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; import org.apache.jackrabbit.oak.segment.file.tar.TarFiles; -import org.apache.jackrabbit.oak.segment.file.tar.TarFiles.CleanupResult; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; @@ -107,8 +103,6 @@ public class FileStore extends AbstractF private static final int MB = 1024 * 1024; - static final String LOCK_FILE_NAME = "repo.lock"; - /** * GC counter for logging purposes */ @@ -122,10 +116,7 @@ public class FileStore extends AbstractF private final TarFiles tarFiles; - private final RandomAccessFile lockFile; - - @Nonnull - private final FileLock lock; + private final SegmentNodeStorePersistence.RepositoryLock repositoryLock; private volatile TarRevisions revisions; @@ -139,7 +130,7 @@ public class FileStore extends AbstractF * not be removed immediately, because they first need to be closed, and the * JVM needs to release the memory mapped file references. */ - private final FileReaper fileReaper = new FileReaper(); + private final FileReaper fileReaper; /** * This flag is periodically updated by calling the {@code SegmentGCOptions} @@ -163,13 +154,8 @@ public class FileStore extends AbstractF FileStore(final FileStoreBuilder builder) throws InvalidFileStoreVersionException, IOException { super(builder); - lockFile = new RandomAccessFile(new File(directory, LOCK_FILE_NAME), "rw"); - try { - lock = lockFile.getChannel().lock(); - } catch (OverlappingFileLockException ex) { - throw new IllegalStateException(directory.getAbsolutePath() - + " is in use by another store.", ex); - } + SegmentNodeStorePersistence persistence = builder.getPersistence(); + repositoryLock = persistence.lockRepository(); this.segmentWriter = defaultSegmentWriterBuilder("sys") .withGeneration(() -> getGcGeneration().nonGC()) @@ -180,24 +166,29 @@ public class FileStore extends AbstractF this.garbageCollector = new GarbageCollector( builder.getGcOptions(), builder.getGcListener(), - new GCJournal(directory), + new GCJournal(persistence.getGCJournalFile()), builder.getCacheManager() .withAccessTracking("COMPACT", builder.getStatsProvider())); - newManifestChecker(directory, builder.getStrictVersionCheck()).checkAndUpdateManifest(); + newManifestChecker(persistence, builder.getStrictVersionCheck()).checkAndUpdateManifest(); this.stats = new FileStoreStats(builder.getStatsProvider(), this, 0); - this.tarFiles = TarFiles.builder() + + TarFiles.Builder tarFilesBuilder = TarFiles.builder() .withDirectory(directory) .withMemoryMapping(memoryMapping) .withTarRecovery(recovery) .withIOMonitor(ioMonitor) .withFileStoreMonitor(stats) .withMaxFileSize(builder.getMaxFileSize() * MB) - .build(); + .withPersistence(builder.getPersistence()); + + this.tarFiles = tarFilesBuilder.build(); long size = this.tarFiles.size(); this.stats.init(size); + this.fileReaper = this.tarFiles.createFileReaper(); + this.snfeListener = builder.getSnfeListener(); fileStoreScheduler.scheduleAtFixedRate(format("TarMK flush [%s]", directory), 5, SECONDS, @@ -448,9 +439,8 @@ public class FileStore extends AbstractF } Closer closer = Closer.create(); - closer.register(lockFile); - closer.register(lock::release); - closer.register(tarFiles); + closer.register(repositoryLock::unlock); + closer.register(tarFiles) ; closer.register(revisions); closeAndLogOnFail(closer); @@ -923,7 +913,7 @@ public class FileStore extends AbstractF * @throws IOException */ @Nonnull - synchronized List<File> cleanup() throws IOException { + synchronized List<String> cleanup() throws IOException { return cleanup(CompactionResult.skipped( lastCompactionType, getGcGeneration(), @@ -938,7 +928,7 @@ public class FileStore extends AbstractF * @throws IOException */ @Nonnull - private List<File> cleanup(@Nonnull CompactionResult compactionResult) + private List<String> cleanup(@Nonnull CompactionResult compactionResult) throws IOException { PrintableStopwatch watch = PrintableStopwatch.createStarted(); @@ -950,7 +940,7 @@ public class FileStore extends AbstractF // to clear stale weak references in the SegmentTracker System.gc(); - CleanupResult cleanupResult = tarFiles.cleanup(newCleanupContext(compactionResult.reclaimer())); + TarFiles.CleanupResult cleanupResult = tarFiles.cleanup(newCleanupContext(compactionResult.reclaimer())); if (cleanupResult.isInterrupted()) { gcListener.info("cleanup interrupted"); } @@ -972,7 +962,7 @@ public class FileStore extends AbstractF return cleanupResult.getRemovableFiles(); } - private String toFileNames(@Nonnull List<File> files) { + private String toFileNames(@Nonnull List<String> files) { if (files.isEmpty()) { return "none"; } else { Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java Tue Feb 13 11:17:42 2018 @@ -40,6 +40,7 @@ import com.google.common.base.Predicate; import org.apache.jackrabbit.oak.segment.CacheWeights.NodeCacheWeigher; import org.apache.jackrabbit.oak.segment.CacheWeights.StringCacheWeigher; import org.apache.jackrabbit.oak.segment.CacheWeights.TemplateCacheWeigher; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; import org.apache.jackrabbit.oak.segment.RecordCache; import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener; @@ -47,6 +48,7 @@ import org.apache.jackrabbit.oak.segment import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; import org.apache.jackrabbit.oak.segment.file.tar.IOMonitor; import org.apache.jackrabbit.oak.segment.file.tar.IOMonitorAdapter; +import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.gc.DelegatingGCMonitor; import org.apache.jackrabbit.oak.spi.gc.GCMonitor; @@ -88,6 +90,8 @@ public class FileStoreBuilder { private boolean memoryMapping = MEMORY_MAPPING_DEFAULT; + private SegmentNodeStorePersistence persistence; + @Nonnull private StatisticsProvider statsProvider = StatisticsProvider.NOOP; @@ -139,6 +143,7 @@ public class FileStoreBuilder { private FileStoreBuilder(@Nonnull File directory) { this.directory = checkNotNull(directory); this.gcListener.registerGCMonitor(new LoggingGCMonitor(LOG)); + this.persistence = new TarPersistence(directory); } /** @@ -314,6 +319,11 @@ public class FileStoreBuilder { return this; } + public FileStoreBuilder withCustomPersistence(SegmentNodeStorePersistence persistence) throws IOException { + this.persistence = persistence; + return this; + } + /** * Create a new {@link FileStore} instance with the settings specified in this * builder. If none of the {@code with} methods have been called before calling @@ -336,7 +346,7 @@ public class FileStoreBuilder { checkState(!built, "Cannot re-use builder"); built = true; directory.mkdirs(); - TarRevisions revisions = new TarRevisions(directory); + TarRevisions revisions = new TarRevisions(persistence); LOG.info("Creating file store {}", this); FileStore store; try { @@ -376,7 +386,7 @@ public class FileStoreBuilder { checkState(directory.exists() && directory.isDirectory(), "%s does not exist or is not a directory", directory); built = true; - ReadOnlyRevisions revisions = new ReadOnlyRevisions(directory); + ReadOnlyRevisions revisions = new ReadOnlyRevisions(persistence); LOG.info("Creating file store {}", this); ReadOnlyFileStore store; try { @@ -443,6 +453,10 @@ public class FileStoreBuilder { return snfeListener; } + SegmentNodeStorePersistence getPersistence() { + return persistence; + } + /** * @return creates or returns the {@code WriterCacheManager} this builder passes or * passed to the store on {@link #build()}. Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java Tue Feb 13 11:17:42 2018 @@ -17,11 +17,11 @@ package org.apache.jackrabbit.oak.segment.file; -import java.io.File; import java.io.IOException; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.SegmentIdProvider; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.SegmentStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,9 +45,9 @@ class FileStoreUtil { * found. * @throws IOException If an I/O error occurs. */ - static RecordId findPersistedRecordId(SegmentStore store, SegmentIdProvider idProvider, File journal) + static RecordId findPersistedRecordId(SegmentStore store, SegmentIdProvider idProvider, SegmentNodeStorePersistence.JournalFile journalFile) throws IOException { - try (JournalReader journalReader = new JournalReader(journal)) { + try (JournalReader journalReader = new JournalReader(journalFile)) { while (journalReader.hasNext()) { JournalEntry entry = journalReader.next(); try { Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java Tue Feb 13 11:17:42 2018 @@ -19,20 +19,10 @@ package org.apache.jackrabbit.oak.segment.file; -import static com.google.common.base.Charsets.UTF_8; import static com.google.common.base.Preconditions.checkNotNull; -import static java.nio.file.Files.newBufferedWriter; -import static java.nio.file.Files.readAllLines; -import static java.nio.file.StandardOpenOption.APPEND; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.DSYNC; -import static java.nio.file.StandardOpenOption.WRITE; import static org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration; -import java.io.BufferedWriter; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -42,13 +32,15 @@ import javax.annotation.Nonnull; import com.google.common.base.Joiner; import org.apache.jackrabbit.oak.segment.RecordId; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.GCJournalFile; import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; +import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Persists the repository size and the reclaimed size following a cleanup - * operation in the {@link #GC_JOURNAL gc journal} file with the format: + * operation in the {@link TarPersistence#GC_JOURNAL gc journal} file with the format: * 'repoSize, reclaimedSize, timestamp, gc generation, gc full generation (since Oak 1.8), * number of nodes compacted, root id (since Oak 1.8)'. */ @@ -56,15 +48,12 @@ public class GCJournal { private static final Logger LOG = LoggerFactory.getLogger(GCJournal.class); - public static final String GC_JOURNAL = "gc.log"; - - @Nonnull - private final File directory; + private final GCJournalFile journalFile; private GCJournalEntry latest; - public GCJournal(@Nonnull File directory) { - this.directory = checkNotNull(directory); + public GCJournal(@Nonnull GCJournalFile journalFile) { + this.journalFile = journalFile; } /** @@ -90,13 +79,8 @@ public class GCJournal { } latest = new GCJournalEntry(repoSize, reclaimedSize, System.currentTimeMillis(), gcGeneration, nodes, checkNotNull(root)); - Path path = new File(directory, GC_JOURNAL).toPath(); try { - try (BufferedWriter w = newBufferedWriter(path, UTF_8, WRITE, - APPEND, CREATE, DSYNC)) { - w.write(latest.toString()); - w.newLine(); - } + journalFile.writeLine(latest.toString()); } catch (IOException e) { LOG.error("Error writing gc journal", e); } @@ -130,13 +114,10 @@ public class GCJournal { } private List<String> readLines() { - File file = new File(directory, GC_JOURNAL); - if (file.exists()) { - try { - return readAllLines(file.toPath(), UTF_8); - } catch (IOException e) { - LOG.error("Error reading gc journal", e); - } + try { + return journalFile.readLines(); + } catch (IOException e) { + LOG.error("Error reading gc journal", e); } return new ArrayList<String>(); } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java Tue Feb 13 11:17:42 2018 @@ -19,14 +19,11 @@ package org.apache.jackrabbit.oak.segment.file; -import static java.nio.charset.Charset.defaultCharset; - import java.io.Closeable; -import java.io.File; import java.io.IOException; import java.util.List; -import org.apache.commons.io.input.ReversedLinesFileReader; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,10 +37,10 @@ import com.google.common.collect.Abstrac public final class JournalReader extends AbstractIterator<JournalEntry> implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(JournalReader.class); - private final ReversedLinesFileReader journal; + private final SegmentNodeStorePersistence.JournalFileReader reader; - public JournalReader(File journalFile) throws IOException { - journal = new ReversedLinesFileReader(journalFile, defaultCharset()); + public JournalReader(SegmentNodeStorePersistence.JournalFile journal) throws IOException { + this.reader = journal.openJournalReader(); } /** @@ -54,7 +51,7 @@ public final class JournalReader extends protected JournalEntry computeNext() { try { String line = null; - while ((line = journal.readLine()) != null) { + while ((line = reader.readLine()) != null) { if (line.indexOf(' ') != -1) { List<String> splits = Splitter.on(' ').splitToList(line); String revision = splits.get(0); @@ -83,6 +80,6 @@ public final class JournalReader extends @Override public void close() throws IOException { - journal.close(); + reader.close(); } } Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java?rev=1824115&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java Tue Feb 13 11:17:42 2018 @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment.file; + +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static com.google.common.base.Charsets.UTF_8; +import static java.nio.file.Files.newBufferedWriter; +import static java.nio.file.Files.readAllLines; +import static java.nio.file.StandardOpenOption.APPEND; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.DSYNC; +import static java.nio.file.StandardOpenOption.WRITE; + +public class LocalGCJournalFile implements SegmentNodeStorePersistence.GCJournalFile { + + private final File file; + + public LocalGCJournalFile(File parent, String name) { + this(new File(parent, name)); + } + + public LocalGCJournalFile(File file) { + this.file = file; + } + + @Override + public void writeLine(String line) throws IOException { + try (BufferedWriter w = newBufferedWriter(file.toPath(), UTF_8, WRITE, APPEND, CREATE, DSYNC)) { + w.write(line); + w.newLine(); + } + } + + @Override + public List<String> readLines() throws IOException { + if (file.exists()) { + return readAllLines(file.toPath(), UTF_8); + } + return new ArrayList<String>(); + } +} Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java?rev=1824115&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java Tue Feb 13 11:17:42 2018 @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment.file; + +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.ManifestFile; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Properties; + +public class LocalManifestFile implements ManifestFile { + + private final File file; + + public LocalManifestFile(File parent, String name) { + this(new File(parent, name)); + } + + public LocalManifestFile(File file) { + this.file = file; + } + + @Override + public boolean exists() { + return file.exists(); + } + + @Override + public Properties load() throws IOException { + Properties properties = new Properties(); + try (FileReader r = new FileReader(file)) { + properties.load(r); + } + return properties; + } + + @Override + public void save(Properties properties) throws IOException { + properties.store(new FileWriter(file), null); + } + +} Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java Tue Feb 13 11:17:42 2018 @@ -17,9 +17,9 @@ package org.apache.jackrabbit.oak.segment.file; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.ManifestFile; + import java.io.IOException; import java.util.Properties; @@ -34,12 +34,8 @@ class Manifest { * @return A manifest file. * @throws IOException If any error occurs when loading the manifest. */ - static Manifest load(File file) throws IOException { - Properties properties = new Properties(); - try (FileReader r = new FileReader(file)) { - properties.load(r); - } - return new Manifest(properties); + static Manifest load(ManifestFile file) throws IOException { + return new Manifest(file.load()); } /** @@ -85,8 +81,8 @@ class Manifest { * @param file The file to save the manifest to. * @throws IOException if an error occurs while saving the manifest. */ - void save(File file) throws IOException { - properties.store(new FileWriter(file), null); + void save(ManifestFile file) throws IOException { + file.save(properties); } private int getIntegerProperty(String name, int otherwise) { Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java Tue Feb 13 11:17:42 2018 @@ -17,21 +17,22 @@ package org.apache.jackrabbit.oak.segment.file; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.ManifestFile; + import static com.google.common.base.Preconditions.checkArgument; -import java.io.File; import java.io.IOException; public class ManifestChecker { - public static ManifestChecker newManifestChecker(File path, boolean shouldExist, int minStoreVersion, int maxStoreVersion) { - checkArgument(path != null, "path"); + public static ManifestChecker newManifestChecker(ManifestFile file, boolean shouldExist, int minStoreVersion, int maxStoreVersion) { + checkArgument(file != null, "file"); checkArgument(minStoreVersion > 0, "minStoreVersion"); checkArgument(maxStoreVersion > 0, "maxStoreVersion"); - return new ManifestChecker(path, shouldExist, minStoreVersion, maxStoreVersion); + return new ManifestChecker(file, shouldExist, minStoreVersion, maxStoreVersion); } - private final File file; + private final ManifestFile file; private final boolean shouldExist; @@ -39,7 +40,7 @@ public class ManifestChecker { private final int maxStoreVersion; - private ManifestChecker(File file, boolean shouldExist, int minStoreVersion, int maxStoreVersion) { + private ManifestChecker(ManifestFile file, boolean shouldExist, int minStoreVersion, int maxStoreVersion) { this.file = file; this.shouldExist = shouldExist; this.minStoreVersion = minStoreVersion; Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java Tue Feb 13 11:17:42 2018 @@ -63,7 +63,7 @@ public class ReadOnlyFileStore extends A ReadOnlyFileStore(FileStoreBuilder builder) throws InvalidFileStoreVersionException, IOException { super(builder); - newManifestChecker(directory, builder.getStrictVersionCheck()).checkManifest(); + newManifestChecker(builder.getPersistence(), builder.getStrictVersionCheck()).checkManifest(); tarFiles = TarFiles.builder() .withDirectory(directory) Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java Tue Feb 13 11:17:42 2018 @@ -23,9 +23,7 @@ import static com.google.common.base.Pre import static org.apache.jackrabbit.oak.segment.file.FileStoreUtil.findPersistedRecordId; import java.io.Closeable; -import java.io.File; import java.io.IOException; -import java.io.RandomAccessFile; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; @@ -34,31 +32,19 @@ import com.google.common.base.Function; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; import org.apache.jackrabbit.oak.segment.SegmentIdProvider; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.SegmentStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ReadOnlyRevisions implements Revisions, Closeable { - private static final Logger LOG = LoggerFactory - .getLogger(ReadOnlyRevisions.class); - - public static final String JOURNAL_FILE_NAME = "journal.log"; - @Nonnull private final AtomicReference<RecordId> head; @Nonnull - private final File directory; + private final SegmentNodeStorePersistence.JournalFile journalFile; - @Nonnull - private final RandomAccessFile journalFile; - - public ReadOnlyRevisions(@Nonnull File directory) throws IOException { - this.directory = checkNotNull(directory); - this.journalFile = new RandomAccessFile(new File(directory, - JOURNAL_FILE_NAME), "r"); - this.journalFile.seek(journalFile.length()); + public ReadOnlyRevisions(@Nonnull SegmentNodeStorePersistence persistence) { + this.journalFile = checkNotNull(persistence).getJournalFile(); this.head = new AtomicReference<>(null); } @@ -74,7 +60,7 @@ public class ReadOnlyRevisions implement if (head.get() != null) { return; } - RecordId persistedId = findPersistedRecordId(store, idProvider, new File(directory, JOURNAL_FILE_NAME)); + RecordId persistedId = findPersistedRecordId(store, idProvider, journalFile); if (persistedId == null) { throw new IllegalStateException("Cannot start readonly store from empty journal"); } @@ -112,14 +98,8 @@ public class ReadOnlyRevisions implement throw new UnsupportedOperationException("ReadOnly Revisions"); } - /** - * Close the underlying journal file. - * - * @throws IOException - */ @Override public void close() throws IOException { - journalFile.close(); + // do nothing } - } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java?rev=1824115&r1=1824114&r2=1824115&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java Tue Feb 13 11:17:42 2018 @@ -19,16 +19,13 @@ package org.apache.jackrabbit.oak.segment.file; -import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static java.lang.Long.MAX_VALUE; import static java.util.concurrent.TimeUnit.DAYS; import static org.apache.jackrabbit.oak.segment.file.FileStoreUtil.findPersistedRecordId; import java.io.Closeable; -import java.io.File; import java.io.IOException; -import java.io.RandomAccessFile; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -44,13 +41,15 @@ import com.google.common.base.Supplier; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; import org.apache.jackrabbit.oak.segment.SegmentIdProvider; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.SegmentStore; +import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This implementation of {@code Revisions} is backed by a - * {@link #JOURNAL_FILE_NAME journal} file where the current head is persisted + * {@link TarPersistence#JOURNAL_FILE_NAME journal} file where the current head is persisted * by calling {@link #tryFlush(Flusher)}. * <p> * The {@link #setHead(Function, Option...)} method supports a timeout @@ -62,8 +61,6 @@ import org.slf4j.LoggerFactory; public class TarRevisions implements Revisions, Closeable { private static final Logger LOG = LoggerFactory.getLogger(TarRevisions.class); - public static final String JOURNAL_FILE_NAME = "journal.log"; - /** * The lock protecting {@link #journalFile}. */ @@ -72,14 +69,15 @@ public class TarRevisions implements Rev @Nonnull private final AtomicReference<RecordId> head; - @Nonnull - private final File directory; + private final SegmentNodeStorePersistence persistence; + + private final SegmentNodeStorePersistence.JournalFile journalFile; /** - * The journal file. It is protected by {@link #journalFileLock}. It becomes + * The journal file writer. It is protected by {@link #journalFileLock}. It becomes * {@code null} after it's closed. */ - private volatile RandomAccessFile journalFile; + private volatile SegmentNodeStorePersistence.JournalFileWriter journalFileWriter; /** * The persisted head of the root journal, used to determine whether the @@ -138,16 +136,15 @@ public class TarRevisions implements Rev /** * Create a new instance placing the journal log file into the passed * {@code directory}. - * @param directory directory of the journal file + * @param persistence object representing the segment persistence * @throws IOException */ - public TarRevisions(@Nonnull File directory) throws IOException { - this.directory = checkNotNull(directory); - this.journalFile = new RandomAccessFile(new File(directory, - JOURNAL_FILE_NAME), "rw"); - this.journalFile.seek(journalFile.length()); + public TarRevisions(SegmentNodeStorePersistence persistence) throws IOException { + this.journalFile = persistence.getJournalFile(); + this.journalFileWriter = journalFile.openJournalWriter(); this.head = new AtomicReference<>(null); this.persistedHead = new AtomicReference<>(null); + this.persistence = persistence; } /** @@ -164,7 +161,7 @@ public class TarRevisions implements Rev if (head.get() != null) { return; } - RecordId persistedId = findPersistedRecordId(store, idProvider, new File(directory, JOURNAL_FILE_NAME)); + RecordId persistedId = findPersistedRecordId(store, idProvider, journalFile); if (persistedId == null) { head.set(writeInitialNode.get()); } else { @@ -226,7 +223,7 @@ public class TarRevisions implements Rev } private void doFlush(Flusher flusher) throws IOException { - if (journalFile == null) { + if (journalFileWriter == null) { LOG.debug("No journal file available, skipping flush"); return; } @@ -238,8 +235,7 @@ public class TarRevisions implements Rev } flusher.flush(); LOG.debug("TarMK journal update {} -> {}", before, after); - journalFile.writeBytes(after.toString10() + " root " + System.currentTimeMillis() + "\n"); - journalFile.getChannel().force(false); + journalFileWriter.writeLine(after.toString10() + " root " + System.currentTimeMillis()); persistedHead.set(after); } @@ -354,11 +350,11 @@ public class TarRevisions implements Rev public void close() throws IOException { journalFileLock.lock(); try { - if (journalFile == null) { + if (journalFileWriter == null) { return; } - journalFile.close(); - journalFile = null; + journalFileWriter.close(); + journalFileWriter = null; } finally { journalFileLock.unlock(); } Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java?rev=1824115&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java Tue Feb 13 11:17:42 2018 @@ -0,0 +1,98 @@ +package org.apache.jackrabbit.oak.segment.file.tar; + +import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.zip.CRC32; + +import static com.google.common.collect.Lists.newArrayListWithCapacity; +import static com.google.common.collect.Maps.newHashMapWithExpectedSize; +import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.GRAPH_MAGIC; + +public final class GraphLoader { + + private static final Logger log = LoggerFactory.getLogger(GraphLoader.class); + + private static final int FOOTER_SIZE = 16; + + private GraphLoader() { + } + + /** + * Loads the optional pre-compiled graph entry from the given tar file. + * + * @return the graph or {@code null} if one was not found + * @throws IOException if the tar file could not be read + */ + public static ByteBuffer loadGraph(ReaderAtEnd readerAtEnd) throws IOException { + ByteBuffer meta = readerAtEnd.readAtEnd(FOOTER_SIZE, FOOTER_SIZE); + + int crc32 = meta.getInt(); + int count = meta.getInt(); + int bytes = meta.getInt(); + int magic = meta.getInt(); + + if (magic != GRAPH_MAGIC) { + log.warn("Invalid graph magic number"); + return null; + } + + if (count < 0) { + log.warn("Invalid number of entries"); + return null; + } + + if (bytes < 4 + count * 34) { + log.warn("Invalid entry size"); + return null; + } + + ByteBuffer graph = readerAtEnd.readAtEnd(bytes, bytes); + + byte[] b = new byte[bytes - FOOTER_SIZE]; + + graph.mark(); + graph.get(b); + graph.reset(); + + CRC32 checksum = new CRC32(); + checksum.update(b); + + if (crc32 != (int) checksum.getValue()) { + log.warn("Invalid graph checksum in tar file"); + return null; + } + + return graph; + } + + public static Map<UUID, List<UUID>> parseGraph(ByteBuffer buffer) { + int nEntries = buffer.getInt(buffer.limit() - 12); + + Map<UUID, List<UUID>> graph = newHashMapWithExpectedSize(nEntries); + + for (int i = 0; i < nEntries; i++) { + long msb = buffer.getLong(); + long lsb = buffer.getLong(); + int nVertices = buffer.getInt(); + + List<UUID> vertices = newArrayListWithCapacity(nVertices); + + for (int j = 0; j < nVertices; j++) { + long vMsb = buffer.getLong(); + long vLsb = buffer.getLong(); + vertices.add(new UUID(vMsb, vLsb)); + } + + graph.put(new UUID(msb, lsb), vertices); + } + + return graph; + } +} Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java?rev=1824115&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java (added) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java Tue Feb 13 11:17:42 2018 @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment.file.tar; + +import org.apache.commons.io.input.ReversedLinesFileReader; +import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +import static java.nio.charset.Charset.defaultCharset; + +public class LocalJournalFile implements SegmentNodeStorePersistence.JournalFile { + + private final File journalFile; + + public LocalJournalFile(File directory, String journalFile) { + this.journalFile = new File(directory, journalFile); + } + + public LocalJournalFile(File journalFile) { + this.journalFile = journalFile; + } + + @Override + public SegmentNodeStorePersistence.JournalFileReader openJournalReader() throws IOException { + return new LocalJournalFileReader(journalFile); + } + + @Override + public SegmentNodeStorePersistence.JournalFileWriter openJournalWriter() throws IOException { + return new LocalJournalFileWriter(journalFile); + } + + @Override + public String getName() { + return journalFile.getName(); + } + + @Override + public boolean exists() { + return journalFile.exists(); + } + + private static class LocalJournalFileReader implements SegmentNodeStorePersistence.JournalFileReader { + + private final ReversedLinesFileReader journal; + + public LocalJournalFileReader(File file) throws IOException { + journal = new ReversedLinesFileReader(file, defaultCharset()); + } + + @Override + public String readLine() throws IOException { + return journal.readLine(); + } + + @Override + public void close() throws IOException { + journal.close(); + } + } + + private static class LocalJournalFileWriter implements SegmentNodeStorePersistence.JournalFileWriter { + + private final RandomAccessFile journalFile; + + public LocalJournalFileWriter(File file) throws IOException { + journalFile = new RandomAccessFile(file, "rw"); + journalFile.seek(journalFile.length()); + } + + @Override + public void truncate() throws IOException { + journalFile.setLength(0); + } + + @Override + public void writeLine(String line) throws IOException { + journalFile.writeBytes(line + "\n"); + journalFile.getChannel().force(false); + } + + @Override + public void close() throws IOException { + journalFile.close(); + } + } +}