This is an automated email from the ASF dual-hosted git repository. amitj pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push: new aada6bd4b0 OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore outputs the wrong path when FlatFileSplitter used (#826) aada6bd4b0 is described below commit aada6bd4b0888677b850b5ffd151b079507a0851 Author: Amit Jain <am...@apache.org> AuthorDate: Mon Jan 23 11:39:13 2023 +0530 OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore outputs the wrong path when FlatFileSplitter used (#826) - System property (oak.indexer.sortedFilePath) defines the folder of the flat file store rather than the current specific file path - Enable buildFlatFiles to use the folder in the system property defined as the place to look for files - In case the folder is split then it uses that to init the split flat file store --- .../indexer/document/DocumentStoreIndexerBase.java | 3 +- .../flatfile/FlatFileNodeStoreBuilder.java | 82 +++++--- .../document/flatfile/FlatFileSplitter.java | 5 +- .../indexer/document/flatfile/FlatFileStore.java | 2 +- .../flatfile/FlatFileNodeStoreBuilderTest.java | 213 +++++++++++++++++---- 5 files changed, 238 insertions(+), 67 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 90478a2db1..974b6524f8 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -217,7 +217,8 @@ public abstract class DocumentStoreIndexerBase implements Closeable{ preferredPathElements.addAll(indexDf.getRelativeNodeNames()); } Predicate<String> predicate = s -> indexDefinitions.stream().anyMatch(indexDef -> indexDef.getPathFilter().filter(s) != PathFilter.Result.EXCLUDE); - FlatFileStore flatFileStore = buildFlatFileStoreList(checkpointedState, null, predicate, preferredPathElements, false, indexDefinitions).get(0); + FlatFileStore flatFileStore = buildFlatFileStoreList(checkpointedState, null, predicate, + preferredPathElements, IndexerConfiguration.parallelIndexEnabled(), indexDefinitions).get(0); log.info("FlatFileStore built at {}. To use this flatFileStore in a reindex step, set System Property-{} with value {}", flatFileStore.getFlatFileStorePath(), OAK_INDEXER_SORTED_FILE_PATH, flatFileStore.getFlatFileStorePath()); return flatFileStore; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index 8cae86fa1a..177bab3446 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile; import com.google.common.collect.Iterables; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.index.IndexHelper; import org.apache.jackrabbit.oak.index.IndexerSupport; @@ -31,6 +32,7 @@ import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; import org.apache.jackrabbit.oak.query.NodeStateNodeTypeInfoProvider; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +40,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; @@ -65,6 +68,9 @@ public class FlatFileNodeStoreBuilder { * Allowed values are the values from enum {@link SortStrategyType} */ public static final String OAK_INDEXER_SORT_STRATEGY_TYPE = "oak.indexer.sortStrategyType"; + /** + * System property to define the existing folder containing the flat file store files + */ public static final String OAK_INDEXER_SORTED_FILE_PATH = "oak.indexer.sortedFilePath"; /** @@ -191,7 +197,8 @@ public class FlatFileNodeStoreBuilder { logFlags(); comparator = new PathElementComparator(preferredPathElements); entryWriter = new NodeStateEntryWriter(blobStore); - FlatFileStore store = new FlatFileStore(blobStore, createdSortedStoreFile(), new NodeStateEntryReader(blobStore), + FlatFileStore store = new FlatFileStore(blobStore, createdSortedStoreFiles().get(0), + new NodeStateEntryReader(blobStore), unmodifiableSet(preferredPathElements), algorithm); if (entryCount > 0) { store.setEntryCount(entryCount); @@ -205,13 +212,18 @@ public class FlatFileNodeStoreBuilder { comparator = new PathElementComparator(preferredPathElements); entryWriter = new NodeStateEntryWriter(blobStore); - File flatStoreFile = createdSortedStoreFile(); + List<File> fileList = createdSortedStoreFiles(); + long start = System.currentTimeMillis(); - NodeStore nodeStore = new MemoryNodeStore(indexerSupport.retrieveNodeStateForCheckpoint()); - FlatFileSplitter splitter = new FlatFileSplitter(flatStoreFile, indexHelper.getWorkDir(), new NodeStateNodeTypeInfoProvider(nodeStore.getRoot()), new NodeStateEntryReader(blobStore), + // If not already split, split otherwise skip splitting + if (!fileList.stream().allMatch(FlatFileSplitter.IS_SPLIT)) { + NodeStore nodeStore = new MemoryNodeStore(indexerSupport.retrieveNodeStateForCheckpoint()); + FlatFileSplitter splitter = new FlatFileSplitter(fileList.get(0), indexHelper.getWorkDir(), + new NodeStateNodeTypeInfoProvider(nodeStore.getRoot()), new NodeStateEntryReader(blobStore), indexDefinitions); - List<File> fileList = splitter.split(); - log.info("Split flat file to result files '{}' is done, took {} ms", fileList, System.currentTimeMillis() - start); + fileList = splitter.split(); + log.info("Split flat file to result files '{}' is done, took {} ms", fileList, System.currentTimeMillis() - start); + } List<FlatFileStore> storeList = new ArrayList<>(); for (File flatFileItem : fileList) { @@ -222,26 +234,52 @@ public class FlatFileNodeStoreBuilder { return storeList; } - private File createdSortedStoreFile() throws IOException, CompositeException { + /** + * Returns the existing list of store files if it can read from system property OAK_INDEXER_SORTED_FILE_PATH which + * defines the existing folder where the flat file store files are present. Will throw an exception if it cannot + * read or the path in the system property is not a directory. + * If the system property OAK_INDEXER_SORTED_FILE_PATH in undefined, or it cannot read relevant files it + * initializes the flat file store. + * + * @return list of flat files + * @throws IOException + * @throws CompositeException + */ + private List<File> createdSortedStoreFiles() throws IOException, CompositeException { + // Check system property defined path String sortedFilePath = System.getProperty(OAK_INDEXER_SORTED_FILE_PATH); - if (sortedFilePath != null) { - File sortedFile = new File(sortedFilePath); - if (sortedFile.exists() && sortedFile.isFile() && sortedFile.canRead()) { - log.info("Reading from provided sorted file [{}] (via system property '{}')", - sortedFile.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH); - return sortedFile; - } else { - String msg = String.format("Cannot read sorted file at [%s] configured via system property '%s'", - sortedFile.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH); - throw new IllegalArgumentException(msg); + if (StringUtils.isNotBlank(sortedFilePath)) { + File sortedDir = new File(sortedFilePath); + log.info("Attempting to read from provided sorted files directory [{}] (via system property '{}')", + sortedDir.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH); + List<File> files = getFiles(sortedDir); + if (files != null) { + return files; + } + } + + // Initialize the flat file store again + + createStoreDir(); + SortStrategy strategy = createSortStrategy(flatFileStoreDir); + File result = strategy.createSortedStoreFile(); + entryCount = strategy.getEntryCount(); + return Collections.singletonList(result); + } + + @Nullable + private List<File> getFiles(File sortedDir) { + if (sortedDir.exists() && sortedDir.canRead() && sortedDir.isDirectory()) { + File[] files = sortedDir.listFiles( + (dir, name) -> name.endsWith(FlatFileStoreUtils.getSortedStoreFileName(algorithm))); + if (files != null && files.length != 0) { + return Arrays.asList(files); } } else { - createStoreDir(); - org.apache.jackrabbit.oak.index.indexer.document.flatfile.SortStrategy strategy = createSortStrategy(flatFileStoreDir); - File result = strategy.createSortedStoreFile(); - entryCount = strategy.getEntryCount(); - return result; + String msg = String.format("Cannot read sorted files directory at [%s]", sortedDir.getAbsolutePath()); + throw new IllegalArgumentException(msg); } + return null; } SortStrategy createSortStrategy(File dir) throws IOException { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java index c99db1fbd6..663fcd4de9 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitter.java @@ -44,6 +44,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.Stack; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; @@ -73,7 +74,9 @@ public class FlatFileSplitter { private Set<String> splitNodeTypeNames; private boolean useCompression = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_USE_ZIP, "true")); private boolean useLZ4 = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_USE_LZ4, "false")); - + + static Predicate<File> IS_SPLIT = path -> path.getParent().endsWith(SPLIT_DIR_NAME); + public FlatFileSplitter(File flatFile, File workdir, NodeTypeInfoProvider infoProvider, NodeStateEntryReader entryReader, Set<IndexDefinition> indexDefinitions) { this.flatFile = flatFile; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java index 5c4b675416..5c31d0323f 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java @@ -57,7 +57,7 @@ public class FlatFileStore implements Iterable<NodeStateEntry>, Closeable { } public String getFlatFileStorePath() { - return storeFile.getAbsolutePath(); + return storeFile.getParentFile().getAbsolutePath(); } public long getEntryCount() { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java index 8fcbb74252..acd655192a 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java @@ -18,23 +18,47 @@ */ package org.apache.jackrabbit.oak.index.indexer.document.flatfile; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; import java.io.IOException; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import org.apache.commons.io.IOUtils; +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.OakInitializer; +import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.index.IndexHelper; import org.apache.jackrabbit.oak.index.IndexerSupport; import org.apache.jackrabbit.oak.index.indexer.document.CompositeException; +import org.apache.jackrabbit.oak.index.indexer.document.IndexerConfiguration; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.plugins.name.NamespaceEditorProvider; +import org.apache.jackrabbit.oak.plugins.nodetype.TypeEditorProvider; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfo; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfoProvider; +import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.junit.Rule; import org.junit.Test; +import org.junit.contrib.java.lang.system.RestoreSystemProperties; import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.mockito.Mockito; import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORTED_FILE_PATH; import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE; import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_TRAVERSE_WITH_SORT; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_USE_LZ4; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_USE_ZIP; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -47,6 +71,9 @@ public class FlatFileNodeStoreBuilderTest { @Rule public TemporaryFolder folder = new TemporaryFolder(new File(BUILD_TARGET_FOLDER)); + @Rule + public final TestRule restoreSystemProperties = new RestoreSystemProperties(); + private final NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory = range -> null; @Test @@ -60,60 +87,162 @@ public class FlatFileNodeStoreBuilderTest { @Test public void sortStrategyBasedOnSystemProperty() throws Exception { - try { - System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString()); - FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()) - .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory); - SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir()); - assertTrue(sortStrategy instanceof TraverseWithSortStrategy); - } finally { - System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE); - } + System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString()); + FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()) + .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory); + SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir()); + assertTrue(sortStrategy instanceof TraverseWithSortStrategy); } @Test public void disableTraverseAndSortStrategyUsingSystemProperty() throws Exception { - try { - System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false"); - FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()) - .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory); - SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir()); - assertTrue(sortStrategy instanceof StoreAndSortStrategy); - } finally { - System.clearProperty(OAK_INDEXER_TRAVERSE_WITH_SORT); - } + System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false"); + FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()) + .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory); + SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir()); + assertTrue(sortStrategy instanceof StoreAndSortStrategy); } @Test public void sortStrategySystemPropertyPrecedence() throws Exception { - try { - System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false"); - System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString()); - FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()) - .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory); - SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir()); - assertTrue(sortStrategy instanceof TraverseWithSortStrategy); - } finally { - System.clearProperty(OAK_INDEXER_SORT_STRATEGY_TYPE); - System.clearProperty(OAK_INDEXER_TRAVERSE_WITH_SORT); - } + System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false"); + System.setProperty(OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.TRAVERSE_WITH_SORT.toString()); + FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()) + .withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory); + SortStrategy sortStrategy = builder.createSortStrategy(builder.createStoreDir()); + assertTrue(sortStrategy instanceof TraverseWithSortStrategy); + } + + @Test + public void testBuild() throws CompositeException, IOException { + System.setProperty(OAK_INDEXER_USE_ZIP, "false"); + File newFlatFile = getFile("simple-split.json", Compression.NONE); + + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath()); + assertBuild(newFlatFile.getParentFile().getAbsolutePath()); + } + + @Test + public void testBuildGZIP() throws CompositeException, IOException { + System.setProperty(OAK_INDEXER_USE_ZIP, "true"); + File newFlatFile = getFile("simple-split.json", Compression.GZIP); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath()); + + assertBuild(newFlatFile.getParentFile().getAbsolutePath()); + } + + @Test + public void testBuildLZ4() throws CompositeException, IOException { + System.setProperty(OAK_INDEXER_USE_ZIP, "true"); + System.setProperty(OAK_INDEXER_USE_LZ4, "true"); + LZ4Compression compression = new LZ4Compression(); + + File newFlatFile = getFile("simple-split.json", compression); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath()); + + assertBuild(newFlatFile.getParentFile().getAbsolutePath()); + } + + @Test + public void testBuildListNoSplit() throws CompositeException, IOException { + System.setProperty(OAK_INDEXER_USE_ZIP, "false"); + + File newFlatFile = getFile("complex-split.json", Compression.NONE); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath()); + + assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), false); + } + + @Test + public void testBuildListSplit() throws CompositeException, IOException { + System.setProperty(OAK_INDEXER_USE_ZIP, "false"); + System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD, "0"); + + File newFlatFile = getFile("complex-split.json", Compression.NONE); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath()); + + assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true); + } + + @Test + public void testBuildListSplitGZIP() throws CompositeException, IOException { + System.setProperty(OAK_INDEXER_USE_ZIP, "true"); + System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD, "0"); + + File newFlatFile = getFile("complex-split.json", Compression.GZIP); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath()); + + assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true); } @Test - public void testBuildList() throws CompositeException, IOException { - try { - File flatFile = new File(getClass().getClassLoader().getResource("simple-split.json").getFile()); - System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, flatFile.getAbsolutePath()); - FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory( - nodeStateEntryTraverserFactory); - IndexHelper indexHelper = mock(IndexHelper.class); - IndexerSupport indexerSupport = mock(IndexerSupport.class); - NodeState rootState = mock(NodeState.class); - when(indexerSupport.retrieveNodeStateForCheckpoint()).thenReturn(rootState); - List<FlatFileStore> storeList = builder.buildList(indexHelper, indexerSupport, null); + public void testBuildListSplitLZ4() throws CompositeException, IOException { + System.setProperty(OAK_INDEXER_USE_ZIP, "true"); + System.setProperty(OAK_INDEXER_USE_LZ4, "true"); + System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD, "0"); + LZ4Compression compression = new LZ4Compression(); + + File newFlatFile = getFile("complex-split.json", compression); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, newFlatFile.getParentFile().getAbsolutePath()); + + assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true); + } + + public void assertBuild(String dir) throws CompositeException, IOException { + FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory( + nodeStateEntryTraverserFactory); + FlatFileStore store = builder.build(); + assertEquals(dir, store.getFlatFileStorePath()); + } + + private File getFile(String dataFile, Compression compression) throws IOException { + File flatFile = new File(getClass().getClassLoader().getResource(dataFile).getFile()); + File newFlatFile = new File(folder.getRoot(), FlatFileStoreUtils.getSortedStoreFileName(compression)); + try (BufferedReader reader = FlatFileStoreUtils.createReader(flatFile, false); + BufferedWriter writer = FlatFileStoreUtils.createWriter(newFlatFile, compression)) { + IOUtils.copy(reader, writer); + } + return newFlatFile; + } + + public void assertBuildList(String dir, boolean split) throws CompositeException, IOException { + FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory( + nodeStateEntryTraverserFactory); + IndexHelper indexHelper = mock(IndexHelper.class); + when(indexHelper.getWorkDir()).thenReturn(new File(dir)); + IndexerSupport indexerSupport = mock(IndexerSupport.class); + NodeState rootState = mock(NodeState.class); + when(indexerSupport.retrieveNodeStateForCheckpoint()).thenReturn(rootState); + + List<FlatFileStore> storeList = builder.buildList(indexHelper, indexerSupport, mockIndexDefns()); + + if (split) { + assertEquals(new File(dir, "split").getAbsolutePath(), storeList.get(0).getFlatFileStorePath()); + assertTrue(storeList.size() > 1); + } else { + assertEquals(dir, storeList.get(0).getFlatFileStorePath()); assertEquals(1, storeList.size()); - } finally { - System.clearProperty(OAK_INDEXER_SORTED_FILE_PATH); } } + + private static Set<IndexDefinition> mockIndexDefns() { + NodeStore store = new MemoryNodeStore(); + EditorHook hook = new EditorHook( + new CompositeEditorProvider(new NamespaceEditorProvider(), new TypeEditorProvider())); + OakInitializer.initialize(store, new InitialContent(), hook); + + Set<IndexDefinition> defns = new HashSet<>(); + IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder(); + defnBuilder.indexRule("dam:Asset"); + defnBuilder.aggregateRule("dam:Asset"); + IndexDefinition defn = IndexDefinition.newBuilder(store.getRoot(), defnBuilder.build(), "/foo").build(); + defns.add(defn); + + NodeTypeInfoProvider mockNodeTypeInfoProvider = Mockito.mock(NodeTypeInfoProvider.class); + NodeTypeInfo mockNodeTypeInfo = Mockito.mock(NodeTypeInfo.class, "dam:Asset"); + Mockito.when(mockNodeTypeInfo.getNodeTypeName()).thenReturn("dam:Asset"); + Mockito.when(mockNodeTypeInfoProvider.getNodeTypeInfo("dam:Asset")).thenReturn(mockNodeTypeInfo); + + return defns; + } }