This is an automated email from the ASF dual-hosted git repository.
nfsantos pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 32135dd8bd OAK-11797 - Document store indexing: start Lucene writer
pool only before indexing, not before download
32135dd8bd is described below
commit 32135dd8bdb835c1e8d9ece8636f5028d21d80fe
Author: Nuno Santos <[email protected]>
AuthorDate: Tue Jul 8 14:08:59 2025 +0200
OAK-11797 - Document store indexing: start Lucene writer pool only before
indexing, not before download
---
.../indexer/document/DocumentStoreIndexerBase.java | 21 ++++++++-------------
.../oak/index/ElasticDocumentStoreIndexer.java | 4 +---
.../indexer/document/DocumentStoreIndexer.java | 3 +--
.../oak/index/DocumentStoreIndexerIT.java | 5 +++--
4 files changed, 13 insertions(+), 20 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
index b17c84640d..a79f20960d 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
@@ -97,16 +97,16 @@ public abstract class DocumentStoreIndexerBase implements
Closeable {
public static final String INDEXER_METRICS_PREFIX = "oak_indexer_";
public static final String METRIC_INDEXING_DURATION_SECONDS =
INDEXER_METRICS_PREFIX + "indexing_duration_seconds";
public static final String METRIC_FULL_INDEX_CREATION_DURATION_SECONDS =
INDEXER_METRICS_PREFIX + "full_index_creation_duration_seconds";
+ private static final int MAX_DOWNLOAD_ATTEMPTS =
Integer.parseInt(System.getProperty("oak.indexer.maxDownloadRetries", "5")) + 1;
- private final Logger log = LoggerFactory.getLogger(getClass());
- private final Logger traversalLog =
LoggerFactory.getLogger(DocumentStoreIndexerBase.class.getName() +
".traversal");
protected final Closer closer = Closer.create();
protected final IndexHelper indexHelper;
+ protected final IndexerSupport indexerSupport;
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final Logger traversalLog =
LoggerFactory.getLogger(DocumentStoreIndexerBase.class.getName() +
".traversal");
private final IndexingReporter indexingReporter;
private final StatisticsProvider statisticsProvider;
- protected NodeStateIndexerProvider indexerProvider;
- protected final IndexerSupport indexerSupport;
- private static final int MAX_DOWNLOAD_ATTEMPTS =
Integer.parseInt(System.getProperty("oak.indexer.maxDownloadRetries", "5")) + 1;
private static final int TOP_SLOWEST_PATHS_TO_LOG =
ConfigHelper.getSystemPropertyAsInt(
"oak.indexer.topSlowestPathsToLog", 20);
@@ -118,10 +118,6 @@ public abstract class DocumentStoreIndexerBase implements
Closeable {
this.statisticsProvider = indexHelper.getStatisticsProvider();
}
- protected void setProvider() throws IOException {
- this.indexerProvider = createProvider();
- }
-
private static class MongoNodeStateEntryTraverserFactory implements
NodeStateEntryTraverserFactory {
/**
@@ -381,8 +377,8 @@ public abstract class DocumentStoreIndexerBase implements
Closeable {
NodeBuilder builder = copyOnWriteStore.getRoot().builder();
INDEXING_PHASE_LOGGER.info("[TASK:INDEXING:START] Starting
indexing");
Stopwatch indexerWatch = Stopwatch.createStarted();
- try {
- try (CompositeIndexer compositeIndexer =
prepareIndexers(copyOnWriteStore, builder, progressReporter)) {
+ try (NodeStateIndexerProvider indexerProvider = createProvider()) {
+ try (CompositeIndexer compositeIndexer =
prepareIndexers(indexerProvider, copyOnWriteStore, builder, progressReporter)) {
preIndexOperations(compositeIndexer.getIndexers());
if (indexStores.size() > 1) {
indexParallel(indexStores, compositeIndexer,
progressReporter);
@@ -413,7 +409,6 @@ public abstract class DocumentStoreIndexerBase implements
Closeable {
ExtractedTextCache extractedTextCache =
indexerProvider.getTextCache();
CacheStats cacheStats = extractedTextCache == null ? null :
extractedTextCache.getCacheStats();
log.info("Text extraction cache statistics: {}", cacheStats ==
null ? "N/A" : cacheStats.cacheInfoAsString());
- indexerProvider.close();
progressReporter.reindexingTraversalEnd();
progressReporter.logReport();
@@ -519,7 +514,7 @@ public abstract class DocumentStoreIndexerBase implements
Closeable {
traversalLog.trace(id);
}
- protected CompositeIndexer prepareIndexers(NodeStore copyOnWriteStore,
NodeBuilder builder,
+ protected CompositeIndexer prepareIndexers(NodeStateIndexerProvider
indexerProvider, NodeStore copyOnWriteStore, NodeBuilder builder,
IndexingProgressReporter
progressReporter) {
NodeState root = copyOnWriteStore.getRoot();
List<NodeStateIndexer> indexers = new ArrayList<>();
diff --git
a/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
b/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
index e180acfe5b..dd21b98188 100644
---
a/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
+++
b/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
@@ -26,7 +26,6 @@ import
org.apache.jackrabbit.oak.index.indexer.document.NodeStateIndexerProvider
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
import
org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticRetryPolicy;
-import java.io.IOException;
import java.util.List;
/*
@@ -45,7 +44,7 @@ public class ElasticDocumentStoreIndexer extends
DocumentStoreIndexerBase {
public ElasticDocumentStoreIndexer(IndexHelper indexHelper, IndexerSupport
indexerSupport,
String indexPrefix, String scheme,
String host, int port,
- String apiKeyId, String apiSecretId)
throws IOException {
+ String apiKeyId, String apiSecretId) {
super(indexHelper, indexerSupport);
this.indexHelper = indexHelper;
this.indexPrefix = indexPrefix;
@@ -55,7 +54,6 @@ public class ElasticDocumentStoreIndexer extends
DocumentStoreIndexerBase {
this.apiKeyId = apiKeyId;
this.apiSecretId = apiSecretId;
this.retryPolicy =
ElasticRetryPolicy.createRetryPolicyFromSystemProperties();
- setProvider();
}
protected NodeStateIndexerProvider createProvider() {
diff --git
a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
index 607bbca49d..e362fef445 100644
---
a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
+++
b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
@@ -29,10 +29,9 @@ public class DocumentStoreIndexer extends
DocumentStoreIndexerBase implements Cl
private final ExtendedIndexHelper extendedIndexHelper;
- public DocumentStoreIndexer(ExtendedIndexHelper extendedIndexHelper,
IndexerSupport indexerSupport) throws IOException {
+ public DocumentStoreIndexer(ExtendedIndexHelper extendedIndexHelper,
IndexerSupport indexerSupport) {
super(extendedIndexHelper, indexerSupport);
this.extendedIndexHelper = extendedIndexHelper;
- setProvider();
}
private NodeStateIndexerProvider createLuceneIndexProvider() throws
IOException {
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
index 9c3ff5d0ac..463ce783e5 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
@@ -31,6 +31,7 @@ import
org.apache.jackrabbit.oak.index.indexer.document.DocumentStoreIndexerBase
import org.apache.jackrabbit.oak.index.indexer.document.IndexerConfiguration;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateIndexer;
+import
org.apache.jackrabbit.oak.index.indexer.document.NodeStateIndexerProvider;
import
org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils;
import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider;
@@ -367,7 +368,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
CollectingIndexer testIndexer = new CollectingIndexer(p ->
p.startsWith("/test"));
DocumentStoreIndexer index = new DocumentStoreIndexer(helper, support)
{
@Override
- protected CompositeIndexer prepareIndexers(NodeStore nodeStore,
NodeBuilder builder,
+ protected CompositeIndexer
prepareIndexers(NodeStateIndexerProvider indexerProvider, NodeStore nodeStore,
NodeBuilder builder,
IndexingProgressReporter progressReporter) {
return new CompositeIndexer(List.of(testIndexer));
}
@@ -448,7 +449,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
CollectingIndexer testIndexer = new CollectingIndexer(p ->
p.startsWith("/test"));
DocumentStoreIndexer index = new DocumentStoreIndexer(helper,
support) {
@Override
- protected CompositeIndexer prepareIndexers(NodeStore
nodeStore, NodeBuilder builder,
+ protected CompositeIndexer
prepareIndexers(NodeStateIndexerProvider indexerProvider, NodeStore nodeStore,
NodeBuilder builder,
IndexingProgressReporter progressReporter) {
return new CompositeIndexer(List.of(testIndexer));
}
