This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6fb315dc9c prevent background merges on the realtime lucene index
(#13050)
6fb315dc9c is described below
commit 6fb315dc9c5f5b2b348cacdca072159cd22b1b10
Author: Christopher Peck <27231838+itschrisp...@users.noreply.github.com>
AuthorDate: Fri May 3 14:02:29 2024 -0700
prevent background merges on the realtime lucene index (#13050)
---
.../segment/creator/impl/text/LuceneTextIndexCreator.java | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
index 2cdbf13f6a..c24778ab37 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
@@ -36,6 +36,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoMergeScheduler;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -131,6 +132,15 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
indexWriterConfig.setCommitOnClose(commit);
indexWriterConfig.setUseCompoundFile(config.isLuceneUseCompoundFile());
+ // For the realtime segment, prevent background merging. The realtime
segment will call .commit()
+ // on the IndexWriter when segment conversion occurs. By default, Lucene
will sometimes choose to
+ // merge segments in the background, which is problematic because the
lucene index directory's
+ // contents is copied to create the immutable segment. If a background
merge occurs during this
+ // copy, a FileNotFoundException will be triggered and segment build
will fail.
+ if (!_commitOnClose) {
+ indexWriterConfig.setMergeScheduler(NoMergeScheduler.INSTANCE);
+ }
+
if (_reuseMutableIndex) {
LOGGER.info("Reusing the realtime lucene index for segment {} and
column {}", segmentIndexDir, column);
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
