This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c64040b017 add additional lucene index configs (#11354)
c64040b017 is described below
commit c64040b0171fd0ac5fd0aa9942228100166b86f7
Author: Christopher Peck <[email protected]>
AuthorDate: Fri Aug 18 18:55:29 2023 -0700
add additional lucene index configs (#11354)
---
.../invertedindex/RealtimeLuceneTextIndex.java | 4 +-
.../creator/impl/text/LuceneTextIndexCreator.java | 13 +++---
.../segment/index/text/TextIndexConfigBuilder.java | 11 +++++
.../local/segment/index/text/TextIndexType.java | 3 +-
.../invertedindex/LuceneMutableTextIndexTest.java | 3 +-
.../NativeAndLuceneMutableTextIndexTest.java | 6 ++-
.../segment/store/FilePerIndexDirectoryTest.java | 8 ++--
.../store/SingleFileIndexDirectoryTest.java | 8 ++--
.../pinot/segment/spi/index/TextIndexConfig.java | 53 +++++++++++++++++++---
.../segment/spi/index/TextIndexConfigTest.java | 14 +++++-
.../apache/pinot/spi/config/table/FieldConfig.java | 2 +
11 files changed, 97 insertions(+), 28 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
index 54a684be04..f5ddd1f76f 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
@@ -61,7 +61,7 @@ public class RealtimeLuceneTextIndex implements
MutableTextIndex {
* @param stopWordsExclude stop words to exclude from default stop words
*/
public RealtimeLuceneTextIndex(String column, File segmentIndexDir, String
segmentName,
- List<String> stopWordsInclude, List<String> stopWordsExclude) {
+ List<String> stopWordsInclude, List<String> stopWordsExclude, boolean
useCompoundFile, int maxBufferSizeMB) {
_column = column;
_segmentName = segmentName;
try {
@@ -75,7 +75,7 @@ public class RealtimeLuceneTextIndex implements
MutableTextIndex {
// for realtime
_indexCreator =
new LuceneTextIndexCreator(column, new
File(segmentIndexDir.getAbsolutePath() + "/" + segmentName),
- false /* commitOnClose */, stopWordsInclude, stopWordsExclude);
+ false /* commitOnClose */, stopWordsInclude, stopWordsExclude,
useCompoundFile, maxBufferSizeMB);
IndexWriter indexWriter = _indexCreator.getIndexWriter();
_searcherManager = new SearcherManager(indexWriter, false, false, null);
} catch (Exception e) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
index 1ff32a2b23..3d96f9b226 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
@@ -50,9 +50,6 @@ import
org.apache.pinot.segment.spi.index.creator.DictionaryBasedInvertedIndexCr
* and realtime from {@link RealtimeLuceneTextIndex}
*/
public class LuceneTextIndexCreator extends AbstractTextIndexCreator {
- // TODO: make buffer size configurable choosing a default value based on the
heap usage results in design doc
- private static final int LUCENE_INDEX_MAX_BUFFER_SIZE_MB = 500;
-
public static final String LUCENE_INDEX_DOC_ID_COLUMN_NAME = "DocID";
private final String _textColumn;
@@ -70,7 +67,6 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
public static final CharArraySet ENGLISH_STOP_WORDS_SET = new
CharArraySet(getDefaultEnglishStopWordsSet(), true);
-
/**
* Called by {@link SegmentColumnarIndexCreator}
* when building an offline segment. Similar to how it creates per column
@@ -97,7 +93,8 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
* @param stopWordsExclude the words to exclude from the default stop word
list
*/
public LuceneTextIndexCreator(String column, File segmentIndexDir, boolean
commit,
- @Nullable List<String> stopWordsInclude, @Nullable List<String>
stopWordsExclude) {
+ @Nullable List<String> stopWordsInclude, @Nullable List<String>
stopWordsExclude, boolean useCompoundFile,
+ int maxBufferSizeMB) {
_textColumn = column;
try {
// segment generation is always in V1 and later we convert (as part of
post creation processing)
@@ -108,8 +105,9 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
StandardAnalyzer standardAnalyzer =
TextIndexUtils.getStandardAnalyzerWithCustomizedStopWords(stopWordsInclude,
stopWordsExclude);
IndexWriterConfig indexWriterConfig = new
IndexWriterConfig(standardAnalyzer);
- indexWriterConfig.setRAMBufferSizeMB(LUCENE_INDEX_MAX_BUFFER_SIZE_MB);
+ indexWriterConfig.setRAMBufferSizeMB(maxBufferSizeMB);
indexWriterConfig.setCommitOnClose(commit);
+ indexWriterConfig.setUseCompoundFile(useCompoundFile);
_indexWriter = new IndexWriter(_indexDirectory, indexWriterConfig);
} catch (Exception e) {
throw new RuntimeException(
@@ -119,7 +117,8 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
public LuceneTextIndexCreator(IndexCreationContext context, TextIndexConfig
indexConfig) {
this(context.getFieldSpec().getName(), context.getIndexDir(),
context.isTextCommitOnClose(),
- indexConfig.getStopWordsInclude(), indexConfig.getStopWordsExclude());
+ indexConfig.getStopWordsInclude(), indexConfig.getStopWordsExclude(),
indexConfig.isLuceneUseCompoundFile(),
+ indexConfig.getLuceneMaxBufferSizeMB());
}
public IndexWriter getIndexWriter() {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexConfigBuilder.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexConfigBuilder.java
index 74d82caca6..980cb04ccc 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexConfigBuilder.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexConfigBuilder.java
@@ -51,6 +51,17 @@ public class TextIndexConfigBuilder extends
TextIndexConfig.AbstractBuilder {
_stopWordsInclude =
TextIndexUtils.extractStopWordsInclude(textIndexProperties);
_stopWordsExclude =
TextIndexUtils.extractStopWordsExclude(textIndexProperties);
+ if
(textIndexProperties.get(FieldConfig.TEXT_INDEX_LUCENE_USE_COMPOUND_FILE) !=
null) {
+ _luceneUseCompoundFile =
+
Boolean.parseBoolean(textIndexProperties.get(FieldConfig.TEXT_INDEX_LUCENE_USE_COMPOUND_FILE));
+ }
+
+ if
(textIndexProperties.get(FieldConfig.TEXT_INDEX_LUCENE_MAX_BUFFER_SIZE_MB) !=
null) {
+ _luceneMaxBufferSizeMB =
+
Integer.parseInt(textIndexProperties.get(FieldConfig.TEXT_INDEX_LUCENE_MAX_BUFFER_SIZE_MB));
+ }
+
+
for (Map.Entry<String, String> entry : textIndexProperties.entrySet()) {
if (entry.getKey().equalsIgnoreCase(FieldConfig.TEXT_FST_TYPE)) {
_fstType = FSTType.NATIVE;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
index 54cd746708..12bea6edbf 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
@@ -190,6 +190,7 @@ public class TextIndexType extends
AbstractIndexType<TextIndexConfig, TextIndexR
throw new IllegalArgumentException("A consumer directory is required");
}
return new RealtimeLuceneTextIndex(context.getFieldSpec().getName(),
context.getConsumerDir(),
- context.getSegmentName(), config.getStopWordsInclude(),
config.getStopWordsExclude());
+ context.getSegmentName(), config.getStopWordsInclude(),
config.getStopWordsExclude(),
+ config.isLuceneUseCompoundFile(), config.getLuceneMaxBufferSizeMB());
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
index 8a807b5e5e..d521a981f2 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
@@ -44,7 +44,8 @@ public class LuceneMutableTextIndexTest {
@BeforeClass
public void setUp()
throws Exception {
- _realtimeLuceneTextIndex = new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME,
INDEX_DIR, "fooBar", null, null);
+ _realtimeLuceneTextIndex =
+ new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR, "fooBar",
null, null, true, 500);
String[][] documents = getTextData();
for (String[] row : documents) {
_realtimeLuceneTextIndex.add(row);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
index 2311943ef7..31a82bd343 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
@@ -68,10 +68,12 @@ public class NativeAndLuceneMutableTextIndexTest {
@BeforeClass
public void setUp()
throws Exception {
- _realtimeLuceneTextIndex = new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME,
INDEX_DIR, "fooBar", null, null);
+ _realtimeLuceneTextIndex =
+ new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR, "fooBar",
null, null, true, 500);
_nativeMutableTextIndex = new NativeMutableTextIndex(TEXT_COLUMN_NAME);
- _realtimeLuceneMVTextIndex = new
RealtimeLuceneTextIndex(MV_TEXT_COLUMN_NAME, INDEX_DIR, "fooBar", null, null);
+ _realtimeLuceneMVTextIndex =
+ new RealtimeLuceneTextIndex(MV_TEXT_COLUMN_NAME, INDEX_DIR, "fooBar",
null, null, true, 500);
_nativeMutableMVTextIndex = new
NativeMutableTextIndex(MV_TEXT_COLUMN_NAME);
String[] documents = getTextData();
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
index 7d039d79be..d2ad5b8295 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
@@ -174,9 +174,9 @@ public class FilePerIndexDirectoryTest {
throws IOException {
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true,
- null, null);
+ null, null, true, 500);
LuceneTextIndexCreator barCreator = new LuceneTextIndexCreator("bar",
TEMP_DIR, true,
- null, null)) {
+ null, null, true, 500)) {
PinotDataBuffer buf = fpi.newBuffer("col1", StandardIndexes.forward(),
1024);
buf.putInt(0, 1);
@@ -237,9 +237,9 @@ public class FilePerIndexDirectoryTest {
// Write sth to buffers and flush them to index files on disk
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true,
- null, null);
+ null, null, true, 500);
LuceneTextIndexCreator barCreator = new LuceneTextIndexCreator("bar",
TEMP_DIR, true,
- null, null)) {
+ null, null, true, 500)) {
PinotDataBuffer buf = fpi.newBuffer("col1", StandardIndexes.forward(),
1024);
buf.putInt(0, 111);
buf = fpi.newBuffer("col2", StandardIndexes.dictionary(), 1024);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
index 070691f92f..32feaa0177 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
@@ -235,9 +235,9 @@ public class SingleFileIndexDirectoryTest {
throws IOException, ConfigurationException {
try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true,
- null, null);
+ null, null, true, 500);
LuceneTextIndexCreator barCreator = new LuceneTextIndexCreator("bar",
TEMP_DIR, true,
- null, null)) {
+ null, null, true, 500)) {
PinotDataBuffer buf = sfd.newBuffer("col1", StandardIndexes.forward(),
1024);
buf.putInt(0, 1);
@@ -340,9 +340,9 @@ public class SingleFileIndexDirectoryTest {
throws Exception {
try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true,
- null, null);
+ null, null, true, 500);
LuceneTextIndexCreator barCreator = new LuceneTextIndexCreator("bar",
TEMP_DIR, true,
- null, null)) {
+ null, null, true, 500)) {
PinotDataBuffer buf = sfd.newBuffer("col1", StandardIndexes.forward(),
1024);
buf.putInt(0, 111);
buf = sfd.newBuffer("col2", StandardIndexes.dictionary(), 1024);
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
index babc5645d0..474bfd67e1 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
@@ -32,8 +32,11 @@ import org.apache.pinot.spi.config.table.IndexConfig;
public class TextIndexConfig extends IndexConfig {
- public static final TextIndexConfig DISABLED = new TextIndexConfig(true,
null, null, false, false,
- Collections.emptyList(), Collections.emptyList());
+ private static final int LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB = 500;
+ private static final boolean LUCENE_INDEX_DEFAULT_USE_COMPOUND_FILE = true;
+ public static final TextIndexConfig DISABLED =
+ new TextIndexConfig(true, null, null, false, false,
Collections.emptyList(), Collections.emptyList(), false,
+ LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB);
private final FSTType _fstType;
@Nullable
private final Object _rawValueForTextIndex;
@@ -41,6 +44,8 @@ public class TextIndexConfig extends IndexConfig {
private final boolean _useANDForMultiTermQueries;
private final List<String> _stopWordsInclude;
private final List<String> _stopWordsExclude;
+ private final boolean _luceneUseCompoundFile;
+ private final int _luceneMaxBufferSizeMB;
@JsonCreator
public TextIndexConfig(
@@ -50,7 +55,9 @@ public class TextIndexConfig extends IndexConfig {
@JsonProperty("queryCache") boolean enableQueryCache,
@JsonProperty("useANDForMultiTermQueries") boolean
useANDForMultiTermQueries,
@JsonProperty("stopWordsInclude") List<String> stopWordsInclude,
- @JsonProperty("stopWordsExclude") List<String> stopWordsExclude) {
+ @JsonProperty("stopWordsExclude") List<String> stopWordsExclude,
+ @JsonProperty("luceneUseCompoundFile") Boolean luceneUseCompoundFile,
+ @JsonProperty("luceneMaxBufferSizeMB") Integer luceneMaxBufferSizeMB) {
super(disabled);
_fstType = fstType;
_rawValueForTextIndex = rawValueForTextIndex;
@@ -58,6 +65,10 @@ public class TextIndexConfig extends IndexConfig {
_useANDForMultiTermQueries = useANDForMultiTermQueries;
_stopWordsInclude = stopWordsInclude;
_stopWordsExclude = stopWordsExclude;
+ _luceneUseCompoundFile =
+ luceneUseCompoundFile == null ? LUCENE_INDEX_DEFAULT_USE_COMPOUND_FILE
: luceneUseCompoundFile;
+ _luceneMaxBufferSizeMB =
+ luceneMaxBufferSizeMB == null ?
LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB : luceneMaxBufferSizeMB;
}
public FSTType getFstType() {
@@ -90,6 +101,20 @@ public class TextIndexConfig extends IndexConfig {
return _stopWordsExclude;
}
+ /**
+ * Whether Lucene IndexWriter uses compound file format. Improves indexing
speed but may cause file descriptor issues
+ */
+ public boolean isLuceneUseCompoundFile() {
+ return _luceneUseCompoundFile;
+ }
+
+ /**
+ * Lucene buffer size. Helps with indexing speed but may cause heap issues
+ */
+ public int getLuceneMaxBufferSizeMB() {
+ return _luceneMaxBufferSizeMB;
+ }
+
public static abstract class AbstractBuilder {
@Nullable
protected FSTType _fstType;
@@ -99,6 +124,8 @@ public class TextIndexConfig extends IndexConfig {
protected boolean _useANDForMultiTermQueries = true;
protected List<String> _stopWordsInclude = new ArrayList<>();
protected List<String> _stopWordsExclude = new ArrayList<>();
+ protected boolean _luceneUseCompoundFile =
LUCENE_INDEX_DEFAULT_USE_COMPOUND_FILE;
+ protected int _luceneMaxBufferSizeMB =
LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB;
public AbstractBuilder(@Nullable FSTType fstType) {
_fstType = fstType;
@@ -110,11 +137,13 @@ public class TextIndexConfig extends IndexConfig {
_useANDForMultiTermQueries = other._useANDForMultiTermQueries;
_stopWordsInclude = new ArrayList<>(other._stopWordsInclude);
_stopWordsExclude = new ArrayList<>(other._stopWordsExclude);
+ _luceneUseCompoundFile = other._luceneUseCompoundFile;
+ _luceneMaxBufferSizeMB = other._luceneMaxBufferSizeMB;
}
public TextIndexConfig build() {
return new TextIndexConfig(false, _fstType, _rawValueForTextIndex,
_enableQueryCache, _useANDForMultiTermQueries,
- _stopWordsInclude, _stopWordsExclude);
+ _stopWordsInclude, _stopWordsExclude, _luceneUseCompoundFile,
_luceneMaxBufferSizeMB);
}
public abstract AbstractBuilder withProperties(@Nullable Map<String,
String> textIndexProperties);
@@ -133,6 +162,16 @@ public class TextIndexConfig extends IndexConfig {
_stopWordsExclude = stopWordsExclude;
return this;
}
+
+ public AbstractBuilder withLuceneUseCompoundFile(boolean useCompoundFile) {
+ _luceneUseCompoundFile = useCompoundFile;
+ return this;
+ }
+
+ public AbstractBuilder withLuceneMaxBufferSizeMB(int maxBufferSizeMB) {
+ _luceneMaxBufferSizeMB = maxBufferSizeMB;
+ return this;
+ }
}
@Override
@@ -150,12 +189,14 @@ public class TextIndexConfig extends IndexConfig {
return _enableQueryCache == that._enableQueryCache &&
_useANDForMultiTermQueries == that._useANDForMultiTermQueries
&& _fstType == that._fstType && Objects.equals(_rawValueForTextIndex,
that._rawValueForTextIndex)
&& Objects.equals(_stopWordsInclude, that._stopWordsInclude) &&
Objects.equals(_stopWordsExclude,
- that._stopWordsExclude);
+ that._stopWordsExclude) && _luceneUseCompoundFile ==
that._luceneUseCompoundFile
+ && _luceneMaxBufferSizeMB == that._luceneMaxBufferSizeMB;
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), _fstType, _rawValueForTextIndex,
_enableQueryCache,
- _useANDForMultiTermQueries, _stopWordsInclude, _stopWordsExclude);
+ _useANDForMultiTermQueries, _stopWordsInclude, _stopWordsExclude,
_luceneUseCompoundFile,
+ _luceneMaxBufferSizeMB);
}
}
diff --git
a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/TextIndexConfigTest.java
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/TextIndexConfigTest.java
index fd7147740f..96f57dcccf 100644
---
a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/TextIndexConfigTest.java
+++
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/TextIndexConfigTest.java
@@ -41,6 +41,8 @@ public class TextIndexConfigTest {
assertFalse(config.isUseANDForMultiTermQueries(), "Unexpected
useANDForMultiTermQueries");
assertNull(config.getStopWordsInclude(), "Unexpected stopWordsInclude");
assertNull(config.getStopWordsExclude(), "Unexpected stopWordsExclude");
+ assertTrue(config.isLuceneUseCompoundFile(), "Unexpected
luceneUseCompoundFile");
+ assertEquals(config.getLuceneMaxBufferSizeMB(), 500, "Unexpected
luceneMaxBufferSize");
}
@Test
@@ -56,6 +58,8 @@ public class TextIndexConfigTest {
assertFalse(config.isUseANDForMultiTermQueries(), "Unexpected
useANDForMultiTermQueries");
assertNull(config.getStopWordsInclude(), "Unexpected stopWordsInclude");
assertNull(config.getStopWordsExclude(), "Unexpected stopWordsExclude");
+ assertTrue(config.isLuceneUseCompoundFile(), "Unexpected
luceneUseCompoundFile");
+ assertEquals(config.getLuceneMaxBufferSizeMB(), 500, "Unexpected
luceneMaxBufferSize");
}
@Test
@@ -71,6 +75,8 @@ public class TextIndexConfigTest {
assertFalse(config.isUseANDForMultiTermQueries(), "Unexpected
useANDForMultiTermQueries");
assertNull(config.getStopWordsInclude(), "Unexpected stopWordsInclude");
assertNull(config.getStopWordsExclude(), "Unexpected stopWordsExclude");
+ assertTrue(config.isLuceneUseCompoundFile(), "Unexpected
luceneUseCompoundFile");
+ assertEquals(config.getLuceneMaxBufferSizeMB(), 500, "Unexpected
luceneMaxBufferSize");
}
@Test
@@ -86,6 +92,8 @@ public class TextIndexConfigTest {
assertFalse(config.isUseANDForMultiTermQueries(), "Unexpected
useANDForMultiTermQueries");
assertNull(config.getStopWordsInclude(), "Unexpected stopWordsInclude");
assertNull(config.getStopWordsExclude(), "Unexpected stopWordsExclude");
+ assertTrue(config.isLuceneUseCompoundFile(), "Unexpected
luceneUseCompoundFile");
+ assertEquals(config.getLuceneMaxBufferSizeMB(), 500, "Unexpected
luceneMaxBufferSize");
}
@Test
@@ -97,7 +105,9 @@ public class TextIndexConfigTest {
+ " \"queryCache\": true,\n"
+ " \"useANDForMultiTermQueries\": true,\n"
+ " \"stopWordsInclude\": [\"a\"],\n"
- + " \"stopWordsExclude\": [\"b\"]\n"
+ + " \"stopWordsExclude\": [\"b\"],\n"
+ + " \"luceneUseCompoundFile\": false,\n"
+ + " \"luceneMaxBufferSizeMB\": 1024\n"
+ "}";
TextIndexConfig config = JsonUtils.stringToObject(confStr,
TextIndexConfig.class);
@@ -108,5 +118,7 @@ public class TextIndexConfigTest {
assertTrue(config.isUseANDForMultiTermQueries(), "Unexpected
useANDForMultiTermQueries");
assertEquals(config.getStopWordsInclude(), Lists.newArrayList("a"),
"Unexpected stopWordsInclude");
assertEquals(config.getStopWordsExclude(), Lists.newArrayList("b"),
"Unexpected stopWordsExclude");
+ assertFalse(config.isLuceneUseCompoundFile(), "Unexpected
luceneUseCompoundFile");
+ assertEquals(config.getLuceneMaxBufferSizeMB(), 1024, "Unexpected
luceneMaxBufferSize");
}
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java
index e567bb6f2e..30a42189a4 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java
@@ -49,6 +49,8 @@ public class FieldConfig extends BaseJsonConfig {
public static final String TEXT_INDEX_DEFAULT_RAW_VALUE = "n";
public static final String TEXT_INDEX_STOP_WORD_INCLUDE_KEY =
"stopWordInclude";
public static final String TEXT_INDEX_STOP_WORD_EXCLUDE_KEY =
"stopWordExclude";
+ public static final String TEXT_INDEX_LUCENE_USE_COMPOUND_FILE =
"luceneUseCompoundFile";
+ public static final String TEXT_INDEX_LUCENE_MAX_BUFFER_SIZE_MB =
"luceneMaxBufferSizeMB";
public static final String TEXT_INDEX_STOP_WORD_SEPERATOR = ",";
// "native" for native, default is Lucene
public static final String TEXT_FST_TYPE = "fstType";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
