This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3c02ec48b9 Adding a new config skipSegmentPreprocess in table
IndexingConfig (#14982)
3c02ec48b9 is described below
commit 3c02ec48b9b4291b07518df519d89f59d6cbaa8b
Author: Xiang Fu <[email protected]>
AuthorDate: Mon Feb 3 23:01:00 2025 -0800
Adding a new config skipSegmentPreprocess in table IndexingConfig (#14982)
---
.../local/indexsegment/immutable/ImmutableSegmentLoader.java | 3 +++
.../segment/local/segment/index/loader/IndexLoadingConfig.java | 4 ++++
.../java/org/apache/pinot/spi/config/table/IndexingConfig.java | 9 +++++++++
3 files changed, 16 insertions(+)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
index 4fe545eba5..4650c832e3 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -282,6 +282,9 @@ public class ImmutableSegmentLoader {
public static boolean needPreprocess(SegmentDirectory segmentDirectory,
IndexLoadingConfig indexLoadingConfig,
@Nullable Schema schema)
throws Exception {
+ if (indexLoadingConfig.isSkipSegmentPreprocess()) {
+ return false;
+ }
if (needConvertSegmentFormat(indexLoadingConfig,
segmentDirectory.getSegmentMetadata())) {
return true;
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
index 77512a2a38..cf9fd5e30d 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
@@ -337,6 +337,10 @@ public class IndexLoadingConfig {
_errorOnColumnBuildFailure = errorOnColumnBuildFailure;
}
+ public boolean isSkipSegmentPreprocess() {
+ return _tableConfig != null &&
_tableConfig.getIndexingConfig().isSkipSegmentPreprocess();
+ }
+
@Nullable
public FieldIndexConfigs getFieldIndexConfig(String columnName) {
if (_indexConfigsByColName == null || _dirty) {
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
index 5beb126e0d..16642e2a7f 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
@@ -66,6 +66,7 @@ public class IndexingConfig extends BaseJsonConfig {
private boolean _aggregateMetrics;
private boolean _nullHandlingEnabled;
private boolean _columnMajorSegmentBuilderEnabled = true;
+ private boolean _skipSegmentPreprocess;
/**
* If `optimizeDictionary` enabled, dictionary is not created for the
high-cardinality
@@ -361,6 +362,14 @@ public class IndexingConfig extends BaseJsonConfig {
_columnMajorSegmentBuilderEnabled = columnMajorSegmentBuilderEnabled;
}
+ public boolean isSkipSegmentPreprocess() {
+ return _skipSegmentPreprocess;
+ }
+
+ public void setSkipSegmentPreprocess(boolean skipSegmentPreprocess) {
+ _skipSegmentPreprocess = skipSegmentPreprocess;
+ }
+
public boolean isOptimizeDictionary() {
return _optimizeDictionary;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
