This is an automated email from the ASF dual-hosted git repository.
ankitsultana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f9af5f1c29 Add option for ignoreCrcMismatch for upsert-compaction task
(#14668)
f9af5f1c29 is described below
commit f9af5f1c29ce5d0f4402e5970c35a86211a5f3a8
Author: Pratik Tibrewal <[email protected]>
AuthorDate: Wed Dec 18 00:25:22 2024 +0530
Add option for ignoreCrcMismatch for upsert-compaction task (#14668)
---
.../java/org/apache/pinot/core/common/MinionConstants.java | 10 ++++++++++
.../tasks/upsertcompaction/UpsertCompactionTaskExecutor.java | 9 ++++++---
.../tasks/upsertcompaction/UpsertCompactionTaskGenerator.java | 3 +++
3 files changed, 19 insertions(+), 3 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
index 08b0eca909..7a276d6254 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
@@ -219,6 +219,16 @@ public class MinionConstants {
*/
public static final String SNAPSHOT = "snapshot";
+ /**
+ * key representing if upsert compaction task executor should ignore crc
mismatch or not during task execution
+ */
+ public static final String IGNORE_CRC_MISMATCH_KEY = "ignoreCrcMismatch";
+
+ /**
+ * default value for the key IGNORE_CRC_MISMATCH_KEY: false
+ */
+ public static final boolean DEFAULT_IGNORE_CRC_MISMATCH = false;
+
/**
* number of segments to query in one batch to fetch valid doc id
metadata, by default 500
*/
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java
index 12f9ee12bb..e5469a22ae 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java
@@ -26,6 +26,7 @@ import
org.apache.pinot.common.metadata.segment.SegmentZKMetadataCustomMapModifi
import org.apache.pinot.common.metrics.MinionMeter;
import org.apache.pinot.common.restlet.resources.ValidDocIdsType;
import org.apache.pinot.core.common.MinionConstants;
+import org.apache.pinot.core.common.MinionConstants.UpsertCompactionTask;
import org.apache.pinot.core.minion.PinotTaskConfig;
import
org.apache.pinot.plugin.minion.tasks.BaseSingleSegmentConversionExecutor;
import org.apache.pinot.plugin.minion.tasks.MinionTaskUtils;
@@ -58,11 +59,13 @@ public class UpsertCompactionTaskExecutor extends
BaseSingleSegmentConversionExe
TableConfig tableConfig = getTableConfig(tableNameWithType);
String validDocIdsTypeStr =
-
configs.getOrDefault(MinionConstants.UpsertCompactionTask.VALID_DOC_IDS_TYPE,
ValidDocIdsType.SNAPSHOT.name());
+ configs.getOrDefault(UpsertCompactionTask.VALID_DOC_IDS_TYPE,
ValidDocIdsType.SNAPSHOT.name());
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
String originalSegmentCrcFromTaskGenerator =
configs.get(MinionConstants.ORIGINAL_SEGMENT_CRC_KEY);
String crcFromDeepStorageSegment = segmentMetadata.getCrc();
- if
(!originalSegmentCrcFromTaskGenerator.equals(crcFromDeepStorageSegment)) {
+ boolean ignoreCrcMismatch =
Boolean.parseBoolean(configs.getOrDefault(UpsertCompactionTask.IGNORE_CRC_MISMATCH_KEY,
+ String.valueOf(UpsertCompactionTask.DEFAULT_IGNORE_CRC_MISMATCH)));
+ if (!ignoreCrcMismatch &&
!originalSegmentCrcFromTaskGenerator.equals(crcFromDeepStorageSegment)) {
String message = String.format("Crc mismatched between ZK and deepstore
copy of segment: %s. Expected crc "
+ "from ZK: %s, crc from deepstore: %s", segmentName,
originalSegmentCrcFromTaskGenerator,
crcFromDeepStorageSegment);
@@ -145,7 +148,7 @@ public class UpsertCompactionTaskExecutor extends
BaseSingleSegmentConversionExe
protected SegmentZKMetadataCustomMapModifier
getSegmentZKMetadataCustomMapModifier(PinotTaskConfig pinotTaskConfig,
SegmentConversionResult segmentConversionResult) {
return new
SegmentZKMetadataCustomMapModifier(SegmentZKMetadataCustomMapModifier.ModifyMode.UPDATE,
-
Collections.singletonMap(MinionConstants.UpsertCompactionTask.TASK_TYPE +
MinionConstants.TASK_TIME_SUFFIX,
+ Collections.singletonMap(UpsertCompactionTask.TASK_TYPE +
MinionConstants.TASK_TIME_SUFFIX,
String.valueOf(System.currentTimeMillis())));
}
}
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
index 2fa814db01..77aaefc069 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
@@ -185,6 +185,9 @@ public class UpsertCompactionTaskGenerator extends
BaseTaskGenerator {
configs.put(MinionConstants.UPLOAD_URL_KEY,
_clusterInfoAccessor.getVipUrl() + "/segments");
configs.put(MinionConstants.ORIGINAL_SEGMENT_CRC_KEY,
String.valueOf(segment.getCrc()));
configs.put(UpsertCompactionTask.VALID_DOC_IDS_TYPE,
validDocIdsType.toString());
+ configs.put(UpsertCompactionTask.IGNORE_CRC_MISMATCH_KEY,
+
taskConfigs.getOrDefault(UpsertCompactionTask.IGNORE_CRC_MISMATCH_KEY,
+ String.valueOf(UpsertCompactionTask.DEFAULT_IGNORE_CRC_MISMATCH)));
pinotTaskConfigs.add(new
PinotTaskConfig(UpsertCompactionTask.TASK_TYPE, configs));
numTasks++;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
