This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c43f05a355f Block UpsertCompactionTask and UpsertCompactMergeTask when
metadataTTL is enabled (#19057)
c43f05a355f is described below
commit c43f05a355fde11ae01125cb18defbf61dd64f2e
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Mon Jul 27 10:57:51 2026 -0700
Block UpsertCompactionTask and UpsertCompactMergeTask when metadataTTL is
enabled (#19057)
---
.../upsertcompaction/UpsertCompactionTaskGenerator.java | 10 ++++++++--
.../upsertcompactmerge/UpsertCompactMergeTaskGenerator.java | 6 ++++++
.../upsertcompaction/UpsertCompactionTaskGeneratorTest.java | 11 +++++++++++
.../UpsertCompactMergeTaskGeneratorTest.java | 12 ++++++++++++
4 files changed, 37 insertions(+), 2 deletions(-)
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
index b9c077a1c7e..45a285588c9 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java
@@ -301,6 +301,14 @@ public class UpsertCompactionTaskGenerator extends
BaseTaskGenerator {
"UpsertCompactionTask only supports realtime tables!");
// check upsert enabled
Preconditions.checkState(tableConfig.isUpsertEnabled(), "Upsert must be
enabled for UpsertCompactionTask");
+ // check metadataTTL is not set: UpsertCompactionTask does not compact
tombstoned rows in a way that is
+ // consistent with metadataTTL-driven cleanup, so enabling them together
can leave stale rows behind or
+ // resurface aged-out keys. Block the combination to avoid silent
correctness issues.
+ UpsertConfig upsertConfig = tableConfig.getUpsertConfig();
+ assert upsertConfig != null;
+ Preconditions.checkState(upsertConfig.getMetadataTTL() <= 0,
+ "UpsertCompactionTask does not support tables with 'metadataTTL'
enabled, got metadataTTL: %s",
+ upsertConfig.getMetadataTTL());
// check no malformed period
if (taskConfigs.containsKey(UpsertCompactionTask.BUFFER_TIME_PERIOD_KEY)) {
@@ -326,8 +334,6 @@ public class UpsertCompactionTaskGenerator extends
BaseTaskGenerator {
"invalidRecordsThresholdPercent or invalidRecordsThresholdCount or
both must be provided");
// validate validDocIdsType default logic
- UpsertConfig upsertConfig = tableConfig.getUpsertConfig();
- assert upsertConfig != null;
MinionTaskUtils.getValidDocIdsType(upsertConfig, taskConfigs,
UpsertCompactionTask.VALID_DOC_IDS_TYPE);
}
}
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGenerator.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGenerator.java
index 45e14405b62..e719a4a7368 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGenerator.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGenerator.java
@@ -481,6 +481,12 @@ public class UpsertCompactMergeTaskGenerator extends
BaseTaskGenerator {
// NOTE: Allow snapshot to be DEFAULT because it might be enabled at
server level.
Preconditions.checkState(upsertConfig.getSnapshot() != Enablement.DISABLE,
"'snapshot' from UpsertConfig must not be 'DISABLE' for %s",
MinionConstants.UpsertCompactMergeTask.TASK_TYPE);
+ // check metadataTTL is not set: UpsertCompactMergeTask does not compact
tombstoned rows in a way that is
+ // consistent with metadataTTL-driven cleanup, so enabling them together
can leave stale rows behind or
+ // resurface aged-out keys. Block the combination to avoid silent
correctness issues.
+ Preconditions.checkState(upsertConfig.getMetadataTTL() <= 0,
+ "%s does not support tables with 'metadataTTL' enabled, got
metadataTTL: %s",
+ MinionConstants.UpsertCompactMergeTask.TASK_TYPE,
upsertConfig.getMetadataTTL());
// check no malformed period
if
(taskConfigs.containsKey(MinionConstants.UpsertCompactMergeTask.BUFFER_TIME_PERIOD_KEY))
{
TimeUtils.convertPeriodToMillis(taskConfigs.get(MinionConstants.UpsertCompactMergeTask.BUFFER_TIME_PERIOD_KEY));
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
index 33f8e4e3d7e..bc6be0b21ee 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
@@ -478,6 +478,17 @@ public class UpsertCompactionTaskGeneratorTest {
.build();
Assert.assertThrows(IllegalStateException.class,
() -> _taskGenerator.validateTaskConfigs(invalidTableConfig, new
Schema(), upsertCompactionTaskConfig5));
+
+ // metadataTTL enabled is not supported with UpsertCompactionTask
+ UpsertConfig ttlUpsertConfig = new UpsertConfig(UpsertConfig.Mode.FULL);
+ ttlUpsertConfig.setSnapshot(Enablement.ENABLE);
+ ttlUpsertConfig.setMetadataTTL(30);
+ TableConfig metadataTtlTableConfig =
+ new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setUpsertConfig(ttlUpsertConfig)
+ .setTaskConfig(new TableTaskConfig(Map.of("UpsertCompactionTask",
upsertCompactionTaskConfig)))
+ .build();
+ Assert.assertThrows(IllegalStateException.class,
+ () -> _taskGenerator.validateTaskConfigs(metadataTtlTableConfig, new
Schema(), upsertCompactionTaskConfig));
}
@Test
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGeneratorTest.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGeneratorTest.java
index 60104b47286..ff5e00211b5 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGeneratorTest.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompactmerge/UpsertCompactMergeTaskGeneratorTest.java
@@ -164,6 +164,18 @@ public class UpsertCompactMergeTaskGeneratorTest {
Map<String, String> upsertCompactMergeTaskConfig1 =
Map.of("bufferTimePeriod", "5hd");
Assert.assertThrows(IllegalArgumentException.class,
() -> _taskGenerator.validateTaskConfigs(validTableConfig, new
Schema(), upsertCompactMergeTaskConfig1));
+
+ // metadataTTL enabled is not supported with UpsertCompactMergeTask
+ UpsertConfig ttlUpsertConfig = new UpsertConfig(UpsertConfig.Mode.FULL);
+ ttlUpsertConfig.setSnapshot(Enablement.ENABLE);
+ ttlUpsertConfig.setMetadataTTL(30);
+ TableConfig metadataTtlTableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME)
+ .setUpsertConfig(ttlUpsertConfig)
+ .setTaskConfig(
+ new
TableTaskConfig(Map.of(MinionConstants.UpsertCompactMergeTask.TASK_TYPE,
upsertCompactMergeTaskConfig)))
+ .build();
+ Assert.assertThrows(IllegalStateException.class,
+ () -> _taskGenerator.validateTaskConfigs(metadataTtlTableConfig, new
Schema(), upsertCompactMergeTaskConfig));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
