shuturmurgh commented on code in PR #18486:
URL: https://github.com/apache/pinot/pull/18486#discussion_r3240606041
##########
pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/SegmentValidationUtils.java:
##########
@@ -57,6 +63,56 @@ public static void validateTimeInterval(SegmentMetadata
segmentMetadata, TableCo
}
}
+ /**
+ * When {@code controller.segment.upload.rejectOutOfRetention.enabled} is
true on the controller, rejects the upload
+ * if segment end time (or index creation time when {@code
controller.retentionManager.enableCreationTimeFallback} is
+ * enabled and end time is invalid) is past the table's {@code
retentionTimeValue}/{@code retentionTimeUnit} window,
+ * using the same boundary as the controller retention manager.
+ *
+ * <p>Controller wiring: {@link
org.apache.pinot.controller.api.resources.PinotSegmentUploadDownloadRestletResource}
+ * invokes this for single-segment upload only. METADATA batch upload
({@code POST /segments/batchUpload}) and
+ * reingested-segment upload do not call it.
+ *
+ * <p>For <b>OFFLINE</b> tables, when batch segment ingestion type is not
<b>APPEND</b>, this method returns without
+ * evaluating retention (aligned with time retention for completed
segments). <b>REALTIME</b> tables always proceed
+ * to the retention comparison when the flag and parseable retention are set.
+ */
+ public static void rejectUploadIfOutOfRetention(SegmentMetadata
segmentMetadata, TableConfig tableConfig,
+ long currentTimeMs, boolean controllerRejectOutOfRetentionEnabled,
+ boolean retentionCreationTimeFallbackEnabled) {
+ if (!controllerRejectOutOfRetentionEnabled) {
+ return;
+ }
+ SegmentsValidationAndRetentionConfig validationConfig =
tableConfig.getValidationConfig();
+ if (validationConfig == null) {
+ return;
+ }
+ if (tableConfig.getTableType() == TableType.OFFLINE
+ &&
!"APPEND".equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig)))
{
+ return;
+ }
+ OptionalLong retentionMsOpt =
RetentionUtils.parseTableDataRetentionMillis(validationConfig);
+ if (retentionMsOpt.isEmpty()) {
+ if (StringUtils.isNotEmpty(validationConfig.getRetentionTimeUnit())
+ && StringUtils.isNotEmpty(validationConfig.getRetentionTimeValue()))
{
+ LOGGER.warn("Invalid retention {} {} for table {}, skipping upload
retention check",
+ validationConfig.getRetentionTimeUnit(),
validationConfig.getRetentionTimeValue(),
+ tableConfig.getTableName());
+ }
+ return;
+ }
+ long retentionMs = retentionMsOpt.getAsLong();
+ if (RetentionUtils.isPurgeable(tableConfig.getTableName(),
segmentMetadata, retentionMs, currentTimeMs,
Review Comment:
Added a metric. This behavior is opt-in so it won't break existing clients.
Maybe if other contributors have feedback for such a use case we can add
another config for no-action mode.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
