This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0fc9ec6e94 Add validations for Pauseless Tables (#15567)
0fc9ec6e94 is described below
commit 0fc9ec6e94866ed2cbee41aef5568c5e34a6c199
Author: Kartik Khare <[email protected]>
AuthorDate: Sat Apr 26 00:46:20 2025 +0530
Add validations for Pauseless Tables (#15567)
* Validate peer download scheme is set when creating a pauseless table
* Fix tests
* Enforce peer download scheme only when rf > 1
---------
Co-authored-by: KKCorps <[email protected]>
---
.../segment/local/utils/TableConfigUtils.java | 23 ++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
index 43358e5019..9bba44a487 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
@@ -303,8 +303,7 @@ public final class TableConfigUtils {
String peerSegmentDownloadScheme =
validationConfig.getPeerSegmentDownloadScheme();
if (peerSegmentDownloadScheme != null) {
- if
(!CommonConstants.HTTP_PROTOCOL.equalsIgnoreCase(peerSegmentDownloadScheme)
- &&
!CommonConstants.HTTPS_PROTOCOL.equalsIgnoreCase(peerSegmentDownloadScheme)) {
+ if (!isValidPeerDownloadScheme(peerSegmentDownloadScheme)) {
throw new IllegalStateException("Invalid value '" +
peerSegmentDownloadScheme
+ "' for peerSegmentDownloadScheme. Must be one of http or https");
}
@@ -317,6 +316,11 @@ public final class TableConfigUtils {
validateRetentionConfig(tableConfig);
}
+ private static boolean isValidPeerDownloadScheme(String
peerSegmentDownloadScheme) {
+ return
CommonConstants.HTTP_PROTOCOL.equalsIgnoreCase(peerSegmentDownloadScheme)
+ ||
CommonConstants.HTTPS_PROTOCOL.equalsIgnoreCase(peerSegmentDownloadScheme);
+ }
+
/**
* Validates the following:
* 1. validity of filter function
@@ -364,6 +368,21 @@ public final class TableConfigUtils {
List<Map<String, String>> streamConfigMaps =
ingestionConfig.getStreamIngestionConfig().getStreamConfigMaps();
Preconditions.checkState(!streamConfigMaps.isEmpty(), "Must have at
least 1 stream in REALTIME table");
// TODO: for multiple stream configs, validate them
+
+ boolean isPauselessEnabled =
ingestionConfig.getStreamIngestionConfig().isPauselessConsumptionEnabled();
+ if (isPauselessEnabled) {
+ int replication = tableConfig.getReplication();
+ // We are checking for this only when replication is greater than 1
because in test environments
+ // users still prefer to create pauseless tables with replication 1
+ if (replication > 1) {
+ String peerSegmentDownloadScheme =
tableConfig.getValidationConfig().getPeerSegmentDownloadScheme();
+
Preconditions.checkState(StringUtils.isNotEmpty(peerSegmentDownloadScheme) &&
isValidPeerDownloadScheme(
+ peerSegmentDownloadScheme),
+ "Must have a valid peerSegmentDownloadScheme set in validation
config for pauseless consumption");
+ } else {
+ LOGGER.warn("It's not recommended to create pauseless tables with
replication 1 for stability reasons.");
+ }
+ }
}
// Filter config
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
