This is an automated email from the ASF dual-hosted git repository. mcvsubbu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push: new 8a3eb43 Add a new table config field for peer segment download. (#5478) 8a3eb43 is described below commit 8a3eb4315df59d6d73a4ec11308d30eab1c51042 Author: Ting Chen <tingc...@uber.com> AuthorDate: Tue Jun 2 14:00:47 2020 -0700 Add a new table config field for peer segment download. (#5478) * Add a new table config field for peer segment download. * Address feedbacks. * Update pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java Co-authored-by: Subbu Subramaniam <mcvsu...@users.noreply.github.com> * Update pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java Co-authored-by: Subbu Subramaniam <mcvsu...@users.noreply.github.com> Co-authored-by: Subbu Subramaniam <mcvsu...@users.noreply.github.com> --- .../pinot/common/utils/config/TableConfigUtils.java | 18 ++++++++++++++++++ .../common/utils/config/TableConfigSerDeTest.java | 11 +++++++++++ .../table/SegmentsValidationAndRetentionConfig.java | 11 +++++++++++ .../pinot/spi/utils/builder/TableConfigBuilder.java | 7 +++++++ 4 files changed, 47 insertions(+) diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java index d93b24b..b7b7d52 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java @@ -181,9 +181,15 @@ public class TableConfigUtils { * Validates the table config with the following rules: * <ul> * <li>Text index column must be raw</li> + * <li>peerSegmentDownloadScheme in ValidationConfig must be http or https</li> * </ul> */ public static void validate(TableConfig tableConfig) { + validateFieldConfigList(tableConfig); + validateValidationConfig(tableConfig); + } + + private static void validateFieldConfigList(TableConfig tableConfig) { List<FieldConfig> fieldConfigList = tableConfig.getFieldConfigList(); if (fieldConfigList != null) { List<String> noDictionaryColumns = tableConfig.getIndexingConfig().getNoDictionaryColumns(); @@ -201,4 +207,16 @@ public class TableConfigUtils { } } } + + private static void validateValidationConfig(TableConfig tableConfig) { + SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); + if (validationConfig != null) { + String peerSegmentDownloadScheme = validationConfig.getPeerSegmentDownloadScheme(); + if (peerSegmentDownloadScheme != null) { + if (!"http".equalsIgnoreCase(peerSegmentDownloadScheme) && !"https".equalsIgnoreCase(peerSegmentDownloadScheme)) { + throw new IllegalStateException("Invalid value '" + peerSegmentDownloadScheme + "' for peerSegmentDownloadScheme. Must be one of http nor https" ); + } + } + } + } } diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java index 4d34012..d2d2ebf 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java @@ -243,6 +243,17 @@ public class TableConfigSerDeTest { checkTableConfigWithUpsertConfig(JsonUtils.stringToObject(tableConfig.toJsonString(), TableConfig.class)); checkTableConfigWithUpsertConfig(TableConfigUtils.fromZNRecord(TableConfigUtils.toZNRecord(tableConfig))); } + { + // with SegmentsValidationAndRetentionConfig + TableConfig tableConfig = tableConfigBuilder.setPeerSegmentDownloadScheme("http").build(); + checkSegmentsValidationAndRetentionConfig(JsonUtils.stringToObject(tableConfig.toJsonString(), TableConfig.class)); + checkSegmentsValidationAndRetentionConfig(TableConfigUtils.fromZNRecord(TableConfigUtils.toZNRecord(tableConfig))); + } + } + + private void checkSegmentsValidationAndRetentionConfig(TableConfig tableConfig) { + // TODO validate other fields of SegmentsValidationAndRetentionConfig. + assertEquals(tableConfig.getValidationConfig().getPeerSegmentDownloadScheme(), "http"); } private void checkDefaultTableConfig(TableConfig tableConfig) { diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java index 4e0f3e2..30451ab 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java @@ -39,6 +39,11 @@ public class SegmentsValidationAndRetentionConfig extends BaseJsonConfig { private String _segmentAssignmentStrategy; private ReplicaGroupStrategyConfig _replicaGroupStrategyConfig; private CompletionConfig _completionConfig; + // Possible values can be http or https. If this field is set, a Pinot server can download segments from peer servers + // using the specified download scheme. Both realtime tables and offline tables can set this field. + // For more usage of this field, please refer to this design doc: + // https://cwiki.apache.org/confluence/display/PINOT/By-passing+deep-store+requirement+for+Realtime+segment+completion#By-passingdeep-storerequirementforRealtimesegmentcompletion-EnablebesteffortsegmentuploadinSplitSegmentCommiteranddownloadsegmentfrompeerservers. + private String _peerSegmentDownloadScheme; // Number of replicas per partition of low-level consumers. This config is used for realtime tables only. private String _replicasPerPartition; @@ -152,4 +157,10 @@ public class SegmentsValidationAndRetentionConfig extends BaseJsonConfig { public int getReplicasPerPartitionNumber() { return Integer.parseInt(_replicasPerPartition); } + + public String getPeerSegmentDownloadScheme() { return _peerSegmentDownloadScheme; } + + public void setPeerSegmentDownloadScheme(String peerSegmentDownloadScheme) { + _peerSegmentDownloadScheme = peerSegmentDownloadScheme; + } } diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java index 63cc43c..dbd0c03 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java @@ -64,6 +64,7 @@ public class TableConfigBuilder { private String _segmentPushFrequency; private String _segmentPushType = DEFAULT_SEGMENT_PUSH_TYPE; private String _segmentAssignmentStrategy = DEFAULT_SEGMENT_ASSIGNMENT_STRATEGY; + private String _peerSegmentDownloadScheme; private ReplicaGroupStrategyConfig _replicaGroupStrategyConfig; private CompletionConfig _completionConfig; @@ -287,6 +288,11 @@ public class TableConfigBuilder { return this; } + public TableConfigBuilder setPeerSegmentDownloadScheme(String peerSegmentDownloadScheme) { + _peerSegmentDownloadScheme = peerSegmentDownloadScheme; + return this; + } + public TableConfig build() { // Validation config SegmentsValidationAndRetentionConfig validationConfig = new SegmentsValidationAndRetentionConfig(); @@ -301,6 +307,7 @@ public class TableConfigBuilder { validationConfig.setCompletionConfig(_completionConfig); validationConfig.setSchemaName(_schemaName); validationConfig.setReplication(_numReplicas); + validationConfig.setPeerSegmentDownloadScheme(_peerSegmentDownloadScheme); if (_isLLC) { validationConfig.setReplicasPerPartition(_numReplicas); } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org