This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch update_table_ingestion_batch_config in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 49b546cc3968af7f79b2d920e2cb6ee930bc3872 Author: Xiang Fu <[email protected]> AuthorDate: Wed Nov 25 01:33:58 2020 -0800 Rename segmentPushType and SegmentPushFrequency to segmentLoadType and segmentLoadFrequency --- .../routing/timeboundary/TimeBoundaryManager.java | 2 +- .../common/utils/config/TableConfigSerDeTest.java | 4 ++-- .../helix/core/retention/RetentionManager.java | 2 +- .../controller/util/TableRetentionValidator.java | 2 +- .../validation/OfflineSegmentIntervalChecker.java | 2 +- .../apache/pinot/core/util/TableConfigUtils.java | 2 +- .../batch/common/SegmentGenerationTaskRunner.java | 4 ++-- .../pinot/hadoop/job/HadoopSegmentCreationJob.java | 2 +- .../hadoop/job/HadoopSegmentPreprocessingJob.java | 4 ++-- .../hadoop/job/mappers/SegmentCreationMapper.java | 4 ++-- .../spark/jobs/SparkSegmentCreationFunction.java | 4 ++-- .../pinot/spark/jobs/SparkSegmentCreationJob.java | 2 +- .../table/ingestion/BatchIngestionConfig.java | 24 +++++++++---------- .../pinot/spi/utils/IngestionConfigUtils.java | 28 +++++++++++----------- .../pinot/spi/utils/IngestionConfigUtilsTest.java | 16 ++++++------- .../segment/converter/SegmentMergeCommand.java | 4 ++-- 16 files changed, 53 insertions(+), 53 deletions(-) diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java index 3097a1b..1aa9f52 100644 --- a/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java +++ b/pinot-broker/src/main/java/org/apache/pinot/broker/routing/timeboundary/TimeBoundaryManager.java @@ -84,7 +84,7 @@ public class TimeBoundaryManager { // For HOURLY table with time unit other than DAYS, use (maxEndTime - 1 HOUR) as the time boundary; otherwise, use // (maxEndTime - 1 DAY) boolean isHourlyTable = CommonConstants.Table.PUSH_FREQUENCY_HOURLY - .equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig)) + .equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig)) && _timeFormatSpec.getColumnUnit() != TimeUnit.DAYS; _timeOffsetMs = isHourlyTable ? TimeUnit.HOURS.toMillis(1) : TimeUnit.DAYS.toMillis(1); diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java index bd7df17..8ad0047 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java @@ -417,8 +417,8 @@ public class TableConfigSerDeTest { assertNotNull(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps()); assertEquals(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps().size(), 1); assertEquals(ingestionConfig.getBatchIngestionConfig().getBatchConfigMaps().get(0).get("batchType"), "s3"); - assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentPushType(), "APPEND"); - assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentPushFrequency(), "HOURLY"); + assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentLoadType(), "APPEND"); + assertEquals(ingestionConfig.getBatchIngestionConfig().getSegmentLoadFrequency(), "HOURLY"); assertNotNull(ingestionConfig.getStreamIngestionConfig()); assertNotNull(ingestionConfig.getStreamIngestionConfig().getStreamConfigMaps()); assertEquals(ingestionConfig.getStreamIngestionConfig().getStreamConfigMaps().size(), 1); diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java index 2dcd3c4..2e522f1 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java @@ -101,7 +101,7 @@ public class RetentionManager extends ControllerPeriodicTask<Void> { // For offline tables, ensure that the segmentPushType is APPEND. SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); - String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig); + String segmentPushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig); if (tableConfig.getTableType() == TableType.OFFLINE && !"APPEND".equalsIgnoreCase(segmentPushType)) { LOGGER.info("Segment push type is not APPEND for table: {}, skip", tableNameWithType); return; diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java index ba44225..f372152 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableRetentionValidator.java @@ -109,7 +109,7 @@ public class TableRetentionValidator { LOGGER.error("Table: {}, \"segmentsConfig\" field is missing in table config", tableName); continue; } - String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig); + String segmentPushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig); if (segmentPushType == null) { LOGGER.error("Table: {}, null push type", tableName); continue; diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java index e21c72e..602aebc 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java @@ -102,7 +102,7 @@ public class OfflineSegmentIntervalChecker extends ControllerPeriodicTask<Void> .warn("Table: {} has {} segments with invalid interval", offlineTableName, numSegmentsWithInvalidIntervals); } Duration frequency = - SegmentIntervalUtils.convertToDuration(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig)); + SegmentIntervalUtils.convertToDuration(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig)); numMissingSegments = computeNumMissingSegments(segmentIntervals, frequency); } // Update the gauge that contains the number of missing segments diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java index e6cafaf..75a353c 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/util/TableConfigUtils.java @@ -116,7 +116,7 @@ public final class TableConfigUtils { String.format("Table: %s, \"segmentsConfig\" field is missing in table config", tableName)); } - String segmentPushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig); + String segmentPushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig); // segmentPushType is not needed for Realtime table if (tableConfig.getTableType() == TableType.OFFLINE && segmentPushType != null && !segmentPushType.isEmpty()) { if (!segmentPushType.equalsIgnoreCase("REFRESH") && !segmentPushType.equalsIgnoreCase("APPEND")) { diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java index 4d15022..f304701 100644 --- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java +++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java @@ -137,8 +137,8 @@ public class SegmentGenerationTaskRunner implements Serializable { } return new NormalizedDateSegmentNameGenerator(tableName, segmentNameGeneratorConfigs.get(SEGMENT_NAME_PREFIX), Boolean.parseBoolean(segmentNameGeneratorConfigs.get(EXCLUDE_SEQUENCE_ID)), - IngestionConfigUtils.getBatchSegmentPushType(tableConfig), - IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), dateTimeFormatSpec); + IngestionConfigUtils.getBatchSegmentLoadType(tableConfig), + IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), dateTimeFormatSpec); default: throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType); } diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java index 2b77efc..66c0ef4 100644 --- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java +++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java @@ -143,7 +143,7 @@ public class HadoopSegmentCreationJob extends SegmentCreationJob { SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); // For APPEND use case, timeColumnName and timeType must be set - if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushType(tableConfig))) { + if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig))) { Preconditions.checkState(validationConfig.getTimeColumnName() != null && validationConfig.getTimeType() != null, "For APPEND use case, time column and type must be set"); } diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java index 9e5e3e1..036adb8 100644 --- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java +++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java @@ -368,7 +368,7 @@ public class HadoopSegmentPreprocessingJob extends SegmentPreprocessingJob { // If the use case is an append use case, check that one time unit is contained in one file. If there is more than one, // the job should be disabled, as we should not resize for these use cases. Therefore, setting the time column name // and value - if (IngestionConfigUtils.getBatchSegmentPushType(_tableConfig).equalsIgnoreCase("APPEND")) { + if (IngestionConfigUtils.getBatchSegmentLoadType(_tableConfig).equalsIgnoreCase("APPEND")) { job.getConfiguration().set(InternalConfigConstants.IS_APPEND, "true"); String timeColumnName = validationConfig.getTimeColumnName(); job.getConfiguration().set(InternalConfigConstants.TIME_COLUMN_CONFIG, timeColumnName); @@ -385,7 +385,7 @@ public class HadoopSegmentPreprocessingJob extends SegmentPreprocessingJob { } } job.getConfiguration().set(InternalConfigConstants.SEGMENT_PUSH_FREQUENCY, - IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig)); + IngestionConfigUtils.getBatchSegmentLoadFrequency(_tableConfig)); try (DataFileStream<GenericRecord> dataStreamReader = getAvroReader(path)) { job.getConfiguration() .set(InternalConfigConstants.TIME_COLUMN_VALUE, dataStreamReader.next().get(timeColumnName).toString()); diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java index 9efb7a3..594b3bd 100644 --- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java +++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java @@ -163,8 +163,8 @@ public class SegmentCreationMapper extends Mapper<LongWritable, Text, LongWritab _segmentNameGenerator = new NormalizedDateSegmentNameGenerator(_rawTableName, _jobConf.get(JobConfigConstants.SEGMENT_NAME_PREFIX), _jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false), - IngestionConfigUtils.getBatchSegmentPushType(_tableConfig), - IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig), dateTimeFormatSpec); + IngestionConfigUtils.getBatchSegmentLoadType(_tableConfig), + IngestionConfigUtils.getBatchSegmentLoadFrequency(_tableConfig), dateTimeFormatSpec); break; default: throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType); diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java index 60d3ef9..a0c055b 100644 --- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java +++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java @@ -131,8 +131,8 @@ public class SparkSegmentCreationFunction implements Serializable { _segmentNameGenerator = new NormalizedDateSegmentNameGenerator(_rawTableName, _jobConf.get(JobConfigConstants.SEGMENT_NAME_PREFIX), _jobConf.getBoolean(JobConfigConstants.EXCLUDE_SEQUENCE_ID, false), - IngestionConfigUtils.getBatchSegmentPushType(_tableConfig), - IngestionConfigUtils.getBatchSegmentPushFrequency(_tableConfig), dateTimeFormatSpec); + IngestionConfigUtils.getBatchSegmentLoadType(_tableConfig), + IngestionConfigUtils.getBatchSegmentLoadFrequency(_tableConfig), dateTimeFormatSpec); break; default: throw new UnsupportedOperationException("Unsupported segment name generator type: " + segmentNameGeneratorType); diff --git a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java index d737020..7abf57b 100644 --- a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java +++ b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java @@ -156,7 +156,7 @@ public class SparkSegmentCreationJob extends SegmentCreationJob { SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); // For APPEND use case, timeColumnName and timeType must be set - if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentPushType(tableConfig))) { + if (APPEND.equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig))) { Preconditions.checkState(validationConfig.getTimeColumnName() != null && validationConfig.getTimeType() != null, "For APPEND use case, time column and type must be set"); } diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java index e9b8718..90e1b8e 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/BatchIngestionConfig.java @@ -35,19 +35,19 @@ public class BatchIngestionConfig extends BaseJsonConfig { @JsonPropertyDescription("Configs for all the batch sources to ingest from") private final List<Map<String, String>> _batchConfigMaps; - @JsonPropertyDescription("Push type APPEND or REFRESH") - private final String _segmentPushType; + @JsonPropertyDescription("Load type APPEND or REFRESH") + private final String _segmentLoadType; - @JsonPropertyDescription("Push frequency HOURLY or DAILY") - private final String _segmentPushFrequency; + @JsonPropertyDescription("Load frequency HOURLY or DAILY") + private final String _segmentLoadFrequency; @JsonCreator public BatchIngestionConfig(@JsonProperty("batchConfigMaps") @Nullable List<Map<String, String>> batchConfigMaps, - @JsonProperty("segmentPushType") String segmentPushType, - @JsonProperty("segmentPushFrequency") String segmentPushFrequency) { + @JsonProperty("segmentLoadType") String segmentLoadType, + @JsonProperty("segmentLoadFrequency") String segmentLoadFrequency) { _batchConfigMaps = batchConfigMaps; - _segmentPushType = segmentPushType; - _segmentPushFrequency = segmentPushFrequency; + _segmentLoadType = segmentLoadType; + _segmentLoadFrequency = segmentLoadFrequency; } @Nullable @@ -55,11 +55,11 @@ public class BatchIngestionConfig extends BaseJsonConfig { return _batchConfigMaps; } - public String getSegmentPushType() { - return _segmentPushType; + public String getSegmentLoadType() { + return _segmentLoadType; } - public String getSegmentPushFrequency() { - return _segmentPushFrequency; + public String getSegmentLoadFrequency() { + return _segmentLoadFrequency; } } diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java index 717aab2..187ad68 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java @@ -57,39 +57,39 @@ public final class IngestionConfigUtils { } /** - * Fetches the configured segmentPushType (APPEND/REFRESH) from the table config + * Fetches the configured segmentLoadType (APPEND/REFRESH) from the table config * First checks in the ingestionConfig. If not found, checks in the segmentsConfig (has been deprecated from here in favor of ingestion config) */ - public static String getBatchSegmentPushType(TableConfig tableConfig) { - String segmentPushType = null; + public static String getBatchSegmentLoadType(TableConfig tableConfig) { + String segmentLoadType = null; if (tableConfig.getIngestionConfig() != null) { BatchIngestionConfig batchIngestionConfig = tableConfig.getIngestionConfig().getBatchIngestionConfig(); if (batchIngestionConfig != null) { - segmentPushType = batchIngestionConfig.getSegmentPushType(); + segmentLoadType = batchIngestionConfig.getSegmentLoadType(); } } - if (segmentPushType == null) { - segmentPushType = tableConfig.getValidationConfig().getSegmentPushType(); + if (segmentLoadType == null) { + segmentLoadType = tableConfig.getValidationConfig().getSegmentPushType(); } - return segmentPushType; + return segmentLoadType; } /** - * Fetches the configured segmentPushFrequency from the table config + * Fetches the configured segmentLoadFrequency from the table config * First checks in the ingestionConfig. If not found, checks in the segmentsConfig (has been deprecated from here in favor of ingestion config) */ - public static String getBatchSegmentPushFrequency(TableConfig tableConfig) { - String segmentPushFrequency = null; + public static String getBatchSegmentLoadFrequency(TableConfig tableConfig) { + String segmentLoadFrequency = null; if (tableConfig.getIngestionConfig() != null) { BatchIngestionConfig batchIngestionConfig = tableConfig.getIngestionConfig().getBatchIngestionConfig(); if (batchIngestionConfig != null) { - segmentPushFrequency = batchIngestionConfig.getSegmentPushFrequency(); + segmentLoadFrequency = batchIngestionConfig.getSegmentLoadFrequency(); } } - if (segmentPushFrequency == null) { - segmentPushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency(); + if (segmentLoadFrequency == null) { + segmentLoadFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency(); } - return segmentPushFrequency; + return segmentLoadFrequency; } } diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java index dee56bb..cbd5c4c 100644 --- a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java +++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java @@ -103,23 +103,23 @@ public class IngestionConfigUtilsTest { TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build(); tableConfig .setIngestionConfig(new IngestionConfig(new BatchIngestionConfig(null, "APPEND", "HOURLY"), null, null, null)); - Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "HOURLY"); + Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), "HOURLY"); // get from ingestion config, even if present in segmentsConfig SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig = new SegmentsValidationAndRetentionConfig(); segmentsValidationAndRetentionConfig.setSegmentPushFrequency("DAILY"); tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig); - Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "HOURLY"); + Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), "HOURLY"); // get from segmentsConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build(); tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig); - Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig), "DAILY"); + Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig), "DAILY"); // present nowhere segmentsValidationAndRetentionConfig.setSegmentPushFrequency(null); - Assert.assertNull(IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig)); + Assert.assertNull(IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig)); } @Test @@ -128,22 +128,22 @@ public class IngestionConfigUtilsTest { TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build(); tableConfig .setIngestionConfig(new IngestionConfig(new BatchIngestionConfig(null, "APPEND", "HOURLY"), null, null, null)); - Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "APPEND"); + Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig), "APPEND"); // get from ingestion config, even if present in segmentsConfig SegmentsValidationAndRetentionConfig segmentsValidationAndRetentionConfig = new SegmentsValidationAndRetentionConfig(); segmentsValidationAndRetentionConfig.setSegmentPushType("REFRESH"); tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig); - Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "APPEND"); + Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig), "APPEND"); // get from segmentsConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("myTable").build(); tableConfig.setValidationConfig(segmentsValidationAndRetentionConfig); - Assert.assertEquals(IngestionConfigUtils.getBatchSegmentPushType(tableConfig), "REFRESH"); + Assert.assertEquals(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig), "REFRESH"); // present nowhere segmentsValidationAndRetentionConfig.setSegmentPushType(null); - Assert.assertNull(IngestionConfigUtils.getBatchSegmentPushType(tableConfig)); + Assert.assertNull(IngestionConfigUtils.getBatchSegmentLoadType(tableConfig)); } } diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java index be2196d..dbbffbe 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/SegmentMergeCommand.java @@ -239,8 +239,8 @@ public class SegmentMergeCommand extends AbstractBaseAdminCommand implements Com // Fetch time related configurations from schema and table config. SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); - String pushFrequency = IngestionConfigUtils.getBatchSegmentPushFrequency(tableConfig); - String pushType = IngestionConfigUtils.getBatchSegmentPushType(tableConfig); + String pushFrequency = IngestionConfigUtils.getBatchSegmentLoadFrequency(tableConfig); + String pushType = IngestionConfigUtils.getBatchSegmentLoadType(tableConfig); String timeColumnName = validationConfig.getTimeColumnName(); DateTimeFormatSpec dateTimeFormatSpec = null; if (timeColumnName != null) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
