This is an automated email from the ASF dual-hosted git repository. FrankChen021 pushed a commit to branch codex/native-compaction-test-matrix in repository https://gitbox.apache.org/repos/asf/druid.git
commit a3da69d4cebf6ee82e51a67c9f9344270498a82c Author: Frank Chen <[email protected]> AuthorDate: Wed Sep 9 17:33:08 2026 +0800 test: select compaction configurations directly --- .../common/task/CompactionTaskRunBase.java | 44 ++++++------- .../common/task/CompactionTaskRunTestCases.java | 75 ++++------------------ .../druid/msq/exec/MSQCompactionTaskRunTest.java | 30 +++------ 3 files changed, 43 insertions(+), 106 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java index 9638839a2b6..620ec738fa2 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java @@ -60,7 +60,7 @@ import org.apache.druid.indexing.common.config.TaskConfigBuilder; import org.apache.druid.indexing.common.task.CompactionTask.Builder; import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.CompactionTest; import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Configuration; -import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Scenario; +import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Selection; import org.apache.druid.indexing.overlord.Segments; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; @@ -336,7 +336,7 @@ public abstract class CompactionTaskRunBase { } - @CompactionTest(Scenario.RUN_WITH_DYNAMIC_PARTITIONING) + @CompactionTest(Selection.ALL) public void testRunWithDynamicPartitioning(Configuration configuration) throws Exception { startCase(configuration); @@ -361,7 +361,7 @@ public abstract class CompactionTaskRunBase ); } - @CompactionTest(Scenario.RUN_WITH_HASH_PARTITIONING) + @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY) public void testRunWithHashPartitioning(Configuration configuration) throws Exception { startCase(configuration); @@ -402,7 +402,7 @@ public abstract class CompactionTaskRunBase } } - @CompactionTest(Scenario.RUN_COMPACTION_TWICE) + @CompactionTest(Selection.TIME_CHUNK_LOCK) public void testRunCompactionTwice(Configuration configuration) throws Exception { startCase(configuration); @@ -453,7 +453,7 @@ public abstract class CompactionTaskRunBase } } - @CompactionTest(Scenario.RUN_COMPACTION_TWICE_WITH_SEGMENT_LOCK) + @CompactionTest(Selection.SEGMENT_LOCK) public void testRunCompactionTwiceWithSegmentLock(Configuration configuration) throws Exception { startCase(configuration); @@ -519,7 +519,7 @@ public abstract class CompactionTaskRunBase } } - @CompactionTest(Scenario.RUN_INDEX_AND_COMPACT_AT_THE_SAME_TIME_FOR_DIFFERENT_INTERVAL) + @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testRunIndexAndCompactAtTheSameTimeForDifferentInterval(Configuration configuration) throws Exception { startCase(configuration); @@ -583,7 +583,7 @@ public abstract class CompactionTaskRunBase ); } - @CompactionTest(Scenario.WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY) public void testWithSegmentGranularityMisalignedInterval(Configuration configuration) throws Exception { startCase(configuration); @@ -603,7 +603,7 @@ public abstract class CompactionTaskRunBase Assertions.assertTrue(e.getMessage().contains(Granularities.WEEK.toString())); } - @CompactionTest(Scenario.WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL_ALLOWED) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY) public void testWithSegmentGranularityMisalignedIntervalAllowed(Configuration configuration) throws Exception { startCase(configuration); @@ -627,7 +627,7 @@ public abstract class CompactionTaskRunBase ); } - @CompactionTest(Scenario.WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL_ALLOWED_2) + @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testWithSegmentGranularityMisalignedIntervalAllowed2(Configuration configuration) throws Exception { startCase(configuration); @@ -656,7 +656,7 @@ public abstract class CompactionTaskRunBase ); } - @CompactionTest(Scenario.COMPACTION_WITH_FILTER_IN_TRANSFORM_SPEC) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY) public void testCompactionWithFilterInTransformSpec(Configuration configuration) throws Exception { startCase(configuration); @@ -713,7 +713,7 @@ public abstract class CompactionTaskRunBase ); } - @CompactionTest(Scenario.COMPACTION_WITH_NEW_METRIC_IN_METRICS_SPEC) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY) public void testCompactionWithNewMetricInMetricsSpec(Configuration configuration) throws Exception { startCase(configuration); @@ -747,7 +747,7 @@ public abstract class CompactionTaskRunBase Assertions.assertEquals(expectedCompactionState, segments.get(0).getLastCompactionState()); } - @CompactionTest(Scenario.WITH_GRANULARITY_SPEC_NON_NULL_QUERY_GRANULARITY) + @CompactionTest(Selection.ALL) public void testWithGranularitySpecNonNullQueryGranularity(Configuration configuration) throws Exception { startCase(configuration); @@ -772,7 +772,7 @@ public abstract class CompactionTaskRunBase ); } - @CompactionTest(Scenario.WITH_GRANULARITY_SPEC_NON_NULL_QUERY_GRANULARITY_AND_COARSE_SEGMENT_GRANULARITY) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testWithGranularitySpecNonNullQueryGranularityAndCoarseSegmentGranularity(Configuration configuration) throws Exception { @@ -802,7 +802,7 @@ public abstract class CompactionTaskRunBase Assertions.assertEquals(new NumberedShardSpec(0, 1), segments.get(0).getShardSpec()); } - @CompactionTest(Scenario.COMPACT_THEN_APPEND) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY) public void testCompactThenAppend(Configuration configuration) throws Exception { startCase(configuration); @@ -829,7 +829,7 @@ public abstract class CompactionTaskRunBase } @CompactionTest( - Scenario.PARTIAL_INTERVAL_COMPACT_WITH_FINER_SEGMENT_GRANULARITY_THAN_FULL_INTERVAL_COMPACT_WITH_DROP_EXISTING_TRUE + Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL ) public void testPartialIntervalCompactWithFinerSegmentGranularityThanFullIntervalCompactWithDropExistingTrue( Configuration configuration @@ -955,7 +955,7 @@ public abstract class CompactionTaskRunBase ); } - @CompactionTest(Scenario.COMPACT_DATASOURCE_OVER_INTERVAL_WITH_ONLY_TOMBSTONES) + @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testCompactDatasourceOverIntervalWithOnlyTombstones(Configuration configuration) throws Exception { startCase(configuration); @@ -1048,7 +1048,7 @@ public abstract class CompactionTaskRunBase } @CompactionTest( - Scenario.PARTIAL_INTERVAL_COMPACT_WITH_FINER_SEGMENT_GRANULARITY_THEN_FULL_INTERVAL_COMPACT_WITH_DROP_EXISTING_FALSE + Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL ) public void testPartialIntervalCompactWithFinerSegmentGranularityThenFullIntervalCompactWithDropExistingFalse( Configuration configuration @@ -1103,7 +1103,7 @@ public abstract class CompactionTaskRunBase } } - @CompactionTest(Scenario.RUN_INDEX_AND_COMPACT_FOR_SAME_SEGMENT_AT_THE_SAME_TIME) + @CompactionTest(Selection.ALL) public void testRunIndexAndCompactForSameSegmentAtTheSameTime(Configuration configuration) throws Exception { startCase(configuration); @@ -1154,7 +1154,7 @@ public abstract class CompactionTaskRunBase Assertions.assertTrue(e.getMessage().contains("not ready")); } - @CompactionTest(Scenario.RUN_INDEX_AND_COMPACT_FOR_SAME_SEGMENT_AT_THE_SAME_TIME_2) + @CompactionTest(Selection.ALL) public void testRunIndexAndCompactForSameSegmentAtTheSameTime2(Configuration configuration) throws Exception { startCase(configuration); @@ -1213,7 +1213,7 @@ public abstract class CompactionTaskRunBase Assertions.assertEquals(TaskState.FAILED, compactionResult.lhs.getStatusCode()); } - @CompactionTest(Scenario.RUN_WITH_SPATIAL_DIMENSIONS) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testRunWithSpatialDimensions(Configuration configuration) throws Exception { startCase(configuration); @@ -1322,7 +1322,7 @@ public abstract class CompactionTaskRunBase Assertions.assertEquals(spatialrows, rowsFromSegment); } - @CompactionTest(Scenario.RUN_WITH_AUTO_CAST_DIMENSIONS) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testRunWithAutoCastDimensions(Configuration configuration) throws Exception { startCase(configuration); @@ -1440,7 +1440,7 @@ public abstract class CompactionTaskRunBase Assertions.assertEquals(rows, rowsFromSegment); } - @CompactionTest(Scenario.RUN_WITH_AUTO_CAST_DIMENSIONS_SORT_BY_DIMENSION) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testRunWithAutoCastDimensionsSortByDimension(Configuration configuration) throws Exception { startCase(configuration); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTestCases.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTestCases.java index b9cd9a850f4..aa6fc97fcb9 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTestCases.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTestCases.java @@ -40,63 +40,16 @@ import java.util.stream.Stream; public final class CompactionTaskRunTestCases { - public enum Scenario - { - RUN_WITH_DYNAMIC_PARTITIONING(Selection.ALL), - RUN_WITH_HASH_PARTITIONING(Selection.NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY), - RUN_COMPACTION_TWICE(Selection.TIME_CHUNK_LOCK), - RUN_COMPACTION_TWICE_WITH_SEGMENT_LOCK(Selection.SEGMENT_LOCK), - RUN_INDEX_AND_COMPACT_AT_THE_SAME_TIME_FOR_DIFFERENT_INTERVAL( - Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL - ), - WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL(Selection.SIX_HOUR_GRANULARITY), - WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL_ALLOWED(Selection.SIX_HOUR_GRANULARITY), - WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL_ALLOWED_2( - Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL - ), - COMPACTION_WITH_FILTER_IN_TRANSFORM_SPEC(Selection.SIX_HOUR_GRANULARITY), - COMPACTION_WITH_NEW_METRIC_IN_METRICS_SPEC(Selection.SIX_HOUR_GRANULARITY), - WITH_GRANULARITY_SPEC_NON_NULL_QUERY_GRANULARITY(Selection.ALL), - WITH_GRANULARITY_SPEC_NON_NULL_QUERY_GRANULARITY_AND_COARSE_SEGMENT_GRANULARITY( - Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL - ), - COMPACT_THEN_APPEND(Selection.SIX_HOUR_GRANULARITY), - PARTIAL_INTERVAL_COMPACT_WITH_FINER_SEGMENT_GRANULARITY_THAN_FULL_INTERVAL_COMPACT_WITH_DROP_EXISTING_TRUE( - Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL - ), - COMPACT_DATASOURCE_OVER_INTERVAL_WITH_ONLY_TOMBSTONES( - Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL - ), - PARTIAL_INTERVAL_COMPACT_WITH_FINER_SEGMENT_GRANULARITY_THEN_FULL_INTERVAL_COMPACT_WITH_DROP_EXISTING_FALSE( - Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL - ), - RUN_INDEX_AND_COMPACT_FOR_SAME_SEGMENT_AT_THE_SAME_TIME(Selection.ALL), - RUN_INDEX_AND_COMPACT_FOR_SAME_SEGMENT_AT_THE_SAME_TIME_2(Selection.ALL), - RUN_WITH_SPATIAL_DIMENSIONS(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL), - RUN_WITH_AUTO_CAST_DIMENSIONS(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL), - RUN_WITH_AUTO_CAST_DIMENSIONS_SORT_BY_DIMENSION(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL); - - private final Selection selection; - - Scenario(Selection selection) - { - this.selection = selection; - } - - public boolean isApplicable(Configuration configuration) - { - return selection.isApplicable(configuration); - } - } - - private enum Selection + public enum Selection { ALL, TIME_CHUNK_LOCK, SEGMENT_LOCK, NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY, + NON_NULL_GRANULARITY_NOT_FINER_THAN_SIX_HOUR, SIX_HOUR_GRANULARITY, SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL, + NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY, NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL; boolean isApplicable(Configuration configuration) @@ -111,11 +64,17 @@ public final class CompactionTaskRunTestCases case NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY: return configuration.getLockGranularity() != LockGranularity.SEGMENT && configuration.getSegmentGranularity() == null; + case NON_NULL_GRANULARITY_NOT_FINER_THAN_SIX_HOUR: + return configuration.getSegmentGranularity() != null + && !configuration.getSegmentGranularity().isFinerThan(Granularities.SIX_HOUR); case SIX_HOUR_GRANULARITY: return Granularities.SIX_HOUR.equals(configuration.getSegmentGranularity()); case SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL: return Granularities.SIX_HOUR.equals(configuration.getSegmentGranularity()) && CompactionTaskRunBase.TEST_INTERVAL.equals(configuration.getInputInterval()); + case NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY: + return configuration.getLockGranularity() != LockGranularity.SEGMENT + && Granularities.SIX_HOUR.equals(configuration.getSegmentGranularity()); case NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL: return configuration.getLockGranularity() != LockGranularity.SEGMENT && Granularities.SIX_HOUR.equals(configuration.getSegmentGranularity()) @@ -208,11 +167,6 @@ public final class CompactionTaskRunTestCases public interface ConfigurationProvider { Stream<Configuration> configurations(); - - default boolean isApplicable(Scenario scenario, Configuration configuration) - { - return scenario.isApplicable(configuration); - } } @Inherited @@ -226,13 +180,13 @@ public final class CompactionTaskRunTestCases @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) @ParameterizedTest(name = "{0}") - @ArgumentsSource(ScenarioArgumentsProvider.class) + @ArgumentsSource(SelectionArgumentsProvider.class) public @interface CompactionTest { - Scenario value(); + Selection value(); } - public static class ScenarioArgumentsProvider implements ArgumentsProvider + public static class SelectionArgumentsProvider implements ArgumentsProvider { @Override public Stream<? extends Arguments> provideArguments(ExtensionContext context) throws Exception @@ -252,10 +206,7 @@ public final class CompactionTaskRunTestCases .getDeclaredConstructor() .newInstance(); return configurationProvider.configurations() - .filter(configuration -> configurationProvider.isApplicable( - compactionTest.value(), - configuration - )) + .filter(compactionTest.value()::isApplicable) .map(Arguments::of); } } diff --git a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java index 6626a33fbfc..e1926577241 100644 --- a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java +++ b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java @@ -54,7 +54,7 @@ import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Compacti import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Configuration; import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.ConfigurationProvider; import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.ConfigurationSource; -import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Scenario; +import org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Selection; import org.apache.druid.indexing.common.task.IndexTask; import org.apache.druid.indexing.common.task.MinorCompactionInputSpec; import org.apache.druid.indexing.common.task.Tasks; @@ -174,20 +174,6 @@ public class MSQCompactionTaskRunTest extends CompactionTaskRunBase } return configurations.stream(); } - - @Override - public boolean isApplicable(Scenario scenario, Configuration configuration) - { - if (scenario == Scenario.COMPACTION_WITH_NEW_METRIC_IN_METRICS_SPEC) { - return configuration.getSegmentGranularity() != null - && !configuration.getSegmentGranularity().isFinerThan(Granularities.SIX_HOUR); - } else if (scenario - == Scenario.PARTIAL_INTERVAL_COMPACT_WITH_FINER_SEGMENT_GRANULARITY_THAN_FULL_INTERVAL_COMPACT_WITH_DROP_EXISTING_TRUE) { - return configuration.getLockGranularity() != LockGranularity.SEGMENT - && Granularities.SIX_HOUR.equals(configuration.getSegmentGranularity()); - } - return ConfigurationProvider.super.isApplicable(scenario, configuration); - } } public static Stream<Configuration> concurrentLockConfigurations() @@ -295,14 +281,14 @@ public class MSQCompactionTaskRunTest extends CompactionTaskRunBase @Override @Disabled("Hash paritioning is not supported in MSQ") - @CompactionTest(Scenario.RUN_WITH_HASH_PARTITIONING) + @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY) public void testRunWithHashPartitioning(Configuration configuration) { } @Override @Disabled("The MSQ compaction test matrix does not support segment locks") - @CompactionTest(Scenario.RUN_COMPACTION_TWICE_WITH_SEGMENT_LOCK) + @CompactionTest(Selection.SEGMENT_LOCK) public void testRunCompactionTwiceWithSegmentLock(Configuration configuration) { } @@ -310,7 +296,7 @@ public class MSQCompactionTaskRunTest extends CompactionTaskRunBase @Override @Disabled("dropExisting must set to true in MSQ") @CompactionTest( - Scenario.PARTIAL_INTERVAL_COMPACT_WITH_FINER_SEGMENT_GRANULARITY_THEN_FULL_INTERVAL_COMPACT_WITH_DROP_EXISTING_FALSE + Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL ) public void testPartialIntervalCompactWithFinerSegmentGranularityThenFullIntervalCompactWithDropExistingFalse( Configuration configuration @@ -320,20 +306,20 @@ public class MSQCompactionTaskRunTest extends CompactionTaskRunBase @Override @Disabled("allowNonAlignedInterval is not supported in MSQ") - @CompactionTest(Scenario.WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL_ALLOWED) + @CompactionTest(Selection.SIX_HOUR_GRANULARITY) public void testWithSegmentGranularityMisalignedIntervalAllowed(Configuration configuration) { } @Override @Disabled("allowNonAlignedInterval is not supported in MSQ") - @CompactionTest(Scenario.WITH_SEGMENT_GRANULARITY_MISALIGNED_INTERVAL_ALLOWED_2) + @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL) public void testWithSegmentGranularityMisalignedIntervalAllowed2(Configuration configuration) { } @Override - @CompactionTest(Scenario.COMPACTION_WITH_NEW_METRIC_IN_METRICS_SPEC) + @CompactionTest(Selection.NON_NULL_GRANULARITY_NOT_FINER_THAN_SIX_HOUR) public void testCompactionWithNewMetricInMetricsSpec(Configuration configuration) throws Exception { startCase(configuration); @@ -364,7 +350,7 @@ public class MSQCompactionTaskRunTest extends CompactionTaskRunBase @Override @CompactionTest( - Scenario.PARTIAL_INTERVAL_COMPACT_WITH_FINER_SEGMENT_GRANULARITY_THAN_FULL_INTERVAL_COMPACT_WITH_DROP_EXISTING_TRUE + Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY ) public void testPartialIntervalCompactWithFinerSegmentGranularityThanFullIntervalCompactWithDropExistingTrue( Configuration configuration --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
