This is an automated email from the ASF dual-hosted git repository.
FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 76ddcf297e4 perf(test): Reduce compaction tests 12min and Re-sharding
test cases to reduce 6min (#20307)
76ddcf297e4 is described below
commit 76ddcf297e4ecf28171ee535f8d0023a377fee9d
Author: Frank Chen <[email protected]>
AuthorDate: Tue Sep 15 13:20:08 2026 +0800
perf(test): Reduce compaction tests 12min and Re-sharding test cases to
reduce 6min (#20307)
---
.github/scripts/test_balancing_calc | 5 +-
.github/workflows/ci.yml | 2 +-
.../common/task/CompactionTaskRunBase.java | 270 ++++++++++++---------
.../common/task/CompactionTaskRunTestCases.java | 211 ++++++++++++++++
.../common/task/NativeCompactionTaskRunTest.java | 90 +++----
.../druid/msq/exec/MSQCompactionTaskRunTest.java | 180 ++++++--------
6 files changed, 484 insertions(+), 274 deletions(-)
diff --git a/.github/scripts/test_balancing_calc
b/.github/scripts/test_balancing_calc
index bbd4c6ec824..5abb51013a0 100755
--- a/.github/scripts/test_balancing_calc
+++ b/.github/scripts/test_balancing_calc
@@ -21,7 +21,8 @@
set -e
-find . -name 'TEST*xml' |
+# QTest runs in dedicated shards and is explicitly excluded from the main
unit-test shards.
+find . -name 'TEST*xml' ! -name 'TEST-org.apache.druid.quidem.QTest.xml' |
# <testsuite ... name="org.apache.druid.query.policy.NoRestrictionPolicyTest"
time="0.071" ...>
xargs sed -nr
'/^<testsuite/s/.*name=\"([^\"]+)\".*time=\"([^\"]+)\".*/\1\t\2/p' |
# org.apache.druid.server.RequestLogLineTest 0.052
@@ -39,7 +40,7 @@ done | sed -r 's/ +/\t/' | sort -k 2 -nr > letter_times
echo "=== Letter times (sorted by duration) ==="
cat letter_times
-NUM_BUCKETS=8
+NUM_BUCKETS=9
echo ""
echo "=== Balanced buckets ($NUM_BUCKETS buckets) ==="
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c8b04b55259..50f9eba512d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
fail-fast: false
matrix:
jdk: [ "25" ]
- pattern: [ "C*", "N*", "D*,H*,L*", "I*,A*,U*", "K*,E*,W*,Z*,Y*,X*",
"M*,P*,O*", "R*,B*,Q*,V*", "S*", "T*,F*,G*,J*" ]
+ pattern: [ "C*", "S*", "K*,U*,Z*,Y*,X*", "N*,Q*", "I*,L*,J*",
"M*,A*,V*,W*", "E*,G*,F*", "R*,B*,P*", "H*,D*,T*,O*" ]
uses: ./.github/workflows/worker.yml
with:
script: .github/scripts/run_unit-tests -Dtest=!QTest,'${{ matrix.pattern
}}' -Dmaven.test.failure.ignore=true
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java
index 6dd77ff2ffe..e6fe17fa852 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunBase.java
@@ -58,6 +58,9 @@ import
org.apache.druid.indexing.common.actions.TaskActionClient;
import org.apache.druid.indexing.common.actions.TaskActionTestKit;
import org.apache.druid.indexing.common.config.TaskConfigBuilder;
import org.apache.druid.indexing.common.task.CompactionTask.Builder;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.CompactionTest;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Configuration;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Selection;
import org.apache.druid.indexing.overlord.Segments;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Intervals;
@@ -68,6 +71,7 @@ import org.apache.druid.java.util.common.concurrent.Execs;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.java.util.common.guava.Comparators;
+import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.query.OrderBy;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
@@ -122,9 +126,6 @@ import org.apache.druid.timeline.partition.PartitionIds;
import org.joda.time.Interval;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Assumptions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import javax.annotation.Nullable;
@@ -193,8 +194,7 @@ public abstract class CompactionTaskRunBase
public static final TemporaryFolderExtension temporaryFolder =
TemporaryFolderExtension.classScoped();
//CHECKSTYLE.ON: ConstantName
- @RegisterExtension
- public TaskActionTestKit taskActionTestKit = new TaskActionTestKit();
+ public TaskActionTestKit taskActionTestKit;
protected ObjectMapper objectMapper;
protected File reportsFile;
@@ -203,45 +203,27 @@ public abstract class CompactionTaskRunBase
protected final CoordinatorClient coordinatorClient;
protected final SegmentCacheManagerFactory segmentCacheManagerFactory;
- protected final LockGranularity lockGranularity;
- protected final boolean useCentralizedDatasourceSchema;
- protected final boolean useConcurrentLocks;
- protected final Interval inputInterval;
- protected final Granularity segmentGranularity;
+ protected LockGranularity lockGranularity;
+ protected boolean useCentralizedDatasourceSchema;
+ protected boolean useConcurrentLocks;
+ protected Interval inputInterval;
+ protected Granularity segmentGranularity;
protected final TestUtils testUtils;
protected ExecutorService exec;
protected File localDeepStorage;
- public CompactionTaskRunBase(
- String name,
- LockGranularity lockGranularity,
- boolean useCentralizedDatasourceSchema,
- boolean batchSegmentAllocation,
- boolean useSegmentMetadataCache,
- boolean useConcurrentLocks,
- Interval inputInterval,
- Granularity segmentGranularity
- )
+ private boolean taskActionTestKitStarted;
+ private boolean baseSetupStarted;
+ private boolean runnerSetupStarted;
+
+ protected CompactionTaskRunBase()
{
- this.lockGranularity = lockGranularity;
- this.useCentralizedDatasourceSchema = useCentralizedDatasourceSchema;
-
taskActionTestKit.setUseCentralizedDatasourceSchema(useCentralizedDatasourceSchema)
- .setUseSegmentMetadataCache(useSegmentMetadataCache)
- .setBatchSegmentAllocation(batchSegmentAllocation);
- this.useConcurrentLocks = useConcurrentLocks;
- this.inputInterval = inputInterval;
- this.segmentGranularity = segmentGranularity;
-
- reportsFile = new File(temporaryFolder.getRoot(), "reports.json");
testUtils = new TestUtils();
- segmentCacheManagerFactory =
SegmentCacheManagerFactory.createWithOwnedPool(TestIndex.INDEX_IO,
testUtils.getTestObjectMapper());
-
- objectMapper = testUtils.getTestObjectMapper();
- objectMapper.registerSubtypes(new NamedType(LocalLoadSpec.class, "local"));
- objectMapper.registerSubtypes(LocalDataSegmentPuller.class);
- objectMapper.registerSubtypes(TombstoneLoadSpec.class);
-
+ segmentCacheManagerFactory =
SegmentCacheManagerFactory.createWithOwnedPool(
+ TestIndex.INDEX_IO,
+ testUtils.getTestObjectMapper()
+ );
overlordClient = new NoopOverlordClient();
coordinatorClient = new NoopCoordinatorClient()
{
@@ -280,22 +262,84 @@ public abstract class CompactionTaskRunBase
};
}
- @BeforeEach
- public void setup() throws IOException
+ private void configure(Configuration configuration)
{
- exec = Execs.multiThreaded(2, "compaction-task-run-test-%d");
- localDeepStorage = temporaryFolder.newFolder();
+ lockGranularity = configuration.lockGranularity();
+ useCentralizedDatasourceSchema =
configuration.useCentralizedDatasourceSchema();
+ useConcurrentLocks = configuration.useConcurrentLocks();
+ inputInterval = configuration.inputInterval();
+ segmentGranularity = configuration.segmentGranularity();
+
+ taskActionTestKit = new TaskActionTestKit()
+ .setUseCentralizedDatasourceSchema(useCentralizedDatasourceSchema)
+ .setUseSegmentMetadataCache(configuration.useSegmentMetadataCache())
+ .setBatchSegmentAllocation(configuration.batchSegmentAllocation());
+
+ objectMapper = testUtils.getTestObjectMapper();
+ objectMapper.registerSubtypes(new NamedType(LocalLoadSpec.class, "local"));
+ objectMapper.registerSubtypes(LocalDataSegmentPuller.class);
+ objectMapper.registerSubtypes(TombstoneLoadSpec.class);
+ }
+
+ protected final void startCase(Configuration configuration) throws Exception
+ {
+ taskActionTestKitStarted = false;
+ baseSetupStarted = false;
+ runnerSetupStarted = false;
+
+ configure(configuration);
+ taskActionTestKit.before();
+ taskActionTestKitStarted = true;
+
+ setup();
+ baseSetupStarted = true;
+
+ setUpRunner();
+ runnerSetupStarted = true;
}
@AfterEach
- public void teardown() throws IOException
+ public void cleanUpCase() throws IOException
+ {
+ try (Closer closer = Closer.create()) {
+ if (taskActionTestKitStarted) {
+ closer.register(taskActionTestKit::after);
+ }
+ if (baseSetupStarted) {
+ closer.register(this::teardown);
+ }
+ if (runnerSetupStarted) {
+ closer.register(this::tearDownRunner);
+ }
+ }
+ }
+
+ protected void setup() throws IOException
+ {
+ localDeepStorage = temporaryFolder.newFolder();
+ reportsFile = new File(temporaryFolder.newFolder(), "reports.json");
+ exec = Execs.multiThreaded(2, "compaction-task-run-test-%d");
+ }
+
+ protected void teardown()
+ {
+ if (exec != null) {
+ exec.shutdownNow();
+ }
+ }
+
+ protected void setUpRunner() throws IOException
+ {
+ }
+
+ protected void tearDownRunner() throws IOException
{
- exec.shutdownNow();
}
- @Test
- public void testRunWithDynamicPartitioning() throws Exception
+ @CompactionTest(Selection.ALL)
+ public void testRunWithDynamicPartitioning(Configuration configuration)
throws Exception
{
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -317,12 +361,10 @@ public abstract class CompactionTaskRunBase
);
}
- @Test
- public void testRunWithHashPartitioning() throws Exception
+ @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY)
+ public void testRunWithHashPartitioning(Configuration configuration) throws
Exception
{
- // Hash partitioning is not supported with segment lock yet
- Assumptions.assumeTrue(lockGranularity != LockGranularity.SEGMENT, "Hash
partitioning is not supported with segment lock yet");
- Assumptions.assumeTrue(segmentGranularity == null, "Test null segment
granularity is sufficient");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -360,10 +402,10 @@ public abstract class CompactionTaskRunBase
}
}
- @Test
- public void testRunCompactionTwice() throws Exception
+ @CompactionTest(Selection.TIME_CHUNK_LOCK)
+ public void testRunCompactionTwice(Configuration configuration) throws
Exception
{
- Assumptions.assumeTrue(lockGranularity == LockGranularity.TIME_CHUNK);
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask1 =
@@ -411,10 +453,10 @@ public abstract class CompactionTaskRunBase
}
}
- @Test
- public void testRunCompactionTwiceWithSegmentLock() throws Exception
+ @CompactionTest(Selection.SEGMENT_LOCK)
+ public void testRunCompactionTwiceWithSegmentLock(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(lockGranularity == LockGranularity.SEGMENT);
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask1 =
@@ -477,11 +519,10 @@ public abstract class CompactionTaskRunBase
}
}
- @Test
- public void testRunIndexAndCompactAtTheSameTimeForDifferentInterval() throws
Exception
+
@CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void
testRunIndexAndCompactAtTheSameTimeForDifferentInterval(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval)
- && lockGranularity != LockGranularity.SEGMENT, "test with defined
segment granularity and interval in this test");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -542,10 +583,10 @@ public abstract class CompactionTaskRunBase
);
}
- @Test
- public void testWithSegmentGranularityMisalignedInterval() throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY)
+ public void testWithSegmentGranularityMisalignedInterval(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity),
"use Granularities.WEEK segment granularity in this test");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
// Test when inputInterval is less than Granularities.WEEK is not allowed
final CompactionTask compactionTask1 =
@@ -562,10 +603,10 @@ public abstract class CompactionTaskRunBase
Assertions.assertTrue(e.getMessage().contains(Granularities.WEEK.toString()));
}
- @Test
- public void testWithSegmentGranularityMisalignedIntervalAllowed() throws
Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY)
+ public void
testWithSegmentGranularityMisalignedIntervalAllowed(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity),
"use Granularities.WEEK segment granularity in this test");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
// Test when inputInterval is less than Granularities.WEEK is allowed
final CompactionTask compactionTask1 =
@@ -586,11 +627,10 @@ public abstract class CompactionTaskRunBase
);
}
- @Test
- public void testWithSegmentGranularityMisalignedIntervalAllowed2() throws
Exception
+
@CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void
testWithSegmentGranularityMisalignedIntervalAllowed2(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval)
- && lockGranularity != LockGranularity.SEGMENT, "test with defined
segment granularity and interval in this test");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
// Test when inputInterval doesn't align with segment granularity
final Interval interval =
Intervals.of("2014-01-01T00:30:00Z/2014-01-01T01:30:00Z");
@@ -616,10 +656,10 @@ public abstract class CompactionTaskRunBase
);
}
- @Test
- public void testCompactionWithFilterInTransformSpec() throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY)
+ public void testCompactionWithFilterInTransformSpec(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity),
"test with six hour granularity is enough");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
compactionTaskBuilder(segmentGranularity)
@@ -673,10 +713,10 @@ public abstract class CompactionTaskRunBase
);
}
- @Test
- public void testCompactionWithNewMetricInMetricsSpec() throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY)
+ public void testCompactionWithNewMetricInMetricsSpec(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity),
"test with six hour granularity is enough");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -707,9 +747,10 @@ public abstract class CompactionTaskRunBase
Assertions.assertEquals(expectedCompactionState,
segments.get(0).getLastCompactionState());
}
- @Test
- public void testWithGranularitySpecNonNullQueryGranularity() throws Exception
+ @CompactionTest(Selection.ALL)
+ public void testWithGranularitySpecNonNullQueryGranularity(Configuration
configuration) throws Exception
{
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
// second queryGranularity
@@ -731,10 +772,11 @@ public abstract class CompactionTaskRunBase
);
}
- @Test
- public void
testWithGranularitySpecNonNullQueryGranularityAndCoarseSegmentGranularity()
throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void
testWithGranularitySpecNonNullQueryGranularityAndCoarseSegmentGranularity(Configuration
configuration)
+ throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval), "test with defined segment granularity
and interval in this test");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
// day segmentGranularity and day queryGranularity
@@ -760,10 +802,10 @@ public abstract class CompactionTaskRunBase
Assertions.assertEquals(new NumberedShardSpec(0, 1),
segments.get(0).getShardSpec());
}
- @Test
- public void testCompactThenAppend() throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY)
+ public void testCompactThenAppend(Configuration configuration) throws
Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity),
"test three hour segment granularity is enough");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -786,13 +828,15 @@ public abstract class CompactionTaskRunBase
Assertions.assertEquals(expectedSegments, usedSegments);
}
- @Test
- public void
testPartialIntervalCompactWithFinerSegmentGranularityThanFullIntervalCompactWithDropExistingTrue()
+ @CompactionTest(
+ Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL
+ )
+ public void
testPartialIntervalCompactWithFinerSegmentGranularityThanFullIntervalCompactWithDropExistingTrue(
+ Configuration configuration
+ )
throws Exception
{
- // This test fails with segment lock because of the bug reported in
https://github.com/apache/druid/issues/10911.
- Assumptions.assumeTrue(lockGranularity != LockGranularity.SEGMENT);
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval), "test with defined segment granularity
and interval in this test");
+ startCase(configuration);
// The following task creates (several, more than three, last time I
checked, six) HOUR segments with intervals of
// - 2014-01-01T00:00:00/2014-01-01T01:00:00
@@ -911,12 +955,10 @@ public abstract class CompactionTaskRunBase
);
}
- @Test
- public void testCompactDatasourceOverIntervalWithOnlyTombstones() throws
Exception
+
@CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void
testCompactDatasourceOverIntervalWithOnlyTombstones(Configuration
configuration) throws Exception
{
- // This test fails with segment lock because of the bug reported in
https://github.com/apache/druid/issues/10911.
- Assumptions.assumeTrue(lockGranularity != LockGranularity.SEGMENT);
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval), "test with defined segment granularity
and interval in this test");
+ startCase(configuration);
// The following task creates (several, more than three, last time I
checked, six) HOUR segments with intervals of
// - 2014-01-01T00:00:00/2014-01-01T01:00:00
@@ -1005,13 +1047,15 @@ public abstract class CompactionTaskRunBase
resultOverOnlyTombstones.rhs.getSegments().forEach(t ->
Assertions.assertTrue(t.isTombstone()));
}
- @Test
- public void
testPartialIntervalCompactWithFinerSegmentGranularityThenFullIntervalCompactWithDropExistingFalse()
+ @CompactionTest(
+ Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL
+ )
+ public void
testPartialIntervalCompactWithFinerSegmentGranularityThenFullIntervalCompactWithDropExistingFalse(
+ Configuration configuration
+ )
throws Exception
{
- // This test fails with segment lock because of the bug reported in
https://github.com/apache/druid/issues/10911.
- Assumptions.assumeTrue(lockGranularity != LockGranularity.SEGMENT);
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval), "test with defined segment granularity
and interval in this test");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final Set<DataSegment> expectedSegments = new HashSet<>(
@@ -1059,9 +1103,10 @@ public abstract class CompactionTaskRunBase
}
}
- @Test
- public void testRunIndexAndCompactForSameSegmentAtTheSameTime() throws
Exception
+ @CompactionTest(Selection.ALL)
+ public void testRunIndexAndCompactForSameSegmentAtTheSameTime(Configuration
configuration) throws Exception
{
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
// make sure that indexTask becomes ready first, then compactionTask
becomes ready, then indexTask runs
@@ -1109,9 +1154,10 @@ public abstract class CompactionTaskRunBase
Assertions.assertTrue(e.getMessage().contains("not ready"));
}
- @Test
- public void testRunIndexAndCompactForSameSegmentAtTheSameTime2() throws
Exception
+ @CompactionTest(Selection.ALL)
+ public void testRunIndexAndCompactForSameSegmentAtTheSameTime2(Configuration
configuration) throws Exception
{
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -1167,10 +1213,10 @@ public abstract class CompactionTaskRunBase
Assertions.assertEquals(TaskState.FAILED,
compactionResult.lhs.getStatusCode());
}
- @Test
- public void testRunWithSpatialDimensions() throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void testRunWithSpatialDimensions(Configuration configuration) throws
Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval), "test with defined segment granularity
and interval in this test");
+ startCase(configuration);
final List<String> spatialrows = ImmutableList.of(
"2014-01-01T00:00:10Z,a,10,100,1\n",
"2014-01-01T00:00:10Z,b,20,110,2\n",
@@ -1276,10 +1322,10 @@ public abstract class CompactionTaskRunBase
Assertions.assertEquals(spatialrows, rowsFromSegment);
}
- @Test
- public void testRunWithAutoCastDimensions() throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void testRunWithAutoCastDimensions(Configuration configuration)
throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval), "test with defined segment granularity
and interval in this test");
+ startCase(configuration);
final List<String> rows = ImmutableList.of(
"2014-01-01T00:00:10Z,a,10,100,1\n",
"2014-01-01T00:00:10Z,b,20,110,2\n",
@@ -1394,10 +1440,10 @@ public abstract class CompactionTaskRunBase
Assertions.assertEquals(rows, rowsFromSegment);
}
- @Test
- public void testRunWithAutoCastDimensionsSortByDimension() throws Exception
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void testRunWithAutoCastDimensionsSortByDimension(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(Granularities.SIX_HOUR.equals(segmentGranularity)
&& TEST_INTERVAL.equals(inputInterval), "test with defined segment granularity
and interval in this test");
+ startCase(configuration);
// Compaction will produce one segment sorted by [x, __time], even though
input rows are sorted by __time.
final List<String> rows = ImmutableList.of(
"2014-01-01T00:00:10Z,a,10,100,1\n",
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTestCases.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTestCases.java
new file mode 100644
index 00000000000..6a4360b65ef
--- /dev/null
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTestCases.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.common.task;
+
+import org.apache.druid.indexing.common.LockGranularity;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
+import org.joda.time.Interval;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.junit.jupiter.params.provider.ArgumentsSource;
+
+import javax.annotation.Nullable;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.stream.Stream;
+
+public final class CompactionTaskRunTestCases
+{
+ public enum Selection
+ {
+ ALL {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return true;
+ }
+ },
+ TIME_CHUNK_LOCK {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.lockGranularity() == LockGranularity.TIME_CHUNK;
+ }
+ },
+ SEGMENT_LOCK {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.lockGranularity() == LockGranularity.SEGMENT;
+ }
+ },
+ CONCURRENT_LOCK {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.useConcurrentLocks();
+ }
+ },
+ CONCURRENT_TIME_CHUNK_LOCK {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.useConcurrentLocks()
+ && configuration.lockGranularity() ==
LockGranularity.TIME_CHUNK;
+ }
+ },
+ NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.lockGranularity() != LockGranularity.SEGMENT
+ && configuration.segmentGranularity() == null;
+ }
+ },
+ NON_NULL_GRANULARITY_NOT_FINER_THAN_SIX_HOUR {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.segmentGranularity() != null
+ &&
!configuration.segmentGranularity().isFinerThan(Granularities.SIX_HOUR);
+ }
+ },
+ SIX_HOUR_GRANULARITY {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return
Granularities.SIX_HOUR.equals(configuration.segmentGranularity());
+ }
+ },
+ SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return
Granularities.SIX_HOUR.equals(configuration.segmentGranularity())
+ &&
CompactionTaskRunBase.TEST_INTERVAL.equals(configuration.inputInterval());
+ }
+ },
+ NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.lockGranularity() != LockGranularity.SEGMENT
+ &&
Granularities.SIX_HOUR.equals(configuration.segmentGranularity());
+ }
+ },
+ NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL {
+ @Override
+ boolean isApplicable(Configuration configuration)
+ {
+ return configuration.lockGranularity() != LockGranularity.SEGMENT
+ &&
Granularities.SIX_HOUR.equals(configuration.segmentGranularity())
+ &&
CompactionTaskRunBase.TEST_INTERVAL.equals(configuration.inputInterval());
+ }
+ };
+
+ /**
+ * Returns whether the given configuration should be included for a test
using this selection. This method is
+ * evaluated while test arguments are generated, so inapplicable
configurations are excluded before per-test
+ * fixtures are initialized.
+ */
+ abstract boolean isApplicable(Configuration configuration);
+ }
+
+ public record Configuration(
+ LockGranularity lockGranularity,
+ boolean useCentralizedDatasourceSchema,
+ boolean batchSegmentAllocation,
+ boolean useSegmentMetadataCache,
+ boolean useConcurrentLocks,
+ Interval inputInterval,
+ @Nullable Granularity segmentGranularity
+ )
+ {
+ @Override
+ public String toString()
+ {
+ return "lockGranularity=" + lockGranularity
+ + ", useCentralizedDatasourceSchema=" +
useCentralizedDatasourceSchema
+ + ", batchSegmentAllocation=" + batchSegmentAllocation
+ + ", useSegmentMetadataCache=" + useSegmentMetadataCache
+ + ", useConcurrentLocks=" + useConcurrentLocks
+ + ", inputInterval=" + inputInterval
+ + ", segmentGranularity=" + segmentGranularity;
+ }
+ }
+
+ public interface ConfigurationProvider
+ {
+ Stream<Configuration> configurations();
+ }
+
+ @Inherited
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.TYPE)
+ public @interface ConfigurationSource
+ {
+ Class<? extends ConfigurationProvider> value();
+ }
+
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ @ParameterizedTest(name = "{0}")
+ @ArgumentsSource(SelectionArgumentsProvider.class)
+ public @interface CompactionTest
+ {
+ Selection value();
+ }
+
+ public static class SelectionArgumentsProvider implements ArgumentsProvider
+ {
+ @Override
+ public Stream<? extends Arguments> provideArguments(ExtensionContext
context) throws Exception
+ {
+ final CompactionTest compactionTest =
context.getRequiredTestMethod().getAnnotation(CompactionTest.class);
+ if (compactionTest == null) {
+ throw new IllegalStateException("Missing @CompactionTest on " +
context.getRequiredTestMethod());
+ }
+
+ final ConfigurationSource configurationSource =
context.getRequiredTestClass()
+
.getAnnotation(ConfigurationSource.class);
+ if (configurationSource == null) {
+ throw new IllegalStateException("Missing @ConfigurationSource on " +
context.getRequiredTestClass());
+ }
+
+ final ConfigurationProvider configurationProvider =
configurationSource.value()
+
.getDeclaredConstructor()
+
.newInstance();
+ return configurationProvider.configurations()
+ .filter(compactionTest.value()::isApplicable)
+ .map(Arguments::of);
+ }
+ }
+
+ private CompactionTaskRunTestCases()
+ {
+ }
+}
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NativeCompactionTaskRunTest.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NativeCompactionTaskRunTest.java
index e9ba9d1c747..dddb2d1c14c 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NativeCompactionTaskRunTest.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/NativeCompactionTaskRunTest.java
@@ -21,82 +21,56 @@ package org.apache.druid.indexing.common.task;
import org.apache.druid.client.indexing.ClientCompactionTaskGranularitySpec;
import org.apache.druid.indexing.common.LockGranularity;
-import org.apache.druid.java.util.common.StringUtils;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Configuration;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.ConfigurationProvider;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.ConfigurationSource;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.joda.time.Interval;
-import org.junit.jupiter.params.ParameterizedClass;
-import org.junit.jupiter.params.provider.MethodSource;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Stream;
-@ParameterizedClass
-@MethodSource("constructorFeeder")
+@ConfigurationSource(NativeCompactionTaskRunTest.NativeConfigurations.class)
public class NativeCompactionTaskRunTest extends CompactionTaskRunBase
{
-
- public static Iterable<Object[]> constructorFeeder()
+ public static class NativeConfigurations implements ConfigurationProvider
{
- final List<Object[]> constructors = new ArrayList<>();
+ @Override
+ public Stream<Configuration> configurations()
+ {
+ final List<Configuration> configurations = new ArrayList<>();
- for (LockGranularity lockGranularity : new
LockGranularity[]{LockGranularity.TIME_CHUNK, LockGranularity.SEGMENT}) {
- for (boolean useCentralizedDatasourceSchema : new boolean[]{true}) {
- for (boolean batchSegmentAllocation : new boolean[]{false, true}) {
- for (boolean useSegmentMetadataCache : new boolean[]{false, true}) {
- for (boolean useConcurrentLocks : new boolean[]{false, true}) {
- for (Interval inputInterval : new Interval[]{TEST_INTERVAL,
TEST_INTERVAL_DAY}) {
- for (Granularity segmentGran : new Granularity[]{null,
Granularities.HOUR, Granularities.SIX_HOUR}) {
- String name = StringUtils.format(
- "lockGranularity=%s, useCentralizedDatasourceSchema=%s,
batchSegmentAllocation=%s, useSegmentMetadataCache=%s, useConcurrentLocks=%s",
- lockGranularity,
- useCentralizedDatasourceSchema,
- batchSegmentAllocation,
- useSegmentMetadataCache,
- useConcurrentLocks
- );
- constructors.add(new Object[]{
- name,
- lockGranularity,
- useCentralizedDatasourceSchema,
- batchSegmentAllocation,
- useSegmentMetadataCache,
- useConcurrentLocks,
- inputInterval,
- segmentGran
- });
+ for (final LockGranularity lockGranularity
+ : new LockGranularity[]{LockGranularity.TIME_CHUNK,
LockGranularity.SEGMENT}) {
+ for (final boolean useCentralizedDatasourceSchema : new
boolean[]{true}) {
+ for (final boolean batchSegmentAllocation : new boolean[]{false,
true}) {
+ for (final boolean useSegmentMetadataCache : new boolean[]{false,
true}) {
+ for (final boolean useConcurrentLocks : new boolean[]{false,
true}) {
+ for (final Interval inputInterval : new
Interval[]{TEST_INTERVAL, TEST_INTERVAL_DAY}) {
+ for (final Granularity segmentGranularity
+ : new Granularity[]{null, Granularities.HOUR,
Granularities.SIX_HOUR}) {
+ configurations.add(
+ new Configuration(
+ lockGranularity,
+ useCentralizedDatasourceSchema,
+ batchSegmentAllocation,
+ useSegmentMetadataCache,
+ useConcurrentLocks,
+ inputInterval,
+ segmentGranularity
+ )
+ );
+ }
}
}
}
}
}
}
-
+ return configurations.stream();
}
- return constructors;
- }
-
- public NativeCompactionTaskRunTest(
- String name,
- LockGranularity lockGranularity,
- boolean useCentralizedDatasourceSchema,
- boolean batchSegmentAllocation,
- boolean useSegmentMetadataCache,
- boolean useConcurrentLocks,
- Interval inputInterval,
- Granularity compactionGranularity
- )
- {
- super(
- name,
- lockGranularity,
- useCentralizedDatasourceSchema,
- batchSegmentAllocation,
- useSegmentMetadataCache,
- useConcurrentLocks,
- inputInterval,
- compactionGranularity
- );
}
@Override
diff --git
a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java
b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java
index f72cb81754b..913bb35d072 100644
---
a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java
+++
b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQCompactionTaskRunTest.java
@@ -50,13 +50,17 @@ import
org.apache.druid.indexing.common.actions.TaskActionClient;
import org.apache.druid.indexing.common.task.CompactionIntervalSpec;
import org.apache.druid.indexing.common.task.CompactionTask;
import org.apache.druid.indexing.common.task.CompactionTaskRunBase;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.CompactionTest;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Configuration;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.ConfigurationProvider;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.ConfigurationSource;
+import
org.apache.druid.indexing.common.task.CompactionTaskRunTestCases.Selection;
import org.apache.druid.indexing.common.task.IndexTask;
import org.apache.druid.indexing.common.task.MinorCompactionInputSpec;
import org.apache.druid.indexing.common.task.Tasks;
import org.apache.druid.indexing.common.task.TuningConfigBuilder;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.concurrent.Execs;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
@@ -105,12 +109,7 @@ import
org.apache.druid.timeline.partition.NumberedShardSpec;
import org.apache.druid.timeline.partition.ShardSpec;
import org.joda.time.Interval;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Assumptions;
-import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedClass;
-import org.junit.jupiter.params.provider.MethodSource;
import java.io.File;
import java.util.ArrayList;
@@ -123,6 +122,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
@@ -132,73 +132,46 @@ import static org.mockito.Mockito.when;
* Tests for CompactionTask using MSQCompactionRunner.
* Extends CompactionTaskRunTest to reuse all test infrastructure.
*/
-@ParameterizedClass
-@MethodSource("constructorFeeder")
+@ConfigurationSource(MSQCompactionTaskRunTest.MsqConfigurations.class)
public class MSQCompactionTaskRunTest extends CompactionTaskRunBase
{
private final ConcurrentHashMap<String, TaskActionClient> taskActionClients
= new ConcurrentHashMap<>();
private Injector injector;
- public static Iterable<Object[]> constructorFeeder()
+ public static class MsqConfigurations implements ConfigurationProvider
{
- final List<Object[]> constructors = new ArrayList<>();
-
- for (LockGranularity lockGranularity : new
LockGranularity[]{LockGranularity.TIME_CHUNK}) {
- for (boolean useCentralizedDatasourceSchema : new boolean[]{false}) {
- for (boolean batchSegmentAllocation : new boolean[]{false, true}) {
- for (boolean useSegmentMetadataCache : new boolean[]{false, true}) {
- for (boolean useConcurrentLocks : new boolean[]{false, true}) {
- for (Interval inputInterval : new Interval[]{TEST_INTERVAL}) {
- for (Granularity segmentGran : new
Granularity[]{Granularities.SIX_HOUR}) {
- String name = StringUtils.format(
- "lockGranularity=%s, useCentralizedDatasourceSchema=%s,
batchSegmentAllocation=%s, useSegmentMetadataCache=%s, useConcurrentLocks=%s",
- lockGranularity,
- useCentralizedDatasourceSchema,
- batchSegmentAllocation,
- useSegmentMetadataCache,
- useConcurrentLocks
- );
- constructors.add(new Object[]{
- name,
- lockGranularity,
- useCentralizedDatasourceSchema,
- batchSegmentAllocation,
- useSegmentMetadataCache,
- useConcurrentLocks,
- inputInterval,
- segmentGran
- });
+ @Override
+ public Stream<Configuration> configurations()
+ {
+ final List<Configuration> configurations = new ArrayList<>();
+
+ for (final LockGranularity lockGranularity : new
LockGranularity[]{LockGranularity.TIME_CHUNK}) {
+ for (final boolean useCentralizedDatasourceSchema : new
boolean[]{false}) {
+ for (final boolean batchSegmentAllocation : new boolean[]{false,
true}) {
+ for (final boolean useSegmentMetadataCache : new boolean[]{false,
true}) {
+ for (final boolean useConcurrentLocks : new boolean[]{false,
true}) {
+ for (final Interval inputInterval : new
Interval[]{TEST_INTERVAL}) {
+ for (final Granularity segmentGranularity : new
Granularity[]{Granularities.SIX_HOUR}) {
+ configurations.add(
+ new Configuration(
+ lockGranularity,
+ useCentralizedDatasourceSchema,
+ batchSegmentAllocation,
+ useSegmentMetadataCache,
+ useConcurrentLocks,
+ inputInterval,
+ segmentGranularity
+ )
+ );
+ }
}
}
}
}
}
}
+ return configurations.stream();
}
- return constructors;
- }
-
- public MSQCompactionTaskRunTest(
- String name,
- LockGranularity lockGranularity,
- boolean useCentralizedDatasourceSchema,
- boolean batchSegmentAllocation,
- boolean useSegmentMetadataCache,
- boolean useConcurrentLocks,
- Interval inputInterval,
- Granularity compactionGranularities
- )
- {
- super(
- name,
- lockGranularity,
- useCentralizedDatasourceSchema,
- batchSegmentAllocation,
- useSegmentMetadataCache,
- useConcurrentLocks,
- inputInterval,
- compactionGranularities
- );
}
@Override
@@ -207,8 +180,8 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
Preconditions.checkState(taskActionClients.put(taskId, taskActionClient)
== null);
}
- @BeforeEach
- public void setUpMSQ()
+ @Override
+ protected void setUpRunner()
{
objectMapper.registerModules(new MSQIndexingModule().getJacksonModules());
ComplexMetrics.registerSerde(NestedDataComplexTypeSerde.TYPE_NAME,
NestedDataComplexTypeSerde.INSTANCE);
@@ -295,38 +268,48 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
@Override
@Disabled("Hash paritioning is not supported in MSQ")
- @Test
- public void testRunWithHashPartitioning()
+ @CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_NULL_GRANULARITY)
+ public void testRunWithHashPartitioning(Configuration configuration)
+ {
+ }
+
+ @Override
+ @Disabled("The MSQ compaction test matrix does not support segment locks")
+ @CompactionTest(Selection.SEGMENT_LOCK)
+ public void testRunCompactionTwiceWithSegmentLock(Configuration
configuration)
{
}
@Override
@Disabled("dropExisting must set to true in MSQ")
- @Test
- public void
testPartialIntervalCompactWithFinerSegmentGranularityThenFullIntervalCompactWithDropExistingFalse()
+ @CompactionTest(
+ Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL
+ )
+ public void
testPartialIntervalCompactWithFinerSegmentGranularityThenFullIntervalCompactWithDropExistingFalse(
+ Configuration configuration
+ )
{
}
@Override
@Disabled("allowNonAlignedInterval is not supported in MSQ")
- @Test
- public void testWithSegmentGranularityMisalignedIntervalAllowed()
+ @CompactionTest(Selection.SIX_HOUR_GRANULARITY)
+ public void
testWithSegmentGranularityMisalignedIntervalAllowed(Configuration configuration)
{
}
@Override
@Disabled("allowNonAlignedInterval is not supported in MSQ")
- @Test
- public void testWithSegmentGranularityMisalignedIntervalAllowed2()
+
@CompactionTest(Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY_AND_TEST_INTERVAL)
+ public void
testWithSegmentGranularityMisalignedIntervalAllowed2(Configuration
configuration)
{
}
@Override
- @Test
- public void testCompactionWithNewMetricInMetricsSpec() throws Exception
+ @CompactionTest(Selection.NON_NULL_GRANULARITY_NOT_FINER_THAN_SIX_HOUR)
+ public void testCompactionWithNewMetricInMetricsSpec(Configuration
configuration) throws Exception
{
- // MSQ doesn't support count aggregator
- Assumptions.assumeTrue(segmentGranularity != null &&
!segmentGranularity.isFinerThan(Granularities.SIX_HOUR));
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -353,17 +336,16 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
}
@Override
- @Test
- public void
testPartialIntervalCompactWithFinerSegmentGranularityThanFullIntervalCompactWithDropExistingTrue()
+ @CompactionTest(
+ Selection.NON_SEGMENT_LOCK_WITH_SIX_HOUR_GRANULARITY
+ )
+ public void
testPartialIntervalCompactWithFinerSegmentGranularityThanFullIntervalCompactWithDropExistingTrue(
+ Configuration configuration
+ )
throws Exception
{
+ startCase(configuration);
// This test is almost identical to base, except for fullCompactionTask,
since MSQ doesn't allow disjoint intervals.
- // This test fails with segment lock because of the bug reported in
https://github.com/apache/druid/issues/10911.
- Assumptions.assumeTrue(lockGranularity != LockGranularity.SEGMENT);
- Assumptions.assumeTrue(
- Granularities.SIX_HOUR.equals(segmentGranularity),
- "test with defined segment granularity in this test"
- );
// The following task creates (several, more than three, last time I
checked, six) HOUR segments with intervals of
// - 2014-01-01T00:00:00/2014-01-01T01:00:00
@@ -472,10 +454,10 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
);
}
- @Test
- public void testMSQCompactionWithConcurrentAppendCompactionLocksFirst()
throws Exception
+ @CompactionTest(Selection.CONCURRENT_LOCK)
+ public void
testMSQCompactionWithConcurrentAppendCompactionLocksFirst(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(useConcurrentLocks);
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -532,11 +514,10 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
verifyTaskSuccessRowsAndSchemaMatch(finalResult, 19);
}
-
- @Test
- public void testMSQCompactionWithConcurrentAppendAppendLocksFirst() throws
Exception
+ @CompactionTest(Selection.CONCURRENT_LOCK)
+ public void
testMSQCompactionWithConcurrentAppendAppendLocksFirst(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(useConcurrentLocks);
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask =
@@ -593,11 +574,10 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
verifyTaskSuccessRowsAndSchemaMatch(finalResult, 19);
}
- @Test
- public void testMinorCompaction() throws Exception
+ @CompactionTest(Selection.CONCURRENT_TIME_CHUNK_LOCK)
+ public void testMinorCompaction(Configuration configuration) throws Exception
{
- Assumptions.assumeTrue(lockGranularity == LockGranularity.TIME_CHUNK);
- Assumptions.assumeTrue(useConcurrentLocks, "Minor compaction depends on
concurrent lock");
+ startCase(configuration);
verifyTaskSuccessRowsAndSchemaMatch(runIndexTask(), TOTAL_TEST_ROWS);
final CompactionTask compactionTask1 =
@@ -649,9 +629,10 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
), usedSegments);
}
- @Test
- public void testMinorCompactionRangePartition() throws Exception
+ @CompactionTest(Selection.CONCURRENT_TIME_CHUNK_LOCK)
+ public void testMinorCompactionRangePartition(Configuration configuration)
throws Exception
{
+ startCase(configuration);
List<String> rows = ImmutableList.of(
"2014-01-01T00:00:10Z,a,1\n",
"2014-01-01T00:00:10Z,b,2\n",
@@ -663,8 +644,6 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
"2014-01-01T02:00:30Z,b,2\n",
"2014-01-01T02:00:30Z,c,3\n"
);
- Assumptions.assumeTrue(lockGranularity == LockGranularity.TIME_CHUNK);
- Assumptions.assumeTrue(useConcurrentLocks, "Minor compaction depends on
concurrent lock");
verifyTaskSuccessRowsAndSchemaMatch(
runTask(buildIndexTask(DEFAULT_TIMESTAMP_SPEC,
DEFAULT_DIMENSIONS_SPEC, DEFAULT_INPUT_FORMAT, rows, inputInterval, false)),
9
@@ -707,11 +686,10 @@ public class MSQCompactionTaskRunTest extends
CompactionTaskRunBase
Assertions.assertEquals(Set.of("range"),
shards.stream().map(ShardSpec::getType).collect(Collectors.toSet()));
}
- @Test
- public void testMinorCompactionOverlappingInterval() throws Exception
+ @CompactionTest(Selection.CONCURRENT_TIME_CHUNK_LOCK)
+ public void testMinorCompactionOverlappingInterval(Configuration
configuration) throws Exception
{
- Assumptions.assumeTrue(lockGranularity == LockGranularity.TIME_CHUNK);
- Assumptions.assumeTrue(useConcurrentLocks, "Minor compaction depends on
concurrent lock");
+ startCase(configuration);
List<String> rows = new ArrayList<>();
rows.add("2014-01-01T00:00:10Z,a1,11\n");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
