This is an automated email from the ASF dual-hosted git repository. szetszwo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ratis.git
commit 9400a1d136329583747336ad6de132660a13d36e Author: Doroszlai, Attila <[email protected]> AuthorDate: Wed Feb 26 18:40:39 2025 +0100 RATIS-2217. Automatically re-try flaky tests in CI (#1229) --- .github/workflows/ci.yaml | 1 + dev-support/checks/unit.sh | 4 +++ pom.xml | 18 ++++++++++ ratis-assembly/pom.xml | 2 ++ ratis-client/pom.xml | 5 +++ .../test/java/org/apache/ratis/test/tag/Flaky.java | 41 ++++++++++++++++++++++ .../java/org/apache/ratis/test/tag/FlakyTest.java | 28 +++++++++++++++ ratis-docs/pom.xml | 6 ++++ .../java/org/apache/ratis/TestMultiRaftGroup.java | 4 +++ ratis-experiments/pom.xml | 5 +++ ratis-metrics-api/pom.xml | 5 +++ ratis-netty/pom.xml | 5 +++ ratis-proto/pom.xml | 2 ++ ratis-replicated-map/pom.xml | 5 +++ ratis-server-api/pom.xml | 5 +++ .../ratis/server/impl/LeaderElectionTests.java | 6 ++-- ratis-shell/pom.xml | 5 +++ .../org/apache/ratis/grpc/TestRaftWithGrpc.java | 2 ++ ratis-tools/pom.xml | 5 +++ 19 files changed, 152 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 55d55336b..cf230d384 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,6 +83,7 @@ jobs: - grpc - server - misc + - flaky fail-fast: false uses: ./.github/workflows/check.yaml with: diff --git a/dev-support/checks/unit.sh b/dev-support/checks/unit.sh index c652136da..c0369898e 100755 --- a/dev-support/checks/unit.sh +++ b/dev-support/checks/unit.sh @@ -36,6 +36,10 @@ mkdir -p "$REPORT_DIR" export MAVEN_OPTS="-Xmx4096m" MAVEN_OPTIONS='-V -B' +if [[ "$@" =~ "-Pflaky-tests" ]]; then + MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.timeout=1200" +fi + if [[ "${FAIL_FAST}" == "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-fast -Dsurefire.skipAfterFailureCount=1" else diff --git a/pom.xml b/pom.xml index 29b2137e4..0a8b1c731 100644 --- a/pom.xml +++ b/pom.xml @@ -222,6 +222,7 @@ <slf4j.version>2.0.7</slf4j.version> <junit-bom.version>5.11.2</junit-bom.version> <jacoco.version>0.8.12</jacoco.version> + <flaky-test-groups>flaky | org.apache.ratis.test.tag.FlakyTest</flaky-test-groups> </properties> <dependencyManagement> @@ -1118,6 +1119,7 @@ <includes> <include>org.apache.ratis.grpc.**</include> </includes> + <excludedGroups>${flaky-test-groups}</excludedGroups> </configuration> </plugin> </plugins> @@ -1135,6 +1137,7 @@ <include>org.apache.ratis.datastream.**</include> <include>org.apache.ratis.server.**</include> </includes> + <excludedGroups>${flaky-test-groups}</excludedGroups> </configuration> </plugin> </plugins> @@ -1153,6 +1156,21 @@ <exclude>org.apache.ratis.grpc.**</exclude> <exclude>org.apache.ratis.server.**</exclude> </excludes> + <excludedGroups>${flaky-test-groups}</excludedGroups> + </configuration> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>flaky-tests</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <groups>${flaky-test-groups}</groups> </configuration> </plugin> </plugins> diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml index 276de0a6b..11058eb32 100644 --- a/ratis-assembly/pom.xml +++ b/ratis-assembly/pom.xml @@ -33,6 +33,8 @@ <test.cache.data>${project.build.directory}/test-classes</test.cache.data> <test.build.classes>${project.build.directory}/test-classes</test.build.classes> <license.bundles.dependencies>true</license.bundles.dependencies> + <!-- no testable code in this module --> + <skipTests>true</skipTests> </properties> <build> diff --git a/ratis-client/pom.xml b/ratis-client/pom.xml index 9a8d228d3..8a5008eaf 100644 --- a/ratis-client/pom.xml +++ b/ratis-client/pom.xml @@ -23,6 +23,11 @@ <artifactId>ratis-client</artifactId> <name>Apache Ratis Client</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <groupId>org.apache.ratis</groupId> diff --git a/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java b/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java new file mode 100644 index 000000000..2d8c63030 --- /dev/null +++ b/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.test.tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.junit.jupiter.api.Tag; + +/** + * Annotation to mark JUnit5 test classes or methods that exhibit intermittent + * issues. These are run separately from the normal tests in CI. In case of + * failure they may be repeated a few times. + * Usage: <code>@Flaky("RATIS-123")</code> + */ +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@Tag("flaky") +public @interface Flaky { + /** + * The issue(s) tracking the flaky test. + */ + String[] value(); +} diff --git a/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java b/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java new file mode 100644 index 000000000..f43bd1698 --- /dev/null +++ b/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.test.tag; + +/** + * Interface to mark JUnit4 test classes or methods that exhibit intermittent + * issues. These are run separately from the normal tests in CI. In case of + * failure they may be repeated a few times. + * Usage: <code>@Category(FlakyTest.class) @Flaky("RATIS-123")</code> + */ +public interface FlakyTest { + // category marker +} diff --git a/ratis-docs/pom.xml b/ratis-docs/pom.xml index 79f9d79b2..08a86fb03 100644 --- a/ratis-docs/pom.xml +++ b/ratis-docs/pom.xml @@ -26,4 +26,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";> <artifactId>ratis-docs</artifactId> <name>Apache Ratis Documentation</name> <packaging>jar</packaging> + + <properties> + <!-- no testable code in this module --> + <skipTests>true</skipTests> + </properties> + </project> diff --git a/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java b/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java index 190f75858..7822b694d 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java +++ b/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java @@ -26,7 +26,9 @@ import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.impl.GroupManagementBaseTest; import org.apache.ratis.server.impl.MiniRaftCluster; import org.apache.ratis.util.Slf4jUtils; +import org.apache.ratis.test.tag.Flaky; import org.apache.ratis.util.function.CheckedBiConsumer; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.event.Level; @@ -35,6 +37,8 @@ import java.io.IOException; import java.util.Collection; import java.util.concurrent.atomic.AtomicInteger; +@Flaky("RATIS-2218") +@Timeout(value = 300) public class TestMultiRaftGroup extends BaseTest { static { Slf4jUtils.setLogLevel(RaftServer.Division.LOG, Level.DEBUG); diff --git a/ratis-experiments/pom.xml b/ratis-experiments/pom.xml index 467ff08eb..5dc169c36 100644 --- a/ratis-experiments/pom.xml +++ b/ratis-experiments/pom.xml @@ -23,6 +23,11 @@ <artifactId>ratis-experiments</artifactId> <name>Apache Ratis Experiments</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <groupId>org.apache.ratis</groupId> diff --git a/ratis-metrics-api/pom.xml b/ratis-metrics-api/pom.xml index 3f30339bd..ba3b36b32 100644 --- a/ratis-metrics-api/pom.xml +++ b/ratis-metrics-api/pom.xml @@ -24,6 +24,11 @@ <artifactId>ratis-metrics-api</artifactId> <name>Apache Ratis Metrics API</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <artifactId>ratis-common</artifactId> diff --git a/ratis-netty/pom.xml b/ratis-netty/pom.xml index bbd6cbe94..b5714a943 100644 --- a/ratis-netty/pom.xml +++ b/ratis-netty/pom.xml @@ -23,6 +23,11 @@ <artifactId>ratis-netty</artifactId> <name>Apache Ratis Netty Support</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <groupId>org.apache.ratis</groupId> diff --git a/ratis-proto/pom.xml b/ratis-proto/pom.xml index c5fc03ac9..9e65096e4 100644 --- a/ratis-proto/pom.xml +++ b/ratis-proto/pom.xml @@ -25,6 +25,8 @@ <properties> <maven.javadoc.skip>true</maven.javadoc.skip> + <!-- no testable code in this module --> + <skipTests>true</skipTests> <spotbugs.skip>true</spotbugs.skip> </properties> diff --git a/ratis-replicated-map/pom.xml b/ratis-replicated-map/pom.xml index a0c4f94c1..486b9cf9c 100644 --- a/ratis-replicated-map/pom.xml +++ b/ratis-replicated-map/pom.xml @@ -23,6 +23,11 @@ <artifactId>ratis-replicated-map</artifactId> <name>Apache Ratis Replicated Map</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <groupId>org.apache.ratis</groupId> diff --git a/ratis-server-api/pom.xml b/ratis-server-api/pom.xml index 7de592030..812da9f02 100644 --- a/ratis-server-api/pom.xml +++ b/ratis-server-api/pom.xml @@ -23,6 +23,11 @@ <artifactId>ratis-server-api</artifactId> <name>Apache Ratis Server API</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <groupId>org.apache.ratis</groupId> diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java index b175ffe29..212938543 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java @@ -39,13 +39,15 @@ import org.apache.ratis.server.metrics.LeaderElectionMetrics; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogTestUtils; import org.apache.ratis.util.ExitUtils; +import org.apache.ratis.test.tag.Flaky; +import org.apache.ratis.thirdparty.com.codahale.metrics.Timer; +import org.apache.ratis.util.CodeInjectionForTesting; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.Timestamp; import org.apache.ratis.util.function.CheckedBiConsumer; -import org.apache.ratis.util.CodeInjectionForTesting; import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -75,7 +77,6 @@ import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.apache.ratis.thirdparty.com.codahale.metrics.Timer; import org.slf4j.event.Level; public abstract class LeaderElectionTests<CLUSTER extends MiniRaftCluster> @@ -743,6 +744,7 @@ public abstract class LeaderElectionTests<CLUSTER extends MiniRaftCluster> } } + @Flaky("RATIS-2108") @Test public void testLeaderLeaseDuringReconfiguration() throws Exception { // use a strict lease diff --git a/ratis-shell/pom.xml b/ratis-shell/pom.xml index 38eed1b98..7b4534a60 100644 --- a/ratis-shell/pom.xml +++ b/ratis-shell/pom.xml @@ -23,6 +23,11 @@ <artifactId>ratis-shell</artifactId> <name>Apache Ratis Shell</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <artifactId>ratis-client</artifactId> diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java index 5821b7f22..c13544a78 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java @@ -28,6 +28,7 @@ import org.apache.ratis.server.raftlog.LogEntryHeader; import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; +import org.apache.ratis.test.tag.Flaky; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.TimeDuration; import org.junit.jupiter.api.Assertions; @@ -55,6 +56,7 @@ public class TestRaftWithGrpc // skip testWithLoad() from parent, called from parameterized testWithLoad(boolean) } + @Flaky("RATIS-2253") @ParameterizedTest @ValueSource(booleans = {true, false}) public void testWithLoad(boolean separateHeartbeat) throws Exception { diff --git a/ratis-tools/pom.xml b/ratis-tools/pom.xml index 49f36f0b7..0fe13f402 100644 --- a/ratis-tools/pom.xml +++ b/ratis-tools/pom.xml @@ -23,6 +23,11 @@ <artifactId>ratis-tools</artifactId> <name>Apache Ratis Tools</name> + <properties> + <!-- no tests in this module so far --> + <skipTests>true</skipTests> + </properties> + <dependencies> <dependency> <groupId>org.apache.ratis</groupId>
