[SUREFIRE-1372] Rerunning failing tests fails in combination with Description#createSuiteDescription
Filtering tests by the tuple of (Class, Method) does not work when a test suite is used as a suite may not have (Class, Method) . Filtering by the description of the failed test should however work. Related Issues: - https://github.com/cucumber/cucumber-jvm/issues/1134 - https://github.com/temyers/cucumber-jvm-parallel-plugin/issues/31 SUREFIRE-1372 Filter junit4 tests to be rerun by description Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/0a81c489 Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/0a81c489 Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/0a81c489 Branch: refs/heads/SUREFIRE-1455 Commit: 0a81c48971e3ee4b79b0858be7b8f4b13ece7287 Parents: 869d6f2 Author: mpkorstanje <[email protected]> Authored: Wed May 10 23:59:31 2017 +0200 Committer: Tibor17 <[email protected]> Committed: Thu Jan 4 00:50:29 2018 +0100 ---------------------------------------------------------------------- .../src/site/apt/examples/cucumber.apt.vm | 115 +++++++++++++++++++ .../apt/examples/rerun-failing-tests.apt.vm | 5 + maven-surefire-plugin/src/site/site.xml | 1 + .../JUnit47RerunFailingTestWithCucumberIT.java | 81 +++++++++++++ .../pom.xml | 77 +++++++++++++ .../org/sample/cucumber/FlakeCucumberTest.java | 29 +++++ .../test/java/org/sample/cucumber/StepDefs.java | 56 +++++++++ .../org/sample/cucumber/Sample.feature | 10 ++ .../common/junit4/JUnit4ProviderUtil.java | 67 ++--------- .../common/junit4/MatchDescriptions.java | 104 +++++++++++++++++ .../common/junit4/JUnit4ProviderUtilTest.java | 30 ++--- .../common/junit48/FailingMethodFilter.java | 85 -------------- .../surefire/common/junit48/FilterFactory.java | 7 +- .../maven/surefire/junit4/JUnit4Provider.java | 38 ++---- .../surefire/junitcore/JUnitCoreProvider.java | 12 +- 15 files changed, 518 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm b/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm new file mode 100644 index 0000000..e0436cc --- /dev/null +++ b/maven-surefire-plugin/src/site/apt/examples/cucumber.apt.vm @@ -0,0 +1,115 @@ + ------ + Using Cucumber + ------ + M.P. Korstanje <[email protected]> + ------ + 2010-10-10 + ------ + + ~~ Licensed to the Apache Software Foundation (ASF) under one + ~~ or more contributor license agreements. See the NOTICE file + ~~ distributed with this work for additional information + ~~ regarding copyright ownership. The ASF licenses this file + ~~ to you under the Apache License, Version 2.0 (the + ~~ "License"); you may not use this file except in compliance + ~~ with the License. You may obtain a copy of the License at + ~~ + ~~ http://www.apache.org/licenses/LICENSE-2.0 + ~~ + ~~ Unless required by applicable law or agreed to in writing, + ~~ software distributed under the License is distributed on an + ~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~~ KIND, either express or implied. See the License for the + ~~ specific language governing permissions and limitations + ~~ under the License. + + ~~ NOTE: For help with the syntax of this file, see: + ~~ http://maven.apache.org/doxia/references/apt-format.html + +Using Cucumber with JUnit + +* Configuring Cucumber JUnit + + To get started with Cucumber, you need to add the required version of Cucumber JUnit to your project: + ++---+ + <dependencies> + [...] + <dependency> + <groupId>io.cucumber</groupId> + <artifactId>cucumber-junit</artifactId> + <version>${cucumber.version}</version> + <scope>test</scope> + </dependency> + [...] + </dependencies> ++---+ + + Then create an empty class that uses the Cucumber JUnit runner. + +#{if}(${project.artifactId}=="maven-surefire-plugin") + ++---+ +package org.sample.cucumber; + +import org.junit.runner.RunWith; + +import cucumber.api.CucumberOptions; +import cucumber.api.junit.Cucumber; + +@RunWith( Cucumber.class ) +public class RunCucumberTest +{ + [...] +} ++---+ + +#{else} + ++---+ +package org.sample.cucumber; + +import org.junit.runner.RunWith; + +import cucumber.api.CucumberOptions; +import cucumber.api.junit.Cucumber; + +@RunWith( Cucumber.class ) +public class RunCucumberIT +{ + [...] +} ++---+ + +#{end} + + This will execute all scenarios in the package of the runner. By default a glue code is assumed to be in the same + package. The <<<@CucumberOptions>>> annotation can be used to provide additional configuration of Cucumber. + +#{if}(${project.artifactId}=="maven-surefire-plugin") + Note that in this example the BDD scenarios are executed by the ${thisPlugin} Plugin in the <<<test>>> phase of the + build lifecycle. + ++---+ +mvn test ++---+ + +#{else} + Note that in this example the BDD scenarios are executed by the ${thisPlugin} Plugin in the <<<integration-test>>> + phase of the build lifecycle. The ${thisPlugin} Plugin can be invoked by calling the <<<verify>>> phase. + ++---+ +mvn verify ++---+ +#{end} + +* Using JUnit Rules + + The Cucumber supports JUnit annotations <<<@ClassRule>>>, <<<@BeforeClass>>> and <<<@AfterClass>>>. These are + invoked around the suite of features. Using these is not recommended as it limits the portability between different + runners. Instead it is recommended to use Cucumbers `Before` and `After` hooks to setup scaffolding. + +* Using other JUnit features + + The Cucumber runner acts like a suite of a JUnit tests. As such other JUnit features such as Categories, Custom JUnit + Listeners and Reporters and re-running failed tests can all be expected to work. http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm index 8a6a8bd..c34e2a6 100644 --- a/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm +++ b/maven-surefire-plugin/src/site/apt/examples/rerun-failing-tests.apt.vm @@ -140,6 +140,11 @@ mvn -D${thisPlugin.toLowerCase()}.rerunFailingTestsCount=2 test The provider <<<surefire-junit4>>> executes individual test class and consequently re-runs failed tests. The provider <<<surefire-junit47>>> executes all test classes and re-runs failed tests afterwards. +* Re-run execution in Cucumber JVM + + Since of 2.21.0 the provider <<<surefire-junit47>>> can rerun scenarios created by <<cucumber-jvm>> + 2.0.0 and higher. + * Re-run and skip execution Since of 2.19.1 you can use parameters <<<skipAfterFailureCount>>> and <<<rerunFailingTestsCount>>> together. http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/maven-surefire-plugin/src/site/site.xml ---------------------------------------------------------------------- diff --git a/maven-surefire-plugin/src/site/site.xml b/maven-surefire-plugin/src/site/site.xml index 2d2e95a..f5497e2 100644 --- a/maven-surefire-plugin/src/site/site.xml +++ b/maven-surefire-plugin/src/site/site.xml @@ -41,6 +41,7 @@ <item name="Using TestNG" href="examples/testng.html"/> <item name="Using JUnit" href="examples/junit.html"/> <item name="Using POJO Tests" href="examples/pojo-test.html"/> + <item name="Using Cucumber" href="examples/cucumber.html"/> <item name="Skipping Tests" href="examples/skipping-tests.html"/> <item name="Skip After Failure" href="examples/skip-after-failure.html"/> <item name="Inclusions and Exclusions of Tests" href="examples/inclusion-exclusion.html"/> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java new file mode 100644 index 0000000..7383783 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/JUnit47RerunFailingTestWithCucumberIT.java @@ -0,0 +1,81 @@ +package org.apache.maven.surefire.its; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static org.apache.commons.lang3.JavaVersion.JAVA_1_7; +import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion; + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests using the JUnit 47 provider to rerun failing tests with the cucumber runner. The main + * problem that the junit4 provider has with the cucumber runner is that the junit Description + * instance created by the runner has a null test class attribute. This requires that tests are + * rerun based on their description. + * + * @author mpkorstanje + */ +public class JUnit47RerunFailingTestWithCucumberIT + extends SurefireJUnit4IntegrationTestCase { + + @Before + public void assumeJdk17() { + assumeJavaVersion(JAVA_1_7); + } + + + private SurefireLauncher unpack() { + return unpack("junit47-rerun-failing-tests-with-cucumber") + .setJUnitVersion("4.12"); + } + + @Test + public void testRerunFailingErrorTestsFalse() { + unpack() + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=" + 0) + .withFailure() + .executeTest() + .assertTestSuiteResults(1, 0, 1, 0, 0); + } + + @Test + public void testRerunFailingErrorTestsWithOneRetry() { + unpack() + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=" + 1) + .withFailure() + .executeTest() + .assertTestSuiteResults(1, 0, 1, 0, 0); + } + + @Test + public void testRerunFailingErrorTestsTwoRetry() { + unpack() + .maven() + .addGoal("-Dsurefire.rerunFailingTestsCount=" + 2) + .executeTest() + .assertTestSuiteResults(1, 0, 0, 0, 2); + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml new file mode 100644 index 0000000..24ee294 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/pom.xml @@ -0,0 +1,77 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>it-parent</artifactId> + <version>1.0</version> + </parent> + + <artifactId>junit47-rerun-failing-tests-with-cucumber</artifactId> + <name>Test for rerun failing cucumber tests in JUnit 47</name> + + <properties> + <cucumber.version>2.0.0</cucumber.version> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <dependencies> + <dependency> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-junit47</artifactId> + <version>${surefire.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </build> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.cucumber</groupId> + <artifactId>cucumber-java</artifactId> + <version>${cucumber.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.cucumber</groupId> + <artifactId>cucumber-junit</artifactId> + <version>${cucumber.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java new file mode 100644 index 0000000..ecda87b --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/FlakeCucumberTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.sample.cucumber; + +import cucumber.api.junit.Cucumber; +import org.junit.runner.RunWith; + +@RunWith( Cucumber.class ) +public class FlakeCucumberTest +{ + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java new file mode 100644 index 0000000..d253222 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/java/org/sample/cucumber/StepDefs.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.sample.cucumber; + +import cucumber.api.java.en.Given; +import cucumber.api.java.en.Then; +import cucumber.api.java.en.When; + +import static org.junit.Assert.fail; + +public class StepDefs +{ + private static int testFailures = 0; + + @Given( "^I have some code$" ) + public void I_have_some_code() + throws Throwable + { + // do nothing + } + + @When( "^I run test$" ) + public void I_run_test() + throws Throwable + { + // do nothing + } + + @Then( "^I get a flake$" ) + public void I_get_a_flake() + throws Throwable + { + // This test will error out with only one retry, but will pass with two + if( testFailures < 2 ) { + testFailures++; + fail( "failing the test on purpose." ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature new file mode 100644 index 0000000..d5b5dac --- /dev/null +++ b/surefire-integration-tests/src/test/resources/junit47-rerun-failing-tests-with-cucumber/src/test/resources/org/sample/cucumber/Sample.feature @@ -0,0 +1,10 @@ +Feature: Sample + + In order to use Maven + As a user + I want to do tests. + + Scenario: Do a flake test + Given I have some code + When I run test + Then I get a flake http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java index 075d9d9..6b94a64 100644 --- a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtil.java @@ -19,19 +19,14 @@ package org.apache.maven.surefire.common.junit4; * under the License. */ -import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import org.apache.maven.surefire.testset.TestSetFailedException; - import org.junit.runner.Description; +import org.junit.runner.manipulation.Filter; import org.junit.runner.notification.Failure; -import static org.apache.maven.surefire.common.junit4.JUnit4Reflector.createRequest; import static org.apache.maven.surefire.util.internal.StringUtils.isBlank; import static org.junit.runner.Description.TEST_MECHANISM; @@ -44,74 +39,30 @@ import static org.junit.runner.Description.TEST_MECHANISM; */ public final class JUnit4ProviderUtil { - private JUnit4ProviderUtil() { throw new IllegalStateException( "Cannot instantiate." ); } /** - * Organize all the failures in previous run into a map between test classes and corresponding failing test methods - * - * @param allFailures all the failures in previous run - * @param testClassLoader ClassLoader used for test classes - * @return a map between failing test classes and their corresponding failing test methods - */ - public static Map<Class<?>, Set<String>> generateFailingTests( List<Failure> allFailures, - ClassLoader testClassLoader ) - throws TestSetFailedException - { - Map<Class<?>, Set<String>> testClassMethods = new HashMap<Class<?>, Set<String>>(); - Set<ClassMethod> failingTests = generateFailingTests( allFailures ); - for ( ClassMethod classMethod: failingTests ) - { - try - { - Class testClassObj = Class.forName( classMethod.getClazz(), false, testClassLoader ); - Set<String> failingMethods = testClassMethods.get( testClassObj ); - if ( failingMethods == null ) - { - failingMethods = new HashSet<String>(); - testClassMethods.put( testClassObj, failingMethods ); - } - failingMethods.add( classMethod.getMethod() ); - } - catch ( ClassNotFoundException e ) - { - throw new TestSetFailedException( "Unable to create test class '" + classMethod.getClazz() + "'", e ); - } - } - return testClassMethods; - } - - /** - * Get all test methods from a list of Failures + * Get all descriptions from a list of Failures * * @param allFailures the list of failures for a given test class - * @return the list of test methods + * @return the list of descriptions */ - public static Set<ClassMethod> generateFailingTests( List<Failure> allFailures ) + public static Set<Description> generateFailingTestDescriptions( List<Failure> allFailures ) { - Set<ClassMethod> failingMethods = new HashSet<ClassMethod>(); + Set<Description> failingTestDescriptions = new HashSet<Description>(); for ( Failure failure : allFailures ) { Description description = failure.getDescription(); if ( description.isTest() && !isFailureInsideJUnitItself( description ) ) { - ClassMethod classMethod = cutTestClassAndMethod( description ); - if ( classMethod.isValid() ) - { - failingMethods.add( classMethod ); - } + failingTestDescriptions.add( description ); } } - return failingMethods; - } - - public static Description createSuiteDescription( Collection<Class<?>> classes ) - { - return createRequest( classes.toArray( new Class[classes.size()] ) ).getRunner().getDescription(); + return failingTestDescriptions; } public static boolean isFailureInsideJUnitItself( Description failure ) @@ -153,4 +104,8 @@ public final class JUnit4ProviderUtil return isBlank( s ) ? null : s; } + public static Filter createMatchAnyDescriptionFilter( Iterable<Description> descriptions ) + { + return new MatchDescriptions( descriptions ); + } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java new file mode 100644 index 0000000..3275f7f --- /dev/null +++ b/surefire-providers/common-junit4/src/main/java/org/apache/maven/surefire/common/junit4/MatchDescriptions.java @@ -0,0 +1,104 @@ +package org.apache.maven.surefire.common.junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.junit.runner.Description; +import org.junit.runner.manipulation.Filter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Only run test methods in the given failure set. + * + * @author mpkorstanje + */ +public final class MatchDescriptions + extends Filter +{ + private final List<Filter> filters = new ArrayList<Filter>(); + + public MatchDescriptions( Iterable<Description> descriptions ) + { + for ( Description description : descriptions ) + { + filters.add( matchDescription( description ) ); + } + } + + @Override + public boolean shouldRun( Description description ) + { + for ( Filter filter : filters ) + { + if ( filter.shouldRun( description ) ) + { + return true; + } + } + return false; + } + + @Override + public String describe() + { + StringBuilder description = new StringBuilder( "Matching description " ); + for ( int i = 0; i < filters.size(); i++ ) + { + description.append( filters.get( i ).describe() ); + if ( i != filters.size() - 1 ) + { + description.append( " OR " ); + } + } + return description.toString(); + } + + private static Filter matchDescription( final Description desiredDescription ) + { + return new Filter() + { + @Override + public boolean shouldRun( Description description ) + { + if ( description.isTest() ) + { + return desiredDescription.equals( description ); + } + + for ( Description each : description.getChildren() ) + { + if ( shouldRun( each ) ) + { + return true; + } + } + + return false; + } + + @Override + public String describe() + { + return String.format( "Method %s", desiredDescription.getDisplayName() ); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java index b05a562..7a2e6da 100644 --- a/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java +++ b/surefire-providers/common-junit4/src/test/java/org/apache/maven/surefire/common/junit4/JUnit4ProviderUtilTest.java @@ -20,14 +20,11 @@ package org.apache.maven.surefire.common.junit4; */ import junit.framework.TestCase; -import org.apache.maven.surefire.testset.TestSetFailedException; import org.junit.runner.Description; import org.junit.runner.notification.Failure; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.*; @@ -38,9 +35,9 @@ import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.*; public class JUnit4ProviderUtilTest extends TestCase { - public void testGenerateFailingTests() throws TestSetFailedException + public void testGenerateFailingTestDescriptions() { - List<Failure> failures = new ArrayList<Failure>( ); + List<Failure> failures = new ArrayList<Failure>(); Description test1Description = Description.createTestDescription( T1.class, "testOne" ); Description test2Description = Description.createTestDescription( T1.class, "testTwo" ); @@ -54,21 +51,14 @@ public class JUnit4ProviderUtilTest failures.add( new Failure( test4Description, new AssertionError() ) ); failures.add( new Failure( test5Description, new RuntimeException() ) ); - Map<Class<?>, Set<String>> result = generateFailingTests( failures, getClass().getClassLoader() ); + Set<Description> result = generateFailingTestDescriptions( failures ); - assertEquals( 2, result.size() ); - Set<String> resultForT1 = result.get( T1.class ); - Set<String> resultForT2 = result.get( T2.class ); + assertEquals( 4, result.size() ); - Set<String> expectedResultForT1 = new HashSet<String>(); - expectedResultForT1.add( "testOne" ); - expectedResultForT1.add( "testTwo" ); - Set<String> expectedResultForT2 = new HashSet<String>(); - expectedResultForT2.add( "testThree" ); - expectedResultForT2.add( "testFour" ); - - assertEquals( expectedResultForT1, resultForT1 ); - assertEquals( expectedResultForT2, resultForT2 ); + assertTrue( result.contains( test1Description) ); + assertTrue( result.contains( test2Description) ); + assertTrue( result.contains( test3Description) ); + assertTrue( result.contains( test4Description) ); } public void testIllegalTestDescription$NegativeTest() @@ -97,12 +87,12 @@ public class JUnit4ProviderUtilTest assertEquals( T1.class.getName(), classMethod.getClazz() ); } - class T1 + private static class T1 { } - class T2 + private static class T2 { } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java deleted file mode 100644 index e69e7d0..0000000 --- a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FailingMethodFilter.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.apache.maven.surefire.common.junit48; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.maven.shared.utils.io.SelectorUtils; -import org.junit.runner.Description; -import org.junit.runner.manipulation.Filter; - -import java.util.Map; -import java.util.Set; - -/** - * Only run test methods in the given input map, indexed by test class - */ -final class FailingMethodFilter - extends Filter -{ - // Map from Class -> List of method names. Are the method names hashed to include the signature? - private final Map<Class<?>, Set<String>> failingClassMethodPatterns; - - public FailingMethodFilter( Map<Class<?>, Set<String>> failingClassMethodPatterns ) - { - this.failingClassMethodPatterns = failingClassMethodPatterns; - } - - @Override - public boolean shouldRun( Description description ) - { - return isDescriptionMatch( description ); - } - - private boolean isDescriptionMatch( Description description ) - { - if ( description.getTestClass() == null || description.getMethodName() == null ) - { - for ( Description childrenDescription : description.getChildren() ) - { - if ( isDescriptionMatch( childrenDescription ) ) - { - return true; - } - } - return false; - } - - Set<String> testMethodPatterns = failingClassMethodPatterns.get( description.getTestClass() ); - String testMethod = description.getMethodName(); - return testMethodPatterns != null && matchMethod( testMethodPatterns, testMethod ); - } - - private static boolean matchMethod( Set<String> patterns, String methodName ) - { - for ( String pattern : patterns ) - { - if ( SelectorUtils.match( pattern, methodName ) ) - { - return true; - } - } - return false; - } - - @Override - public String describe() - { - return "By failing class method"; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java ---------------------------------------------------------------------- diff --git a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java index 228ca6d..6ebff47 100644 --- a/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java +++ b/surefire-providers/common-junit48/src/main/java/org/apache/maven/surefire/common/junit48/FilterFactory.java @@ -19,14 +19,15 @@ package org.apache.maven.surefire.common.junit48; * under the License. */ +import org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil; import org.apache.maven.surefire.group.match.GroupMatcher; import org.apache.maven.surefire.group.parse.GroupMatcherParser; import org.apache.maven.surefire.group.parse.ParseException; import org.apache.maven.surefire.testset.TestListResolver; +import org.junit.runner.Description; import org.junit.runner.manipulation.Filter; import java.util.Map; -import java.util.Set; import static org.apache.maven.surefire.booter.ProviderParameterNames.TESTNG_EXCLUDEDGROUPS_PROP; import static org.apache.maven.surefire.booter.ProviderParameterNames.TESTNG_GROUPS_PROP; @@ -118,9 +119,9 @@ public class FilterFactory return new MethodFilter( resolver ); } - public Filter createFailingMethodFilter( Map<Class<?>, Set<String>> failingClassMethodMap ) + public Filter createMatchAnyDescriptionFilter( Iterable<Description> descriptions ) { - return new FailingMethodFilter( failingClassMethodMap ); + return JUnit4ProviderUtil.createMatchAnyDescriptionFilter( descriptions ); } public Filter and( Filter filter1, Filter filter2 ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index ad29224..c31a019 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -22,7 +22,6 @@ package org.apache.maven.surefire.junit4; import org.apache.maven.surefire.booter.Command; import org.apache.maven.surefire.booter.CommandListener; import org.apache.maven.surefire.booter.CommandReader; -import org.apache.maven.surefire.common.junit4.ClassMethod; import org.apache.maven.surefire.common.junit4.JUnit4RunListener; import org.apache.maven.surefire.common.junit4.JUnit4TestChecker; import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener; @@ -46,7 +45,6 @@ import org.junit.runner.Request; import org.junit.runner.Result; import org.junit.runner.Runner; import org.junit.runner.manipulation.Filter; -import org.junit.runner.notification.RunNotifier; import org.junit.runner.notification.StoppedByUserException; import java.util.Collection; @@ -55,7 +53,9 @@ import java.util.Set; import static java.lang.reflect.Modifier.isAbstract; import static java.lang.reflect.Modifier.isInterface; import static org.apache.maven.surefire.booter.CommandReader.getReader; -import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTests; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.createMatchAnyDescriptionFilter; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTestDescriptions; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.isFailureInsideJUnitItself; import static org.apache.maven.surefire.common.junit4.JUnit4Reflector.createDescription; import static org.apache.maven.surefire.common.junit4.JUnit4Reflector.createIgnored; import static org.apache.maven.surefire.common.junit4.JUnit4RunListener.rethrowAnyTestMechanismFailures; @@ -67,7 +67,6 @@ import static org.apache.maven.surefire.testset.TestListResolver.optionallyWildc import static org.apache.maven.surefire.util.TestsToRun.fromClass; import static org.apache.maven.surefire.util.internal.ObjectUtils.systemProps; import static org.junit.runner.Request.aClass; -import static org.junit.runner.Request.method; /** * @author Kristian Rosenvold @@ -261,7 +260,6 @@ public class JUnit4Provider } private void executeWithRerun( Class<?> clazz, Notifier notifier ) - throws TestSetFailedException { JUnitTestFailureListener failureListener = new JUnitTestFailureListener(); notifier.addListener( failureListener ); @@ -286,12 +284,10 @@ public class JUnit4Provider notifier.copyListenersTo( rerunNotifier ); for ( int i = 0; i < rerunFailingTestsCount && !failureListener.getAllFailures().isEmpty(); i++ ) { - Set<ClassMethod> failedTests = generateFailingTests( failureListener.getAllFailures() ); + Set<Description> failures = generateFailingTestDescriptions( failureListener.getAllFailures() ); failureListener.reset(); - if ( !failedTests.isEmpty() ) - { - executeFailedMethod( rerunNotifier, failedTests ); - } + Filter failureDescriptionFilter = createMatchAnyDescriptionFilter( failures ); + execute( clazz, rerunNotifier, failureDescriptionFilter ); } } } @@ -371,24 +367,6 @@ public class JUnit4Provider } } - private void executeFailedMethod( RunNotifier notifier, Set<ClassMethod> failedMethods ) - throws TestSetFailedException - { - for ( ClassMethod failedMethod : failedMethods ) - { - try - { - Class<?> methodClass = Class.forName( failedMethod.getClazz(), true, testClassLoader ); - String methodName = failedMethod.getMethod(); - method( methodClass, methodName ).getRunner().run( notifier ); - } - catch ( ClassNotFoundException e ) - { - throw new TestSetFailedException( "Unable to create test class '" + failedMethod.getClazz() + "'", e ); - } - } - } - /** * JUnit error: test count includes one test-class as a suite which has filtered out all children. * Then the child test has a description "initializationError0(org.junit.runner.manipulation.Filter)" @@ -422,6 +400,10 @@ public class JUnit4Provider private static boolean hasFilteredOutAllChildren( Description description ) { + if ( isFailureInsideJUnitItself( description ) ) + { + return true; + } String name = description.getDisplayName(); // JUnit 4.0: initializationError0; JUnit 4.12: initializationError. if ( name == null ) http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/0a81c489/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java index a4c061e..a652515 100644 --- a/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java +++ b/surefire-providers/surefire-junit47/src/main/java/org/apache/maven/surefire/junitcore/JUnitCoreProvider.java @@ -39,16 +39,15 @@ import org.apache.maven.surefire.util.RunOrderCalculator; import org.apache.maven.surefire.util.ScanResult; import org.apache.maven.surefire.util.ScannerFilter; import org.apache.maven.surefire.util.TestsToRun; +import org.junit.runner.Description; import org.junit.runner.manipulation.Filter; -import org.junit.runner.notification.Failure; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import static org.apache.maven.surefire.booter.CommandReader.getReader; -import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTests; +import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.generateFailingTestDescriptions; import static org.apache.maven.surefire.common.junit4.JUnit4RunListenerFactory.createCustomListeners; import static org.apache.maven.surefire.common.junit4.Notifier.pureNotifier; import static org.apache.maven.surefire.junitcore.ConcurrentRunListener.createInstance; @@ -167,12 +166,11 @@ public class JUnitCoreProvider JUnitCoreWrapper rerunCore = new JUnitCoreWrapper( rerunNotifier, jUnitCoreParameters, consoleStream ); for ( int i = 0; i < rerunFailingTestsCount && !testFailureListener.getAllFailures().isEmpty(); i++ ) { - List<Failure> failures = testFailureListener.getAllFailures(); - Map<Class<?>, Set<String>> failingTests = generateFailingTests( failures, testClassLoader ); + Set<Description> failures = generateFailingTestDescriptions( testFailureListener.getAllFailures() ); testFailureListener.reset(); FilterFactory filterFactory = new FilterFactory( testClassLoader ); - Filter failingMethodsFilter = filterFactory.createFailingMethodFilter( failingTests ); - rerunCore.execute( testsToRun, failingMethodsFilter ); + Filter failureDescriptionFilter = filterFactory.createMatchAnyDescriptionFilter( failures ); + rerunCore.execute( testsToRun, failureDescriptionFilter ); } } }
