Repository: maven-surefire Updated Branches: refs/heads/master 391bab043 -> 7bd22e0ad
[SUREFIRE-654] Support for TestNG successPercentage TestNG has the concept of "successPercentage" where a probabilistic test can be run some number of invocations and then if the successPercentage is met, the test is considered passing. Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/2340c019 Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/2340c019 Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/2340c019 Branch: refs/heads/master Commit: 2340c019d6e4c217415ed52b0d3a0031c944744b Parents: 391bab0 Author: Jon Todd <jonrt...@gmail.com> Authored: Tue Sep 2 17:29:08 2014 -0700 Committer: Andreas Gudian <agud...@apache.org> Committed: Sun Oct 26 22:27:43 2014 +0100 ---------------------------------------------------------------------- .../surefire/its/TestNgSuccessPercentageIT.java | 27 ++++++ .../testng-succes-percentage-fail/pom.xml | 90 ++++++++++++++++++++ .../java/testng/TestNGSuccessPercentTest.java | 44 ++++++++++ .../testng-succes-percentage-pass/pom.xml | 90 ++++++++++++++++++++ .../java/testng/TestNGSuccessPercentTest.java | 44 ++++++++++ .../maven/surefire/testng/TestNGReporter.java | 2 +- 6 files changed, 296 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/2340c019/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java new file mode 100644 index 0000000..0df5cb6 --- /dev/null +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/TestNgSuccessPercentageIT.java @@ -0,0 +1,27 @@ +package org.apache.maven.surefire.its; + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * Test that TestNG's @Test(successPercentage = n, invocationCount=n) passes so long as successPercentage tests + * have passed. + * + * @author Jon Todd + */ +public class TestNgSuccessPercentageIT extends SurefireJUnit4IntegrationTestCase { + @Test + public void testPassesWhenFailuresLessThanSuccessPercentage() + { + OutputValidator validator = unpack("/testng-succes-percentage-pass").executeTest(); + validator.assertTestSuiteResults(4, 0, 0, 0); + } + + @Test + public void testFailsWhenFailuresMoreThanSuccessPercentage() + { + OutputValidator validator = unpack("/testng-succes-percentage-fail").executeTest(); + validator.assertTestSuiteResults(4, 0, 1, 0); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/2340c019/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml b/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml new file mode 100644 index 0000000..42933f2 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/pom.xml @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.maven.plugins.surefire</groupId> + <artifactId>junit4</artifactId> + <version>1.0-SNAPSHOT</version> + <name>Test for Testng</name> + + + <properties> + <testNgVersion>5.7</testNgVersion> + <testNgClassifier>jdk15</testNgClassifier> + </properties> + + <profiles> + <profile> + <id>testng-old</id> + <activation> + <property><name>testNgClassifier</name></property> + </activation> + <dependencies> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>${testNgVersion}</version> + <classifier>${testNgClassifier}</classifier> + </dependency> + </dependencies> + </profile> + <profile> + <id>testng-new</id> + <activation> + <property><name>!testNgClassifier</name></property> + </activation> + <dependencies> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>${testNgVersion}</version> + </dependency> + </dependencies> + </profile> + </profiles> + + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.3.2</version> + <configuration> + <source>1.5</source> + <target>1.5</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <test>TestNGSuccessPercentTest</test> + </configuration> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/2340c019/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java b/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java new file mode 100644 index 0000000..a5b7a25 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/testng-succes-percentage-fail/src/test/java/testng/TestNGSuccessPercentTest.java @@ -0,0 +1,44 @@ +package testng; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.testng.annotations.*; +import static org.testng.Assert.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class TestNGSuccessPercentTest +{ + + private static AtomicInteger counter = new AtomicInteger(0); + + // Pass 2 of 4 tests, expect this test to fail when 60% success is required + @Test(invocationCount = 4, successPercentage = 60) + public void testFailure() { + if (isOdd(counter.get())) { + assertTrue(false); // Fail + } + counter.addAndGet(1); + } + + private boolean isOdd(int number) { + return number % 2 == 0; + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/2340c019/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml b/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml new file mode 100644 index 0000000..42933f2 --- /dev/null +++ b/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/pom.xml @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.maven.plugins.surefire</groupId> + <artifactId>junit4</artifactId> + <version>1.0-SNAPSHOT</version> + <name>Test for Testng</name> + + + <properties> + <testNgVersion>5.7</testNgVersion> + <testNgClassifier>jdk15</testNgClassifier> + </properties> + + <profiles> + <profile> + <id>testng-old</id> + <activation> + <property><name>testNgClassifier</name></property> + </activation> + <dependencies> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>${testNgVersion}</version> + <classifier>${testNgClassifier}</classifier> + </dependency> + </dependencies> + </profile> + <profile> + <id>testng-new</id> + <activation> + <property><name>!testNgClassifier</name></property> + </activation> + <dependencies> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>${testNgVersion}</version> + </dependency> + </dependencies> + </profile> + </profiles> + + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.3.2</version> + <configuration> + <source>1.5</source> + <target>1.5</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire.version}</version> + <configuration> + <test>TestNGSuccessPercentTest</test> + </configuration> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/2340c019/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java b/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java new file mode 100644 index 0000000..775587d --- /dev/null +++ b/surefire-integration-tests/src/test/resources/testng-succes-percentage-pass/src/test/java/testng/TestNGSuccessPercentTest.java @@ -0,0 +1,44 @@ +package testng; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.testng.annotations.*; +import static org.testng.Assert.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class TestNGSuccessPercentTest +{ + + private static AtomicInteger counter = new AtomicInteger(0); + + // Pass 2 of 4 tests, expect this test to pass when 50% success is required + @Test(invocationCount = 4, successPercentage = 50) + public void testSuccess() { + if (isOdd(counter.get())) { + assertTrue(false); // Fail + } + counter.addAndGet(1); + } + + private boolean isOdd(int number) { + return number % 2 == 0; + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/2340c019/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java ---------------------------------------------------------------------- diff --git a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java index b4770d6..e8ff18c 100644 --- a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java +++ b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGReporter.java @@ -125,7 +125,7 @@ public class TestNGReporter result.getMethod().getMethodName(), result.getThrowable() ) ); - reporter.testError( report ); + reporter.testSucceeded( report ); } public void onStart( ITestContext context )
