http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncherTest.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncherTest.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncherTest.java new file mode 100644 index 0000000..b5df941 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncherTest.java @@ -0,0 +1,43 @@ +package org.apache.maven.surefire.its.fixture; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import org.apache.maven.it.VerificationException; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author Kristian Rosenvold + */ +public class SurefireLauncherTest +{ + @Test + public void launcherGetsProperMethodName() + throws IOException, VerificationException + { + MavenLauncher mavenLauncher = new MavenLauncher( SurefireLauncherTest.class, "foo", "" ); + String method = new SurefireLauncher( mavenLauncher ).getTestMethodName(); + assertEquals( "launcherGetsProperMethodName", method ); + + } +}
http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireVerifierException.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireVerifierException.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireVerifierException.java new file mode 100644 index 0000000..5201503 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireVerifierException.java @@ -0,0 +1,37 @@ +package org.apache.maven.surefire.its.fixture; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * @author Kristian Rosenvold + */ +public class SurefireVerifierException + extends RuntimeException +{ + public SurefireVerifierException( String message, Throwable cause ) + { + super( message, cause ); + } + + public SurefireVerifierException( Throwable cause ) + { + super( cause ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFile.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFile.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFile.java new file mode 100644 index 0000000..cf6ad84 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFile.java @@ -0,0 +1,155 @@ +package org.apache.maven.surefire.its.fixture; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.Charset; +import java.util.List; +import org.apache.commons.io.FileUtils; + +import junit.framework.Assert; + +import static junit.framework.Assert.assertTrue; + +/** + * @author Kristian Rosenvold + */ +public class TestFile +{ + private final File file; + + private final Charset encoding; + + private final OutputValidator surefireVerifier; + + public TestFile( File file, OutputValidator surefireVerifier ) + { + this( file, Charset.defaultCharset(), surefireVerifier); + } + + public TestFile( File file, Charset charset, OutputValidator surefireVerifier ) + { + this.file = file; + this.encoding = charset; + this.surefireVerifier = surefireVerifier; + } + + public OutputValidator assertFileExists() + { + assertTrue( "File doesn't exist: " + file.getAbsolutePath(), file.exists() ); + return surefireVerifier; + } + + public OutputValidator assertFileNotExists() + { + assertTrue( "File doesn't exist: " + file.getAbsolutePath(), !file.exists() ); + return surefireVerifier; + } + + public void delete() + { + //noinspection ResultOfMethodCallIgnored + file.delete(); + } + + public String getAbsolutePath() + { + return file.getAbsolutePath(); + } + + public boolean exists() + { + return file.exists(); + } + + public FileInputStream getFileInputStream() + throws FileNotFoundException + { + return new FileInputStream( file ); + } + + public String slurpFile() + { + try + { + StringBuilder sb = new StringBuilder(); + BufferedReader reader; + reader = new BufferedReader( new FileReader( file ) ); + for ( String line = reader.readLine(); line != null; line = reader.readLine() ) + { + sb.append( line ); + } + reader.close(); + return sb.toString(); + } + catch ( IOException e ) + { + throw new SurefireVerifierException( e ); + } + + } + + public String readFileToString() + { + try + { + return FileUtils.readFileToString( file ); + } + catch ( IOException e ) + { + throw new SurefireVerifierException( e ); + } + } + + public boolean isFile() + { + return file.isFile(); + } + + public TestFile assertContainsText( String text ) + { + final List<String> list = surefireVerifier.loadFile( file, encoding ); + for ( String line : list ) + { + if ( line.contains( text ) ) + { + return this; + } + } + Assert.fail( "Did not find expected message in log" ); + return null; + } + + public URI toURI() + { + return file.toURI(); + } + + public File getFile() + { + return file; + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFramework.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFramework.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFramework.java new file mode 100644 index 0000000..c544347 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFramework.java @@ -0,0 +1,29 @@ +package org.apache.maven.surefire.its.fixture; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @since 2.19 + */ +public enum TestFramework +{ + JUNIT4, JUNIT47, TestNG +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1024VerifyFailsafeIfTestedIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1024VerifyFailsafeIfTestedIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1024VerifyFailsafeIfTestedIT.java new file mode 100644 index 0000000..886cd07 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1024VerifyFailsafeIfTestedIT.java @@ -0,0 +1,48 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * "verify" goal ignores "dependenciesToScan" parameter when checking tests existence + * <p> + * Found in Surefire 2.16. + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1024">SUREFIRE-1024</a> + * @since 2.19 + */ +public class Surefire1024VerifyFailsafeIfTestedIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void shouldScanAndRunTestsInDependencyJars() throws Exception { + SurefireLauncher launcher = unpack( "surefire-1024" ); + launcher.executeVerify() + .verifyTextInLog( "class jiras.surefire1024.A1IT#test() dependency to scan" ); + + launcher.getSubProjectValidator( "jiras-surefire-1024-it" ) + .assertIntegrationTestSuiteResults( 1, 0, 0, 0 ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1028UnableToRunSingleIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1028UnableToRunSingleIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1028UnableToRunSingleIT.java new file mode 100644 index 0000000..ec63a8a --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1028UnableToRunSingleIT.java @@ -0,0 +1,59 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * Plugin Configuration: parallel=classes + * <br> + * With Surefire 2.15 + * {@code $ mvn test -Dtest=MyTest#testFoo} + * Results: + * Tests run: 1, Failures: 0, Errors: 0, Skipped: 0 + * <br> + * With Surefire 2.16 + * {@code $ mvn test -Dtest=MyTest#testFoo} + * <br> + * Results: + * Tests run: 0, Failures: 0, Errors: 0, Skipped: 0 + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1028">SUREFIRE-1028</a> + * @since 2.18 + */ +public class Surefire1028UnableToRunSingleIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void methodFilteringParallelExecution() + { + unpack().setTestToRun( "SomeTest#test" ).parallelClasses().useUnlimitedThreads() + .executeTest().verifyErrorFree( 1 ).verifyTextInLog( "OK!" ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1028-unable-to-run-single-test" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1036NonFilterableJUnitRunnerWithCategoriesIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1036NonFilterableJUnitRunnerWithCategoriesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1036NonFilterableJUnitRunnerWithCategoriesIT.java new file mode 100644 index 0000000..0d6d8b6 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1036NonFilterableJUnitRunnerWithCategoriesIT.java @@ -0,0 +1,71 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.it.VerificationException; +import org.apache.maven.shared.utils.xml.Xpp3Dom; +import org.apache.maven.shared.utils.xml.Xpp3DomBuilder; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.apache.maven.surefire.its.fixture.TestFile; +import org.junit.Test; + +import java.io.FileNotFoundException; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1036">SUREFIRE-1036</a> + * @since 2.18 + */ +public class Surefire1036NonFilterableJUnitRunnerWithCategoriesIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void test() + throws VerificationException, FileNotFoundException + { + OutputValidator validator = unpack().maven().executeTest(); + validator.assertTestSuiteResults( 1, 0, 0, 0 ); + assertFalse( validator.getSurefireReportsXmlFile( + "TEST-jiras.surefire1036.TestSomethingWithMockitoRunner.xml" ).exists() ); + assertFalse( validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1036.TestSomeUnit.xml" ).exists() ); + TestFile reportFile = + validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1036.TestSomeIntegration.xml" ); + assertTestCount( reportFile, 1 ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1036-NonFilterableJUnitRunnerWithCategories" ); + } + + private void assertTestCount( TestFile reportFile, int tests ) + throws FileNotFoundException + { + assertTrue( reportFile.exists() ); + Xpp3Dom testResult = Xpp3DomBuilder.build( reportFile.getFileInputStream(), "UTF-8" ); + Xpp3Dom[] children = testResult.getChildren( "testcase" ); + assertEquals( tests, children.length ); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1041FailingJUnitRunnerIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1041FailingJUnitRunnerIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1041FailingJUnitRunnerIT.java new file mode 100644 index 0000000..bb65682 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1041FailingJUnitRunnerIT.java @@ -0,0 +1,38 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * SUREFIRE-1041: An error in a JUnit runner should not lead to an error in Surefire + * + * @author Andreas Gudian + */ +public class Surefire1041FailingJUnitRunnerIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void reportErrorInJUnitRunnerAsTestError() + { + unpack( "surefire-1041-exception-in-junit-runner" ).mavenTestFailureIgnore( true ).executeTest().assertTestSuiteResults( 1, 1, 0, 0 ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1053SystemPropertiesIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1053SystemPropertiesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1053SystemPropertiesIT.java new file mode 100644 index 0000000..b3ed744 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1053SystemPropertiesIT.java @@ -0,0 +1,72 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1053">SUREFIRE-1053</a> + * @since 2.18 + */ +public class Surefire1053SystemPropertiesIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void checkWarningsFileEncoding() + { + unpack().sysProp( "file.encoding", "ISO-8859-1" ) + .executeTest() + .verifyErrorFree( 1 ) + .verifyTextInLog( "file.encoding cannot be set as system property, use <argLine>-D" + + "file.encoding=...</argLine> instead" ); + } + @Test + public void checkWarningsSysPropTwice() throws Exception + { + OutputValidator validator = unpack() + .argLine( "-DmyArg=myVal2 -Dfile.encoding=ISO-8859-1" ) + .sysProp( "file.encoding", "ISO-8859-1" ) + .executeTest() + .verifyErrorFree( 1 ) + .verifyTextInLog( "The system property myArg is configured twice! " + + "The property appears in <argLine/> and any of <systemPropertyVariables/>, " + + "<systemProperties/> or user property." ); + + for ( String line : validator.loadLogLines() ) + { + assertFalse( "no warning for file.encoding not in argLine", + line.contains( "file.encoding cannot be set as system property, use <argLine>" ) ); + assertFalse( "no warning for double definition of file.encoding", + line.contains( "The system property file.encoding is configured twice!" ) ); + } + + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1053-system-properties" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1055CorrectConcurrentTestCountIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1055CorrectConcurrentTestCountIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1055CorrectConcurrentTestCountIT.java new file mode 100644 index 0000000..0293a75 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1055CorrectConcurrentTestCountIT.java @@ -0,0 +1,41 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * SUREFIRE-1055 Correct test count in parallel run mode. + * + * @author Kristian Rosenvold + */ +public class Surefire1055CorrectConcurrentTestCountIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void testTestNgAndJUnitTogether() + { + OutputValidator outputValidator = unpack( "surefire-1055-parallelTestCount" ).executeTest(); + outputValidator.assertTestSuiteResults( 21, 0, 0, 0 ); + + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java new file mode 100644 index 0000000..d6a38fa --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1080ParallelForkDoubleTestIT.java @@ -0,0 +1,54 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * Description of SUREFIRE-1080: <br> + * <br> + * There are 9 tests in total in the attached project, and mvn test will show 9 tests run. + * When I use the command " mvn test -Dparallel=classes -DforkCount=2 -DuseUnlimitedThreads=true", it shows 13 tests + * run (and sometimes 16), and some tests are run more than once. + * If I remove forkCount, or parallel, everything will be fine. But it is problematic when combining together. + * Apache Maven 3.2.2-SNAPSHOT + * Surefire 2.18-SNAPSHOT + * JUnit 4.11 + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1080">SUREFIRE-1080</a> + * @since 2.18 + */ +public class Surefire1080ParallelForkDoubleTestIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void test() + { + unpack().executeTest().assertTestSuiteResults( 9 ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1080-parallel-fork-double-test" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1082ParallelJUnitParameterizedIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1082ParallelJUnitParameterizedIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1082ParallelJUnitParameterizedIT.java new file mode 100644 index 0000000..2669b2f --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1082ParallelJUnitParameterizedIT.java @@ -0,0 +1,212 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.it.VerificationException; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.apache.maven.surefire.its.fixture.TestFile; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.junit.Test; + +import java.nio.charset.Charset; +import java.util.Collection; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +import static org.hamcrest.core.AnyOf.anyOf; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1082">SUREFIRE-1082</a> + * @since 2.18 + */ +public class Surefire1082ParallelJUnitParameterizedIT + extends SurefireJUnit4IntegrationTestCase +{ + private static Set<String> printOnlyTestLinesFromConsole( OutputValidator validator ) + throws VerificationException + { + return printOnlyTestLines( validator.loadLogLines() ); + } + + private static Set<String> printOnlyTestLinesFromOutFile( OutputValidator validator ) + throws VerificationException + { + TestFile report = validator.getSurefireReportsFile( "jiras.surefire1082.Jira1082Test-output.txt" ); + report.assertFileExists(); + return printOnlyTestLines( validator.loadFile( report.getFile(), Charset.forName( "UTF-8" ) ) ); + } + + private static Set<String> printOnlyTestLines( Collection<String> logs ) + throws VerificationException + { + Set<String> log = new TreeSet<String>(); + for ( String line : logs ) + { + if ( line.startsWith( "class jiras.surefire1082." ) ) + { + log.add( line ); + } + } + return log; + } + + private static Matcher<Set<String>> regex( Set<String> r ) + { + return new IsRegex( r ); + } + + private static void assertParallelRun( Set<String> log ) + { + assertThat( log.size(), is( 4 ) ); + + Set<String> expectedLogs1 = new TreeSet<String>(); + expectedLogs1.add( "class jiras.surefire1082.Jira1082Test a 0 pool-[\\d]+-thread-1" ); + expectedLogs1.add( "class jiras.surefire1082.Jira1082Test b 0 pool-[\\d]+-thread-1" ); + expectedLogs1.add( "class jiras.surefire1082.Jira1082Test a 1 pool-[\\d]+-thread-2" ); + expectedLogs1.add( "class jiras.surefire1082.Jira1082Test b 1 pool-[\\d]+-thread-2" ); + + Set<String> expectedLogs2 = new TreeSet<String>(); + expectedLogs2.add( "class jiras.surefire1082.Jira1082Test a 1 pool-[\\d]+-thread-1" ); + expectedLogs2.add( "class jiras.surefire1082.Jira1082Test b 1 pool-[\\d]+-thread-1" ); + expectedLogs2.add( "class jiras.surefire1082.Jira1082Test a 0 pool-[\\d]+-thread-2" ); + expectedLogs2.add( "class jiras.surefire1082.Jira1082Test b 0 pool-[\\d]+-thread-2" ); + + assertThat( log, anyOf( regex( expectedLogs1 ), regex( expectedLogs2 ) ) ); + } + + @Test + public void checkClassesRunParallel() + throws VerificationException + { + OutputValidator validator = unpack().setTestToRun( "Jira1082Test" ) + .parallelClasses() + .useUnlimitedThreads() + .executeTest() + .verifyErrorFree( 4 ); + + validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1082Test.xml" ) + .assertFileExists(); + + validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1082Test" ), is( 1 ) ); + + Set<String> log = printOnlyTestLinesFromConsole( validator ); + assertParallelRun( log ); + } + + @Test + public void checkOutFileClassesRunParallel() + throws VerificationException + { + OutputValidator validator = unpack().redirectToFile( true ) + .setTestToRun( "Jira1082Test" ) + .parallelClasses() + .useUnlimitedThreads() + .executeTest() + .verifyErrorFree( 4 ); + + validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1082Test.xml" ) + .assertFileExists(); + + validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1082Test" ), is( 1 ) ); + + Set<String> log = printOnlyTestLinesFromOutFile( validator ); + assertParallelRun( log ); + } + + @Test + public void shouldRunTwo() throws VerificationException + { + OutputValidator validator = unpack().redirectToFile( true ) + .parallelClasses() + .useUnlimitedThreads() + .executeTest() + .verifyErrorFree( 8 ); + + validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1082Test.xml" ) + .assertFileExists(); + + validator.getSurefireReportsXmlFile( "TEST-jiras.surefire1082.Jira1264Test.xml" ) + .assertFileExists(); + + validator.getSurefireReportsFile( "jiras.surefire1082.Jira1082Test-output.txt" ) + .assertFileExists(); + + validator.getSurefireReportsFile( "jiras.surefire1082.Jira1264Test-output.txt" ) + .assertFileExists(); + + validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1082Test" ), is( 1 ) ); + + validator.assertThatLogLine( containsString( "Running jiras.surefire1082.Jira1264Test" ), is( 1 ) ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1082-parallel-junit-parameterized" ); + } + + private static class IsRegex + extends BaseMatcher<Set<String>> + { + private final Set<String> expectedRegex; + + IsRegex( Set<String> expectedRegex ) + { + this.expectedRegex = expectedRegex; + } + + @Override + public boolean matches( Object o ) + { + if ( o != null && o instanceof Set ) + { + Set<String> actual = (Set<String>) o; + boolean matches = actual.size() == expectedRegex.size(); + Iterator<String> regex = expectedRegex.iterator(); + for ( String s : actual ) + { + if ( s == null || !regex.hasNext() || !s.matches( regex.next() ) ) + { + matches = false; + } + } + return matches; + } + else + { + return false; + } + } + + @Override + public void describeTo( Description description ) + { + description.appendValue( expectedRegex ); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1095NpeInRunListener.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1095NpeInRunListener.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1095NpeInRunListener.java new file mode 100644 index 0000000..f894265 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1095NpeInRunListener.java @@ -0,0 +1,94 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +@SuppressWarnings( { "javadoc", "checkstyle:javadoctype" } ) +/** + * + * In the surefire plugin, it is possible to specify one or more RunListener when running tests with JUnit. + * However, it does not look like the listener is properly called by the plugin. In particular, there is a problem + * with the method: + * <pre> + * public void testRunStarted(Description description) + * </pre> + * it's javadoc at + * <a href="http://junit.sourceforge.net/javadoc/org/junit/runner/notification/RunListener.html#testRunStarted%28org.junit.runner.Description%29"/> + * states: + * "Parameters: + * description - describes the tests to be run " + * however, in all maven projects I tried ("mvn test"), the surefire plugin seems like passing a null reference instead + * of a Description instance that "describes the tests to be run " + * Note: other methods in the RunListener I tested seems fine (i.e., they get a valid Description object as input) + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1095}"/> + * @since 2.18 + */ +public final class Surefire1095NpeInRunListener + extends SurefireJUnit4IntegrationTestCase +{ + + /** + * Method Request.classes( String, Class[] ); exists in JUnit 4.0 - 4.4 + * See JUnit4Reflector. + */ + @Test + public void testRunStartedWithJUnit40() + { + unpack().setJUnitVersion( "4.0" ) + .executeTest() + .verifyErrorFree( 1 ) + .verifyTextInLog( "Running JUnit 4.0" ) + .verifyTextInLog( "testRunStarted [jiras.surefire1095.SomeTest]" ); + } + + /** + * Method Request.classes( Class[] ); Since of JUnit 4.5 + * See JUnit4Reflector. + */ + @Test + public void testRunStartedWithJUnit45() + { + unpack().setJUnitVersion( "4.5" ) + .executeTest() + .verifyErrorFree( 1 ) + .verifyTextInLog( "Running JUnit 4.5" ) + .verifyTextInLog( "testRunStarted [jiras.surefire1095.SomeTest]" ); + } + + @Test + public void testRunStartedWithJUnit47() + { + unpack().setJUnitVersion( "4.7" ) + .executeTest() + .verifyErrorFree( 1 ) + .verifyTextInLog( "Running JUnit 4.7" ) + .verifyTextInLog( "testRunStarted [jiras.surefire1095.SomeTest]" ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1095-npe-in-runlistener" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1098BalancedRunOrderIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1098BalancedRunOrderIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1098BalancedRunOrderIT.java new file mode 100644 index 0000000..e81f066 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1098BalancedRunOrderIT.java @@ -0,0 +1,119 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.it.VerificationException; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static org.junit.Assert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.AnyOf.anyOf; + +/** + * The purpose of this IT is to assert that the run order of test classes is according to the settings:<br> + * + * runOrder=balanced<br> + * parallel=classes<br> + * threadCount=2<br> + * perCoreThreadCount=false<br> + * <br> + * The list of tests should be reordered to (DTest, CTest, BTest, ATest) in the second run. + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1098">SUREFIRE-1098</a> + * @since 2.18 + */ +public class Surefire1098BalancedRunOrderIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void reorderedParallelClasses() + throws VerificationException + { + SurefireLauncher launcher = unpack(); + + launcher + // .runOrder( "balanced" ) call it in 3.x and remove it in surefire-1098-balanced-runorder/pom.xml + // as soon as there is prefix available "failsafe" and "surefire" in system property for this parameter. + .parallelClasses().threadCount( 2 ).disablePerCoreThreadCount() + .executeTest().verifyErrorFree( 4 ); + + OutputValidator validator = + launcher + // .runOrder( "balanced" ) call it in 3.x and remove it in surefire-1098-balanced-runorder/pom.xml + // as soon as there is prefix available "failsafe" and "surefire" in system property for this parameter. + .parallelClasses().threadCount( 2 ).disablePerCoreThreadCount() + .executeTest().verifyErrorFree( 4 ); + + List<String> log = printOnlyTestLines( validator ); + assertThat( log.size(), is( 4 ) ); + Collections.sort( log ); + final int[] threadPoolIdsOfLongestTest = extractThreadPoolIds( log.get( 3 ) ); + final int pool = threadPoolIdsOfLongestTest[0]; + int thread = threadPoolIdsOfLongestTest[1]; + assertThat( thread, anyOf( is( 1 ), is( 2 ) ) ); + thread = thread == 1 ? 2 : 1; + // If the longest test class DTest is running in pool-2-thread-1, the others should run in pool-2-thread-2 + // and vice versa. + assertThat( log.get( 0 ), is( testLine( "A", pool, thread ) ) ); + assertThat( log.get( 1 ), is( testLine( "B", pool, thread ) ) ); + assertThat( log.get( 2 ), is( testLine( "C", pool, thread ) ) ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1098-balanced-runorder" ); + } + + private static List<String> printOnlyTestLines( OutputValidator validator ) + throws VerificationException + { + List<String> log = new ArrayList<String>( validator.loadLogLines() ); + for ( Iterator<String> it = log.iterator(); it.hasNext(); ) { + String line = it.next(); + if ( !line.startsWith( "class jiras.surefire1098." ) ) { + it.remove(); + } + } + return log; + } + + private static int[] extractThreadPoolIds(String logLine) + { + //Example to parse "class jiras.surefire1098.DTest pool-2-thread-1" into {2, 1}. + String t = logLine.split( " " )[2]; + String[] ids = t.split( "-" ); + return new int[]{ Integer.parseInt( ids[1] ), Integer.parseInt( ids[3] )}; + } + + private String testLine(String test, int pool, int thread) + { + return String.format( "class jiras.surefire1098.%sTest pool-%d-thread-%d", test, pool, thread ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1122ParallelAndFlakyTestsIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1122ParallelAndFlakyTestsIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1122ParallelAndFlakyTestsIT.java new file mode 100644 index 0000000..2f9dca8 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1122ParallelAndFlakyTestsIT.java @@ -0,0 +1,48 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * @author agudian + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1122">SUREFIRE-1122</a> + */ +public class Surefire1122ParallelAndFlakyTestsIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void nonParallelCreatesCorrectReport() + { + unpack( "surefire-1122-parallel-and-flakyTests" ) + .executeTest() + .assertTestSuiteResults( 2, 0, 0, 0, 1 ); + } + + @Test + public void parallelCreatesCorrectReport() + { + unpack( "surefire-1122-parallel-and-flakyTests" ) + .activateProfile( "parallel" ) + .executeTest() + .assertTestSuiteResults( 2, 0, 0, 0, 1 ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1135ImproveIgnoreMessageForTestNGIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1135ImproveIgnoreMessageForTestNGIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1135ImproveIgnoreMessageForTestNGIT.java new file mode 100644 index 0000000..f7324c9 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1135ImproveIgnoreMessageForTestNGIT.java @@ -0,0 +1,152 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static org.apache.maven.shared.utils.xml.Xpp3DomBuilder.build; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertThat; + +import java.io.FileNotFoundException; + +import org.apache.maven.shared.utils.xml.Xpp3Dom; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * Test surefire-report on TestNG test + * + * @author <a href="mailto:michal.bo...@gmail.com">Michal Bocek</a> + */ +public class Surefire1135ImproveIgnoreMessageForTestNGIT + extends SurefireJUnit4IntegrationTestCase +{ + + private enum ResultType + { + SKIPPED( "skipped" ), FAILURE( "failure" ); + + private final String type; + + ResultType(String type) + { + this.type = type; + } + + public String getType() { + return type; + } + } + + @Test + public void testNgReport688() throws Exception { + testNgReport( "6.8.8", null, ResultType.SKIPPED, + "Skip test", + /*"org.testng.SkipException"*/ null, + /*"SkipExceptionReportTest.java:30"*/ null ); + } + + @Test + public void testNgReport57() throws Exception { + testNgReport( "5.7", "jdk15", ResultType.SKIPPED, + "Skip test", + /*"org.testng.SkipException"*/ null, + /*"SkipExceptionReportTest.java:30"*/ null ); + } + + private void testNgReport( String version, String classifier, ResultType resultType, String message, String type, + String stackTrace ) + throws Exception + { + OutputValidator outputValidator = + runTest( version, classifier, resultType, "/surefire-1135-improve-ignore-message-for-testng" ); + + Xpp3Dom[] children = readTests( outputValidator, "testng.SkipExceptionReportTest" ); + assertThat( "Report should contains only one test case", children.length, is( 1 ) ); + + Xpp3Dom test = children[0]; + assertThat( "Not expected classname", test.getAttribute( "classname" ), + is( "testng.SkipExceptionReportTest" ) ); + + assertThat( "Not expected test name", test.getAttribute( "name" ), is( "testSkipException" ) ); + + children = test.getChildren( resultType.getType() ); + assertThat( "Test should contains only one " + resultType.getType() + " element", children, + is( arrayWithSize( 1 ) ) ); + + Xpp3Dom result = children[0]; + if ( message == null ) + { + assertThat( "Subelement message attribute must be null", result.getAttribute( "message" ), + is( nullValue() ) ); + } + else + { + assertThat( "Subelement should contains message attribute", result.getAttribute( "message" ), + is( message ) ); + } + + if ( type == null ) + { + assertThat( "Subelement type attribute must be null", result.getAttribute( "type" ), is( nullValue() ) ); + } + else + { + assertThat( "Subelement should contains type attribute", result.getAttribute( "type" ), is( type ) ); + } + + if ( stackTrace == null ) + { + assertThat( "Element body must be null", result.getValue() , isEmptyOrNullString() ); + } + else + { + assertThat( "Element body must contains", result.getValue(), containsString( stackTrace ) ); + } + } + + private OutputValidator runTest( String version, String classifier, ResultType resultType, String resource ) + { + int skipped = ResultType.SKIPPED.equals( resultType ) ? 1 : 0; + int failure = ResultType.FAILURE.equals( resultType ) ? 1 : 0; + + SurefireLauncher launcher = unpack( resource ).sysProp( "testNgVersion", version ); + + if ( classifier != null ) + { + launcher.sysProp( "testNgClassifier", classifier ); + } + + return launcher.addSurefireReportGoal() + .executeCurrentGoals() + .assertTestSuiteResults( 1, 0, failure, skipped ); + } + + private static Xpp3Dom[] readTests( OutputValidator validator, String className ) + throws FileNotFoundException + { + Xpp3Dom testResult = + build( validator.getSurefireReportsXmlFile( "TEST-" + className + ".xml" ).getFileInputStream(), + "UTF-8" + ); + return testResult.getChildren( "testcase" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1136CwdPropagationInForkedModeIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1136CwdPropagationInForkedModeIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1136CwdPropagationInForkedModeIT.java new file mode 100644 index 0000000..8aaecea --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1136CwdPropagationInForkedModeIT.java @@ -0,0 +1,43 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * SUREFIRE-1136 Correct current working directory propagation in forked mode + * + * Note: variables expansion behaves differently on MVN 2.x since not existing variables + * are resolved to 'null' value so that ${surefire.forkNumber} cannot work. + * + * @author Norbert Wnuk + */ +public class Surefire1136CwdPropagationInForkedModeIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void testTestNgAndJUnitTogether() + { + OutputValidator outputValidator = unpack( "surefire-1136-cwd-propagation-in-forked-mode" ).executeTest(); + outputValidator.assertTestSuiteResults( 1, 0, 0, 0 ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java new file mode 100644 index 0000000..d81d7c2 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1144XmlRunTimeIT.java @@ -0,0 +1,58 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.plugins.surefire.report.ReportTestSuite; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +import java.util.List; + +import static org.apache.maven.surefire.its.fixture.HelperAssertions.extractReports; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +/** + * Test that runtime reported on console matches runtime in XML + * + * @author <a href="mailto:elous...@illinois.edu">Lamyaa Eloussi</a> + */ +public class Surefire1144XmlRunTimeIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void testXmlRunTime() + throws Exception + { + OutputValidator outputValidator = unpack( "/surefire-1144-xml-runtime" ).forkOnce().executeTest(); + + List<ReportTestSuite> reports = extractReports( outputValidator.getBaseDir() ); + assertThat( reports, hasSize( 1 ) ); + + ReportTestSuite report = reports.get( 0 ); + float xmlTime = report.getTimeElapsed(); + + assertThat( xmlTime, is(greaterThanOrEqualTo( 1.6f ) ) ); //include beforeClass and afterClass + outputValidator.verifyTextInLog( Float.toString( xmlTime ) ); //same time in console + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1146RerunFailedAndParameterized.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1146RerunFailedAndParameterized.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1146RerunFailedAndParameterized.java new file mode 100644 index 0000000..f78c16a --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1146RerunFailedAndParameterized.java @@ -0,0 +1,75 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1146">SUREFIRE-1146</a> + */ +public class Surefire1146RerunFailedAndParameterized + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void testsAreRerun() + { + OutputValidator outputValidator = unpack( "surefire-1146-rerunFailingTests-with-Parameterized" ).executeTest(); + verify(outputValidator, 8, 0, 0, 0, 5); + } + + private void verify( OutputValidator outputValidator, int run, int failures, int errors, int skipped, int flakes ) + { + outputValidator.verifyTextInLog( "Flakes:" ); + outputValidator.verifyTextInLog( "jiras.surefire1146.CustomDescriptionParameterizedTest.flakyTest[0: (Test11); Test12; Test13;](jiras.surefire1146.CustomDescriptionParameterizedTest)" ); + outputValidator.verifyTextInLog( "Run 1: CustomDescriptionParameterizedTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 2: CustomDescriptionParameterizedTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 3: PASS" ); + + outputValidator.verifyTextInLog( "jiras.surefire1146.CustomDescriptionWithCommaParameterizedTest.flakyTest[0: (Test11), Test12, Test13;](jiras.surefire1146.CustomDescriptionWithCommaParameterizedTest)" ); + outputValidator.verifyTextInLog( "Run 1: CustomDescriptionWithCommaParameterizedTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 2: CustomDescriptionWithCommaParameterizedTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 3: PASS" ); + + outputValidator.verifyTextInLog( "jiras.surefire1146.CustomDescriptionWithCommaParameterizedTest.flakyTest[2: (Test31), Test32, Test33;](jiras.surefire1146.CustomDescriptionWithCommaParameterizedTest)" ); + outputValidator.verifyTextInLog( "Run 1: CustomDescriptionWithCommaParameterizedTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 2: PASS" ); + + outputValidator.verifyTextInLog( "jiras.surefire1146.SimpleParameterizedTest.flakyTest[0](jiras.surefire1146.SimpleParameterizedTest)" ); + outputValidator.verifyTextInLog( "Run 1: SimpleParameterizedTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 2: SimpleParameterizedTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 3: PASS" ); + + outputValidator.verifyTextInLog( "jiras.surefire1146.StandardTest.flakyTest(jiras.surefire1146.StandardTest)" ); + outputValidator.verifyTextInLog( "Run 1: StandardTest.flakyTest:" ); + outputValidator.verifyTextInLog( "Run 2: PASS" ); + + verifyStatistics( outputValidator, run, failures, errors, skipped, flakes ); + } + + private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int skipped, + int flakes ) + { + outputValidator.verifyTextInLog( "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + + ", Skipped: " + skipped + ", Flakes: " + flakes ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java new file mode 100644 index 0000000..58e951a --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1152RerunFailingTestsInSuiteIT.java @@ -0,0 +1,64 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * SUREFIRE-1152 Assert rerunFailingTestsCount works with test suites + * + * @author Sean Flanigan + */ +public class Surefire1152RerunFailingTestsInSuiteIT + extends SurefireJUnit4IntegrationTestCase +{ + private static final String RUNNING_WITH_PROVIDER47 = + "Using configured provider org.apache.maven.surefire.junitcore.JUnitCoreProvider"; + + public OutputValidator runMethodPattern( String projectName, String... goals ) + { + SurefireLauncher launcher = unpack( projectName ); + for ( String goal : goals ) + { + launcher.addGoal( goal ); + } + OutputValidator outputValidator = launcher.showErrorStackTraces().debugLogging().executeVerify(); + outputValidator.assertTestSuiteResults( 3, 0, 0, 0, 3 ); + outputValidator.assertIntegrationTestSuiteResults( 1, 0, 0, 0 ); + return outputValidator; + } + + @Test + public void testJUnit48Provider4() + { + runMethodPattern( "surefire-1152-rerunFailingTestsCount-suite", "-P surefire-junit4" ); + } + + @Test + public void testJUnit48Provider47() + { + runMethodPattern( "surefire-1152-rerunFailingTestsCount-suite", "-P surefire-junit47" ) + .verifyTextInLog( RUNNING_WITH_PROVIDER47 ); + } + +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1153IncludesAndSpecifiedTestIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1153IncludesAndSpecifiedTestIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1153IncludesAndSpecifiedTestIT.java new file mode 100644 index 0000000..3a2c826 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1153IncludesAndSpecifiedTestIT.java @@ -0,0 +1,46 @@ +package org.apache.maven.surefire.its.jiras; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; + +import org.junit.Test; + +public class Surefire1153IncludesAndSpecifiedTestIT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void testSpecifiedTestInIncludes() + { + unpack( "/surefire-1153-includesAndSpecifiedTest" ) + .setTestToRun( "#testIncluded" ) + .executeTest() + .verifyErrorFree( 1 ); + } + + @Test + public void testSpecifiedTestNotInIncludes() + { + unpack( "/surefire-1153-includesAndSpecifiedTest" ) + .setTestToRun( "#testNotIncluded" ) + .executeTest() + .verifyErrorFree( 1 ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1158RemoveInfoLinesIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1158RemoveInfoLinesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1158RemoveInfoLinesIT.java new file mode 100644 index 0000000..706f6b5 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1158RemoveInfoLinesIT.java @@ -0,0 +1,142 @@ +package org.apache.maven.surefire.its.jiras; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireVerifierException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; + +import static org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase.unpack; +import static org.junit.runners.Parameterized.*; +import static org.junit.Assert.*; + +/** + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1158">SUREFIRE-1158</a> + * @since 2.19 + */ +@RunWith( Parameterized.class ) +public class Surefire1158RemoveInfoLinesIT +{ + + @Parameters(name = "{0}") + public static Iterable<Object[]> data() + { + ArrayList<Object[]> args = new ArrayList<Object[]>(); + args.add( new Object[] { "junit-option-ff", "JUnitTest", "-ff", "surefire-junit47", false, true } ); + args.add( new Object[] { "testng-option-ff", "TestNGSuiteTest", "-ff", "surefire-testng", false, false } ); + args.add( new Object[] { "junit-option-X", "JUnitTest", "-X", "surefire-junit47", true, true } ); + args.add( new Object[] { "testng-option-X", "TestNGSuiteTest", "-X", "surefire-testng", true, false } ); + args.add( new Object[] { "junit-option-e", "JUnitTest", "-e", "surefire-junit47", true, true } ); + args.add( new Object[] { "testng-option-e", "TestNGSuiteTest", "-e", "surefire-testng", true, false } ); + return args; + } + + @Parameter(0) + public String description; + + @Parameter(1) + public String testToRun; + + @Parameter(2) + public String cliOption; + + @Parameter(3) + public String provider; + + @Parameter(4) + public boolean printsInfoLines; + + @Parameter(5) + public boolean isJUnit; + + @Test + public void shouldRunWithCliOption() + throws Exception + { + OutputValidator validator = assertTest(); + if ( isJUnit ) + { + assertJUnitTestLogs( validator ); + } + else + { + assertTestNGTestLogs( validator ); + } + } + + private OutputValidator assertTest() + throws Exception + { + final String[] cli = {"--batch-mode"}; + return unpack( getClass(), "/surefire-1158-remove-info-lines", "_" + description, cli ) + .sysProp( "provider", provider ).addGoal( cliOption ).setTestToRun( testToRun ) + .executeTest() + .verifyErrorFreeLog().assertTestSuiteResults( 1, 0, 0, 0 ); + } + + private void assertJUnitTestLogs( OutputValidator validator ) + { + try + { + validator.verifyTextInLog( "Surefire report directory:" ); + validator.verifyTextInLog( "Using configured provider org.apache.maven.surefire.junitcore.JUnitCoreProvider" ); + validator.verifyTextInLog( "parallel='none', perCoreThreadCount=true, threadCount=0, " + + "useUnlimitedThreads=false, threadCountSuites=0, threadCountClasses=0, " + + "threadCountMethods=0, parallelOptimized=true" ); + if ( !printsInfoLines ) + { + fail(); + } + } + catch ( SurefireVerifierException e ) + { + if ( printsInfoLines ) + { + fail(); + } + } + } + + private void assertTestNGTestLogs( OutputValidator validator ) + { + try + { + validator.verifyTextInLog( "Surefire report directory:" ); + validator.verifyTextInLog( "Using configured provider org.apache.maven.surefire.testng.TestNGProvider" ); + validator.verifyTextInLog( "Configuring TestNG with: TestNGMapConfigurator" ); + if ( !printsInfoLines ) + { + fail(); + } + } + catch ( SurefireVerifierException e ) + { + if ( printsInfoLines ) + { + fail(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1177TestngParallelSuitesIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1177TestngParallelSuitesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1177TestngParallelSuitesIT.java new file mode 100644 index 0000000..8c2c213 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1177TestngParallelSuitesIT.java @@ -0,0 +1,59 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.it.VerificationException; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; + +/** + * IT for https://issues.apache.org/jira/browse/SUREFIRE-1177 + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @since 2.19 + */ +public class Surefire1177TestngParallelSuitesIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void shouldRunTwoSuitesInParallel() + throws VerificationException + { + assumeJavaVersion( 1.7d ); + + unpack().executeTest() + .verifyErrorFree( 2 ) + .assertThatLogLine( containsString( "ShouldNotRunTest#shouldNotRun()" ), is( 0 ) ) + .assertThatLogLine( startsWith( "TestNGSuiteTest#shouldRunAndPrintItself()" ), is( 2 ) ) + .assertThatLogLine( is( "TestNGSuiteTest#shouldRunAndPrintItself() 1." ), is( 1 ) ) + .assertThatLogLine( is( "TestNGSuiteTest#shouldRunAndPrintItself() 2." ), is( 1 ) ); + } + + private SurefireLauncher unpack() + { + return unpack( "testng-parallel-suites" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1179IT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1179IT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1179IT.java new file mode 100644 index 0000000..e4e8e31 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1179IT.java @@ -0,0 +1,43 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * Fix for TestNG parameter -dataproviderthreadcount. + */ +public class Surefire1179IT + extends SurefireJUnit4IntegrationTestCase +{ + + @Test + public void suiteXmlForkCountTwoReuse() + { + unpack().executeTest().verifyErrorFreeLog().verifyTextInLog( " CONCURRENCY=30." ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1179-testng-parallel-dataprovider" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1185DoNotSpawnTestsIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1185DoNotSpawnTestsIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1185DoNotSpawnTestsIT.java new file mode 100644 index 0000000..74a4e46 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1185DoNotSpawnTestsIT.java @@ -0,0 +1,62 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.it.VerificationException; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +/** + * Surefire 2.19 spawns unnecessary tests in surefire-junit4 provider. + * https://issues.apache.org/jira/browse/SUREFIRE-1185 + * Example, UnlistedTest is the problem here because it runs with filtered out methods: + * + * Running pkg.UnlistedTest + * Tests run: 0, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.001 sec - in pkg.UnlistedTest + * Running pkg.RunningTest + * Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.001 sec - in pkg.RunningTest + * + * Results: + * + * Tests run: 1, Failures: 0, Errors: 0, Skipped: 0 + */ +public class Surefire1185DoNotSpawnTestsIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void doNotSpawnUnwantedTests() + throws VerificationException + { + unpack().setTestToRun( "RunningTest#test" ) + .executeTest() + .assertTestSuiteResults( 1 ) + .assertThatLogLine( containsString( "in pkg.RunningTest" ), is( 1 ) ) + .assertThatLogLine( containsString( "in pkg.UnlistedTest" ), is( 0 ) ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1185" ); + } +} http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/09f0eef8/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1202RerunAndSkipIT.java ---------------------------------------------------------------------- diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1202RerunAndSkipIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1202RerunAndSkipIT.java new file mode 100644 index 0000000..f886446 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1202RerunAndSkipIT.java @@ -0,0 +1,58 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.it.VerificationException; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; + +/** + * Allow rerunFailingTestsCount, skipAfterFailureCount together + * + * @author <a href="mailto:tibordig...@apache.org">Tibor Digana (tibor17)</a> + * @see <a href="https://issues.apache.org/jira/browse/SUREFIRE-1202">SUREFIRE-1202</a> + * @since 2.19.1 + */ +public class Surefire1202RerunAndSkipIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void junit47() + throws VerificationException + { + unpack().executeTest() + .assertTestSuiteResults( 5, 0, 0, 3, 4 ); + } + + @Test + public void junit4() + throws VerificationException + { + unpack().addGoal( "-Pjunit4" ) + .executeTest() + .assertTestSuiteResults( 5, 0, 0, 3, 4 ); + } + + private SurefireLauncher unpack() + { + return unpack( "surefire-1202-rerun-and-failfast" ); + } +}