[SUREFIRE-1281] Fix permanently failing Surefire Windows Build
Project: http://git-wip-us.apache.org/repos/asf/maven-surefire/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-surefire/commit/f37a8d4d Tree: http://git-wip-us.apache.org/repos/asf/maven-surefire/tree/f37a8d4d Diff: http://git-wip-us.apache.org/repos/asf/maven-surefire/diff/f37a8d4d Branch: refs/heads/junit5 Commit: f37a8d4dcb59b0ea8fd982f3d8db9b158088fa28 Parents: e0ce6cb Author: Tibor17 <tibo...@lycos.com> Authored: Fri Oct 7 07:46:16 2016 +0200 Committer: Tibor17 <tibo...@lycos.com> Committed: Fri Oct 7 07:46:16 2016 +0200 ---------------------------------------------------------------------- surefire-integration-tests/pom.xml | 1 + ...refire747MethodParallelWithSuiteCountIT.java | 86 ++++++++++++++------ .../junit47-parallel-with-suite/pom.xml | 5 +- .../src/test/java/surefire747/SuiteTest1.java | 8 +- .../src/test/java/surefire747/SuiteTest2.java | 8 +- 5 files changed, 76 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f37a8d4d/surefire-integration-tests/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/pom.xml b/surefire-integration-tests/pom.xml index 43d4b0a..beab2be 100644 --- a/surefire-integration-tests/pom.xml +++ b/surefire-integration-tests/pom.xml @@ -116,6 +116,7 @@ <testBuildDirectory>${project.build.testOutputDirectory}</testBuildDirectory> <verifier.forkMode>${verifier.forkMode}</verifier.forkMode> </systemPropertyVariables> + <redirectTestOutputToFile>false</redirectTestOutputToFile> </configuration> <dependencies> <dependency> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f37a8d4d/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire747MethodParallelWithSuiteCountIT.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire747MethodParallelWithSuiteCountIT.java b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire747MethodParallelWithSuiteCountIT.java index 6166b3e..c5dbc84 100644 --- a/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire747MethodParallelWithSuiteCountIT.java +++ b/surefire-integration-tests/src/test/java/org/apache/maven/surefire/its/jiras/Surefire747MethodParallelWithSuiteCountIT.java @@ -23,12 +23,18 @@ import org.apache.maven.it.VerificationException; import org.apache.maven.surefire.its.fixture.OutputValidator; import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Before; import org.junit.Test; +import java.text.Format; +import java.text.NumberFormat; import java.util.Iterator; import java.util.Set; import java.util.TreeSet; +import static java.lang.String.format; +import static java.math.RoundingMode.DOWN; +import static java.util.Locale.ROOT; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.endsWith; @@ -40,11 +46,15 @@ import static org.junit.Assert.assertTrue; * @author Kristian Rosenvold */ public class Surefire747MethodParallelWithSuiteCountIT - extends SurefireJUnit4IntegrationTestCase + extends SurefireJUnit4IntegrationTestCase { + // if you want to change his constant, change it in SuiteTest1.java and SuiteTest2.java as well + private static final int PERFORMANCE_TEST_MULTIPLICATION_FACTOR = 4; + + private Format lowerScaleFormatter, noFractionalDigitsFormatter; private static Set<String> printTestLines( OutputValidator validator, String pattern ) - throws VerificationException + throws VerificationException { Set<String> log = new TreeSet<String>( validator.loadLogLines() ); for ( Iterator<String> it = log.iterator(); it.hasNext(); ) @@ -63,9 +73,25 @@ public class Surefire747MethodParallelWithSuiteCountIT return Integer.decode( logLine.split( "=" )[1] ); } + @Before + public void init() + { + NumberFormat lowScaleFormatter = NumberFormat.getInstance( ROOT ); + lowScaleFormatter.setRoundingMode( DOWN ); + lowScaleFormatter.setMinimumFractionDigits( 1 ); + lowScaleFormatter.setMaximumFractionDigits( 1 ); + this.lowerScaleFormatter = lowScaleFormatter; + + NumberFormat noFractionalDigitsFormatter = NumberFormat.getInstance( ROOT ); + noFractionalDigitsFormatter.setRoundingMode( DOWN ); + noFractionalDigitsFormatter.setMinimumFractionDigits( 0 ); + noFractionalDigitsFormatter.setMaximumFractionDigits( 0 ); + this.noFractionalDigitsFormatter = noFractionalDigitsFormatter; + } + @Test public void testMethodsParallelWithSuite() - throws VerificationException + throws VerificationException { OutputValidator validator = unpack().executeTest().verifyErrorFree( 6 ); Set<String> testLines = printTestLines( validator, "test finished after duration=" ); @@ -73,37 +99,45 @@ public class Surefire747MethodParallelWithSuiteCountIT for ( String testLine : testLines ) { long duration = duration( testLine ); - long min = 250, max = 750; - assertTrue( String.format( "duration %d should be between %d and %d ms", duration, min, max ), - duration > min && duration < max ); + long min = 250 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + long max = 750 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + assertTrue( format( "duration %d should be between %d and %d ms", duration, min, max ), + duration > min && duration < max + ); } Set<String> suiteLines = printTestLines( validator, "suite finished after duration=" ); assertThat( suiteLines.size(), is( 1 ) ); long duration = duration( suiteLines.iterator().next() ); - long min = 750, max = 1250; - assertTrue( String.format( "duration %d should be between %d and %d ms", duration, min, max ), - duration > min && duration < max ); + long min = 750 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + long max = 1250 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + assertTrue( format( "duration %d should be between %d and %d ms", duration, min, max ), + duration > min && duration < max + ); + + String delayMin = lowerScaleFormatter.format( 0.98 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR * 0.5 ); + String delayMax = noFractionalDigitsFormatter.format( PERFORMANCE_TEST_MULTIPLICATION_FACTOR * 0.5 ) + "."; for ( String line : validator.loadLogLines() ) { if ( line.startsWith( "Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:" ) ) { - assertThat( line, anyOf( // 0.5 sec, the delta -1.0/+0.3 can be varying depending on CI jobs - containsString( "Time elapsed: 0.4" ), - containsString( "Time elapsed: 0.5" ), - containsString( "Time elapsed: 0.6" ), - containsString( "Time elapsed: 0.7" ), - containsString( "Time elapsed: 0.8" ) ) ); + assertThat( line, anyOf( // 1.9xx to 2.xxx can vary depending on CI jobs + containsString( "Time elapsed: " + delayMin ), + containsString( "Time elapsed: " + delayMax ) + ) + ); assertThat( line, anyOf( - endsWith(" s - in surefire747.SuiteTest1" ), - endsWith(" s - in surefire747.SuiteTest2" ) ) ); + endsWith( " s - in surefire747.SuiteTest1" ), + endsWith( " s - in surefire747.SuiteTest2" ) + ) + ); } } } @Test public void testClassesParallelWithSuite() - throws VerificationException + throws VerificationException { OutputValidator validator = unpack().parallelClasses().executeTest().verifyErrorFree( 6 ); Set<String> testLines = printTestLines( validator, "test finished after duration=" ); @@ -111,16 +145,20 @@ public class Surefire747MethodParallelWithSuiteCountIT for ( String testLine : testLines ) { long duration = duration( testLine ); - long min = 1250, max = 1750; - assertTrue( String.format( "duration %d should be between %d and %d ms", duration, min, max ), - duration > min && duration < max ); + long min = 1450 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + long max = 1750 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + assertTrue( format( "duration %d should be between %d and %d ms", duration, min, max ), + duration > min && duration < max + ); } Set<String> suiteLines = printTestLines( validator, "suite finished after duration=" ); assertThat( suiteLines.size(), is( 1 ) ); long duration = duration( suiteLines.iterator().next() ); - long min = 1250, max = 1750; - assertTrue( String.format( "duration %d should be between %d and %d ms", duration, min, max ), - duration > min && duration < max ); + long min = 1450 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + long max = 1750 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR; + assertTrue( format( "duration %d should be between %d and %d ms", duration, min, max ), + duration > min && duration < max + ); } public SurefireLauncher unpack() http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f37a8d4d/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/pom.xml ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/pom.xml b/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/pom.xml index 403b56b..eb68f3e 100644 --- a/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/pom.xml +++ b/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/pom.xml @@ -5,7 +5,7 @@ <artifactId>junit4-test</artifactId> <packaging>jar</packaging> <version>1.0-SNAPSHOT</version> - <name>surefire-747-parallelo-method-skips-test</name> + <name>surefire-747-parallel-method-skips-test</name> <url>http://maven.apache.org</url> <dependencies> <dependency> @@ -32,7 +32,8 @@ <artifactId>maven-surefire-plugin</artifactId> <version>${surefire.version}</version> <configuration> - <threadCount>10</threadCount> + <perCoreThreadCount>false</perCoreThreadCount> + <threadCount>5</threadCount> <includes> <include>**/TestSuite.java</include> </includes> http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f37a8d4d/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest1.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest1.java b/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest1.java index 96138c8..9ac0b90 100644 --- a/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest1.java +++ b/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest1.java @@ -30,6 +30,8 @@ import org.junit.Test; */ public class SuiteTest1 { + private static final int PERFORMANCE_TEST_MULTIPLICATION_FACTOR = 4; + private static long startedAt; public SuiteTest1() @@ -67,7 +69,7 @@ public class SuiteTest1 throws InterruptedException { System.out.println( "begin SuiteTest1.first" ); - Thread.sleep( 500 ); + Thread.sleep( 500 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR ); System.out.println( "end SuiteTest1.first" ); } @@ -76,7 +78,7 @@ public class SuiteTest1 throws InterruptedException { System.out.println( "begin SuiteTest1.second" ); - Thread.sleep( 500 ); + Thread.sleep( 500 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR ); System.out.println( "end SuiteTest1.second" ); } @@ -85,7 +87,7 @@ public class SuiteTest1 throws InterruptedException { System.out.println( "begin SuiteTest1.third" ); - Thread.sleep( 500 ); + Thread.sleep( 500 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR ); System.out.println( "end SuiteTest1.third" ); } } http://git-wip-us.apache.org/repos/asf/maven-surefire/blob/f37a8d4d/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest2.java ---------------------------------------------------------------------- diff --git a/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest2.java b/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest2.java index b5eb77c..99ad7cd 100644 --- a/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest2.java +++ b/surefire-integration-tests/src/test/resources/junit47-parallel-with-suite/src/test/java/surefire747/SuiteTest2.java @@ -30,6 +30,8 @@ import org.junit.Test; */ public class SuiteTest2 { + private static final int PERFORMANCE_TEST_MULTIPLICATION_FACTOR = 4; + private static long startedAt; public SuiteTest2() @@ -67,7 +69,7 @@ public class SuiteTest2 throws InterruptedException { System.out.println( "begin SuiteTest2.first" ); - Thread.sleep( 500 ); + Thread.sleep( 500 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR ); System.out.println( "end SuiteTest2.first" ); } @@ -76,7 +78,7 @@ public class SuiteTest2 throws InterruptedException { System.out.println( "begin SuiteTest2.second" ); - Thread.sleep( 500 ); + Thread.sleep( 500 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR ); System.out.println( "end SuiteTest2.second" ); } @@ -85,7 +87,7 @@ public class SuiteTest2 throws InterruptedException { System.out.println( "begin SuiteTest2.third" ); - Thread.sleep( 500 ); + Thread.sleep( 500 * PERFORMANCE_TEST_MULTIPLICATION_FACTOR ); System.out.println( "end SuiteTest2.third" ); } }