Author: markt Date: Thu Feb 5 15:47:45 2015 New Revision: 1657610 URL: http://svn.apache.org/r1657610 Log: Back-port performance and timing test changes in an attempt to reduce false failures on the ASF CI system.
Added: tomcat/tc8.0.x/trunk/test/org/apache/catalina/mapper/TestMapperPerformance.java - copied unchanged from r1657584, tomcat/trunk/test/org/apache/catalina/mapper/TestMapperPerformance.java Modified: tomcat/tc8.0.x/trunk/ (props changed) tomcat/tc8.0.x/trunk/BUILDING.txt tomcat/tc8.0.x/trunk/build.xml tomcat/tc8.0.x/trunk/test/org/apache/catalina/connector/TestResponsePerformance.java tomcat/tc8.0.x/trunk/test/org/apache/catalina/mapper/TestMapper.java tomcat/tc8.0.x/trunk/test/org/apache/catalina/valves/TesterAccessLogValve.java tomcat/tc8.0.x/trunk/webapps/docs/changelog.xml Propchange: tomcat/tc8.0.x/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Feb 5 15:47:45 2015 @@ -1 +1 @@ -/tomcat/trunk:1636524,1637156,1637176,1637188,1637331,1637684,1637695,1638720-1638725,1639653,1640010,1640083-1640084,1640088,1640275,1640322,1640347,1640361,1640365,1640403,1640410,1640652,1640655-1640658,1640688,1640700-1640883,1640903,1640976,1640978,1641000,1641026,1641038-1641039,1641051-1641052,1641058,1641064,1641300,1641369,1641374,1641380,1641486,1641634,1641656-1641692,1641704,1641707-1641718,1641720-1641722,1641735,1641981,1642233,1642280,1642554,1642564,1642595,1642606,1642668,1642679,1642697,1642699,1642766,1643002,1643045,1643054-1643055,1643066,1643121,1643128,1643206,1643209-1643210,1643216,1643249,1643270,1643283,1643309-1643310,1643323,1643365-1643366,1643370-1643371,1643465,1643474,1643536,1643570,1643634,1643649,1643651,1643654,1643675,1643731,1643733-1643734,1643761,1643766,1643814,1643937,1643963,1644017,1644169,1644201-1644203,1644321,1644323,1644516,1644523,1644529,1644535,1644730,1644768,1644784-1644785,1644790,1644793,1644815,1644884,1644886,1644890,1644892 ,1644910,1644924,1644929-1644930,1644935,1644989,1645011,1645247,1645355,1645357-1645358,1645455,1645465,1645469,1645471,1645473,1645475,1645486-1645488,1645626,1645641,1645685,1645743,1645763,1645951-1645953,1645955,1645993,1646098-1646106,1646178,1646220,1646302,1646304,1646420,1646470-1646471,1646476,1646559,1646717-1646723,1646773,1647026,1647042,1647530,1647655,1648304,1648815,1648907,1650081,1650365,1651116,1651120,1651280,1651470,1652938,1652970,1653041,1653471,1653550,1653574,1653797,1653815-1653816,1653819,1653840,1653857,1653888,1653972,1654013,1654030,1654050,1654123,1654148,1654159,1654513,1654515,1654517,1654522,1654524,1654725,1654735,1654766,1654785,1654851-1654852,1654978,1655122-1655124,1655126-1655127,1655129-1655130,1655132-1655133,1655312,1655438,1655441,1655454,1655558,1656087,1656299,1656319,1656331,1656345,1656350,1656590,1656648-1656650,1656657,1657041,1657054,1657374 +/tomcat/trunk:1636524,1637156,1637176,1637188,1637331,1637684,1637695,1638720-1638725,1639653,1640010,1640083-1640084,1640088,1640275,1640322,1640347,1640361,1640365,1640403,1640410,1640652,1640655-1640658,1640688,1640700-1640883,1640903,1640976,1640978,1641000,1641026,1641038-1641039,1641051-1641052,1641058,1641064,1641300,1641369,1641374,1641380,1641486,1641634,1641656-1641692,1641704,1641707-1641718,1641720-1641722,1641735,1641981,1642233,1642280,1642554,1642564,1642595,1642606,1642668,1642679,1642697,1642699,1642766,1643002,1643045,1643054-1643055,1643066,1643121,1643128,1643206,1643209-1643210,1643216,1643249,1643270,1643283,1643309-1643310,1643323,1643365-1643366,1643370-1643371,1643465,1643474,1643536,1643570,1643634,1643649,1643651,1643654,1643675,1643731,1643733-1643734,1643761,1643766,1643814,1643937,1643963,1644017,1644169,1644201-1644203,1644321,1644323,1644516,1644523,1644529,1644535,1644730,1644768,1644784-1644785,1644790,1644793,1644815,1644884,1644886,1644890,1644892 ,1644910,1644924,1644929-1644930,1644935,1644989,1645011,1645247,1645355,1645357-1645358,1645455,1645465,1645469,1645471,1645473,1645475,1645486-1645488,1645626,1645641,1645685,1645743,1645763,1645951-1645953,1645955,1645993,1646098-1646106,1646178,1646220,1646302,1646304,1646420,1646470-1646471,1646476,1646559,1646717-1646723,1646773,1647026,1647042,1647530,1647655,1648304,1648815,1648907,1650081,1650365,1651116,1651120,1651280,1651470,1652938,1652970,1653041,1653471,1653550,1653574,1653797,1653815-1653816,1653819,1653840,1653857,1653888,1653972,1654013,1654030,1654050,1654123,1654148,1654159,1654513,1654515,1654517,1654522,1654524,1654725,1654735,1654766,1654785,1654851-1654852,1654978,1655122-1655124,1655126-1655127,1655129-1655130,1655132-1655133,1655312,1655438,1655441,1655454,1655558,1656087,1656299,1656319,1656331,1656345,1656350,1656590,1656648-1656650,1656657,1657041,1657054,1657374,1657492,1657510,1657565,1657580,1657584,1657586,1657589,1657592,1657607,1657609 Modified: tomcat/tc8.0.x/trunk/BUILDING.txt URL: http://svn.apache.org/viewvc/tomcat/tc8.0.x/trunk/BUILDING.txt?rev=1657610&r1=1657609&r2=1657610&view=diff ============================================================================== --- tomcat/tc8.0.x/trunk/BUILDING.txt (original) +++ tomcat/tc8.0.x/trunk/BUILDING.txt Thu Feb 5 15:47:45 2015 @@ -401,6 +401,20 @@ For example: Apache License v1.1. See http://cobertura.sf.net for details. Using it during Tomcat build is optional and is off by default. + 6. The performance tests are written to run reasonably powerful machines (such + as a developer may use day to day) assuming no other resource hungry + processes are running. These assumptions are not always true (e.g. on CI + systems running ina virtual machine) so the performance tests may be + disabled by using the following property: + + test.excludePerformance=true + + 7. Some tests include checks that the access log valve entries are as expected. + These checks include timings. On slower / loaded systems these checks will + often fail. The checks may be relaxed by using the following proeprty: + + test.relaxTiming=true + (8) Source code checks Modified: tomcat/tc8.0.x/trunk/build.xml URL: http://svn.apache.org/viewvc/tomcat/tc8.0.x/trunk/build.xml?rev=1657610&r1=1657609&r2=1657610&view=diff ============================================================================== --- tomcat/tc8.0.x/trunk/build.xml (original) +++ tomcat/tc8.0.x/trunk/build.xml Thu Feb 5 15:47:45 2015 @@ -168,6 +168,7 @@ <!-- Tests To Run --> <property name="test.name" value="**/Test*.java"/> <property name="test.formatter" value="-Dorg.apache.juli.formatter=java.util.logging.SimpleFormatter"/> + <property name="test.relaxTiming" value="false"/> <!-- Cobertura code coverage settings --> <property name="cobertura.out" value="${tomcat.output}/coverage"/> @@ -1419,6 +1420,7 @@ <sysproperty key="tomcat.test.accesslog" value="${test.accesslog}" /> <sysproperty key="tomcat.test.reports" value="${test.reports}" /> <sysproperty key="tomcat.test.openssl.path" value="${test.openssl.path}" /> + <sysproperty key="tomcat.test.relaxTiming" value="${test.relaxTiming}" /> <!-- File for Cobertura to write coverage results to --> <sysproperty key="net.sourceforge.cobertura.datafile" file="${cobertura.datafile}" /> @@ -1442,7 +1444,9 @@ <!-- Exclude the tests known to fail --> <exclude name="org/apache/catalina/tribes/test/**" /> <!-- Exclude the OpenSSL tests unless OpenSSL is available --> - <exclude name="org/apache/tomcat/util/net/jsse/openssl/**" unless="${test.openssl.exists}"/> + <exclude name="org/apache/tomcat/util/net/jsse/openssl/**" unless="${test.openssl.exists}" /> + <!-- Exclude performance tests. E.g. on systems with slow/inconsistent timing --> + <exclude name="**/*Performance.java" if="${test.excludePerformance}" /> </fileset> </batchtest> </junit> Modified: tomcat/tc8.0.x/trunk/test/org/apache/catalina/connector/TestResponsePerformance.java URL: http://svn.apache.org/viewvc/tomcat/tc8.0.x/trunk/test/org/apache/catalina/connector/TestResponsePerformance.java?rev=1657610&r1=1657609&r2=1657610&view=diff ============================================================================== --- tomcat/tc8.0.x/trunk/test/org/apache/catalina/connector/TestResponsePerformance.java (original) +++ tomcat/tc8.0.x/trunk/test/org/apache/catalina/connector/TestResponsePerformance.java Thu Feb 5 15:47:45 2015 @@ -18,32 +18,61 @@ package org.apache.catalina.connector; import java.net.URI; -import static org.junit.Assert.assertTrue; +import org.junit.Assert; import org.junit.Test; -public class TestResponsePerformance { +import org.apache.catalina.startup.LoggingBaseTest; + +public class TestResponsePerformance extends LoggingBaseTest { + + private final int ITERATIONS = 100000; + @Test public void testToAbsolutePerformance() throws Exception { Request req = new TesterRequest(); Response resp = new Response(); resp.setRequest(req); + // Warm up + doHomebrew(resp); + doUri(); + + // To allow for timing differences between runs, a "best of n" approach + // is taken for this test + final int bestOf = 5; + final int winTarget = (bestOf + 1) / 2; + int homebrewWin = 0; + int count = 0; + + while (count < bestOf && homebrewWin < winTarget) { + long homebrew = doHomebrew(resp); + long uri = doUri(); + log.info("Current 'home-brew': " + homebrew + "ms, Using URI: " + uri + "ms"); + if (homebrew < uri) { + homebrewWin++; + } + count++; + } + Assert.assertTrue(homebrewWin == winTarget); + } + + + private long doHomebrew(Response resp) { long start = System.currentTimeMillis(); - for (int i = 0; i < 100000; i++) { + for (int i = 0; i < ITERATIONS; i++) { resp.toAbsolute("bar.html"); } - long homebrew = System.currentTimeMillis() - start; + return System.currentTimeMillis() - start; + } + - start = System.currentTimeMillis(); - for (int i = 0; i < 100000; i++) { + private long doUri() { + long start = System.currentTimeMillis(); + for (int i = 0; i < ITERATIONS; i++) { URI base = URI.create( "http://localhost:8080/level1/level2/foo.html"); base.resolve(URI.create("bar.html")).toASCIIString(); } - long uri = System.currentTimeMillis() - start; - - System.out.println("Current 'home-brew': " + homebrew + - "ms, Using URI: " + uri + "ms"); - assertTrue(homebrew < uri); + return System.currentTimeMillis() - start; } } Modified: tomcat/tc8.0.x/trunk/test/org/apache/catalina/mapper/TestMapper.java URL: http://svn.apache.org/viewvc/tomcat/tc8.0.x/trunk/test/org/apache/catalina/mapper/TestMapper.java?rev=1657610&r1=1657609&r2=1657610&view=diff ============================================================================== --- tomcat/tc8.0.x/trunk/test/org/apache/catalina/mapper/TestMapper.java (original) +++ tomcat/tc8.0.x/trunk/test/org/apache/catalina/mapper/TestMapper.java Thu Feb 5 15:47:45 2015 @@ -41,7 +41,7 @@ import org.apache.tomcat.util.buf.Messag public class TestMapper extends LoggingBaseTest { - private Mapper mapper; + protected Mapper mapper; private HashMap<String, Host> hostMap = new HashMap<>(); @@ -538,38 +538,4 @@ public class TestMapper extends LoggingB assertEquals("/foo/bar/bla", mappingData.contextPath.toString()); } } - - @Test - public void testPerformance() throws Exception { - // Takes ~1s on markt's laptop. If this takes more than 5s something - // probably needs looking at. If this fails repeatedly then we may need - // to increase this limit. - final long maxTime = 5000; - long time = testPerformanceImpl(); - if (time >= maxTime) { - // Rerun to reject occasional failures, e.g. because of gc - log.warn("testPerformance() test completed in " + time + " ms"); - time = testPerformanceImpl(); - log.warn("testPerformance() test rerun completed in " + time + " ms"); - } - assertTrue(String.valueOf(time), time < maxTime); - } - - private long testPerformanceImpl() throws Exception { - MappingData mappingData = new MappingData(); - MessageBytes host = MessageBytes.newInstance(); - host.setString("iowejoiejfoiew"); - MessageBytes uri = MessageBytes.newInstance(); - uri.setString("/foo/bar/blah/bobou/foo"); - uri.toChars(); - uri.getCharChunk().setLimit(-1); - - long start = System.currentTimeMillis(); - for (int i = 0; i < 1000000; i++) { - mappingData.recycle(); - mapper.map(host, uri, null, mappingData); - } - long time = System.currentTimeMillis() - start; - return time; - } } Modified: tomcat/tc8.0.x/trunk/test/org/apache/catalina/valves/TesterAccessLogValve.java URL: http://svn.apache.org/viewvc/tomcat/tc8.0.x/trunk/test/org/apache/catalina/valves/TesterAccessLogValve.java?rev=1657610&r1=1657609&r2=1657610&view=diff ============================================================================== --- tomcat/tc8.0.x/trunk/test/org/apache/catalina/valves/TesterAccessLogValve.java (original) +++ tomcat/tc8.0.x/trunk/test/org/apache/catalina/valves/TesterAccessLogValve.java Thu Feb 5 15:47:45 2015 @@ -31,8 +31,10 @@ import org.apache.catalina.connector.Res public class TesterAccessLogValve extends ValveBase implements AccessLog { - // Timing tests need a small error margin to prevent failures - private static final long ERROR_MARGIN = 100; + private static final boolean RELAX_TIMING = Boolean.getBoolean("tomcat.test.relaxTiming"); + + // Timing tests need an error margin to prevent failures. + private static final long ERROR_MARGIN = RELAX_TIMING ? 1000 : 100; private final List<Entry> entries = new ArrayList<>(); Modified: tomcat/tc8.0.x/trunk/webapps/docs/changelog.xml URL: http://svn.apache.org/viewvc/tomcat/tc8.0.x/trunk/webapps/docs/changelog.xml?rev=1657610&r1=1657609&r2=1657610&view=diff ============================================================================== --- tomcat/tc8.0.x/trunk/webapps/docs/changelog.xml (original) +++ tomcat/tc8.0.x/trunk/webapps/docs/changelog.xml Thu Feb 5 15:47:45 2015 @@ -146,6 +146,12 @@ Guard the digester from MbeansDescriptorsDigesterSource with its own lock object. (fschumacher) </fix> + <fix> + Refactor the unit tests and add some new test properties to make it + easier to exclude performance tests and relax timing tests. This is + primarily for the ASF CI system where these tests frequently fail. + (markt) + </fix> </changelog> </subsection> </section> --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@tomcat.apache.org For additional commands, e-mail: dev-h...@tomcat.apache.org