Repository: incubator-brooklyn Updated Branches: refs/heads/master ca89ed4f5 -> 2b23266eb
Improve performance tests - Extracts code from AbstractPerformanceTest into PerformanceMeasurer, making it more configurable and usable from more places. - Adds a default persister to write files to ~/brooklyn-performance/ (while we figure out where to write those to in anger). Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/3384c7eb Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/3384c7eb Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/3384c7eb Branch: refs/heads/master Commit: 3384c7eb46694b56ce9f1269d253c5f7d21605a0 Parents: ca89ed4 Author: Aled Sage <[email protected]> Authored: Thu Nov 12 23:52:11 2015 +0000 Committer: Aled Sage <[email protected]> Committed: Fri Nov 13 14:54:06 2015 +0000 ---------------------------------------------------------------------- .../qa/performance/AbstractPerformanceTest.java | 47 ++++- .../qa/performance/EntityPerformanceTest.java | 84 +++++--- .../FilePersistencePerformanceTest.java | 146 +++++++++---- .../GroovyYardStickPerformanceTest.groovy | 7 +- .../JavaYardStickPerformanceTest.java | 35 ++-- .../SubscriptionPerformanceTest.java | 58 ++---- .../qa/performance/TaskPerformanceTest.java | 63 ++---- .../BlobStorePersistencePerformanceTest.java | 39 ++-- .../brooklyn/test/PerformanceTestUtils.java | 82 +------- .../test/performance/FilePersister.java | 85 ++++++++ .../brooklyn/test/performance/Histogram.java | 89 ++++++++ .../performance/MeasurementResultPersister.java | 29 +++ .../test/performance/PerformanceMeasurer.java | 156 ++++++++++++++ .../performance/PerformanceTestDescriptor.java | 208 +++++++++++++++++++ .../test/performance/PerformanceTestResult.java | 62 ++++++ .../test/performance/PerformanceTestUtils.java | 107 ++++++++++ 16 files changed, 1041 insertions(+), 256 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/AbstractPerformanceTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/AbstractPerformanceTest.java b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/AbstractPerformanceTest.java index a5a3568..739877b 100644 --- a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/AbstractPerformanceTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/AbstractPerformanceTest.java @@ -20,6 +20,7 @@ package org.apache.brooklyn.core.test.qa.performance; import static org.testng.Assert.assertTrue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.brooklyn.api.mgmt.ManagementContext; @@ -27,6 +28,9 @@ import org.apache.brooklyn.core.entity.Entities; import org.apache.brooklyn.core.entity.factory.ApplicationBuilder; import org.apache.brooklyn.core.location.SimulatedLocation; import org.apache.brooklyn.core.test.entity.TestApplication; +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; +import org.apache.brooklyn.test.performance.PerformanceTestResult; +import org.apache.brooklyn.test.performance.PerformanceMeasurer; import org.apache.brooklyn.util.internal.DoubleSystemProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,11 +84,42 @@ public class AbstractPerformanceTest { public void tearDown() throws Exception { if (app != null) Entities.destroyAll(app.getManagementContext()); } - + + protected PerformanceTestResult measure(PerformanceTestDescriptor options) { + PerformanceTestResult result = PerformanceMeasurer.run(options); + System.out.println("test="+options+"; result="+result); + return result; + } + + /** + * @deprecated since 0.9.0; use {@link #measure(PerformanceTestDescriptor)} + */ + @Deprecated protected void measureAndAssert(String prefix, int numIterations, double minRatePerSec, Runnable r) { - measureAndAssert(prefix, numIterations, minRatePerSec, r, null); + measure(PerformanceTestDescriptor.create() + .summary(prefix) + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(r)); + } + + /** + * @deprecated since 0.9.0; use {@link #measure(PerformanceTestDescriptor)} + */ + @Deprecated + protected void measureAndAssert(String prefix, int numIterations, double minRatePerSec, Runnable r, CountDownLatch completionLatch) { + measure(PerformanceTestDescriptor.create() + .summary(prefix) + .iterations(numIterations) + .completionLatch(completionLatch) + .minAcceptablePerSecond(minRatePerSec) + .job(r)); } + /** + * @deprecated since 0.9.0; use {@link #measure(PerformanceTestDescriptor)} + */ + @Deprecated protected void measureAndAssert(String prefix, int numIterations, double minRatePerSec, Runnable r, Runnable postIterationPhase) { long durationMillis = measure(prefix, numIterations, r); long postIterationDurationMillis = (postIterationPhase != null) ? measure(postIterationPhase) : 0; @@ -102,6 +137,10 @@ public class AbstractPerformanceTest { assertTrue(numPerSecIncludingPostIteration >= minRatePerSec, msg1+msg2); } + /** + * @deprecated since 0.9.0; use {@link #measure(PerformanceTestDescriptor)} + */ + @Deprecated protected long measure(String prefix, int numIterations, Runnable r) { final int logInterval = 5*1000; long nextLogTime = logInterval; @@ -128,6 +167,10 @@ public class AbstractPerformanceTest { return stopwatch.elapsed(TimeUnit.MILLISECONDS); } + /** + * @deprecated since 0.9.0; use {@link #measure(PerformanceTestDescriptor)} + */ + @Deprecated protected long measure(Runnable r) { Stopwatch stopwatch = Stopwatch.createStarted(); r.run(); http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/EntityPerformanceTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/EntityPerformanceTest.java b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/EntityPerformanceTest.java index 16707d0..c73b344 100644 --- a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/EntityPerformanceTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/EntityPerformanceTest.java @@ -30,6 +30,7 @@ import org.apache.brooklyn.api.sensor.SensorEventListener; import org.apache.brooklyn.core.entity.Entities; import org.apache.brooklyn.core.test.entity.TestEntity; import org.apache.brooklyn.test.Asserts; +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.exceptions.Exceptions; import org.testng.annotations.BeforeMethod; @@ -69,10 +70,14 @@ public class EntityPerformanceTest extends AbstractPerformanceTest { double minRatePerSec = 1000 * PERFORMANCE_EXPECTATION; final AtomicInteger i = new AtomicInteger(); - measureAndAssert("updateAttribute", numIterations, minRatePerSec, new Runnable() { - public void run() { - entity.sensors().set(TestEntity.SEQUENCE, i.getAndIncrement()); - }}); + measure(PerformanceTestDescriptor.create() + .summary("EntityPerformanceTest.testUpdateAttributeWhenNoListeners") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + entity.sensors().set(TestEntity.SEQUENCE, i.getAndIncrement()); + }})); } @Test(groups={"Integration", "Acceptance"}) @@ -87,10 +92,14 @@ public class EntityPerformanceTest extends AbstractPerformanceTest { lastVal.set(event.getValue()); }}); - measureAndAssert("updateAttributeWithListener", numIterations, minRatePerSec, new Runnable() { - public void run() { - entity.sensors().set(TestEntity.SEQUENCE, (i.getAndIncrement())); - }}); + measure(PerformanceTestDescriptor.create() + .summary("EntityPerformanceTest.testUpdateAttributeWithNoopListeners") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + entity.sensors().set(TestEntity.SEQUENCE, (i.getAndIncrement())); + }})); Asserts.succeedsEventually(MutableMap.of("timeout", TIMEOUT_MS), new Runnable() { public void run() { @@ -103,10 +112,14 @@ public class EntityPerformanceTest extends AbstractPerformanceTest { int numIterations = numIterations(); double minRatePerSec = 1000 * PERFORMANCE_EXPECTATION; - measureAndAssert("invokeEffector", numIterations, minRatePerSec, new Runnable() { - public void run() { - entity.myEffector(); - }}); + measure(PerformanceTestDescriptor.create() + .summary("EntityPerformanceTest.testInvokeEffector") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + entity.myEffector(); + }})); } @Test(groups={"Integration", "Acceptance"}) @@ -114,31 +127,38 @@ public class EntityPerformanceTest extends AbstractPerformanceTest { int numIterations = numIterations(); double minRatePerSec = 1000 * PERFORMANCE_EXPECTATION; - measureAndAssert("invokeEffectorAsyncAndGet", numIterations, minRatePerSec, new Runnable() { - public void run() { - Task<?> task = entity.invoke(TestEntity.MY_EFFECTOR, MutableMap.<String,Object>of()); - try { - task.get(); - } catch (Exception e) { - throw Exceptions.propagate(e); - } - }}); + measure(PerformanceTestDescriptor.create() + .summary("EntityPerformanceTest.testAsyncEffectorInvocation") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + Task<?> task = entity.invoke(TestEntity.MY_EFFECTOR, MutableMap.<String,Object>of()); + try { + task.get(); + } catch (Exception e) { + throw Exceptions.propagate(e); + } + }})); } - // TODO but surely parallel should be much faster?! @Test(groups={"Integration", "Acceptance"}) public void testMultiEntityConcurrentEffectorInvocation() { int numIterations = numIterations(); - double minRatePerSec = 100 * PERFORMANCE_EXPECTATION; // i.e. 1000 invocations + double minRatePerSec = 100 * PERFORMANCE_EXPECTATION; // i.e. 1000 invocations (because 10 entities per time) - measureAndAssert("invokeEffectorMultiEntityConcurrentAsyncAndGet", numIterations, minRatePerSec, new Runnable() { - public void run() { - Task<?> task = Entities.invokeEffector(app, entities, TestEntity.MY_EFFECTOR); - try { - task.get(); - } catch (Exception e) { - throw Exceptions.propagate(e); - } - }}); + measure(PerformanceTestDescriptor.create() + .summary("EntityPerformanceTest.testMultiEntityConcurrentEffectorInvocation") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + Task<?> task = Entities.invokeEffector(app, entities, TestEntity.MY_EFFECTOR); + try { + task.get(); + } catch (Exception e) { + throw Exceptions.propagate(e); + } + }})); } } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/FilePersistencePerformanceTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/FilePersistencePerformanceTest.java b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/FilePersistencePerformanceTest.java index 594b102..364a673 100644 --- a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/FilePersistencePerformanceTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/FilePersistencePerformanceTest.java @@ -24,10 +24,12 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.brooklyn.core.mgmt.persist.FileBasedStoreObjectAccessor; +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; import org.apache.brooklyn.util.collections.MutableList; import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.core.internal.ssh.process.ProcessTool; import org.apache.brooklyn.util.exceptions.Exceptions; +import org.apache.brooklyn.util.io.FileUtil; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -71,10 +73,14 @@ public class FilePersistencePerformanceTest extends AbstractPerformanceTest { double minRatePerSec = 100 * PERFORMANCE_EXPECTATION; final AtomicInteger i = new AtomicInteger(); - measureAndAssert("FileBasedStoreObjectAccessor.put", numIterations, minRatePerSec, new Runnable() { - public void run() { - fileAccessor.put(""+i.incrementAndGet()); - }}); + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testFileBasedStoreObjectPuts") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + fileAccessor.put(""+i.incrementAndGet()); + }})); } @Test(groups={"Integration", "Acceptance"}) @@ -83,10 +89,14 @@ public class FilePersistencePerformanceTest extends AbstractPerformanceTest { int numIterations = numIterations(); double minRatePerSec = 100 * PERFORMANCE_EXPECTATION; - measureAndAssert("FileBasedStoreObjectAccessor.get", numIterations, minRatePerSec, new Runnable() { - public void run() { - fileAccessor.get(); - }}); + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testFileBasedStoreObjectGet") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + fileAccessor.get(); + }})); } @Test(groups={"Integration", "Acceptance"}) @@ -105,12 +115,16 @@ public class FilePersistencePerformanceTest extends AbstractPerformanceTest { final AtomicInteger i = new AtomicInteger(); try { - measureAndAssert("FileBasedStoreObjectAccessor.delete", numIterations, minRatePerSec, new Runnable() { - public void run() { - File file = files.get(i.getAndIncrement()); - FileBasedStoreObjectAccessor fileAccessor = new FileBasedStoreObjectAccessor(file, "mytmpextension"); - fileAccessor.delete(); - }}); + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testFileBasedStoreObjectDelete") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + File file = files.get(i.getAndIncrement()); + FileBasedStoreObjectAccessor fileAccessor = new FileBasedStoreObjectAccessor(file, "mytmpextension"); + fileAccessor.delete(); + }})); } finally { for (File file : files) { if (file != null) file.delete(); @@ -118,17 +132,51 @@ public class FilePersistencePerformanceTest extends AbstractPerformanceTest { } } + @Test(groups={"Integration", "Acceptance"}) + public void testFileUtilSetFilePermissions() throws IOException { + int numIterations = numIterations(); + double minRatePerSec = 10 * PERFORMANCE_EXPECTATION; + + final File file = File.createTempFile("filePermissions", ".txt"); + + try { + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testFileUtilSetFilePermissions") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + int i = 0; + public void run() { + try { + if (i % 2 == 0) { + FileUtil.setFilePermissionsTo600(file); + } else { + FileUtil.setFilePermissionsTo700(file); + } + } catch (Exception e) { + throw Exceptions.propagate(e); + } + }})); + } finally { + file.delete(); + } + } + // fileAccessor.put() is implemented with an execCommands("mv") so look at performance of just that piece @Test(groups={"Integration", "Acceptance"}) public void testProcessToolExecCommand() { int numIterations = numIterations(); double minRatePerSec = 10 * PERFORMANCE_EXPECTATION; - measureAndAssert("ProcessTool.exec", numIterations, minRatePerSec, new Runnable() { - public void run() { - String cmd = "true"; - new ProcessTool().execCommands(MutableMap.<String,String>of(), MutableList.of(cmd), null); - }}); + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testProcessToolExecCommand") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + String cmd = "true"; + new ProcessTool().execCommands(MutableMap.<String,String>of(), MutableList.of(cmd), null); + }})); } @Test(groups={"Integration", "Acceptance"}) @@ -139,12 +187,16 @@ public class FilePersistencePerformanceTest extends AbstractPerformanceTest { final File parentDir = file.getParentFile(); final AtomicInteger i = new AtomicInteger(); - measureAndAssert("java.util.File.rename", numIterations, minRatePerSec, new Runnable() { - public void run() { - File newFile = new File(parentDir, "fileRename-"+i.incrementAndGet()+".txt"); - file.renameTo(newFile); - file = newFile; - }}); + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testJavaUtilFileRenames") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + File newFile = new File(parentDir, "fileRename-"+i.incrementAndGet()+".txt"); + file.renameTo(newFile); + file = newFile; + }})); } @Test(groups={"Integration", "Acceptance"}) @@ -154,14 +206,18 @@ public class FilePersistencePerformanceTest extends AbstractPerformanceTest { final AtomicInteger i = new AtomicInteger(); - measureAndAssert("guava.Files.write", numIterations, minRatePerSec, new Runnable() { - public void run() { - try { - Files.write(""+i.incrementAndGet(), file, Charsets.UTF_8); - } catch (IOException e) { - throw Exceptions.propagate(e); - } - }}); + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testGuavaFileWrites") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + try { + Files.write(""+i.incrementAndGet(), file, Charsets.UTF_8); + } catch (IOException e) { + throw Exceptions.propagate(e); + } + }})); } @Test(groups={"Integration", "Acceptance"}) @@ -172,15 +228,19 @@ public class FilePersistencePerformanceTest extends AbstractPerformanceTest { final File parentDir = file.getParentFile(); final AtomicInteger i = new AtomicInteger(); - measureAndAssert("guava.Files.move", numIterations, minRatePerSec, new Runnable() { - public void run() { - File newFile = new File(parentDir, "fileRename-"+i.incrementAndGet()+".txt"); - try { - Files.move(file, newFile); - } catch (IOException e) { - throw Exceptions.propagate(e); - } - file = newFile; - }}); + measure(PerformanceTestDescriptor.create() + .summary("FilePersistencePerformanceTest.testGuavaFileMoves") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + File newFile = new File(parentDir, "fileRename-"+i.incrementAndGet()+".txt"); + try { + Files.move(file, newFile); + } catch (IOException e) { + throw Exceptions.propagate(e); + } + file = newFile; + }})); } } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/GroovyYardStickPerformanceTest.groovy ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/GroovyYardStickPerformanceTest.groovy b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/GroovyYardStickPerformanceTest.groovy index c269816..3940b49 100644 --- a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/GroovyYardStickPerformanceTest.groovy +++ b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/GroovyYardStickPerformanceTest.groovy @@ -24,6 +24,7 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; import org.slf4j.Logger import org.slf4j.LoggerFactory import org.testng.annotations.AfterMethod @@ -56,7 +57,11 @@ public class GroovyYardStickPerformanceTest extends AbstractPerformanceTest { double minRatePerSec = 1000000 * PERFORMANCE_EXPECTATION; AtomicInteger i = new AtomicInteger(); - measureAndAssert("noop-groovy", numIterations, minRatePerSec, { i.incrementAndGet() }); + measure(MeasurementOptions.create() + .summary("GroovyYardStickPerformanceTest.noop") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job({ i.incrementAndGet() })); assertTrue(i.get() >= numIterations, "i="+i); } } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/JavaYardStickPerformanceTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/JavaYardStickPerformanceTest.java b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/JavaYardStickPerformanceTest.java index e5d4807..84c60ca 100644 --- a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/JavaYardStickPerformanceTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/JavaYardStickPerformanceTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -53,10 +54,14 @@ public class JavaYardStickPerformanceTest extends AbstractPerformanceTest { int numIterations = 1000000; double minRatePerSec = 1000000 * PERFORMANCE_EXPECTATION; final int[] i = {0}; - measureAndAssert("noop-java", numIterations, minRatePerSec, new Runnable() { - @Override public void run() { - i[0] = i[0] + 1; - }}); + measure(PerformanceTestDescriptor.create() + .summary("JavaYardStickPerformanceTest.noop") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + @Override public void run() { + i[0] = i[0] + 1; + }})); assertTrue(i[0] >= numIterations, "i="+i); } @@ -66,15 +71,19 @@ public class JavaYardStickPerformanceTest extends AbstractPerformanceTest { int numIterations = 100000; double minRatePerSec = 100000 * PERFORMANCE_EXPECTATION; final int[] i = {0}; - measureAndAssert("scheduleExecuteAndGet-java", numIterations, minRatePerSec, new Runnable() { - @Override public void run() { - Future<?> future = executor.submit(new Runnable() { public void run() { i[0] = i[0] + 1; }}); - try { - future.get(); - } catch (Exception e) { - throw Throwables.propagate(e); - } - }}); + measure(PerformanceTestDescriptor.create() + .summary("JavaYardStickPerformanceTest.scheduleExecuteAndGet") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + @Override public void run() { + Future<?> future = executor.submit(new Runnable() { public void run() { i[0] = i[0] + 1; }}); + try { + future.get(); + } catch (Exception e) { + throw Throwables.propagate(e); + } + }})); assertTrue(i[0] >= numIterations, "i="+i); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/SubscriptionPerformanceTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/SubscriptionPerformanceTest.java b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/SubscriptionPerformanceTest.java index 154c0e6..6f66f20 100644 --- a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/SubscriptionPerformanceTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/SubscriptionPerformanceTest.java @@ -18,11 +18,8 @@ */ package org.apache.brooklyn.core.test.qa.performance; -import static org.testng.Assert.assertTrue; - import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -31,8 +28,8 @@ import org.apache.brooklyn.api.mgmt.SubscriptionManager; import org.apache.brooklyn.api.sensor.SensorEvent; import org.apache.brooklyn.api.sensor.SensorEventListener; import org.apache.brooklyn.core.test.entity.TestEntity; +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; import org.apache.brooklyn.util.collections.MutableMap; -import org.apache.brooklyn.util.exceptions.Exceptions; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -41,7 +38,6 @@ import com.google.common.collect.Lists; public class SubscriptionPerformanceTest extends AbstractPerformanceTest { - private static final long LONG_TIMEOUT_MS = 30*1000; private static final int NUM_ITERATIONS = 10000; TestEntity entity; @@ -81,22 +77,15 @@ public class SubscriptionPerformanceTest extends AbstractPerformanceTest { }}); } - measureAndAssert("updateAttributeWithManyPublishedOneSubscriber", numIterations, minRatePerSec, - new Runnable() { + measure(PerformanceTestDescriptor.create() + .summary("SubscriptionPerformanceTest.testManyPublishedOneSubscriber") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { public void run() { entity.sensors().set(TestEntity.SEQUENCE, (iter.getAndIncrement())); - } - }, - new Runnable() { - public void run() { - try { - completionLatch.await(LONG_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw Exceptions.propagate(e); - } - assertTrue(completionLatch.getCount() <= 0); - } - }); + }}) + .completionLatch(completionLatch)); } @Test(groups={"Integration", "Acceptance"}) @@ -118,20 +107,15 @@ public class SubscriptionPerformanceTest extends AbstractPerformanceTest { }}); } - measureAndAssert("updateAttributeWithManyListeners", numIterations, minRatePerSec, - new Runnable() { + measure(PerformanceTestDescriptor.create() + .summary("SubscriptionPerformanceTest.testManyListenersForSensorEvent") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { @Override public void run() { entity.sensors().set(TestEntity.SEQUENCE, (iter.getAndIncrement())); - }}, - new Runnable() { - public void run() { - try { - completionLatch.await(LONG_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw Exceptions.propagate(e); - } - assertTrue(completionLatch.getCount() <= 0); - }}); + }}) + .completionLatch(completionLatch)); } @Test(groups={"Integration", "Acceptance"}) @@ -155,10 +139,14 @@ public class SubscriptionPerformanceTest extends AbstractPerformanceTest { }}); } - measureAndAssert("updateAttributeWithUnrelatedListeners", numIterations, minRatePerSec, new Runnable() { - @Override public void run() { - entity.sensors().set(TestEntity.SEQUENCE, (iter.incrementAndGet())); - }}); + measure(PerformanceTestDescriptor.create() + .summary("SubscriptionPerformanceTest.testUpdateAttributeWithNoListenersButManyUnrelatedListeners") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + @Override public void run() { + entity.sensors().set(TestEntity.SEQUENCE, (iter.incrementAndGet())); + }})); if (exception.get() != null) { throw exception.get(); http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/TaskPerformanceTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/TaskPerformanceTest.java b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/TaskPerformanceTest.java index c80b27b..b0f9a07 100644 --- a/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/TaskPerformanceTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/test/qa/performance/TaskPerformanceTest.java @@ -18,18 +18,18 @@ */ package org.apache.brooklyn.core.test.qa.performance; -import static org.testng.Assert.assertTrue; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.core.task.BasicExecutionManager; import org.apache.brooklyn.util.core.task.SingleThreadedScheduler; import org.apache.brooklyn.util.exceptions.Exceptions; +import org.apache.brooklyn.util.time.Duration; import org.apache.brooklyn.util.time.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,8 +43,6 @@ public class TaskPerformanceTest extends AbstractPerformanceTest { private static final Logger LOG = LoggerFactory.getLogger(TaskPerformanceTest.class); - private static final long LONG_TIMEOUT_MS = 30*1000; - BasicExecutionManager executionManager; @BeforeMethod(alwaysRun=true) @@ -71,20 +69,15 @@ public class TaskPerformanceTest extends AbstractPerformanceTest { if (val >= numIterations) completionLatch.countDown(); }}; - measureAndAssert("executeSimplestRunnable", numIterations, minRatePerSec, - new Runnable() { + measure(PerformanceTestDescriptor.create() + .summary("TaskPerformanceTest.testExecuteSimplestRunnable") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { public void run() { executionManager.submit(work); - }}, - new Runnable() { - public void run() { - try { - completionLatch.await(LONG_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw Exceptions.propagate(e); - } - assertTrue(completionLatch.getCount() <= 0); - }}); + }}) + .completionLatch(completionLatch)); } @Test(groups={"Integration", "Acceptance"}) @@ -102,20 +95,15 @@ public class TaskPerformanceTest extends AbstractPerformanceTest { final Map<String, ?> flags = MutableMap.of("tags", ImmutableList.of("a","b")); - measureAndAssert("testExecuteRunnableWithTags", numIterations, minRatePerSec, - new Runnable() { + measure(PerformanceTestDescriptor.create() + .summary("TaskPerformanceTest.testExecuteRunnableWithTags") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { public void run() { executionManager.submit(flags, work); - }}, - new Runnable() { - public void run() { - try { - completionLatch.await(LONG_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw Exceptions.propagate(e); - } - assertTrue(completionLatch.getCount() <= 0); - }}); + }}) + .completionLatch(completionLatch)); } @Test(groups={"Integration", "Acceptance"}) @@ -146,8 +134,11 @@ public class TaskPerformanceTest extends AbstractPerformanceTest { } }; - measureAndAssert("testExecuteWithSingleThreadedScheduler", numIterations, minRatePerSec, - new Runnable() { + measure(PerformanceTestDescriptor.create() + .summary("TaskPerformanceTest.testExecuteWithSingleThreadedScheduler") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { public void run() { while (submitCount.get() > counter.get() + 5000) { LOG.info("delaying because "+submitCount.get()+" submitted and only "+counter.get()+" run"); @@ -155,16 +146,8 @@ public class TaskPerformanceTest extends AbstractPerformanceTest { } executionManager.submit(MutableMap.of("tags", ImmutableList.of("singlethreaded")), work); submitCount.incrementAndGet(); - }}, - new Runnable() { - public void run() { - try { - completionLatch.await(LONG_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw Exceptions.propagate(e); - } - assertTrue(completionLatch.getCount() <= 0); - }}); + }}) + .completionLatch(completionLatch)); if (exceptions.size() > 0) throw exceptions.get(0); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStorePersistencePerformanceTest.java ---------------------------------------------------------------------- diff --git a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStorePersistencePerformanceTest.java b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStorePersistencePerformanceTest.java index 9690c00..bcba953 100644 --- a/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStorePersistencePerformanceTest.java +++ b/locations/jclouds/src/test/java/org/apache/brooklyn/core/mgmt/persist/jclouds/BlobStorePersistencePerformanceTest.java @@ -26,6 +26,7 @@ import org.apache.brooklyn.core.mgmt.persist.PersistMode; import org.apache.brooklyn.core.mgmt.persist.PersistenceObjectStore.StoreObjectAccessor; import org.apache.brooklyn.core.mgmt.persist.jclouds.JcloudsBlobStoreBasedObjectStore; import org.apache.brooklyn.core.test.qa.performance.AbstractPerformanceTest; +import org.apache.brooklyn.test.performance.PerformanceTestDescriptor; import org.apache.brooklyn.util.text.Identifiers; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -75,10 +76,14 @@ public class BlobStorePersistencePerformanceTest extends AbstractPerformanceTest double minRatePerSec = 10 * PERFORMANCE_EXPECTATION; final AtomicInteger i = new AtomicInteger(); - measureAndAssert("StoreObjectAccessor.put", numIterations, minRatePerSec, new Runnable() { - public void run() { - blobstoreAccessor.put(""+i.incrementAndGet()); - }}); + measure(PerformanceTestDescriptor.create() + .summary("StoreObjectAccessor.put") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + blobstoreAccessor.put(""+i.incrementAndGet()); + }})); } @Test(groups={"Live", "Acceptance"}) @@ -87,10 +92,14 @@ public class BlobStorePersistencePerformanceTest extends AbstractPerformanceTest int numIterations = numIterations(); double minRatePerSec = 10 * PERFORMANCE_EXPECTATION; - measureAndAssert("FileBasedStoreObjectAccessor.get", numIterations, minRatePerSec, new Runnable() { - public void run() { - blobstoreAccessor.get(); - }}); + measure(PerformanceTestDescriptor.create() + .summary("FileBasedStoreObjectAccessor.get") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + blobstoreAccessor.get(); + }})); } @Test(groups={"Live", "Acceptance"}) @@ -107,11 +116,15 @@ public class BlobStorePersistencePerformanceTest extends AbstractPerformanceTest final AtomicInteger i = new AtomicInteger(); try { - measureAndAssert("FileBasedStoreObjectAccessor.delete", numIterations, minRatePerSec, new Runnable() { - public void run() { - StoreObjectAccessor blobstoreAccessor = blobstoreAccessors.get(i.getAndIncrement()); - blobstoreAccessor.delete(); - }}); + measure(PerformanceTestDescriptor.create() + .summary("FileBasedStoreObjectAccessor.delete") + .iterations(numIterations) + .minAcceptablePerSecond(minRatePerSec) + .job(new Runnable() { + public void run() { + StoreObjectAccessor blobstoreAccessor = blobstoreAccessors.get(i.getAndIncrement()); + blobstoreAccessor.delete(); + }})); } finally { for (StoreObjectAccessor blobstoreAccessor : blobstoreAccessors) { blobstoreAccessor.delete(); http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/PerformanceTestUtils.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/PerformanceTestUtils.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/PerformanceTestUtils.java index da38791..c0e9a71 100644 --- a/usage/test-support/src/main/java/org/apache/brooklyn/test/PerformanceTestUtils.java +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/PerformanceTestUtils.java @@ -18,81 +18,9 @@ */ package org.apache.brooklyn.test; -import java.lang.management.ManagementFactory; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import javax.management.MBeanServer; -import javax.management.ObjectName; - -import org.apache.brooklyn.util.time.Duration; -import org.apache.brooklyn.util.time.Time; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Stopwatch; - -public class PerformanceTestUtils { - - private static final Logger LOG = LoggerFactory.getLogger(PerformanceTestUtils.class); - - private static boolean hasLoggedProcessCpuTimeUnavailable; - - public static long getProcessCpuTime() { - try { - MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer(); - ObjectName osMBeanName = ObjectName.getInstance(ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME); - return (Long) mbeanServer.getAttribute(osMBeanName, "ProcessCpuTime"); - } catch (Exception e) { - if (!hasLoggedProcessCpuTimeUnavailable) { - hasLoggedProcessCpuTimeUnavailable = true; - LOG.warn("ProcessCPuTime not available in local JVM MXBean "+ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME+" (only available in sun JVM?)"); - } - return -1; - } - } - - /** - * Creates a background thread that will log.info the CPU fraction usage repeatedly, sampling at the given period. - * Callers <em>must</em> cancel the returned future, e.g. {@code future.cancel(true)}, otherwise it will keep - * logging until the JVM exits. - */ - public static Future<?> sampleProcessCpuTime(final Duration period, final String loggingContext) { - final ExecutorService executor = Executors.newSingleThreadExecutor(new ThreadFactory() { - @Override public Thread newThread(Runnable r) { - Thread thread = new Thread(r, "brooklyn-sampleProcessCpuTime-"+loggingContext); - thread.setDaemon(true); // let the JVM exit - return thread; - }}); - Future<?> future = executor.submit(new Runnable() { - @Override public void run() { - try { - long prevCpuTime = getProcessCpuTime(); - if (prevCpuTime == -1) { - LOG.warn("ProcessCPuTime not available; cannot sample; aborting"); - return; - } - while (true) { - Stopwatch stopwatch = Stopwatch.createStarted(); - Thread.sleep(period.toMilliseconds()); - long currentCpuTime = getProcessCpuTime(); - - long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); - double fractionCpu = (elapsedTime > 0) ? ((double)currentCpuTime-prevCpuTime) / TimeUnit.MILLISECONDS.toNanos(elapsedTime) : -1; - prevCpuTime = currentCpuTime; - - LOG.info("CPU fraction over last {} was {} ({})", new Object[] { - Time.makeTimeStringRounded(elapsedTime), fractionCpu, loggingContext}); - } - } catch (InterruptedException e) { - return; // graceful termination - } finally { - executor.shutdownNow(); - } - }}); - return future; - } +/** + * @deprecated since 0.9.0; see {@link org.apache.brooklyn.test.performance.PerformanceTestUtils}. + */ +@Deprecated +public class PerformanceTestUtils extends org.apache.brooklyn.test.performance.PerformanceTestUtils { } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/FilePersister.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/FilePersister.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/FilePersister.java new file mode 100644 index 0000000..3d4eaf6 --- /dev/null +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/FilePersister.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.test.performance; + +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.apache.brooklyn.util.time.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.Beta; +import com.google.common.base.Charsets; +import com.google.common.io.Files; + +@Beta +public class FilePersister implements MeasurementResultPersister { + + private static final Logger LOG = LoggerFactory.getLogger(PerformanceTestUtils.class); + + private final File dir; + + public FilePersister(File dir) { + this.dir = dir; + } + + @Override + public void persist(Date date, PerformanceTestDescriptor options, PerformanceTestResult result) { + try { + String dateStr = new SimpleDateFormat(Time.DATE_FORMAT_PREFERRED).format(date); + + dir.mkdirs(); + + File file = new File(dir, "auto-test-results.txt"); + file.createNewFile(); + Files.append("date="+dateStr+"; test="+options+"; result="+result+"\n", file, Charsets.UTF_8); + + File summaryFile = new File(dir, "auto-test-summary.txt"); + summaryFile.createNewFile(); + Files.append( + dateStr + +"\t"+options.summary + +"\t"+roundToSignificantFigures(result.ratePerSecond, 6) + +"\t"+result.duration + +(result.cpuTotalFraction != null ? "\t"+"cpu="+roundToSignificantFigures(result.cpuTotalFraction, 3) : "") + +"\n", + summaryFile, Charsets.UTF_8); + + } catch (IOException e) { + LOG.warn("Failed to persist performance results to "+dir+" (continuing)", e); + } + } + + // Code copied from http://stackoverflow.com/questions/202302/rounding-to-an-arbitrary-number-of-significant-digits + private double roundToSignificantFigures(double num, int n) { + if(num == 0) { + return 0; + } + + final double d = Math.ceil(Math.log10(num < 0 ? -num: num)); + final int power = n - (int) d; + + final double magnitude = Math.pow(10, power); + final long shifted = Math.round(num*magnitude); + return shifted/magnitude; + } +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/Histogram.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/Histogram.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/Histogram.java new file mode 100644 index 0000000..9c60eb0 --- /dev/null +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/Histogram.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.test.performance; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.brooklyn.util.collections.MutableList; +import org.apache.brooklyn.util.time.Duration; +import org.apache.brooklyn.util.time.Time; + +import com.google.common.annotations.Beta; +import com.google.common.collect.Maps; + +/** + * A simplistic histogram to store times in a number of buckets. + * The buckets are in nanoseconds, increasing in size in powers of two. + */ +@Beta +public class Histogram { + + // TODO Currently just does toString to get the values back out. + + private final Map<Integer, Integer> counts = Maps.newLinkedHashMap(); + + public void add(long val, TimeUnit unit) { + add(unit.toNanos(val)); + } + + public void add(Duration val) { + add(val.toNanoseconds()); + } + + protected void add(long val) { + if (val < 0) throw new UnsupportedOperationException("Negative numbers not accepted: "+val); + int pow = getPower(val); + Integer count = counts.get(pow); + counts.put(pow, (count == null) ? 1 : count+1); + } + + protected int getPower(long val) { + for (int i = 0; i < 64; i++) { + if (val < Math.pow(2, i)) { + return i; + } + } + return 64; + } + + @Override + public String toString() { + if (counts.isEmpty()) return "<empty>"; + + StringBuilder result = new StringBuilder("{"); + List<Integer> sortedPows = MutableList.copyOf(counts.keySet()); + Collections.sort(sortedPows); + int minPow = sortedPows.get(0); + int maxPow = sortedPows.get(sortedPows.size()-1); + for (int i = minPow; i <= maxPow; i++) { + if (i != minPow) result.append(", "); + long lower = i == 0 ? 0 : (long) Math.pow(2, i-1); + long upper = (long) Math.pow(2, i); + Integer count = counts.get(i); + result.append(Time.makeTimeStringRounded(lower, TimeUnit.NANOSECONDS) + + "-" + Time.makeTimeStringRounded(upper, TimeUnit.NANOSECONDS) + + ": " + (count == null ? 0 : count)); + } + result.append("}"); + return result.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/MeasurementResultPersister.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/MeasurementResultPersister.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/MeasurementResultPersister.java new file mode 100644 index 0000000..cdf00e8 --- /dev/null +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/MeasurementResultPersister.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.test.performance; + +import java.util.Date; + +import com.google.common.annotations.Beta; + +@Beta +public interface MeasurementResultPersister { + + void persist(Date time, PerformanceTestDescriptor options, PerformanceTestResult result); +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java new file mode 100644 index 0000000..31802db --- /dev/null +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceMeasurer.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.test.performance; + +import static org.testng.Assert.fail; + +import java.util.Date; +import java.util.List; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.brooklyn.util.exceptions.Exceptions; +import org.apache.brooklyn.util.time.Duration; +import org.apache.brooklyn.util.time.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.Beta; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; + +/** + * For running simplistic performance tests, to measure the number of operations per second. + * + * With a short run, this is "good enough" for eye-balling performance, to spot if it goes + * horrendously wrong. + * + * However, good performance measurement involves much more warm up (e.g. to ensure java HotSpot + * optimisation have been applied), and running the test for a reasonable length of time. + * + * Longevity tests are also important for to check if object creation is going to kill + * performance in the long-term, etc. + */ +@Beta +public class PerformanceMeasurer { + + private static final Logger LOG = LoggerFactory.getLogger(PerformanceMeasurer.class); + + /** + * Runs a performance test. Repeatedly executes the given job. It measuring either the time it takes for + * many iterations, or the number of iterations it can execute in a fixed time. + */ + public static PerformanceTestResult run(PerformanceTestDescriptor options) { + options.seal(); + long nextLogTime = (options.logInterval == null) ? Long.MAX_VALUE : options.logInterval.toMilliseconds(); + + // Try to force garbage collection before the test, so it interferes less with the measurement. + System.gc(); System.gc(); + + // Run some warm-up cycles. + Stopwatch warmupWatch = Stopwatch.createStarted(); + int warmupCounter = 0; + + while ((options.warmup != null) ? options.warmup.isLongerThan(warmupWatch) : warmupCounter < options.warmupIterations) { + if (warmupWatch.elapsed(TimeUnit.MILLISECONDS) >= nextLogTime) { + LOG.info("Warm-up "+options.summary+" iteration="+warmupCounter+" at "+Time.makeTimeStringRounded(warmupWatch)); + nextLogTime += options.logInterval.toMilliseconds(); + } + options.job.run(); + warmupCounter++; + } + warmupWatch.stop(); + + // Run the actual test (for the given duration / iterations); then wait for completionLatch (if supplied). + nextLogTime = (options.logInterval == null) ? Long.MAX_VALUE : options.logInterval.toMilliseconds(); + int counter = 0; + Histogram histogram = new Histogram(); + List<Double> cpuSampleFractions = Lists.newLinkedList(); + Future<?> sampleCpuFuture = null; + if (options.sampleCpuInterval != null) { + sampleCpuFuture = PerformanceTestUtils.sampleProcessCpuTime(options.sampleCpuInterval, options.summary, cpuSampleFractions); + } + + try { + long preCpuTime = PerformanceTestUtils.getProcessCpuTime(); + Stopwatch watch = Stopwatch.createStarted(); + + while ((options.duration != null) ? options.duration.isLongerThan(watch) : counter < options.iterations) { + if (warmupWatch.elapsed(TimeUnit.MILLISECONDS) >= nextLogTime) { + LOG.info(options.summary+" iteration="+counter+" at "+Time.makeTimeStringRounded(watch)); + nextLogTime += options.logInterval.toMilliseconds(); + } + long before = watch.elapsed(TimeUnit.NANOSECONDS); + options.job.run(); + if (options.histogram) { + histogram.add(watch.elapsed(TimeUnit.NANOSECONDS) - before, TimeUnit.NANOSECONDS); + } + counter++; + } + + if (options.completionLatch != null) { + try { + boolean success = options.completionLatch.await(options.completionLatchTimeout.toMilliseconds(), TimeUnit.MILLISECONDS); + if (!success) { + fail("Timeout waiting for completionLatch: test="+options+"; counter="+counter); + } + } catch (InterruptedException e) { + throw Exceptions.propagate(e); + } + } + watch.stop(); + long postCpuTime = PerformanceTestUtils.getProcessCpuTime(); + + // Generate the results + PerformanceTestResult result = new PerformanceTestResult(); + result.warmup = Duration.of(warmupWatch); + result.warmupIterations = warmupCounter; + result.duration = Duration.of(watch); + result.iterations = counter; + result.ratePerSecond = (((double)counter) / watch.elapsed(TimeUnit.MILLISECONDS)) * 1000; + result.cpuTotalFraction = (watch.elapsed(TimeUnit.NANOSECONDS) > 0 && preCpuTime >= 0) + ? ((double)postCpuTime-preCpuTime) / watch.elapsed(TimeUnit.NANOSECONDS) + : -1; + if (options.histogram) { + result.histogram = histogram; + } + if (options.sampleCpuInterval != null) { + result.cpuSampleFractions = cpuSampleFractions; + } + result.minAcceptablePerSecond = options.minAcceptablePerSecond; + + // Persist the results + if (options.persister != null) { + options.persister.persist(new Date(), options, result); + } + + // Fail if we didn't meet the minimum performance requirements + if (options.minAcceptablePerSecond != null && options.minAcceptablePerSecond > result.ratePerSecond) { + fail("Performance too low: test="+options+"; result="+result); + } + + return result; + + } finally { + if (sampleCpuFuture != null) { + sampleCpuFuture.cancel(true); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java new file mode 100644 index 0000000..89225c9 --- /dev/null +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestDescriptor.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.test.performance; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.io.File; +import java.util.concurrent.CountDownLatch; + +import org.apache.brooklyn.util.time.Duration; +import org.apache.commons.io.FileUtils; + +import com.google.common.annotations.Beta; +import com.google.common.base.Objects; + +/** + * For building up a description of what to measure. + * <p> + * Users are strongly encouraged to call the setter methods, rather than accessing the fields + * directly. The fields may be made protected in a future release. + * <p> + * The following fields are compulsory: + * <ul> + * <li>{@link #job(Runnable)} + * <li>Exactly one of {@link #duration(Duration)} or {@link #iterations(int)} + * </ul> + * + * See {@link PerformanceTestUtils#run(PerformanceTestDescriptor)}. + */ +@Beta +public class PerformanceTestDescriptor { + public String summary; + public Duration warmup; + public Integer warmupIterations; + public Duration duration; + public Integer iterations; + public Runnable job; + public CountDownLatch completionLatch; + public Duration completionLatchTimeout = Duration.FIVE_MINUTES; + public Double minAcceptablePerSecond; + public Duration sampleCpuInterval = Duration.ONE_SECOND; + public Duration logInterval = Duration.FIVE_SECONDS; + public boolean histogram = true; + public MeasurementResultPersister persister = new FilePersister(new File(FileUtils.getUserDirectory(), "brooklyn-performance")); + public boolean sealed; + + public static PerformanceTestDescriptor create() { + return new PerformanceTestDescriptor(); + } + + public static PerformanceTestDescriptor create(String summary) { + return create().summary(summary); + } + + public PerformanceTestDescriptor summary(String val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.summary = val; return this; + } + + /** + * The length of time to repeatedly execute the job for, before doing the proper performance + * test. At most one of {@link #warmup(Duration)} or {@link #warmupIterations(int)} should be + * set - if neither is set, the warmup defaults to one tenth of the test duration. + */ + public PerformanceTestDescriptor warmup(Duration val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.warmup = val; return this; + } + + /** + * See {@link #warmup(Duration)}. + */ + public PerformanceTestDescriptor warmupIterations(int val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.warmupIterations = val; return this; + } + + /** + * The length of time to repeatedly execute the job for, when measuring the performance. + * Exactly one of {@link #duration(Duration)} or {@link #iterations(int)} should be + * set. + */ + public PerformanceTestDescriptor duration(Duration val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.duration = val; return this; + } + + /** + * See {@link #duration(Duration)}. + */ + public PerformanceTestDescriptor iterations(int val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.iterations = val; return this; + } + + /** + * The job to be repeatedly executed. + */ + public PerformanceTestDescriptor job(Runnable val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.job = val; return this; + } + + /** + * If non-null, the performance test will wait for this latch before stopping the timer. + * This is useful for asynchronous work. For example, 1000 iterations of the job might + * be executed that each submits work asynchronously, and then the latch signals when all + * of those 1000 tasks have completed. + */ + public PerformanceTestDescriptor completionLatch(CountDownLatch val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.completionLatch = val; return this; + } + + /** + * The maximum length of time to wait for the {@link #completionLatch(CountDownLatch)}, after + * executing the designated number of jobs. If the latch has not completed within this time, + * then the test will fail. + */ + public PerformanceTestDescriptor completionLatchTimeout(Duration val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.completionLatchTimeout = val; return this; + } + + /** + * If non-null, the measured jobs-per-second will be compared against this number. If the + * jobs-per-second is not high enough, then the test wil fail. + */ + public PerformanceTestDescriptor minAcceptablePerSecond(Double val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.minAcceptablePerSecond = val; return this; + } + + /** + * Whether to collect a histogram of the individual job times. This histogram stores the count + * in buckets (e.g. number of jobs that took 1-2ms, number that took 2-4ms, number that took + * 4-8ms, etc). + */ + public PerformanceTestDescriptor histogram(boolean val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.histogram = val; return this; + } + + /** + * How often to log progress (e.g. number of iterations completed so far). If null, then no + * progress will be logged. + */ + public PerformanceTestDescriptor logInterval(Duration val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.logInterval = val; return this; + } + + /** + * How often to calculate + record the fraction of CPU being used. If null, then CPU usage + * will not be recorded. + */ + public PerformanceTestDescriptor sampleCpuInterval(Duration val) { + if (sealed) throw new IllegalStateException("Should not modify after sealed (e.g. after use)"); + this.sampleCpuInterval = val; return this; + } + + public void seal() { + sealed = true; + assertNotNull(job, "Job must be supplied: "+toString()); + assertTrue(duration != null ^ iterations != null, "Exactly one of duration or iterations must be set: "+toString()); + assertFalse(warmup != null && warmupIterations != null, "At most one of duration and iterations must be set: "+toString()); + if (warmup == null && warmupIterations == null) { + if (duration != null) warmup = Duration.millis(duration.toMilliseconds() / 10); + if (iterations != null) warmupIterations = iterations / 10; + } + if (summary == null) { + summary = job.toString(); + } + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .omitNullValues() + .add("summary", summary) + .add("duration", duration) + .add("warmup", warmup) + .add("iterations", iterations) + .add("warmupIterations", warmupIterations) + .add("job", job) + .add("completionLatch", completionLatch) + .add("minAcceptablePerSecond", minAcceptablePerSecond) + .toString(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestResult.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestResult.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestResult.java new file mode 100644 index 0000000..ee7362e --- /dev/null +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestResult.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.test.performance; + +import java.util.List; + +import org.apache.brooklyn.util.time.Duration; + +import com.google.common.annotations.Beta; +import com.google.common.base.Objects; + +/** + * The results of a performance test. + * + * See {@link PerformanceTestUtils#run(MeasurementOptions)}. + */ +@Beta +public class PerformanceTestResult { + public String summary; + public Duration warmup; + public int warmupIterations; + public Duration duration; + public int iterations; + public double ratePerSecond; + public Histogram histogram; + public Double minAcceptablePerSecond; + public Double cpuTotalFraction; + public List<Double> cpuSampleFractions; + + @Override + public String toString() { + return Objects.toStringHelper(this) + .omitNullValues() + .add("summary", summary) + .add("duration", duration) + .add("warmup", warmup) + .add("iterations", iterations) + .add("warmupIterations", warmupIterations) + .add("ratePerSecond", ratePerSecond) + .add("histogram", histogram) + .add("cpuTotalFraction", cpuTotalFraction) + .add("cpuSampleFractions", cpuSampleFractions) + .add("minAcceptablePerSecond", minAcceptablePerSecond) + .toString(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/3384c7eb/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestUtils.java ---------------------------------------------------------------------- diff --git a/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestUtils.java b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestUtils.java new file mode 100644 index 0000000..c747e8b --- /dev/null +++ b/usage/test-support/src/main/java/org/apache/brooklyn/test/performance/PerformanceTestUtils.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.test.performance; + +import java.lang.management.ManagementFactory; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import org.apache.brooklyn.util.time.Duration; +import org.apache.brooklyn.util.time.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Stopwatch; + +public class PerformanceTestUtils { + + private static final Logger LOG = LoggerFactory.getLogger(PerformanceTestUtils.class); + + private static boolean hasLoggedProcessCpuTimeUnavailable; + + public static long getProcessCpuTime() { + try { + MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName osMBeanName = ObjectName.getInstance(ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME); + return (Long) mbeanServer.getAttribute(osMBeanName, "ProcessCpuTime"); + } catch (Exception e) { + if (!hasLoggedProcessCpuTimeUnavailable) { + hasLoggedProcessCpuTimeUnavailable = true; + LOG.warn("ProcessCPuTime not available in local JVM MXBean "+ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME+" (only available in sun JVM?)"); + } + return -1; + } + } + + /** + * Creates a background thread that will log.info the CPU fraction usage repeatedly, sampling at the given period. + * Callers <em>must</em> cancel the returned future, e.g. {@code future.cancel(true)}, otherwise it will keep + * logging until the JVM exits. + */ + public static Future<?> sampleProcessCpuTime(final Duration period, final String loggingContext) { + return sampleProcessCpuTime(period, loggingContext, null); + } + + public static Future<?> sampleProcessCpuTime(final Duration period, final String loggingContext, final List<Double> cpuFractions) { + final ExecutorService executor = Executors.newSingleThreadExecutor(new ThreadFactory() { + @Override public Thread newThread(Runnable r) { + Thread thread = new Thread(r, "brooklyn-sampleProcessCpuTime-"+loggingContext); + thread.setDaemon(true); // let the JVM exit + return thread; + }}); + Future<?> future = executor.submit(new Runnable() { + @Override public void run() { + try { + long prevCpuTime = getProcessCpuTime(); + if (prevCpuTime == -1) { + LOG.warn("ProcessCPuTime not available; cannot sample; aborting"); + return; + } + while (true) { + Stopwatch stopwatch = Stopwatch.createStarted(); + Thread.sleep(period.toMilliseconds()); + long currentCpuTime = getProcessCpuTime(); + + long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + double fractionCpu = (elapsedTime > 0) ? ((double)currentCpuTime-prevCpuTime) / TimeUnit.MILLISECONDS.toNanos(elapsedTime) : -1; + prevCpuTime = currentCpuTime; + + LOG.info("CPU fraction over last {} was {} ({})", new Object[] { + Time.makeTimeStringRounded(elapsedTime), fractionCpu, loggingContext}); + + if (cpuFractions != null) { + cpuFractions.add(fractionCpu); + } + } + } catch (InterruptedException e) { + return; // graceful termination + } finally { + executor.shutdownNow(); + } + }}); + return future; + } +}
