tests for volumes read and written, plus conveniences for cleaning
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/94495f16 Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/94495f16 Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/94495f16 Branch: refs/heads/master Commit: 94495f16340fa82eed2a1bcc19c5c1493023d7d3 Parents: 522d339 Author: Alex Heneveld <alex.henev...@cloudsoftcorp.com> Authored: Thu Jun 5 16:09:16 2014 +0100 Committer: Alex Heneveld <alex.henev...@cloudsoftcorp.com> Committed: Thu Jun 5 16:09:16 2014 +0100 ---------------------------------------------------------------------- .../rebind/persister/InMemoryObjectStore.java | 4 +- ...ntoPersisterInMemorySizeIntegrationTest.java | 75 ++++++++ .../BrooklynMementoPersisterInMemoryTest.java | 8 - .../rebind/persister/ListeningObjectStore.java | 185 +++++++++++++++++++ .../persister/jclouds/BlobStoreCleaner.java | 54 ++++++ 5 files changed, 316 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java b/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java index 85fd1b0..b89d611 100644 --- a/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java +++ b/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java @@ -22,11 +22,11 @@ public class InMemoryObjectStore implements PersistenceObjectStore { Map<String,String> filesByName = MutableMap.<String,String>of(); boolean prepared = false; - + public InMemoryObjectStore() { log.info("Using memory-based objectStore"); } - + @Override public String getSummaryName() { return "in-memory (test) persistence store"; http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java new file mode 100644 index 0000000..bd7fa04 --- /dev/null +++ b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java @@ -0,0 +1,75 @@ +package brooklyn.entity.rebind.persister; + +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import brooklyn.entity.basic.EntityInternal; +import brooklyn.entity.rebind.RebindTestUtils; +import brooklyn.entity.rebind.persister.ListeningObjectStore.RecordingTransactionListener; +import brooklyn.management.ManagementContext; +import brooklyn.test.entity.TestEntity; +import brooklyn.util.text.Identifiers; +import brooklyn.util.time.Duration; +import brooklyn.util.time.Time; + +/** uses recorder to ensure not too much data is written */ +@Test +public class BrooklynMementoPersisterInMemorySizeIntegrationTest extends BrooklynMementoPersisterTestFixture { + + protected RecordingTransactionListener recorder; + + protected ManagementContext newPersistingManagementContext() { + recorder = new RecordingTransactionListener("in-mem-test-"+Identifiers.makeRandomId(4)); + return RebindTestUtils.managementContextBuilder(classLoader, + new ListeningObjectStore(new InMemoryObjectStore(), recorder)) + .persistPeriod(Duration.millis(100)).buildStarted(); + } + + public void testPersistenceVolumeFast() throws IOException, TimeoutException, InterruptedException { + doTestPersistenceVolume(50*1000, false); + } + @Test(groups="Integration") + public void testPersistenceVolumeWaiting() throws IOException, TimeoutException, InterruptedException { + // by waiting we ensure there aren't extra writes going on + doTestPersistenceVolume(50*1000, true); + } + + protected void doTestPersistenceVolume(int bigBlockSize, boolean forceDelay) throws IOException, TimeoutException, InterruptedException { + if (forceDelay) Time.sleep(Duration.FIVE_SECONDS); + else recorder.blockUntilDataWrittenExceeds(512, Duration.FIVE_SECONDS); + + long out1 = recorder.getBytesOut(); + int filesOut1 = recorder.getCountDataOut(); + Assert.assertTrue(out1>512, "should have written at least 0.5k, only wrote "+out1); + Assert.assertTrue(out1<20*1000, "should have written less than 20k, wrote "+out1); + Assert.assertTrue(filesOut1<20, "should have written fewer than 20 files, wrote "+out1); + + ((EntityInternal)app).setAttribute(TestEntity.NAME, "hello world"); + if (forceDelay) Time.sleep(Duration.FIVE_SECONDS); + else recorder.blockUntilDataWrittenExceeds(out1+10, Duration.FIVE_SECONDS); + + long out2 = recorder.getBytesOut(); + Assert.assertTrue(out2-out1>10, "should have written more data"); + int filesOut2 = recorder.getCountDataOut(); + Assert.assertTrue(filesOut2>filesOut1, "should have written more files"); + + Assert.assertTrue(out2<50*1000, "should have written less than 50k, wrote "+out1); + Assert.assertTrue(filesOut2<40, "should have written fewer than 40 files, wrote "+out1); + + ((EntityInternal)entity).setAttribute(TestEntity.NAME, Identifiers.makeRandomId(bigBlockSize)); + if (forceDelay) Time.sleep(Duration.FIVE_SECONDS); + else recorder.blockUntilDataWrittenExceeds(out2+bigBlockSize, Duration.FIVE_SECONDS); + + long out3 = recorder.getBytesOut(); + Assert.assertTrue(out3-out2 > bigBlockSize, "should have written 50k more data, only wrote "+out3+" compared with "+out2); + int filesOut3 = recorder.getCountDataOut(); + Assert.assertTrue(filesOut3>filesOut2, "should have written more files"); + + Assert.assertTrue(out2<100*1000+bigBlockSize, "should have written less than 100k+block, wrote "+out1); + Assert.assertTrue(filesOut2<60, "should have written fewer than 60 files, wrote "+out1); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java index 04d0810..5ac0595 100644 --- a/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java +++ b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java @@ -1,24 +1,16 @@ package brooklyn.entity.rebind.persister; -import java.io.File; - import org.testng.annotations.Test; import brooklyn.entity.rebind.RebindTestUtils; import brooklyn.management.ManagementContext; import brooklyn.util.time.Duration; -/** - * @author Andrea Turli - */ @Test public class BrooklynMementoPersisterInMemoryTest extends BrooklynMementoPersisterTestFixture { - protected File mementoDir; - protected ManagementContext newPersistingManagementContext() { return RebindTestUtils.managementContextBuilder(classLoader, new InMemoryObjectStore()) .persistPeriod(Duration.millis(10)).buildStarted(); } - } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java b/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java new file mode 100644 index 0000000..82037de --- /dev/null +++ b/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java @@ -0,0 +1,185 @@ +package brooklyn.entity.rebind.persister; + +import java.util.List; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import brooklyn.management.ManagementContext; +import brooklyn.util.text.Strings; +import brooklyn.util.time.CountdownTimer; +import brooklyn.util.time.Duration; + +import com.google.common.base.Preconditions; + +public class ListeningObjectStore implements PersistenceObjectStore { + + protected final PersistenceObjectStore delegate; + protected final ObjectStoreTransactionListener listener; + + public static interface ObjectStoreTransactionListener { + public void recordQueryOut(String summary, int size); + public void recordDataOut(String summary, int size); + public void recordDataIn(String summary, int size); + } + + public static class RecordingTransactionListener implements ObjectStoreTransactionListener { + private static final Logger log = LoggerFactory.getLogger(ListeningObjectStore.RecordingTransactionListener.class); + + protected final String prefix; + protected final AtomicLong bytesIn = new AtomicLong(); + protected final AtomicLong bytesOut = new AtomicLong(); + protected final AtomicInteger countQueriesOut = new AtomicInteger(); + protected final AtomicInteger countDataOut = new AtomicInteger(); + protected final AtomicInteger countDataIn = new AtomicInteger(); + + public RecordingTransactionListener(String prefix) { + this.prefix = prefix; + } + + public long getBytesIn() { + return bytesIn.get(); + } + + public long getBytesOut() { + return bytesOut.get(); + } + + public int getCountQueriesOut() { + return countQueriesOut.get(); + } + + public int getCountDataOut() { + return countDataOut.get(); + } + + public int getCountDataIn() { + return countDataIn.get(); + } + + public String getTotalString() { + return "totals: out="+Strings.makeSizeString(bytesOut.get())+" in="+Strings.makeSizeString(bytesIn.get()); + } + + @Override + public void recordQueryOut(String summary, int size) { + synchronized (this) { this.notifyAll(); } + bytesOut.addAndGet(size); + countQueriesOut.incrementAndGet(); + log.info(prefix+" "+summary+" -->"+size+"; "+getTotalString()); + } + + @Override + public void recordDataOut(String summary, int size) { + synchronized (this) { this.notifyAll(); } + bytesOut.addAndGet(size); + countDataOut.incrementAndGet(); + log.info(prefix+" "+summary+" -->"+size+"; "+getTotalString()); + } + + @Override + public void recordDataIn(String summary, int size) { + synchronized (this) { this.notifyAll(); } + bytesIn.addAndGet(size); + countDataIn.incrementAndGet(); + log.info(prefix+" "+summary+" <--"+size+"; "+getTotalString()); + } + + public void blockUntilDataWrittenExceeds(long count, Duration timeout) throws InterruptedException, TimeoutException { + CountdownTimer timer = CountdownTimer.newInstanceStarted(timeout); + synchronized (this) { + while (bytesOut.get()<count) { + if (timer.isExpired()) + throw new TimeoutException(); + timer.waitOnForExpiry(this); + } + } + } + } + + public ListeningObjectStore(PersistenceObjectStore delegate, ObjectStoreTransactionListener listener) { + this.delegate = Preconditions.checkNotNull(delegate); + this.listener = Preconditions.checkNotNull(listener); + } + + public String getSummaryName() { + return delegate.getSummaryName(); + } + + public StoreObjectAccessor newAccessor(String path) { + return new ListeningAccessor(path, delegate.newAccessor(path)); + } + + public void createSubPath(String subPath) { + listener.recordQueryOut("creating path "+subPath, 1+subPath.length()); + delegate.createSubPath(subPath); + } + + public List<String> listContentsWithSubPath(String subPath) { + listener.recordQueryOut("requesting list "+subPath, 1+subPath.length()); + List<String> result = delegate.listContentsWithSubPath(subPath); + listener.recordDataIn("receiving list "+subPath, result.toString().length()); + return result; + } + + public void backupContents(String sourceSubPath, String targetSubPathForBackups) { + listener.recordQueryOut("backing up "+sourceSubPath+" to "+targetSubPathForBackups, + 1+sourceSubPath.length()+targetSubPathForBackups.length()); + delegate.backupContents(sourceSubPath, targetSubPathForBackups); + } + + public void close() { + delegate.close(); + } + + public void prepareForUse(ManagementContext managementContext, PersistMode persistMode) { + delegate.prepareForUse(managementContext, persistMode); + } + + public void deleteCompletely() { + listener.recordDataOut("deleting completely", 1); + delegate.deleteCompletely(); + } + + public class ListeningAccessor implements StoreObjectAccessor { + + protected final String path; + protected final StoreObjectAccessor delegate; + + public ListeningAccessor(String path, StoreObjectAccessor delegate) { + this.path = path; + this.delegate = delegate; + } + public boolean exists() { + return delegate.exists(); + } + public void writeAsync(String val) { + listener.recordDataOut("writing "+path, val.length()); + delegate.writeAsync(val); + } + public void append(String s) { + listener.recordDataOut("appending "+path, s.length()); + delegate.append(s); + } + public void deleteAsync() { + listener.recordQueryOut("deleting "+path, path.length()); + delegate.deleteAsync(); + } + public String read() { + listener.recordQueryOut("requesting "+path, path.length()); + String result = delegate.read(); + listener.recordDataIn("reading "+path, result.length()); + return result; + } + public void waitForWriteCompleted(Duration timeout) throws InterruptedException, TimeoutException { + delegate.waitForWriteCompleted(timeout); + } + + + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java ---------------------------------------------------------------------- diff --git a/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java new file mode 100644 index 0000000..1107913 --- /dev/null +++ b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java @@ -0,0 +1,54 @@ +package brooklyn.entity.rebind.persister.jclouds; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import brooklyn.config.BrooklynProperties; +import brooklyn.location.basic.LocationConfigKeys; +import brooklyn.location.cloud.CloudLocationConfig; +import brooklyn.location.jclouds.JcloudsLocation; +import brooklyn.test.entity.LocalManagementContextForTests; + +/** Utility for cleaning up after test leaks. Most should not leak of course, but if they do... */ +public class BlobStoreCleaner { + + private static String locationSpec = BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC; + + private static final Logger log = LoggerFactory.getLogger(BlobStoreCleaner.class); + + public static void main(String[] args) { + LocalManagementContextForTests mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault()); + JcloudsLocation location = (JcloudsLocation) mgmt.getLocationRegistry().resolve(locationSpec); + + String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity must not be null"); + String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL), "credential must not be null"); + String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null"); + String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT); + + BlobStoreContext context = ContextBuilder.newBuilder(provider) + .credentials(identity, credential) + .endpoint(endpoint) + .buildView(BlobStoreContext.class); + + PageSet<? extends StorageMetadata> containers = context.getBlobStore().list(); + for (StorageMetadata container: containers) { + if (container.getName().matches("brooklyn.*-test.*") + // to kill all containers here +// || container.getName().matches(".*") + ) { + log.info("killing - "+container.getName()); + context.getBlobStore().deleteContainer(container.getName()); + } else { + log.info("KEEPING - "+container.getName()); + } + } + context.close(); + } + +}
