Author: amitj Date: Wed Sep 2 08:16:35 2015 New Revision: 1700727 URL: http://svn.apache.org/r1700727 Log: OAK-3312: [Blob GC] Test case for GC / OAK-3167
Test cases for no blob GC when the blob age less than some time interval Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1700727&r1=1700726&r2=1700727&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java Wed Sep 2 08:16:35 2015 @@ -33,7 +33,6 @@ import java.util.concurrent.TimeUnit; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.MoreExecutors; import com.mongodb.BasicDBObject; import com.mongodb.DBCollection; import junit.framework.Assert; @@ -56,9 +55,7 @@ import org.junit.Test; public class MongoBlobGCTest extends AbstractMongoConnectionTest { private Clock clock; - public HashSet<String> setUp(boolean deleteDirect) throws Exception { - HashSet<String> set = new HashSet<String>(); - + public DataStoreState setUp(boolean deleteDirect) throws Exception { DocumentNodeStore s = mk.getNodeStore(); NodeBuilder a = s.getRoot().builder(); @@ -73,14 +70,18 @@ public class MongoBlobGCTest extends Abs processed.add(n); } } + + DataStoreState state = new DataStoreState(); for (int i = 0; i < number; i++) { Blob b = s.createBlob(randomStream(i, 16516)); - if (!processed.contains(i)) { - Iterator<String> idIter = - ((GarbageCollectableBlobStore) s.getBlobStore()) - .resolveChunks(b.toString()); - while (idIter.hasNext()) { - set.add(idIter.next()); + Iterator<String> idIter = + ((GarbageCollectableBlobStore) s.getBlobStore()) + .resolveChunks(b.toString()); + while (idIter.hasNext()) { + String chunk = idIter.next(); + state.blobsAdded.add(chunk); + if (!processed.contains(i)) { + state.blobsPresent.add(chunk); } } a.child("c" + i).setProperty("x", b); @@ -106,9 +107,14 @@ public class MongoBlobGCTest extends Abs Assert.assertEquals(processed.size(), stats.deletedDocGCCount); } - return set; + return state; } - + + private class DataStoreState { + Set<String> blobsAdded = Sets.newHashSet(); + Set<String> blobsPresent = Sets.newHashSet(); + } + public HashSet<String> addInlined() throws Exception { HashSet<String> set = new HashSet<String>(); DocumentNodeStore s = mk.getNodeStore(); @@ -130,29 +136,41 @@ public class MongoBlobGCTest extends Abs @Test public void gcDirectMongoDelete() throws Exception { - HashSet<String> set = setUp(true); - gc(set); + DataStoreState state = setUp(true); + Set<String> existingAfterGC = gc(0); + assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty()); } + + @Test + public void noGc() throws Exception { + DataStoreState state = setUp(true); + Set<String> existingAfterGC = gc(86400); + assertTrue(Sets.symmetricDifference(state.blobsAdded, existingAfterGC).isEmpty()); + } @Test public void gcVersionDelete() throws Exception { - HashSet<String> set = setUp(false); - gc(set); + DataStoreState state = setUp(false); + Set<String> existingAfterGC = gc(0); + assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty()); } @Test public void gcDirectMongoDeleteWithInlined() throws Exception { - HashSet<String> set = setUp(true); + DataStoreState state = setUp(true); addInlined(); - gc(set); + Set<String> existingAfterGC = gc(0); + assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty()); } @Test public void gcVersionDeleteWithInlined() throws Exception { - HashSet<String> set = setUp(false); + DataStoreState state = setUp(false); addInlined(); - gc(set); + Set<String> existingAfterGC = gc(0); + assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty()); } - private void gc(HashSet<String> remaining) throws Exception { + + private Set<String> gc(int blobGcMaxAgeInSecs) throws Exception { DocumentNodeStore store = mk.getNodeStore(); String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { @@ -164,14 +182,12 @@ public class MongoBlobGCTest extends Abs ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector( new DocumentBlobReferenceRetriever(store), - (GarbageCollectableBlobStore) store.getBlobStore(), executor, "./target", 5, 0, repoId); + (GarbageCollectableBlobStore) store.getBlobStore(), executor, "./target", 5, blobGcMaxAgeInSecs, repoId); Thread.sleep(4000); gc.collectGarbage(false); assertEquals(0, executor.getTaskCount()); - Set<String> existingAfterGC = iterate(); - boolean empty = Sets.symmetricDifference(remaining, existingAfterGC).isEmpty(); - assertTrue(empty); + return iterate(); } protected Set<String> iterate() throws Exception { Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java?rev=1700727&r1=1700726&r2=1700727&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java Wed Sep 2 08:16:35 2015 @@ -43,7 +43,6 @@ import javax.annotation.Nonnull; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.MoreExecutors; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector; @@ -114,17 +113,15 @@ public class SegmentDataStoreBlobGCTest return new File("target", "DataStoreBlobGCTest"); } - public HashSet<String> setUp() throws Exception { + public DataStoreState setUp() throws Exception { blobStore = DataStoreUtils.getBlobStore(); nodeStore = getNodeStore(blobStore); startDate = new Date(); - - HashSet<String> set = new HashSet<String>(); - + NodeBuilder a = nodeStore.getRoot().builder(); /* Create garbage by creating in-lined blobs (size < 16KB) */ - int number = 10000; + int number = 4000; NodeBuilder content = a.child("content"); for (int i = 0; i < number; i++) { NodeBuilder c = content.child("x" + i); @@ -149,32 +146,32 @@ public class SegmentDataStoreBlobGCTest /* Create and delete nodes with blobs stored in DS*/ int maxDeleted = 5; - number = 10; - // track the number of the assets to be deleted + int numBlobs = 10; List<Integer> processed = Lists.newArrayList(); Random rand = new Random(); for (int i = 0; i < maxDeleted; i++) { - int n = rand.nextInt(number); + int n = rand.nextInt(numBlobs); if (!processed.contains(n)) { processed.add(n); } } - - List<String> createdBlobs = Lists.newArrayList(); - for (int i = 0; i < number; i++) { + + DataStoreState state = new DataStoreState(); + for (int i = 0; i < numBlobs; i++) { SegmentBlob b = (SegmentBlob) nodeStore.createBlob(randomStream(i, 16516)); - createdBlobs.add(b.getBlobId()); - if (!processed.contains(i)) { - Iterator<String> idIter = blobStore - .resolveChunks(b.getBlobId()); - while (idIter.hasNext()) { - set.add(idIter.next()); + Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId()); + while (idIter.hasNext()) { + String chunk = idIter.next(); + state.blobsAdded.add(chunk); + if (!processed.contains(i)) { + state.blobsPresent.add(chunk); } } a.child("c" + i).setProperty("x", b); } + nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY); - log.info("Created blobs : {}", createdBlobs.size()); + log.info("Created blobs : {}", state.blobsAdded.size()); for (int id : processed) { delete("c" + id); @@ -186,19 +183,40 @@ public class SegmentDataStoreBlobGCTest store.maybeCompact(false); store.cleanup(); - return set; + return state; } - + + private class DataStoreState { + Set<String> blobsAdded = Sets.newHashSet(); + Set<String> blobsPresent = Sets.newHashSet(); + } + private void delete(String nodeId) throws CommitFailedException { NodeBuilder builder = nodeStore.getRoot().builder(); builder.child(nodeId).remove(); nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); } - + @Test public void gc() throws Exception { - HashSet<String> remaining = setUp(); + DataStoreState state = setUp(); + log.info("{} blobs that should remain after gc : {}", state.blobsPresent.size(), state.blobsPresent); + log.info("{} blobs for nodes which are deleted : {}", state.blobsPresent.size(), state.blobsPresent); + Set<String> existingAfterGC = gcInternal(0); + assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty()); + } + + @Test + public void noGc() throws Exception { + DataStoreState state = setUp(); + log.info("{} blobs that should remain after gc : {}", state.blobsAdded.size(), state.blobsAdded); + log.info("{} blobs for nodes which are deleted : {}", state.blobsPresent.size(), state.blobsPresent); + Set<String> existingAfterGC = gcInternal(86400); + assertTrue(Sets.symmetricDifference(state.blobsAdded, existingAfterGC).isEmpty()); + } + + private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception { String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { repoId = ClusterRepositoryInfo.createId(nodeStore); @@ -210,16 +228,14 @@ public class SegmentDataStoreBlobGCTest ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector( new SegmentBlobReferenceRetriever(store.getTracker()), - (GarbageCollectableBlobStore) store.getBlobStore(), executor, - "./target", 2048, 0, repoId); + (GarbageCollectableBlobStore) store.getBlobStore(), executor, "./target", 2048, maxBlobGcInSecs, + repoId); gc.collectGarbage(false); assertEquals(0, executor.getTaskCount()); Set<String> existingAfterGC = iterate(); - log.info("{} blobs that should have remained after gc : {}", remaining.size(), remaining); log.info("{} blobs existing after gc : {}", existingAfterGC.size(), existingAfterGC); - - assertTrue(Sets.symmetricDifference(remaining, existingAfterGC).isEmpty()); + return existingAfterGC; } protected Set<String> iterate() throws Exception {