Author: amitj Date: Thu Oct 6 06:41:28 2016 New Revision: 1763500 URL: http://svn.apache.org/viewvc?rev=1763500&view=rev Log: OAK-4869: Enable benchmarks for DataStore
- Added ability to initialize S3DataStore - Rename fixture suffixes from FDS to DS - Minor change to remove usage of deleteAllOlderThan in tear - SmallFileWriteTest - file size configurable through system property "file.size" Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java (with props) Modified: jackrabbit/oak/trunk/oak-run/README.md jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileWriteTest.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java Modified: jackrabbit/oak/trunk/oak-run/README.md URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/README.md (original) +++ jackrabbit/oak/trunk/oak-run/README.md Thu Oct 6 06:41:28 2016 @@ -399,10 +399,10 @@ to be used. The following fixtures are c | Oak-Memory | Oak with default in-memory storage | | Oak-MemoryNS | Oak with default in-memory NodeStore | | Oak-Mongo | Oak with the default Mongo backend | -| Oak-Mongo-FDS | Oak with the default Mongo backend and FileDataStore | +| Oak-Mongo-DS | Oak with the default Mongo backend and DataStore | | Oak-MongoNS | Oak with the Mongo NodeStore | | Oak-Tar | Oak with the Tar backend (aka Segment NodeStore) | -| Oak-Tar-FDS | Oak with the Tar backend and FileDataStore | +| Oak-Tar-DS | Oak with the Tar backend and DataStore | Jackrabbit fixture requires [Oak Runnable JR2 jar](#jr2) @@ -502,16 +502,18 @@ that we used to produce earlier. Finally the benchmark runner supports the following repository fixtures: -| Fixture | Description | -|---------------|-------------------------------------------------------| -| Jackrabbit | Jackrabbit with the default embedded Derby bundle PM | -| Oak-Memory | Oak with default in-memory storage | -| Oak-MemoryNS | Oak with default in-memory NodeStore | -| Oak-Mongo | Oak with the default Mongo backend | -| Oak-Mongo-FDS | Oak with the default Mongo backend and FileDataStore | -| Oak-MongoNS | Oak with the Mongo NodeStore | -| Oak-Tar | Oak with the Tar backend (aka Segment NodeStore) | -| Oak-RDB | Oak with the DocumentMK/RDB persistence | +| Fixture | Description | +|---------------|----------------------------------------------------------------| +| Jackrabbit | Jackrabbit with the default embedded Derby bundle PM | +| Oak-Memory | Oak with default in-memory storage | +| Oak-MemoryNS | Oak with default in-memory NodeStore | +| Oak-Mongo | Oak with the default Mongo backend | +| Oak-Mongo-DS | Oak with the default Mongo backend and DataStore | +| Oak-MongoNS | Oak with the Mongo NodeStore | +| Oak-Tar | Oak with the Tar backend (aka Segment NodeStore) | +| Oak-Tar-DS | Oak with the Tar backend (aka Segment NodeStore) and DataStore | +| Oak-RDB | Oak with the DocumentMK/RDB persistence | +| Oak-RDB-DS | Oak with the DocumentMK/RDB persistence and DataStore | (Note that for Oak-RDB, the required JDBC drivers either need to be embedded into oak-run, or be specified separately in the class path. Furthermode, @@ -677,15 +679,17 @@ suites in the scalability command line, Finally the scalability runner supports the following repository fixtures: -| Fixture | Description | -|---------------|-------------------------------------------------------| -| Oak-Memory | Oak with default in-memory storage | -| Oak-MemoryNS | Oak with default in-memory NodeStore | -| Oak-Mongo | Oak with the default Mongo backend | -| Oak-Mongo-FDS | Oak with the default Mongo backend and FileDataStore | -| Oak-MongoNS | Oak with the Mongo NodeStore | -| Oak-Tar | Oak with the Tar backend (aka Segment NodeStore) | -| Oak-RDB | Oak with the DocumentMK/RDB persistence | +| Fixture | Description | +|---------------|----------------------------------------------------------------| +| Oak-Memory | Oak with default in-memory storage | +| Oak-MemoryNS | Oak with default in-memory NodeStore | +| Oak-Mongo | Oak with the default Mongo backend | +| Oak-Mongo-DS | Oak with the default Mongo backend and DataStore | +| Oak-MongoNS | Oak with the Mongo NodeStore | +| Oak-Tar | Oak with the Tar backend (aka Segment NodeStore) | +| Oak-Tar-DS | Oak with the Tar backend (aka Segment NodeStore) and DataStore | +| Oak-RDB | Oak with the DocumentMK/RDB persistence | +| Oak-RDB-DS | Oak with the DocumentMK/RDB persistence and DataStore | (Note that for Oak-RDB, the required JDBC drivers either need to be embedded into oak-run, or be specified separately in the class path.) Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java Thu Oct 6 06:41:28 2016 @@ -174,7 +174,7 @@ public class BenchmarkRunner { OakRepositoryFixture.getMemoryNS(cacheSize * MB), OakRepositoryFixture.getMongo(uri, dropDBAfterTest.value(options), cacheSize * MB), - OakRepositoryFixture.getMongoWithFDS(uri, + OakRepositoryFixture.getMongoWithDS(uri, dropDBAfterTest.value(options), cacheSize * MB, base.value(options), @@ -184,14 +184,16 @@ public class BenchmarkRunner { cacheSize * MB), OakRepositoryFixture.getTar( base.value(options), 256, cacheSize, mmap.value(options)), - OakRepositoryFixture.getTarWithBlobStore( - base.value(options), 256, cacheSize, mmap.value(options)), - OakRepositoryFixture.getSegmentTar(base.value(options), 256, cacheSize, mmap.value(options)), - OakRepositoryFixture.getSegmentTarWithBlobStore(base.value(options), 256, cacheSize, mmap.value(options)), + OakRepositoryFixture.getTarWithBlobStore(base.value(options), 256, cacheSize, + mmap.value(options), fdsCache.value(options)), + OakRepositoryFixture.getSegmentTar(base.value(options), 256, cacheSize, + mmap.value(options)), + OakRepositoryFixture.getSegmentTarWithBlobStore(base.value(options), 256, cacheSize, + mmap.value(options), fdsCache.value(options)), OakRepositoryFixture.getRDB(rdbjdbcuri.value(options), rdbjdbcuser.value(options), rdbjdbcpasswd.value(options), rdbjdbctableprefix.value(options), dropDBAfterTest.value(options), cacheSize * MB) }; - OakRepositoryFixture.getRDBWithFDS(rdbjdbcuri.value(options), rdbjdbcuser.value(options), + OakRepositoryFixture.getRDBWithDS(rdbjdbcuri.value(options), rdbjdbcuser.value(options), rdbjdbcpasswd.value(options), rdbjdbctableprefix.value(options), dropDBAfterTest.value(options), cacheSize * MB, base.value(options), fdsCache.value(options)); Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileWriteTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileWriteTest.java?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileWriteTest.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileWriteTest.java Thu Oct 6 06:41:28 2016 @@ -26,7 +26,7 @@ public class SmallFileWriteTest extends private static final int FILE_COUNT = 100; - private static final int FILE_SIZE = 10; + private static final int FILE_SIZE = Integer.getInteger("file.size", 10); private Session session; Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java Thu Oct 6 06:41:28 2016 @@ -21,14 +21,19 @@ package org.apache.jackrabbit.oak.fixtur import java.io.Closeable; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Dictionary; +import java.util.Enumeration; import java.util.Map; import javax.annotation.CheckForNull; +import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.commons.io.FileUtils; +import org.apache.felix.cm.file.ConfigurationHandler; import org.apache.jackrabbit.core.data.DataStore; -import org.apache.jackrabbit.core.data.DataStoreException; import org.apache.jackrabbit.core.data.FileDataStore; import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.plugins.blob.BlobStoreStats; @@ -39,6 +44,8 @@ import org.apache.jackrabbit.oak.spi.blo import org.apache.jackrabbit.oak.stats.StatisticsProvider; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.fixture.DataStoreUtils.cleanup; +import static org.apache.jackrabbit.oak.fixture.DataStoreUtils.configureIfS3DataStore; public abstract class BlobStoreFixture implements Closeable{ private final String name; @@ -69,16 +76,18 @@ public abstract class BlobStoreFixture i */ @CheckForNull public static BlobStoreFixture create(File basedir, boolean fallbackToFDS, + int fdsCacheInMB, StatisticsProvider statisticsProvider) { - String className = System.getProperty("dataStore"); - if (className != null) { - return getDataStore(); - } - if(basedir == null){ + if(basedir == null) { basedir = FileUtils.getTempDirectory(); } + String className = System.getProperty("dataStore"); + if (className != null) { + return getDataStore(basedir, fdsCacheInMB); + } + String blobStore = System.getProperty("blobStoreType"); if ("FDS".equals(blobStore) || (blobStore == null && fallbackToFDS)) { return getFileDataStore(basedir, DataStoreBlobStore.DEFAULT_CACHE_SIZE, statisticsProvider); @@ -169,10 +178,12 @@ public abstract class BlobStoreFixture i }; } - public static BlobStoreFixture getDataStore() { + public static BlobStoreFixture getDataStore(final File basedir, final int fdsCacheInMB) { return new BlobStoreFixture("DS") { private DataStore dataStore; private BlobStore blobStore; + private File storeDir; + private Map<String, ?> config; @Override public BlobStore setUp() { @@ -180,9 +191,12 @@ public abstract class BlobStoreFixture i checkNotNull(className, "No system property named 'dataStore' defined"); try { dataStore = Class.forName(className).asSubclass(DataStore.class).newInstance(); - configure(dataStore); - dataStore.init(null); - blobStore = new DataStoreBlobStore(dataStore); + config = getConfig(); + configure(dataStore, config); + dataStore = configureIfS3DataStore(className, dataStore, config, unique.toLowerCase()); + storeDir = new File(basedir, unique); + dataStore.init(storeDir.getAbsolutePath()); + blobStore = new DataStoreBlobStore(dataStore, true, fdsCacheInMB); configure(blobStore); return blobStore; } catch (Exception e) { @@ -193,11 +207,10 @@ public abstract class BlobStoreFixture i @Override public void tearDown() { if (blobStore instanceof DataStoreBlobStore) { - ((DataStoreBlobStore) blobStore).clearInUse(); try { - ((DataStoreBlobStore) blobStore).deleteAllOlderThan( - System.currentTimeMillis() + 10000000); - } catch (DataStoreException e) { + ((DataStoreBlobStore) blobStore).close(); + cleanup(storeDir, config); + } catch (Exception e) { throw new RuntimeException(e); } } @@ -221,8 +234,32 @@ public abstract class BlobStoreFixture i PropertiesUtil.populate(o, getConfig(), false); } - private static Map<String, ?> getConfig() { + private static void configure(Object o, Map<String, ?> config) { + PropertiesUtil.populate(o, config, false); + } + + public static Map<String, Object> loadAndTransformProps(String cfgPath) throws IOException { + Dictionary dict = ConfigurationHandler.read(new FileInputStream(cfgPath)); + Map<String, Object> props = Maps.newHashMap(); + Enumeration keys = dict.keys(); + while (keys.hasMoreElements()) { + String key = (String) keys.nextElement(); + props.put(key, dict.get(key)); + } + return props; + } + + public static Map<String, ?> getConfig() { + // try loading the props from the config file if configured + String cfgFile = System.getProperty("ds.config"); Map<String, Object> result = Maps.newHashMap(); + if (!Strings.isNullOrEmpty(cfgFile)) { + try { + result = loadAndTransformProps(cfgFile); + } catch (IOException e) { + } + } + for (Map.Entry<String, ?> e : Maps.fromProperties(System.getProperties()).entrySet()) { String key = e.getKey(); if (key.startsWith("ds.") || key.startsWith("bs.")) { Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java?rev=1763500&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java (added) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java Thu Oct 6 06:41:28 2016 @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.fixture; + +import java.io.File; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.google.common.base.Strings; +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.core.data.DataStore; +import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore; +import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants; +import org.apache.jackrabbit.oak.blob.cloud.s3.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Extension to {@link DataStoreUtils} to enable S3 extensions for cleaning and initialization. + */ +public class DataStoreUtils { + private static final Logger log = LoggerFactory.getLogger(DataStoreUtils.class); + + private static Class JR2_S3 = SharedS3DataStore.class; + + public static boolean isS3DataStore(String dsName) { + return (dsName != null) && (dsName.equals(JR2_S3.getName())); + } + + public static DataStore configureIfS3DataStore(String className, DataStore ds, + Map<String, ?> config, String bucket) throws Exception { + // Add bucket info + Properties props = new Properties(); + props.putAll(config); + props.setProperty(S3Constants.S3_BUCKET, bucket); + + // Set the props object + if (JR2_S3.getName().equals(className)) { + ((org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore) ds).setProperties(props); + } + + return ds; + } + + /** + * Clean directory and if S3 bucket configured delete that. + * + * @param storeDir the local directory + * @param config the datastore config + * @throws Exception + */ + public static void cleanup(File storeDir, Map<String, ?> config) throws Exception { + FileUtils.deleteQuietly(storeDir); + String bucket = null; + if (config.containsKey(S3Constants.S3_BUCKET)) { + bucket = (String) config.get(S3Constants.S3_BUCKET); + if (!Strings.isNullOrEmpty(bucket)) { + deleteBucket(bucket, config, new Date()); + } + } + } + + public static void deleteBucket(String bucket, Map<String, ?> map, Date date) throws Exception { + log.info("cleaning bucket [" + bucket + "]"); + Properties props = new Properties(); + props.putAll(map); + AmazonS3Client s3service = Utils.openService(props); + TransferManager tmx = new TransferManager(s3service); + if (s3service.doesBucketExist(bucket)) { + for (int i = 0; i < 4; i++) { + tmx.abortMultipartUploads(bucket, date); + ObjectListing prevObjectListing = s3service.listObjects(bucket); + while (prevObjectListing != null ) { + List<DeleteObjectsRequest.KeyVersion> + deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); + for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { + deleteList.add(new DeleteObjectsRequest.KeyVersion( + s3ObjSumm.getKey())); + } + if (deleteList.size() > 0) { + DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest( + bucket); + delObjsReq.setKeys(deleteList); + s3service.deleteObjects(delObjsReq); + } + if (!prevObjectListing.isTruncated()) break; + prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing); + } + } + s3service.deleteBucket(bucket); + log.info("bucket [ " + bucket + "] cleaned"); + } else { + log.info("bucket [" + bucket + "] doesn't exists"); + } + tmx.shutdownNow(); + s3service.shutdown(); + } +} Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java Thu Oct 6 06:41:28 2016 @@ -44,18 +44,17 @@ public abstract class OakFixture { public static final String OAK_MEMORY_NS = "Oak-MemoryNS"; public static final String OAK_MONGO = "Oak-Mongo"; - public static final String OAK_MONGO_FDS = "Oak-Mongo-FDS"; + public static final String OAK_MONGO_DS = "Oak-Mongo-DS"; public static final String OAK_MONGO_NS = "Oak-MongoNS"; public static final String OAK_RDB = "Oak-RDB"; - public static final String OAK_RDB_FDS = "Oak-RDB-FDS"; + public static final String OAK_RDB_DS = "Oak-RDB-DS"; public static final String OAK_TAR = "Oak-Tar"; - public static final String OAK_TAR_FDS = "Oak-Tar-FDS"; + public static final String OAK_TAR_DS = "Oak-Tar-DS"; public static final String OAK_SEGMENT_TAR = "Oak-Segment-Tar"; - - public static final String OAK_SEGMENT_TAR_FDS = "Oak-Segment-Tar-FDS"; + public static final String OAK_SEGMENT_TAR_DS = "Oak-Segment-Tar-DS"; private final String name; @@ -156,8 +155,8 @@ public abstract class OakFixture { public static OakFixture getMongo(final String name, final String uri, final boolean dropDBAfterTest, final long cacheSize, - final boolean useFileDataStore, - final File base, final int fdsCacheInMB) { + final boolean useDataStore, + final File base, final int dsCacheInMB) { return new OakFixture(name) { private DocumentMK[] kernels; private BlobStoreFixture blobStoreFixture; @@ -229,10 +228,9 @@ public abstract class OakFixture { return; } - if (useFileDataStore) { - blobStoreFixture = BlobStoreFixture.getFileDataStore(base, fdsCacheInMB, statsProvider); - } else { - blobStoreFixture = BlobStoreFixture.create(base, false, statsProvider); + if (useDataStore) { + blobStoreFixture = + BlobStoreFixture.create(base, true, dsCacheInMB, statsProvider); } } }; @@ -245,7 +243,7 @@ public abstract class OakFixture { public static OakFixture getRDB(final String name, final String jdbcuri, final String jdbcuser, final String jdbcpasswd, final String tablePrefix, final boolean dropDBAfterTest, final long cacheSize, - final boolean useFileDataStore, final File base, final int fdsCacheInMB) { + final boolean useDataStore, final File base, final int dsCacheInMB) { return new OakFixture(name) { private DocumentMK[] kernels; private BlobStoreFixture blobStoreFixture; @@ -256,7 +254,7 @@ public abstract class OakFixture { private BlobStore getBlobStore(StatisticsProvider statsProvider) { try { - if (useFileDataStore) { + if (useDataStore) { initializeBlobStoreFixture(statsProvider); return blobStoreFixture.setUp(); } else { @@ -325,8 +323,8 @@ public abstract class OakFixture { } private void initializeBlobStoreFixture(StatisticsProvider statsProvider) { - if (useFileDataStore && blobStoreFixture == null) { - blobStoreFixture = BlobStoreFixture.getFileDataStore(base, fdsCacheInMB, statsProvider); + if (useDataStore && blobStoreFixture == null) { + blobStoreFixture = BlobStoreFixture.create(base, true, dsCacheInMB, statsProvider); } } }; @@ -335,11 +333,25 @@ public abstract class OakFixture { public static OakFixture getTar( final String name, final File base, final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping, final boolean useBlobStore) { - return new SegmentFixture(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore); + return getTar(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore, 0); + } + + public static OakFixture getSegmentTar(final String name, final File base, + final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping, + final boolean useBlobStore) { + return getSegmentTar(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore, 0); + } + + public static OakFixture getTar( + final String name, final File base, final int maxFileSizeMB, final int cacheSizeMB, + final boolean memoryMapping, final boolean useBlobStore, int dsCacheInMB) { + return new SegmentFixture(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore, dsCacheInMB); } - public static OakFixture getSegmentTar(final String name, final File base, final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping, final boolean useBlobStore) { - return new SegmentTarFixture(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore); + public static OakFixture getSegmentTar(final String name, final File base, + final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping, + final boolean useBlobStore, final int dsCacheInMB) { + return new SegmentTarFixture(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore, dsCacheInMB); } public static class SegmentFixture extends OakFixture { @@ -350,15 +362,17 @@ public abstract class OakFixture { private final int cacheSizeMB; private final boolean memoryMapping; private final boolean useBlobStore; + private final int dsCacheSizeInMB; public SegmentFixture(String name, File base, int maxFileSizeMB, int cacheSizeMB, - boolean memoryMapping, boolean useBlobStore) { + boolean memoryMapping, boolean useBlobStore, int dsCacheSizeInMB) { super(name); this.base = base; this.maxFileSizeMB = maxFileSizeMB; this.cacheSizeMB = cacheSizeMB; this.memoryMapping = memoryMapping; this.useBlobStore = useBlobStore; + this.dsCacheSizeInMB = dsCacheSizeInMB; } @Override @@ -382,7 +396,8 @@ public abstract class OakFixture { for (int i = 0; i < cluster.length; i++) { BlobStore blobStore = null; if (useBlobStore) { - blobStoreFixtures[i] = BlobStoreFixture.create(base, true, statsProvider); + blobStoreFixtures[i] = + BlobStoreFixture.create(base, true, dsCacheSizeInMB, statsProvider); blobStore = blobStoreFixtures[i].setUp(); } Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java Thu Oct 6 06:41:28 2016 @@ -45,17 +45,17 @@ public class OakRepositoryFixture implem return getMongoNS(uri, dropDBAfterTest, cacheSize); } - public static RepositoryFixture getMongoWithFDS(String host, int port, String database, + public static RepositoryFixture getMongoWithDS(String host, int port, String database, boolean dropDBAfterTest, long cacheSize, final File base, int fdsCacheInMB) { - return getMongo(OakFixture.OAK_MONGO_FDS, host, port, database, + return getMongo(OakFixture.OAK_MONGO_DS, host, port, database, dropDBAfterTest, cacheSize, true, base, fdsCacheInMB); } - public static RepositoryFixture getMongoWithFDS(String uri, + public static RepositoryFixture getMongoWithDS(String uri, boolean dropDBAfterTest, long cacheSize, final File base, int fdsCacheInMB) { - return new OakRepositoryFixture(OakFixture.getMongo(OakFixture.OAK_MONGO_FDS, uri, dropDBAfterTest, + return new OakRepositoryFixture(OakFixture.getMongo(OakFixture.OAK_MONGO_DS, uri, dropDBAfterTest, cacheSize, true, base, fdsCacheInMB)); } @@ -79,32 +79,46 @@ public class OakRepositoryFixture implem cacheSize, useFileDataStore, base, fdsCacheInMB)); } - public static RepositoryFixture getRDB(String jdbcuri, String jdbcuser, String jdbcpasswd, String jdbctableprefix, - boolean dropDBAfterTest, long cacheSize) { - return new OakRepositoryFixture(OakFixture.getRDB(OakFixture.OAK_RDB, jdbcuri, jdbcuser, jdbcpasswd, jdbctableprefix, + public static RepositoryFixture getRDB(String jdbcuri, String jdbcuser, String jdbcpasswd, + String jdbctableprefix, boolean dropDBAfterTest, long cacheSize) { + return new OakRepositoryFixture(OakFixture + .getRDB(OakFixture.OAK_RDB, jdbcuri, jdbcuser, jdbcpasswd, jdbctableprefix, dropDBAfterTest, cacheSize)); } - public static RepositoryFixture getRDBWithFDS(String jdbcuri, String jdbcuser, String jdbcpasswd, String jdbctableprefix, - boolean dropDBAfterTest, long cacheSize, final File base, final int fdsCacheInMB) { - return new OakRepositoryFixture(OakFixture.getRDB(OakFixture.OAK_RDB_FDS, jdbcuri, jdbcuser, jdbcpasswd, jdbctableprefix, + public static RepositoryFixture getRDBWithDS(String jdbcuri, String jdbcuser, String jdbcpasswd, + String jdbctableprefix, boolean dropDBAfterTest, long cacheSize, final File base, + final int fdsCacheInMB) { + return new OakRepositoryFixture(OakFixture + .getRDB(OakFixture.OAK_RDB_DS, jdbcuri, jdbcuser, jdbcpasswd, jdbctableprefix, dropDBAfterTest, cacheSize, true, base, fdsCacheInMB)); } - public static RepositoryFixture getTar(File base, int maxFileSizeMB, int cacheSizeMB, boolean memoryMapping) { - return new OakRepositoryFixture(OakFixture.getTar(OakFixture.OAK_TAR ,base, maxFileSizeMB, cacheSizeMB, memoryMapping, false)); - } - - public static RepositoryFixture getTarWithBlobStore(File base, int maxFileSizeMB, int cacheSizeMB, boolean memoryMapping) { - return new OakRepositoryFixture(OakFixture.getTar(OakFixture.OAK_TAR_FDS,base, maxFileSizeMB, cacheSizeMB, memoryMapping, true)); - } - - public static RepositoryFixture getSegmentTar(File base, int maxFileSizeMB, int cacheSizeMB, boolean memoryMapping) { - return new OakRepositoryFixture(OakFixture.getSegmentTar(OakFixture.OAK_SEGMENT_TAR, base, maxFileSizeMB, cacheSizeMB, memoryMapping, false)); - } - - public static RepositoryFixture getSegmentTarWithBlobStore(File base, int maxFileSizeMB, int cacheSizeMB, boolean memoryMapping) { - return new OakRepositoryFixture(OakFixture.getSegmentTar(OakFixture.OAK_SEGMENT_TAR_FDS, base, maxFileSizeMB, cacheSizeMB, memoryMapping, true)); + public static RepositoryFixture getTar(File base, int maxFileSizeMB, int cacheSizeMB, + boolean memoryMapping) { + return new OakRepositoryFixture(OakFixture + .getTar(OakFixture.OAK_TAR, base, maxFileSizeMB, cacheSizeMB, memoryMapping, false)); + } + + public static RepositoryFixture getTarWithBlobStore(File base, int maxFileSizeMB, + int cacheSizeMB, boolean memoryMapping, int dsCacheInMB) { + return new OakRepositoryFixture(OakFixture + .getTar(OakFixture.OAK_TAR_DS, base, maxFileSizeMB, cacheSizeMB, memoryMapping, true, + dsCacheInMB)); + } + + public static RepositoryFixture getSegmentTar(File base, int maxFileSizeMB, int cacheSizeMB, + boolean memoryMapping) { + return new OakRepositoryFixture(OakFixture + .getSegmentTar(OakFixture.OAK_SEGMENT_TAR, base, maxFileSizeMB, cacheSizeMB, + memoryMapping, false)); + } + + public static RepositoryFixture getSegmentTarWithBlobStore(File base, int maxFileSizeMB, + int cacheSizeMB, boolean memoryMapping, int dsCacheInMB) { + return new OakRepositoryFixture(OakFixture + .getSegmentTar(OakFixture.OAK_SEGMENT_TAR_DS, base, maxFileSizeMB, cacheSizeMB, + memoryMapping, true, dsCacheInMB)); } private final OakFixture oakFixture; Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java Thu Oct 6 06:41:28 2016 @@ -45,13 +45,17 @@ class SegmentTarFixture extends OakFixtu private final boolean useBlobStore; - public SegmentTarFixture(String name, File base, int maxFileSizeMB, int cacheSizeMB, boolean memoryMapping, boolean useBlobStore) { + private final int dsCacheSizeInMB; + + public SegmentTarFixture(String name, File base, int maxFileSizeMB, int cacheSizeMB, + boolean memoryMapping, boolean useBlobStore, int dsCacheSizeInMB) { super(name); this.base = base; this.maxFileSizeMB = maxFileSizeMB; this.cacheSizeMB = cacheSizeMB; this.memoryMapping = memoryMapping; this.useBlobStore = useBlobStore; + this.dsCacheSizeInMB = dsCacheSizeInMB; } @Override @@ -75,7 +79,7 @@ class SegmentTarFixture extends OakFixtu for (int i = 0; i < cluster.length; i++) { BlobStore blobStore = null; if (useBlobStore) { - blobStoreFixtures[i] = BlobStoreFixture.create(base, true, statsProvider); + blobStoreFixtures[i] = BlobStoreFixture.create(base, true, dsCacheSizeInMB, statsProvider); blobStore = blobStoreFixtures[i].setUp(); } Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java?rev=1763500&r1=1763499&r2=1763500&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java Thu Oct 6 06:41:28 2016 @@ -131,7 +131,7 @@ public class ScalabilityRunner { host.value(options), port.value(options), dbName.value(options), dropDBAfterTest.value(options), cacheSize * MB), - OakRepositoryFixture.getMongoWithFDS( + OakRepositoryFixture.getMongoWithDS( host.value(options), port.value(options), dbName.value(options), dropDBAfterTest.value(options), cacheSize * MB, @@ -143,16 +143,16 @@ public class ScalabilityRunner { cacheSize * MB), OakRepositoryFixture.getTar( base.value(options), 256, cacheSize, mmap.value(options)), - OakRepositoryFixture.getTarWithBlobStore( - base.value(options), 256, cacheSize, mmap.value(options)), + OakRepositoryFixture.getTarWithBlobStore(base.value(options), 256, cacheSize, + mmap.value(options), fdsCache.value(options)), OakRepositoryFixture.getSegmentTar( - base.value(options), 256, cacheSize, mmap.value(options)), - OakRepositoryFixture.getSegmentTarWithBlobStore( - base.value(options), 256, cacheSize, mmap.value(options)), + base.value(options), 256, cacheSize, mmap.value(options)), + OakRepositoryFixture.getSegmentTarWithBlobStore(base.value(options), 256, cacheSize, + mmap.value(options), fdsCache.value(options)), OakRepositoryFixture.getRDB(rdbjdbcuri.value(options), rdbjdbcuser.value(options), rdbjdbcpasswd.value(options), rdbjdbctableprefix.value(options), dropDBAfterTest.value(options), cacheSize * MB), - OakRepositoryFixture.getRDBWithFDS(rdbjdbcuri.value(options), rdbjdbcuser.value(options), + OakRepositoryFixture.getRDBWithDS(rdbjdbcuri.value(options), rdbjdbcuser.value(options), rdbjdbcpasswd.value(options), rdbjdbctableprefix.value(options), dropDBAfterTest.value(options), cacheSize * MB, base.value(options), fdsCache.value(options))