Murtadha Hubail has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/2362
Change subject: [ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets ...................................................................... [ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets - user model changes: no - storage format changes: no - interface changes: no Details: - Introduce number of metadata datasets parameter and use it to reserve memory for all metadata datasets. - Default metadata dataset memory component number of pages to 32 pages. - Introduce max active writable datasets and use it to automatically calculate the number of pages of a user dataset memory component. - Remve the assumpation of reserving an NC core for heartbeats. - Validate memory configuration early. Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java M asterixdb/asterix-app/src/main/resources/cc.conf M asterixdb/asterix-app/src/main/resources/cc2.conf M asterixdb/asterix-app/src/main/resources/cc3.conf M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java M asterixdb/asterix-app/src/test/resources/cc-multipart.conf M asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf M asterixdb/asterix-app/src/test/resources/cc.conf M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java 10 files changed, 51 insertions(+), 54 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/62/2362/1 diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index a23a763..d33ee42 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -224,19 +224,9 @@ @Override public NodeCapacity getCapacity() { StorageProperties storageProperties = runtimeContext.getStorageProperties(); - // Deducts the reserved buffer cache size and memory component size from the maxium heap size, - // and deducts one core for processing heartbeats. - long memorySize = Runtime.getRuntime().maxMemory() - storageProperties.getBufferCacheSize() - - storageProperties.getMemoryComponentGlobalBudget(); - if (memorySize <= 0) { - throw new IllegalStateException("Invalid node memory configuration, more memory budgeted than available " - + "in JVM. Runtime max memory: " + Runtime.getRuntime().maxMemory() + " Buffer cache size: " - + storageProperties.getBufferCacheSize() + " Memory component global budget: " - + storageProperties.getMemoryComponentGlobalBudget()); - } + final long memorySize = storageProperties.getJobExecutionMemoryBudget(); int allCores = Runtime.getRuntime().availableProcessors(); - int maximumCoresForComputation = allCores > 1 ? allCores - 1 : allCores; - return new NodeCapacity(memorySize, maximumCoresForComputation); + return new NodeCapacity(memorySize, allCores); } private void performLocalCleanUp() throws HyracksDataException { diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf index c5a3fdb..deebe41 100644 --- a/asterixdb/asterix-app/src/main/resources/cc.conf +++ b/asterixdb/asterix-app/src/main/resources/cc.conf @@ -37,7 +37,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf index 941b6c1..2026ee7 100644 --- a/asterixdb/asterix-app/src/main/resources/cc2.conf +++ b/asterixdb/asterix-app/src/main/resources/cc2.conf @@ -37,7 +37,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf index 01383ef..91ddb7a 100644 --- a/asterixdb/asterix-app/src/main/resources/cc3.conf +++ b/asterixdb/asterix-app/src/main/resources/cc3.conf @@ -37,7 +37,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java index f029669..e752493 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java @@ -87,8 +87,8 @@ Assert.assertTrue(getValue(row) == maxHeap / 4); matchCount++; } - if (row.contains("storage.memorycomponent.numpages")) { - Assert.assertTrue(getValue(row) == maxHeap / (131072 * 64)); + if (row.contains("storage.max.active.writable.datasets")) { + Assert.assertTrue(getValue(row) == 8); matchCount++; } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java index 92e403c..05e5aad 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java @@ -48,10 +48,7 @@ @Before public void setUp() throws Exception { - // Read default test configurations - // Write test config file - integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET, "128MB"); - integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_NUMPAGES, 32); + integrationUtil.addOption(StorageProperties.Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS, 20); integrationUtil.setGracefulShutdown(false); integrationUtil.init(true, TEST_CONFIG_FILE_PATH); } diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf index f28545d..5cf1bbe 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf @@ -38,7 +38,6 @@ storage.subdir=test_storage storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=32 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf index 0b66cfe..0b2ddbd 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf @@ -35,8 +35,8 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 -storage.memorycomponent.globalbudget=512MB +storage.max.active.writable.datasets=50 +storage.memorycomponent.globalbudget=128MB [cc] address = 127.0.0.1 diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf index 3e4ecd9..5cf1bbe 100644 --- a/asterixdb/asterix-app/src/test/resources/cc.conf +++ b/asterixdb/asterix-app/src/test/resources/cc.conf @@ -38,7 +38,6 @@ storage.subdir=test_storage storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java index 963f0ca..23e7d75 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java @@ -42,16 +42,11 @@ STORAGE_BUFFERCACHE_MAXOPENFILES(INTEGER, Integer.MAX_VALUE), STORAGE_MEMORYCOMPONENT_GLOBALBUDGET(LONG_BYTE_UNIT, Runtime.getRuntime().maxMemory() / 4), STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)), - STORAGE_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig, Integer>) accessor -> - // By default, uses 1/16 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET for the write buffer - // budget for a dataset, including data and indexes. - (int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (16 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE)))), STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(INTEGER, 2), - STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig, Integer>) accessor -> - // By default, uses the min of 1/64 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET and 256 pages - // for the write buffer budget for a metadata dataset, including data and indexes. - Math.min((int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (64 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE))), 256)), - STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d); + STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, 32), + STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d), + STORAGE_METADATA_DATASETS(INTEGER, 14), + STORAGE_MAX_ACTIVE_WRITABLE_DATASETS(INTEGER, 8); private final IOptionType interpreter; private final Object defaultValue; @@ -59,11 +54,6 @@ <T> Option(IOptionType<T> interpreter, T defaultValue) { this.interpreter = interpreter; this.defaultValue = defaultValue; - } - - <T> Option(IOptionType<T> interpreter, Function<IApplicationConfig, T> defaultValueFunction) { - this.interpreter = interpreter; - this.defaultValue = defaultValueFunction; } @Override @@ -86,17 +76,16 @@ + "of the memory component page size"; case STORAGE_MEMORYCOMPONENT_PAGESIZE: return "The page size in bytes for pages allocated to memory components"; - case STORAGE_MEMORYCOMPONENT_NUMPAGES: - return "The number of pages to allocate for a memory component. This budget is shared by all " - + "the memory components of the primary index and all its secondary indexes across all I/O " - + "devices on a node. Note: in-memory components usually has fill factor of 75% since " - + "the pages are 75% full and the remaining 25% is un-utilized"; case STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS: return "The number of memory components to be used per lsm index"; case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES: return "The number of pages to allocate for a metadata memory component"; case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE: return "The maximum acceptable false positive rate for bloom filters associated with LSM indexes"; + case STORAGE_METADATA_DATASETS: + return "The number of metadata datasets"; + case STORAGE_MAX_ACTIVE_WRITABLE_DATASETS: + return "The maximum number of datasets that can be concurrently modified"; default: throw new IllegalStateException("NYI: " + this); } @@ -114,21 +103,16 @@ @Override public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) { - switch (this) { - case STORAGE_MEMORYCOMPONENT_NUMPAGES: - return "1/16th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) - + " value"; - case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES: - return "1/64th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) - + " value or 256, whichever is larger"; - default: - return null; + if (this == STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES) { + return "32 pages"; } + return null; } } public StorageProperties(PropertiesAccessor accessor) { super(accessor); + validateMemoryConfig(); } public int getBufferCachePageSize() { @@ -148,7 +132,10 @@ } public int getMemoryComponentNumPages() { - return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_NUMPAGES); + final long metadataReservedMem = getMetadataReservedMemory(); + final long globalUserDatasetMem = getMemoryComponentGlobalBudget() - metadataReservedMem; + final long userDatasetMem = globalUserDatasetMem / getMaxActiveWritableDatasets(); + return (int) (userDatasetMem / getMemoryComponentPageSize()); } public int getMetadataMemoryComponentNumPages() { @@ -170,4 +157,32 @@ public int getBufferCacheNumPages() { return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES)); } + + public long getJobExecutionMemoryBudget() { + return Runtime.getRuntime().maxMemory() - getBufferCacheSize() + getMemoryComponentGlobalBudget(); + } + + public int getMaxActiveWritableDatasets() { + return accessor.getInt(Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS); + } + + public int getMetadataDatasets() { + return accessor.getInt(Option.STORAGE_METADATA_DATASETS); + } + + private long getMetadataReservedMemory() { + return (getMetadataMemoryComponentNumPages() * (long) getMemoryComponentPageSize()) * getMetadataDatasets(); + } + + private void validateMemoryConfig() { + final long remainingMemory = + Runtime.getRuntime().maxMemory() - getBufferCacheSize() - getMemoryComponentGlobalBudget(); + if (remainingMemory <= 0) { + final String msg = String.format( + "Invalid node memory configuration, more memory budgeted than available in JVM. Runtime max memory:" + + " (%o), Buffer cache memory (%o), memory component global budget (%o)", + Runtime.getRuntime().maxMemory(), getBufferCacheSize(), getMemoryComponentGlobalBudget()); + throw new IllegalStateException(msg); + } + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/2362 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Murtadha Hubail <mhub...@apache.org>