Murtadha Hubail has submitted this change and it was merged. Change subject: [ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets ......................................................................
[ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets - user model changes: no - storage format changes: no - interface changes: no Details: - Introduce number of metadata datasets parameter and use it to reserve memory for all metadata datasets. - Default metadata dataset memory component number of pages to 32 pages. - Introduce max active writable datasets and use it to automatically calculate the number of pages of a user dataset memory component. - Remve the assumpation of reserving an NC core for heartbeats. - Fix for [ASTERIXDB-2279] by making the expected result order deterministic. Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2362 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Michael Blow <mb...@apache.org> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java M asterixdb/asterix-app/src/main/resources/cc.conf M asterixdb/asterix-app/src/main/resources/cc2.conf M asterixdb/asterix-app/src/main/resources/cc3.conf M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java M asterixdb/asterix-app/src/test/resources/cc-multipart.conf M asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf M asterixdb/asterix-app/src/test/resources/cc.conf M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java 14 files changed, 56 insertions(+), 56 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; No violations found; ; Verified Michael Blow: Looks good to me, approved diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index 1220462..b0382f7 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -222,19 +222,9 @@ @Override public NodeCapacity getCapacity() { StorageProperties storageProperties = runtimeContext.getStorageProperties(); - // Deducts the reserved buffer cache size and memory component size from the maxium heap size, - // and deducts one core for processing heartbeats. - long memorySize = Runtime.getRuntime().maxMemory() - storageProperties.getBufferCacheSize() - - storageProperties.getMemoryComponentGlobalBudget(); - if (memorySize <= 0) { - throw new IllegalStateException("Invalid node memory configuration, more memory budgeted than available " - + "in JVM. Runtime max memory: " + Runtime.getRuntime().maxMemory() + " Buffer cache size: " - + storageProperties.getBufferCacheSize() + " Memory component global budget: " - + storageProperties.getMemoryComponentGlobalBudget()); - } + final long memorySize = storageProperties.getJobExecutionMemoryBudget(); int allCores = Runtime.getRuntime().availableProcessors(); - int maximumCoresForComputation = allCores > 1 ? allCores - 1 : allCores; - return new NodeCapacity(memorySize, maximumCoresForComputation); + return new NodeCapacity(memorySize, allCores); } private void performLocalCleanUp() throws HyracksDataException { diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf index 914b7b6..fe6cf64 100644 --- a/asterixdb/asterix-app/src/main/resources/cc.conf +++ b/asterixdb/asterix-app/src/main/resources/cc.conf @@ -37,7 +37,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf index 6c01386..113497d 100644 --- a/asterixdb/asterix-app/src/main/resources/cc2.conf +++ b/asterixdb/asterix-app/src/main/resources/cc2.conf @@ -37,7 +37,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf index 933e4af..e210bf62 100644 --- a/asterixdb/asterix-app/src/main/resources/cc3.conf +++ b/asterixdb/asterix-app/src/main/resources/cc3.conf @@ -37,7 +37,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java index f029669..e752493 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java @@ -87,8 +87,8 @@ Assert.assertTrue(getValue(row) == maxHeap / 4); matchCount++; } - if (row.contains("storage.memorycomponent.numpages")) { - Assert.assertTrue(getValue(row) == maxHeap / (131072 * 64)); + if (row.contains("storage.max.active.writable.datasets")) { + Assert.assertTrue(getValue(row) == 8); matchCount++; } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java index 92e403c..05e5aad 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java @@ -48,10 +48,7 @@ @Before public void setUp() throws Exception { - // Read default test configurations - // Write test config file - integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET, "128MB"); - integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_NUMPAGES, 32); + integrationUtil.addOption(StorageProperties.Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS, 20); integrationUtil.setGracefulShutdown(false); integrationUtil.init(true, TEST_CONFIG_FILE_PATH); } diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf index f28545d..5cf1bbe 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf @@ -38,7 +38,6 @@ storage.subdir=test_storage storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=32 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf index 0b66cfe..6cd3b87 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf @@ -35,8 +35,7 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 -storage.memorycomponent.globalbudget=512MB +storage.memorycomponent.globalbudget=128MB [cc] address = 127.0.0.1 @@ -53,4 +52,5 @@ messaging.frame.size=4096 messaging.frame.count=512 txn.log.partitionsize=2MB -txn.log.checkpoint.pollfrequency=2147483647 \ No newline at end of file +txn.log.checkpoint.pollfrequency=2147483647 +storage.max.active.writable.datasets=50 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf index 3e4ecd9..5cf1bbe 100644 --- a/asterixdb/asterix-app/src/test/resources/cc.conf +++ b/asterixdb/asterix-app/src/test/resources/cc.conf @@ -38,7 +38,6 @@ storage.subdir=test_storage storage.buffercache.pagesize=32KB storage.buffercache.size=48MB -storage.memorycomponent.numpages=16 storage.memorycomponent.globalbudget=512MB [cc] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp index 6856726..cd1bcfc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp @@ -23,4 +23,5 @@ SELECT ELEMENT ( WITH z AS (FROM Accounts AS y WHERE y.id=x SELECT VALUE y) SELECT x, z -); +) +ORDER BY x; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 85b121e..59057b1 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -28,6 +28,7 @@ "replication\.log\.buffer\.pagesize" : 131072, "replication\.strategy" : "none", "replication\.timeout" : 30, + "storage.max.active.writable.datasets" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.job\.recovery\.memorysize" : 67108864, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index 7017b06..13dacb8 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -28,6 +28,7 @@ "replication\.log\.buffer\.pagesize" : 131072, "replication\.strategy" : "none", "replication\.timeout" : 30, + "storage.max.active.writable.datasets" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.job\.recovery\.memorysize" : 67108864, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 00e4d67..9f18638 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -28,6 +28,7 @@ "replication\.log\.buffer\.pagesize" : 131072, "replication\.strategy" : "none", "replication\.timeout" : 30, + "storage.max.active.writable.datasets" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.job\.recovery\.memorysize" : 67108864, diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java index 963f0ca..e01198f 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java @@ -42,16 +42,11 @@ STORAGE_BUFFERCACHE_MAXOPENFILES(INTEGER, Integer.MAX_VALUE), STORAGE_MEMORYCOMPONENT_GLOBALBUDGET(LONG_BYTE_UNIT, Runtime.getRuntime().maxMemory() / 4), STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)), - STORAGE_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig, Integer>) accessor -> - // By default, uses 1/16 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET for the write buffer - // budget for a dataset, including data and indexes. - (int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (16 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE)))), STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(INTEGER, 2), - STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig, Integer>) accessor -> - // By default, uses the min of 1/64 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET and 256 pages - // for the write buffer budget for a metadata dataset, including data and indexes. - Math.min((int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (64 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE))), 256)), - STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d); + STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, 32), + STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d), + STORAGE_METADATA_DATASETS(INTEGER, 14), + STORAGE_MAX_ACTIVE_WRITABLE_DATASETS(INTEGER, 8); private final IOptionType interpreter; private final Object defaultValue; @@ -61,13 +56,11 @@ this.defaultValue = defaultValue; } - <T> Option(IOptionType<T> interpreter, Function<IApplicationConfig, T> defaultValueFunction) { - this.interpreter = interpreter; - this.defaultValue = defaultValueFunction; - } - @Override public Section section() { + if (this == STORAGE_MAX_ACTIVE_WRITABLE_DATASETS) { + return Section.COMMON; + } return Section.NC; } @@ -86,17 +79,16 @@ + "of the memory component page size"; case STORAGE_MEMORYCOMPONENT_PAGESIZE: return "The page size in bytes for pages allocated to memory components"; - case STORAGE_MEMORYCOMPONENT_NUMPAGES: - return "The number of pages to allocate for a memory component. This budget is shared by all " - + "the memory components of the primary index and all its secondary indexes across all I/O " - + "devices on a node. Note: in-memory components usually has fill factor of 75% since " - + "the pages are 75% full and the remaining 25% is un-utilized"; case STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS: return "The number of memory components to be used per lsm index"; case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES: return "The number of pages to allocate for a metadata memory component"; case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE: return "The maximum acceptable false positive rate for bloom filters associated with LSM indexes"; + case STORAGE_METADATA_DATASETS: + return "The number of metadata datasets"; + case STORAGE_MAX_ACTIVE_WRITABLE_DATASETS: + return "The maximum number of datasets that can be concurrently modified"; default: throw new IllegalStateException("NYI: " + this); } @@ -114,16 +106,10 @@ @Override public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) { - switch (this) { - case STORAGE_MEMORYCOMPONENT_NUMPAGES: - return "1/16th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) - + " value"; - case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES: - return "1/64th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) - + " value or 256, whichever is larger"; - default: - return null; + if (this == STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES) { + return "32 pages"; } + return null; } } @@ -148,7 +134,10 @@ } public int getMemoryComponentNumPages() { - return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_NUMPAGES); + final long metadataReservedMem = getMetadataReservedMemory(); + final long globalUserDatasetMem = getMemoryComponentGlobalBudget() - metadataReservedMem; + final long userDatasetMem = globalUserDatasetMem / getMaxActiveWritableDatasets(); + return (int) (userDatasetMem / getMemoryComponentPageSize()); } public int getMetadataMemoryComponentNumPages() { @@ -170,4 +159,29 @@ public int getBufferCacheNumPages() { return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES)); } + + public long getJobExecutionMemoryBudget() { + final long jobExecutionMemory = + Runtime.getRuntime().maxMemory() - getBufferCacheSize() - getMemoryComponentGlobalBudget(); + if (jobExecutionMemory <= 0) { + final String msg = String.format( + "Invalid node memory configuration, more memory budgeted than available in JVM. Runtime max memory:" + + " (%d), Buffer cache memory (%d), memory component global budget (%d)", + Runtime.getRuntime().maxMemory(), getBufferCacheSize(), getMemoryComponentGlobalBudget()); + throw new IllegalStateException(msg); + } + return jobExecutionMemory; + } + + public int getMaxActiveWritableDatasets() { + return accessor.getInt(Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS); + } + + private int getMetadataDatasets() { + return accessor.getInt(Option.STORAGE_METADATA_DATASETS); + } + + private long getMetadataReservedMemory() { + return (getMetadataMemoryComponentNumPages() * (long) getMemoryComponentPageSize()) * getMetadataDatasets(); + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/2362 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269 Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Murtadha Hubail <mhub...@apache.org> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Luo Chen <cl...@uci.edu> Gerrit-Reviewer: Michael Blow <mb...@apache.org> Gerrit-Reviewer: Murtadha Hubail <mhub...@apache.org> Gerrit-Reviewer: Till Westmann <ti...@apache.org> Gerrit-Reviewer: abdullah alamoudi <bamou...@gmail.com>
