Taewoo Kim has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/1064
Change subject: ASTERIXDB-1569: the config file now supports unit, comma, and decimal. ...................................................................... ASTERIXDB-1569: the config file now supports unit, comma, and decimal. - This patch fixes an issue that the config file only supports byte value for the byte required parameters. Now, we can use unit (KB, MB, GB ...), comma, and decimal. Change-Id: I763d1b06e93f75775c0294da0d609c2bf3e102c9 --- M asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java M asterixdb/asterix-docker/docker/asterix-configuration.xml M asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml M asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml M asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml M asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml M asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java 14 files changed, 183 insertions(+), 84 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/64/1064/1 diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml index 1d693f9..cbd5c93 100644 --- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml +++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml @@ -52,19 +52,19 @@ </property> <property> <name>compiler.framesize</name> - <value>32768</value> + <value>32KB</value> </property> <property> <name>compiler.sortmemory</name> - <value>327680</value> + <value>320KB</value> </property> <property> <name>compiler.groupmemory</name> - <value>163840</value> + <value>160KB</value> </property> <property> <name>compiler.joinmemory</name> - <value>163840</value> + <value>160KB</value> </property> <property> <name>compiler.pregelix.home</name> @@ -72,14 +72,14 @@ </property> <property> <name>storage.buffercache.pagesize</name> - <value>32768</value> + <value>32KB</value> <description>The page size in bytes for pages in the buffer cache. (Default = "32768" // 32KB) </description> </property> <property> <name>storage.buffercache.size</name> - <value>33554432</value> + <value>32MB</value> <description>The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size(Default = "33554432" // 32MB) diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java index 42f612a..5bfb5f8 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java @@ -40,22 +40,22 @@ public long getSortMemorySize() { return accessor.getProperty(COMPILER_SORTMEMORY_KEY, COMPILER_SORTMEMORY_DEFAULT, - PropertyInterpreters.getLongPropertyInterpreter()); + PropertyInterpreters.getLongBytePropertyInterpreter()); } public long getJoinMemorySize() { return accessor.getProperty(COMPILER_JOINMEMORY_KEY, COMPILER_JOINMEMORY_DEFAULT, - PropertyInterpreters.getLongPropertyInterpreter()); + PropertyInterpreters.getLongBytePropertyInterpreter()); } public long getGroupMemorySize() { return accessor.getProperty(COMPILER_GROUPMEMORY_KEY, COMPILER_GROUPMEMORY_DEFAULT, - PropertyInterpreters.getLongPropertyInterpreter()); + PropertyInterpreters.getLongBytePropertyInterpreter()); } public int getFrameSize() { return accessor.getProperty(COMPILER_FRAMESIZE_KEY, COMPILER_FRAMESIZE_DEFAULT, - PropertyInterpreters.getIntegerPropertyInterpreter()); + PropertyInterpreters.getIntegerBytePropertyInterpreter()); } public String getPregelixHome() { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java index e4d2913..0f3951e 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java @@ -41,7 +41,7 @@ public long getMemoryComponentGlobalBudget() { return accessor.getProperty(FEED_MEMORY_GLOBALBUDGET_KEY, FEED_MEMORY_GLOBALBUDGET_DEFAULT, - PropertyInterpreters.getLongPropertyInterpreter()); + PropertyInterpreters.getLongBytePropertyInterpreter()); } public long getMemoryAvailableWaitTimeout() { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java index 7f51bbd..eec4947 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java @@ -260,7 +260,7 @@ public int getLogBufferPageSize() { return accessor.getProperty(REPLICATION_LOG_BUFFER_PAGE_SIZE_KEY, REPLICATION_LOG_BUFFER_PAGE_SIZE_DEFAULT, - PropertyInterpreters.getIntegerPropertyInterpreter()); + PropertyInterpreters.getIntegerBytePropertyInterpreter()); } public int getLogBufferNumOfPages() { @@ -270,6 +270,6 @@ public int getLogBatchSize() { return accessor.getProperty(REPLICATION_LOG_BATCH_SIZE_KEY, REPLICATION_LOG_BATCH_SIZE_DEFAULT, - PropertyInterpreters.getIntegerPropertyInterpreter()); + PropertyInterpreters.getIntegerBytePropertyInterpreter()); } } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java index 9fb4e94..5fdbc1c 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java @@ -55,12 +55,12 @@ public int getBufferCachePageSize() { return accessor.getProperty(STORAGE_BUFFERCACHE_PAGESIZE_KEY, STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT, - PropertyInterpreters.getIntegerPropertyInterpreter()); + PropertyInterpreters.getIntegerBytePropertyInterpreter()); } public long getBufferCacheSize() { return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, STORAGE_BUFFERCACHE_SIZE_DEFAULT, - PropertyInterpreters.getLongPropertyInterpreter()); + PropertyInterpreters.getLongBytePropertyInterpreter()); } public int getBufferCacheNumPages() { @@ -74,7 +74,7 @@ public int getMemoryComponentPageSize() { return accessor.getProperty(STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY, STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT, - PropertyInterpreters.getIntegerPropertyInterpreter()); + PropertyInterpreters.getIntegerBytePropertyInterpreter()); } public int getMemoryComponentNumPages() { @@ -96,7 +96,7 @@ public long getMemoryComponentGlobalBudget() { return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY, - STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongPropertyInterpreter()); + STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongBytePropertyInterpreter()); } public double getBloomFilterFalsePositiveRate() { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java index 356dad3..00bcdc9 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java @@ -81,12 +81,12 @@ public int getLogBufferPageSize() { return accessor.getProperty(TXN_LOG_BUFFER_PAGESIZE_KEY, TXN_LOG_BUFFER_PAGESIZE_DEFAULT, - PropertyInterpreters.getIntegerPropertyInterpreter()); + PropertyInterpreters.getIntegerBytePropertyInterpreter()); } public long getLogPartitionSize() { return accessor.getProperty(TXN_LOG_PARTITIONSIZE_KEY, TXN_LOG_PARTITIONSIZE_DEFAULT, - PropertyInterpreters.getLongPropertyInterpreter()); + PropertyInterpreters.getLongBytePropertyInterpreter()); } public int getCheckpointLSNThreshold() { @@ -131,6 +131,6 @@ public long getJobRecoveryMemorySize() { return accessor.getProperty(TXN_JOB_RECOVERY_MEMORY_SIZE_KEY, TXN_JOB_RECOVERY_MEMORY_SIZE_DEFAULT, - PropertyInterpreters.getLongPropertyInterpreter()); + PropertyInterpreters.getLongBytePropertyInterpreter()); } } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java index b54bcc3..4dd07d8 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java @@ -20,6 +20,8 @@ import java.util.logging.Level; +import org.apache.hyracks.util.StorageUtil; + public class PropertyInterpreters { public static IPropertyInterpreter<Integer> getIntegerPropertyInterpreter() { @@ -88,4 +90,30 @@ }; } + public static IPropertyInterpreter<Long> getLongBytePropertyInterpreter() { + return new IPropertyInterpreter<Long>() { + @Override + public Long interpret(String s) throws IllegalArgumentException { + try { + return StorageUtil.getByteValue(s); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + } + }; + } + + public static IPropertyInterpreter<Integer> getIntegerBytePropertyInterpreter() { + return new IPropertyInterpreter<Integer>() { + @Override + public Integer interpret(String s) throws IllegalArgumentException { + try { + return (int) StorageUtil.getByteValue(s); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + } + }; + } + } diff --git a/asterixdb/asterix-docker/docker/asterix-configuration.xml b/asterixdb/asterix-docker/docker/asterix-configuration.xml index dd8d571..ea43da9 100644 --- a/asterixdb/asterix-docker/docker/asterix-configuration.xml +++ b/asterixdb/asterix-docker/docker/asterix-configuration.xml @@ -66,7 +66,7 @@ <property> <name>storage.buffercache.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages in the buffer cache. (Default = "131072" // 128KB) </description> @@ -74,7 +74,7 @@ <property> <name>storage.buffercache.size</name> - <value>536870912</value> + <value>512MB</value> <description>The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size(Default = "536870912" // 512MB) @@ -91,7 +91,7 @@ <property> <name>storage.memorycomponent.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages allocated to memory components. (Default = "131072" // 128KB) </description> @@ -122,7 +122,7 @@ <property> <name>storage.memorycomponent.globalbudget</name> - <value>536870912</value> + <value>512MB</value> <description>The total size of memory in bytes that the sum of all open memory components cannot exceed. (Default = "536870192" // 512MB) @@ -146,7 +146,7 @@ <property> <name>txn.log.buffer.pagesize</name> - <value>524288</value> + <value>512KB</value> <description>The size of pages in the in-memory log buffer. (Default = "524288" // 512KB) </description> @@ -154,7 +154,7 @@ <property> <name>txn.log.partitionsize</name> - <value>2147483648</value> + <value>2GB</value> <description>The maximum size of a log file partition allowed before rotating the log to the next partition. (Default = "2147483648" // 2GB) @@ -221,7 +221,7 @@ <property> <name>compiler.sortmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to sort operations. (Default = "33554432" // 32mb) </description> @@ -229,7 +229,7 @@ <property> <name>compiler.joinmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to join operations. (Default = "33554432" // 32mb) </description> @@ -237,7 +237,7 @@ <property> <name>compiler.framesize</name> - <value>131072</value> + <value>128KB</value> <description>The Hyracks frame size that the compiler configures per job. (Default = "131072" // 128KB) </description> diff --git a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml index af37d0b..c642cbb 100644 --- a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml +++ b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml @@ -37,7 +37,7 @@ <property> <name>storage.buffercache.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages in the buffer cache. (Default = "131072" // 128KB) </description> @@ -45,7 +45,7 @@ <property> <name>storage.buffercache.size</name> - <value>3221225472</value> + <value>3GB</value> <description>[3GB] The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size(Default = "536870912" // 512MB) @@ -63,7 +63,7 @@ <!-- Buffer size per dataset for in-memory components: 1GB --> <property> <name>storage.memorycomponent.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages allocated to memory components. (Default = "131072" // 128KB) </description> @@ -95,8 +95,8 @@ <property> <name>storage.memorycomponent.globalbudget</name> - <value>4399824896</value> - <description>[4GB + 8MB]The total size of memory in bytes that the sum of all + <value>4196MB</value> + <description>[4GB + 100MB]The total size of memory in bytes that the sum of all open memory components cannot exceed. (Default = "536870192" // 512MB) </description> @@ -119,7 +119,7 @@ <property> <name>txn.log.buffer.pagesize</name> - <value>8388608</value> + <value>8MB</value> <description>[8MB]The size of pages in the in-memory log buffer. (Default = "524288" // 512KB) </description> @@ -127,7 +127,7 @@ <property> <name>txn.log.partitionsize</name> - <value>2147483648</value> + <value>2GB</value> <description>The maximum size of a log file partition allowed before rotating the log to the next partition. (Default = "2147483648" // 2GB) @@ -194,7 +194,7 @@ <property> <name>compiler.sortmemory</name> - <value>67108864</value> + <value>64MB</value> <description>[64MB]The amount of memory in bytes given to sort operations. (Default = "33554432" // 32mb) </description> @@ -202,7 +202,7 @@ <property> <name>compiler.joinmemory</name> - <value>67108864</value> + <value>64MB</value> <description>[64MB]The amount of memory in bytes given to join operations. (Default = "33554432" // 32mb) </description> @@ -210,7 +210,7 @@ <property> <name>compiler.framesize</name> - <value>131072</value> + <value>128KB</value> <description>The Hyracks frame size that the compiler configures per job. (Default = "131072" // 128KB) </description> diff --git a/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml b/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml index f6da341..a7c5c6d 100644 --- a/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml +++ b/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml @@ -41,7 +41,7 @@ <property> <name>storage.buffercache.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages in the buffer cache. (Default = "131072" // 128KB) </description> @@ -49,7 +49,7 @@ <property> <name>storage.buffercache.size</name> - <value>536870912</value> + <value>512MB</value> <description>The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size(Default = "536870912" // 512MB) @@ -66,7 +66,7 @@ <property> <name>storage.memorycomponent.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages allocated to memory components. (Default = "131072" // 128KB) </description> @@ -98,7 +98,7 @@ <property> <name>storage.memorycomponent.globalbudget</name> - <value>1073741824</value> + <value>1GB</value> <description>The total size of memory in bytes that the sum of all open memory components cannot exceed. (Default = "536870192" // 512MB) @@ -122,7 +122,7 @@ <property> <name>txn.log.buffer.pagesize</name> - <value>524288</value> + <value>512KB</value> <description>The size of pages in the in-memory log buffer. (Default = "524288" // 512KB) </description> @@ -130,7 +130,7 @@ <property> <name>txn.log.partitionsize</name> - <value>2147483648</value> + <value>2GB</value> <description>The maximum size of a log file partition allowed before rotating the log to the next partition. (Default = "2147483648" // 2GB) @@ -197,7 +197,7 @@ <property> <name>txn.job.recovery.memorysize</name> - <value>67108864</value> + <value>64MB</value> <description>The memory allocated per job during recovery. (Default = "67108864" // 64MB) </description> @@ -205,23 +205,23 @@ <property> <name>compiler.sortmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to sort operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.joinmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to join operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.framesize</name> - <value>131072</value> + <value>128KB</value> <description>The Hyracks frame size that the compiler configures per job. (Default = "131072" // 128KB) </description> diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml b/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml index 4cd7fff..e36c33b 100644 --- a/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml +++ b/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml @@ -41,7 +41,7 @@ <property> <name>storage.buffercache.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages in the buffer cache. (Default = "131072" // 128KB) </description> @@ -49,7 +49,7 @@ <property> <name>storage.buffercache.size</name> - <value>536870912</value> + <value>512MB</value> <description>The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size(Default = "536870912" // 512MB) @@ -66,7 +66,7 @@ <property> <name>storage.memorycomponent.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages allocated to memory components. (Default = "131072" // 128KB) </description> @@ -98,7 +98,7 @@ <property> <name>storage.memorycomponent.globalbudget</name> - <value>1073741824</value> + <value>1GB</value> <description>The total size of memory in bytes that the sum of all open memory components cannot exceed. (Default = "536870192" // 512MB) @@ -122,7 +122,7 @@ <property> <name>txn.log.buffer.pagesize</name> - <value>524288</value> + <value>512KB</value> <description>The size of pages in the in-memory log buffer. (Default = "524288" // 512KB) </description> @@ -130,7 +130,7 @@ <property> <name>txn.log.partitionsize</name> - <value>2147483648</value> + <value>2GB</value> <description>The maximum size of a log file partition allowed before rotating the log to the next partition. (Default = "2147483648" // 2GB) @@ -197,23 +197,31 @@ <property> <name>compiler.sortmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to sort operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.joinmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to join operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) + </description> + </property> + + <property> + <name>compiler.groupmemory</name> + <value>32MB</value> + <description>The amount of memory in bytes given to group-by operations. + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.framesize</name> - <value>131072</value> + <value>128KB</value> <description>The Hyracks frame size that the compiler configures per job. (Default = "131072" // 128KB) </description> diff --git a/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml b/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml index 778e51f..788cb76 100644 --- a/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml +++ b/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml @@ -41,7 +41,7 @@ <property> <name>storage.buffercache.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages in the buffer cache. (Default = "131072" // 128KB) </description> @@ -49,7 +49,7 @@ <property> <name>storage.buffercache.size</name> - <value>536870912</value> + <value>512MB</value> <description>The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size(Default = "536870912" // 512MB) @@ -66,7 +66,7 @@ <property> <name>storage.memorycomponent.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages allocated to memory components. (Default = "131072" // 128KB) </description> @@ -98,7 +98,7 @@ <property> <name>storage.memorycomponent.globalbudget</name> - <value>536870912</value> + <value>512MB</value> <description>The total size of memory in bytes that the sum of all open memory components cannot exceed. (Default = "536870192" // 512MB) @@ -122,7 +122,7 @@ <property> <name>txn.log.buffer.pagesize</name> - <value>524288</value> + <value>512KB</value> <description>The size of pages in the in-memory log buffer. (Default = "524288" // 512KB) </description> @@ -130,7 +130,7 @@ <property> <name>txn.log.partitionsize</name> - <value>2147483648</value> + <value>2GB</value> <description>The maximum size of a log file partition allowed before rotating the log to the next partition. (Default = "2147483648" // 2GB) @@ -197,23 +197,23 @@ <property> <name>compiler.sortmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to sort operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.joinmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to join operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.framesize</name> - <value>131072</value> + <value>128KB</value> <description>The Hyracks frame size that the compiler configures per job. (Default = "131072" // 128KB) </description> diff --git a/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml b/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml index 778e51f..788cb76 100644 --- a/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml +++ b/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml @@ -41,7 +41,7 @@ <property> <name>storage.buffercache.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages in the buffer cache. (Default = "131072" // 128KB) </description> @@ -49,7 +49,7 @@ <property> <name>storage.buffercache.size</name> - <value>536870912</value> + <value>512MB</value> <description>The size of memory allocated to the disk buffer cache. The value should be a multiple of the buffer cache page size(Default = "536870912" // 512MB) @@ -66,7 +66,7 @@ <property> <name>storage.memorycomponent.pagesize</name> - <value>131072</value> + <value>128KB</value> <description>The page size in bytes for pages allocated to memory components. (Default = "131072" // 128KB) </description> @@ -98,7 +98,7 @@ <property> <name>storage.memorycomponent.globalbudget</name> - <value>536870912</value> + <value>512MB</value> <description>The total size of memory in bytes that the sum of all open memory components cannot exceed. (Default = "536870192" // 512MB) @@ -122,7 +122,7 @@ <property> <name>txn.log.buffer.pagesize</name> - <value>524288</value> + <value>512KB</value> <description>The size of pages in the in-memory log buffer. (Default = "524288" // 512KB) </description> @@ -130,7 +130,7 @@ <property> <name>txn.log.partitionsize</name> - <value>2147483648</value> + <value>2GB</value> <description>The maximum size of a log file partition allowed before rotating the log to the next partition. (Default = "2147483648" // 2GB) @@ -197,23 +197,23 @@ <property> <name>compiler.sortmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to sort operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.joinmemory</name> - <value>33554432</value> + <value>32MB</value> <description>The amount of memory in bytes given to join operations. - (Default = "33554432" // 32mb) + (Default = "33554432" // 32MB) </description> </property> <property> <name>compiler.framesize</name> - <value>131072</value> + <value>128KB</value> <description>The Hyracks frame size that the compiler configures per job. (Default = "131072" // 128KB) </description> diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java index 4607214..87a7055 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java @@ -71,6 +71,69 @@ } } + public static double getSizeInBytes(final double size, final StorageUnit unit) { + switch (unit) { + case BYTE: + return size; + case KILOBYTE: + return size * KB; + case MEGABYTE: + return size * MB; + case GIGABYTE: + return size * GB; + case TERABYTE: + return size * TB; + case PETABYTE: + return size * PB; + default: + throw new IllegalStateException("Unsupported unti: " + unit); + } + } + + /** + * Helper function to return byte value for the given string (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...) + */ + public static long getByteValue(String s) { + try { + // step1. strip all commas from the given number and + // convert each character to a corresponding lowercase. + s = s.replaceAll(",", "").toLowerCase(); + + // Default type + StorageUtil.StorageUnit unitType = StorageUnit.BYTE; + + // Check the suffix + switch (s.substring(s.length() - 2)) { + case "kb": + unitType = StorageUnit.KILOBYTE; + break; + case "mb": + unitType = StorageUnit.MEGABYTE; + break; + case "gb": + unitType = StorageUnit.GIGABYTE; + break; + case "tb": + unitType = StorageUnit.TERABYTE; + break; + case "pb": + unitType = StorageUnit.PETABYTE; + break; + default: + // Other suffixes will be treated as byte. + break; + } + + // Strip all unit suffixes. + s = s.replaceAll("[^\\.0123456789]", ""); + + // Return the byte value for the given numbers. + return (long) getSizeInBytes(Double.parseDouble(s), unitType); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + } + /** * Returns a human readable value in storage units rounded up to two decimal places. * e.g. toHumanReadableSize(1024L * 1024L * 1024l * 10L *) + 1024l * 1024l * 59) returns 1.06 GB -- To view, visit https://asterix-gerrit.ics.uci.edu/1064 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I763d1b06e93f75775c0294da0d609c2bf3e102c9 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Taewoo Kim <wangs...@yahoo.com>