This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit ad75b9b1427eaa579fb3e22ef2eecc616ce02754 Author: Kang <kxiao.ti...@gmail.com> AuthorDate: Fri Apr 19 13:04:21 2024 +0800 [opt](auto bucket) add fe config autobucket_max_buckets (#33842) --- .../main/java/org/apache/doris/common/Config.java | 8 +++++- .../apache/doris/common/util/AutoBucketUtils.java | 3 +- .../suites/autobucket/test_autobucket.groovy | 33 +++++++++++++++++++--- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8dff0e3f4db..3525d411b66 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2341,12 +2341,18 @@ public class Config extends ConfigBase { }) public static long analyze_record_limit = 20000; - @ConfField(description = { + @ConfField(mutable = true, description = { "Auto Buckets中最小的buckets数目", "min buckets of auto bucket" }) public static int autobucket_min_buckets = 1; + @ConfField(mutable = true, description = { + "Auto Buckets中最大的buckets数目", + "max buckets of auto bucket" + }) + public static int autobucket_max_buckets = 128; + @ConfField(description = {"Arrow Flight Server中所有用户token的缓存上限,超过后LRU淘汰,默认值为512, " + "并强制限制小于 qe_max_connection/2, 避免`Reach limit of connections`, " + "因为arrow flight sql是无状态的协议,连接通常不会主动断开," diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java index 294250fd213..55a07b89eb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java @@ -20,6 +20,7 @@ package org.apache.doris.common.util; import org.apache.doris.catalog.DiskInfo; import org.apache.doris.catalog.DiskInfo.DiskState; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; @@ -85,7 +86,7 @@ public class AutoBucketUtils { public static int getBucketsNum(long partitionSize) { int bucketsNumByPartitionSize = convertParitionSizeToBucketsNum(partitionSize); int bucketsNumByBE = getBucketsNumByBEDisks(); - int bucketsNum = Math.min(128, Math.min(bucketsNumByPartitionSize, bucketsNumByBE)); + int bucketsNum = Math.min(Config.autobucket_max_buckets, Math.min(bucketsNumByPartitionSize, bucketsNumByBE)); int beNum = getBENum(); logger.debug("AutoBucketsUtil: bucketsNumByPartitionSize {}, bucketsNumByBE {}, bucketsNum {}, beNum {}", bucketsNumByPartitionSize, bucketsNumByBE, bucketsNum, beNum); diff --git a/regression-test/suites/autobucket/test_autobucket.groovy b/regression-test/suites/autobucket/test_autobucket.groovy index d3ba70d0df3..615ede5675b 100644 --- a/regression-test/suites/autobucket/test_autobucket.groovy +++ b/regression-test/suites/autobucket/test_autobucket.groovy @@ -40,7 +40,8 @@ suite("test_autobucket") { sql "drop table if exists autobucket_test" - + // set min to 5 + sql "ADMIN SET FRONTEND CONFIG ('autobucket_min_buckets' = '5')" sql "drop table if exists autobucket_test_min_buckets" result = sql """ CREATE TABLE `autobucket_test_min_buckets` ( @@ -55,11 +56,35 @@ suite("test_autobucket") { ) """ - default_min_buckets = 1 // in Config.java result = sql "show partitions from autobucket_test_min_buckets" logger.info("${result}") // XXX: buckets at pos(8), next maybe impl by sql meta - assertEquals(Integer.valueOf(result.get(0).get(8)), default_min_buckets) - + assertEquals(Integer.valueOf(result.get(0).get(8)), 5) + // set back to default + sql "ADMIN SET FRONTEND CONFIG ('autobucket_min_buckets' = '1')" sql "drop table if exists autobucket_test_min_buckets" + + // set max to 4 + sql "ADMIN SET FRONTEND CONFIG ('autobucket_max_buckets' = '4')" + sql "drop table if exists autobucket_test_max_buckets" + result = sql """ + CREATE TABLE `autobucket_test_max_buckets` ( + `user_id` largeint(40) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`user_id`) BUCKETS AUTO + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "estimate_partition_size" = "100000G" + ) + """ + + result = sql "show partitions from autobucket_test_max_buckets" + logger.info("${result}") + // XXX: buckets at pos(8), next maybe impl by sql meta + assertEquals(Integer.valueOf(result.get(0).get(8)), 4) + // set back to default + sql "ADMIN SET FRONTEND CONFIG ('autobucket_max_buckets' = '128')" + sql "drop table if exists autobucket_test_max_buckets" } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org