This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 86686da7bc7 [branch-3.0]Remove cpu/memory sum check limit (#52057)
86686da7bc7 is described below
commit 86686da7bc765b0d419fddf3060a551c8f0880ee
Author: wangbo <[email protected]>
AuthorDate: Thu Jun 26 10:48:02 2025 +0800
[branch-3.0]Remove cpu/memory sum check limit (#52057)
### What problem does this PR solve?
1 Remove cpu/memory sum check limit.
2 remove cpu_hard_limit switch, support cpu hard/soft limit at same
time.
---
be/src/agent/workload_group_listener.cpp | 7 +-----
be/src/runtime/workload_group/workload_group.cpp | 25 ++--------------------
be/src/runtime/workload_group/workload_group.h | 1 -
.../workload_group/workload_group_manager.h | 6 ------
.../main/java/org/apache/doris/common/Config.java | 6 +++---
.../resource/workloadgroup/WorkloadGroup.java | 9 --------
.../resource/workloadgroup/WorkloadGroupMgr.java | 13 +----------
7 files changed, 7 insertions(+), 60 deletions(-)
diff --git a/be/src/agent/workload_group_listener.cpp
b/be/src/agent/workload_group_listener.cpp
index f0f57869f25..50a8b8e509c 100644
--- a/be/src/agent/workload_group_listener.cpp
+++ b/be/src/agent/workload_group_listener.cpp
@@ -54,10 +54,6 @@ void WorkloadGroupListener::handle_topic_info(const
std::vector<TopicInfo>& topi
auto wg =
_exec_env->workload_group_mgr()->get_or_create_workload_group(workload_group_info);
- // 3 set cpu soft hard limit switch
- _exec_env->workload_group_mgr()->_enable_cpu_hard_limit.store(
- workload_group_info.enable_cpu_hard_limit);
-
// 4 create and update task scheduler
wg->upsert_task_scheduler(&workload_group_info, _exec_env);
@@ -65,8 +61,7 @@ void WorkloadGroupListener::handle_topic_info(const
std::vector<TopicInfo>& topi
wg->upsert_scan_io_throttle(&workload_group_info);
LOG(INFO) << "[topic_publish_wg]update workload group finish, wg info="
- << wg->debug_string() << ", enable_cpu_hard_limit="
- << (_exec_env->workload_group_mgr()->enable_cpu_hard_limit()
? "true" : "false")
+ << wg->debug_string()
<< ", cgroup cpu_shares=" <<
workload_group_info.cgroup_cpu_shares
<< ", cgroup cpu_hard_limit=" <<
workload_group_info.cgroup_cpu_hard_limit
<< ", cgroup home path=" << config::doris_cgroup_cpu_path
diff --git a/be/src/runtime/workload_group/workload_group.cpp
b/be/src/runtime/workload_group/workload_group.cpp
index b07548f9970..d2018b0b28e 100644
--- a/be/src/runtime/workload_group/workload_group.cpp
+++ b/be/src/runtime/workload_group/workload_group.cpp
@@ -363,12 +363,6 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info(
enable_memory_overcommit =
tworkload_group_info.enable_memory_overcommit;
}
- // 8 cpu soft limit or hard limit
- bool enable_cpu_hard_limit = false;
- if (tworkload_group_info.__isset.enable_cpu_hard_limit) {
- enable_cpu_hard_limit = tworkload_group_info.enable_cpu_hard_limit;
- }
-
// 9 scan thread num
int scan_thread_num = config::doris_scanner_thread_pool_thread_num;
if (tworkload_group_info.__isset.scan_thread_num &&
tworkload_group_info.scan_thread_num > 0) {
@@ -422,7 +416,6 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info(
.enable_memory_overcommit = enable_memory_overcommit,
.version = version,
.cpu_hard_limit = cpu_hard_limit,
- .enable_cpu_hard_limit = enable_cpu_hard_limit,
.scan_thread_num = scan_thread_num,
.max_remote_scan_thread_num = max_remote_scan_thread_num,
.min_remote_scan_thread_num = min_remote_scan_thread_num,
@@ -437,7 +430,6 @@ void
WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e
std::string tg_name = tg_info->name;
int cpu_hard_limit = tg_info->cpu_hard_limit;
uint64_t cpu_shares = tg_info->cpu_share;
- bool enable_cpu_hard_limit = tg_info->enable_cpu_hard_limit;
int scan_thread_num = tg_info->scan_thread_num;
int max_remote_scan_thread_num = tg_info->max_remote_scan_thread_num;
int min_remote_scan_thread_num = tg_info->min_remote_scan_thread_num;
@@ -551,21 +543,8 @@ void
WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e
// step 6: update cgroup cpu if needed
if (_cgroup_cpu_ctl) {
- if (enable_cpu_hard_limit) {
- if (cpu_hard_limit > 0) {
- _cgroup_cpu_ctl->update_cpu_hard_limit(cpu_hard_limit);
- _cgroup_cpu_ctl->update_cpu_soft_limit(
- CgroupCpuCtl::cpu_soft_limit_default_value());
- } else {
- LOG(INFO) << "[upsert wg thread pool] enable cpu hard limit
but value is "
- "illegal: "
- << cpu_hard_limit << ", gid=" << tg_id;
- }
- } else {
- _cgroup_cpu_ctl->update_cpu_soft_limit(cpu_shares);
- _cgroup_cpu_ctl->update_cpu_hard_limit(
- CPU_HARD_LIMIT_DEFAULT_VALUE); // disable cpu hard limit
- }
+ _cgroup_cpu_ctl->update_cpu_hard_limit(cpu_hard_limit);
+ _cgroup_cpu_ctl->update_cpu_soft_limit(cpu_shares);
_cgroup_cpu_ctl->get_cgroup_cpu_info(&(tg_info->cgroup_cpu_shares),
&(tg_info->cgroup_cpu_hard_limit));
}
diff --git a/be/src/runtime/workload_group/workload_group.h
b/be/src/runtime/workload_group/workload_group.h
index b4d1406a94d..97d90300a81 100644
--- a/be/src/runtime/workload_group/workload_group.h
+++ b/be/src/runtime/workload_group/workload_group.h
@@ -264,7 +264,6 @@ struct WorkloadGroupInfo {
const bool enable_memory_overcommit = false;
const int64_t version = 0;
const int cpu_hard_limit = 0;
- const bool enable_cpu_hard_limit = false;
const int scan_thread_num = 0;
const int max_remote_scan_thread_num = 0;
const int min_remote_scan_thread_num = 0;
diff --git a/be/src/runtime/workload_group/workload_group_manager.h
b/be/src/runtime/workload_group/workload_group_manager.h
index db2444e0bc1..3cd00ebdb44 100644
--- a/be/src/runtime/workload_group/workload_group_manager.h
+++ b/be/src/runtime/workload_group/workload_group_manager.h
@@ -54,12 +54,6 @@ public:
void stop();
- std::atomic<bool> _enable_cpu_hard_limit = false;
-
- bool enable_cpu_soft_limit() { return !_enable_cpu_hard_limit.load(); }
-
- bool enable_cpu_hard_limit() { return _enable_cpu_hard_limit.load(); }
-
void refresh_wg_weighted_memory_limit();
void get_wg_resource_usage(vectorized::Block* block);
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 0ff3300c6fa..3fd035fdcb9 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1920,6 +1920,9 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_workload_group = true;
+ @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
+ public static boolean enable_wg_memory_sum_limit = true;
+
@ConfField(mutable = true)
public static boolean enable_query_queue = true;
@@ -1930,9 +1933,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static long query_queue_update_interval_ms = 5000;
- @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
- public static boolean enable_cpu_hard_limit = false;
-
@ConfField(mutable = true, description = {
"当BE内存用量大于该值时，查询会进入排队逻辑，默认值为-1，代表该值不生效。取值范围0~1的小数",
"When be memory usage bigger than this value, query could queue, "
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
index 408895dd12f..4fff8388dec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
@@ -19,7 +19,6 @@ package org.apache.doris.resource.workloadgroup;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.Pair;
@@ -593,14 +592,6 @@ public class WorkloadGroup implements Writable,
GsonPostProcessable {
if (memOvercommitStr != null) {
tWorkloadGroupInfo.setEnableMemoryOvercommit(Boolean.valueOf(memOvercommitStr));
}
- // enable_cpu_hard_limit = true, using cpu hard limit
- // enable_cpu_hard_limit = false, using cpu soft limit
- tWorkloadGroupInfo.setEnableCpuHardLimit(Config.enable_cpu_hard_limit);
-
- if (Config.enable_cpu_hard_limit && cpuHardLimit <= 0) {
- LOG.warn("enable_cpu_hard_limit=true but cpuHardLimit value not
illegal,"
- + "id=" + id + ",name=" + name);
- }
String scanThreadNumStr = properties.get(SCAN_THREAD_NUM);
if (scanThreadNumStr != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
index 442b1bd783c..1e81c91bcea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
@@ -390,7 +390,6 @@ public class WorkloadGroupMgr extends MasterDaemon
implements Writable, GsonPost
for (String newWgOneTag : newWgTagSet) {
double sumOfAllMemLimit = 0;
- int sumOfAllCpuHardLimit = 0;
// 1 get sum value of all wg which has same tag without current wg
for (Map.Entry<Long, WorkloadGroup> entry :
idToWorkloadGroup.entrySet()) {
@@ -409,9 +408,6 @@ public class WorkloadGroupMgr extends MasterDaemon
implements Writable, GsonPost
continue;
}
- if (wg.getCpuHardLimitWhenCalSum() > 0) {
- sumOfAllCpuHardLimit += wg.getCpuHardLimitWhenCalSum();
- }
if (wg.getMemoryLimitPercentWhenCalSum() > 0) {
sumOfAllMemLimit += wg.getMemoryLimitPercentWhenCalSum();
}
@@ -419,22 +415,15 @@ public class WorkloadGroupMgr extends MasterDaemon
implements Writable, GsonPost
// 2 sum current wg value
sumOfAllMemLimit += newWg.getMemoryLimitPercentWhenCalSum();
- sumOfAllCpuHardLimit += newWg.getCpuHardLimitWhenCalSum();
// 3 check total sum
- if (sumOfAllMemLimit > 100.0 + 1e-6) {
+ if (Config.enable_wg_memory_sum_limit && sumOfAllMemLimit > 100.0
+ 1e-6) {
throw new DdlException(
"The sum of all workload group " +
WorkloadGroup.MEMORY_LIMIT + " within tag " + (
newWgTag.isPresent() ? newWgTag.get() : "")
+ " cannot be greater than 100.0%. current sum
val:" + sumOfAllMemLimit);
}
- if (sumOfAllCpuHardLimit > 100) {
- throw new DdlException(
- "sum of all workload group " +
WorkloadGroup.CPU_HARD_LIMIT + " within tag " + (
- newWgTag.isPresent()
- ? newWgTag.get() : "") + " can not be
greater than 100% ");
- }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
