This is an automated email from the ASF dual-hosted git repository.
jacktengg pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/spill_and_reserve by this push:
new 36b9bbd9d37 remove stale code
36b9bbd9d37 is described below
commit 36b9bbd9d3724a6270ceed014fc73c6025f98b39
Author: jacktengg <[email protected]>
AuthorDate: Fri Dec 6 16:47:48 2024 +0800
remove stale code
---
be/src/pipeline/pipeline_task.cpp | 2 -
be/src/runtime/query_context.h | 43 ----------------------
be/src/runtime/workload_group/workload_group.h | 3 --
.../workload_group/workload_group_manager.cpp | 2 +-
4 files changed, 1 insertion(+), 49 deletions(-)
diff --git a/be/src/pipeline/pipeline_task.cpp
b/be/src/pipeline/pipeline_task.cpp
index a36e37fdeeb..0ae87287d96 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -395,8 +395,6 @@ Status PipelineTask::execute(bool* eos) {
Status::Error<INTERNAL_ERROR>("fault_inject pipeline_task
executing failed");
return status;
});
- // Every loop should check if memory is not enough.
- // _state->get_query_ctx()->update_low_memory_mode();
if (_pending_block) [[unlikely]] {
LOG(INFO) << "Query: " << print_id(query_id)
diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h
index f4bc5e482d4..b011e8d850f 100644
--- a/be/src/runtime/query_context.h
+++ b/be/src/runtime/query_context.h
@@ -306,49 +306,6 @@ public:
return _using_brpc_stubs;
}
- // Query will run in low memory mode when
- // 1. the query is enable spill and wg's low water mark reached, if not
release buffer, it will trigger spill disk, it is very expensive.
- // 2. the query is not enable spill, but wg's high water mark reached, if
not release buffer, the query will be cancelled.
- // 3. the process reached soft mem_limit, if not release these, if not
release buffer, the query will be cancelled.
- // 4. If the query reserve memory failed.
- // Under low memory mode, the query should release some buffers such as
scan operator block queue, union operator queue, exchange buffer size,
streaming agg
- void update_low_memory_mode() {
- if (_low_memory_mode) {
- return;
- }
-
- // If less than 100MB left, then it is low memory mode
- if (doris::GlobalMemoryArbitrator::is_exceed_soft_mem_limit(100 * 1024
* 1024)) {
- _low_memory_mode = true;
- LOG(INFO) << "Query " << print_id(_query_id)
- << " goes to low memory mode due to exceed process soft
memory limit";
- return;
- }
-
- if (_workload_group) {
- bool is_low_watermark = false;
- bool is_high_watermark = false;
- _workload_group->check_mem_used(&is_low_watermark,
&is_high_watermark);
- // If the wg is not enable hard limit, this will also take effect
to lower down the memory usage.
- if (is_high_watermark) {
- LOG(INFO)
- << "Query " << print_id(_query_id)
- << " goes to low memory mode due to workload group
high water mark reached";
- _low_memory_mode = true;
- return;
- }
-
- if (is_low_watermark &&
- (_query_options.__isset.enable_spill &&
_query_options.enable_spill)) {
- LOG(INFO) << "Query " << print_id(_query_id)
- << " goes to low memory mode due to workload group
low water mark "
- "reached and the query enable spill";
- _low_memory_mode = true;
- return;
- }
- }
- }
-
void set_low_memory_mode() { _low_memory_mode = true; }
bool low_memory_mode() { return _low_memory_mode; }
diff --git a/be/src/runtime/workload_group/workload_group.h
b/be/src/runtime/workload_group/workload_group.h
index 9911782b09b..1d617b22bfe 100644
--- a/be/src/runtime/workload_group/workload_group.h
+++ b/be/src/runtime/workload_group/workload_group.h
@@ -108,9 +108,6 @@ public:
}
void set_weighted_memory_ratio(double ratio);
- int spill_threshold_high_water_mark() const {
- return _memory_high_watermark.load(std::memory_order_relaxed);
- }
int total_query_slot_count() const {
return _total_query_slot_count.load(std::memory_order_relaxed);
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp
b/be/src/runtime/workload_group/workload_group_manager.cpp
index 02de6bd5f0e..558765a04b7 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -761,7 +761,7 @@ void
WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool enable_ha
bool is_high_watermark = false;
wg->check_mem_used(&is_low_watermark, &is_high_watermark);
int64_t wg_high_water_mark_limit =
- (int64_t)(wg_mem_limit * wg->spill_threshold_high_water_mark() *
1.0 / 100);
+ (int64_t)(wg_mem_limit * wg->memory_high_watermark() * 1.0 / 100);
int64_t memtable_usage = wg->write_buffer_size();
int64_t wg_high_water_mark_except_load = wg_high_water_mark_limit;
if (memtable_usage > wg->write_buffer_limit()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
