This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e84a77bfd0b [bugfix](scanmemory)modify scan memory profile (#41435)
e84a77bfd0b is described below
commit e84a77bfd0b3d429b53f30236750a8036fccbe48
Author: yiguolei <676222...@qq.com>
AuthorDate: Sat Sep 28 16:04:04 2024 +0800
[bugfix](scanmemory)modify scan memory profile (#41435)
1. memory usage in scan operator is empty.
2. peak memory usage is not in scan operator , it is in scanner.
---------
Co-authored-by: yiguolei <yiguo...@gmail.com>
---
be/src/pipeline/exec/operator.h | 4 ++--
be/src/pipeline/exec/scan_operator.cpp | 8 ++------
be/src/pipeline/exec/scan_operator.h | 1 -
be/src/vec/exec/scan/scanner_context.cpp | 3 +++
be/src/vec/exec/scan/scanner_context.h | 2 ++
5 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h
index 48f8a2d1836..04d8f0dc736 100644
--- a/be/src/pipeline/exec/operator.h
+++ b/be/src/pipeline/exec/operator.h
@@ -199,7 +199,7 @@ protected:
RuntimeProfile::Counter* _projection_timer = nullptr;
RuntimeProfile::Counter* _exec_timer = nullptr;
// Account for peak memory used by this node
- RuntimeProfile::Counter* _peak_memory_usage_counter = nullptr;
+ RuntimeProfile::HighWaterMarkCounter* _peak_memory_usage_counter = nullptr;
RuntimeProfile::Counter* _init_timer = nullptr;
RuntimeProfile::Counter* _open_timer = nullptr;
RuntimeProfile::Counter* _close_timer = nullptr;
@@ -371,7 +371,7 @@ protected:
RuntimeProfile::Counter* _wait_for_finish_dependency_timer = nullptr;
RuntimeProfile::Counter* _exec_timer = nullptr;
RuntimeProfile::Counter* _memory_used_counter = nullptr;
- RuntimeProfile::Counter* _peak_memory_usage_counter = nullptr;
+ RuntimeProfile::HighWaterMarkCounter* _peak_memory_usage_counter = nullptr;
std::shared_ptr<QueryStatistics> _query_statistics = nullptr;
};
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index 507039b1f5e..e46274398b5 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1052,17 +1052,13 @@ Status ScanLocalState<Derived>::_init_profile() {
_total_throughput_counter =
profile()->add_rate_counter("TotalReadThroughput",
_rows_read_counter);
_num_scanners = ADD_COUNTER(_runtime_profile, "NumScanners", TUnit::UNIT);
+ _scanner_peak_memory_usage = _peak_memory_usage_counter;
+ //_runtime_profile->AddHighWaterMarkCounter("PeakMemoryUsage",
TUnit::BYTES);
// 2. counters for scanners
_scanner_profile.reset(new RuntimeProfile("VScanner"));
profile()->add_child(_scanner_profile.get(), true, nullptr);
- _memory_usage_counter = ADD_LABEL_COUNTER_WITH_LEVEL(_scanner_profile,
"MemoryUsage", 1);
- _free_blocks_memory_usage =
- _scanner_profile->AddHighWaterMarkCounter("FreeBlocks",
TUnit::BYTES, "MemoryUsage", 1);
- _scanner_peak_memory_usage =
- _scanner_profile->AddHighWaterMarkCounter("PeakMemoryUsage",
TUnit::BYTES);
-
_newly_create_free_blocks_num =
ADD_COUNTER(_scanner_profile, "NewlyCreateFreeBlocksNum",
TUnit::UNIT);
_scale_up_scanners_counter = ADD_COUNTER(_scanner_profile,
"NumScaleUpScanners", TUnit::UNIT);
diff --git a/be/src/pipeline/exec/scan_operator.h
b/be/src/pipeline/exec/scan_operator.h
index fed1e4015d8..28dbd01280f 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -119,7 +119,6 @@ protected:
// time of filter output block from scanner
RuntimeProfile::Counter* _filter_timer = nullptr;
RuntimeProfile::Counter* _memory_usage_counter = nullptr;
- RuntimeProfile::HighWaterMarkCounter* _free_blocks_memory_usage = nullptr;
RuntimeProfile::Counter* _scale_up_scanners_counter = nullptr;
// rows read from the scanner (including those discarded by (pre)filters)
RuntimeProfile::Counter* _rows_read_counter = nullptr;
diff --git a/be/src/vec/exec/scan/scanner_context.cpp
b/be/src/vec/exec/scan/scanner_context.cpp
index 79bbcd94b8c..276da7e0972 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -133,6 +133,7 @@ Status ScannerContext::init() {
_scanner_wait_batch_timer = _local_state->_scanner_wait_batch_timer;
_scanner_ctx_sched_time = _local_state->_scanner_ctx_sched_time;
_scale_up_scanners_counter = _local_state->_scale_up_scanners_counter;
+ _scanner_memory_used_counter = _local_state->_memory_used_counter;
#ifndef BE_TEST
// 3. get thread token
@@ -172,6 +173,7 @@ vectorized::BlockUPtr ScannerContext::get_free_block(bool
force) {
if (_free_blocks.try_dequeue(block)) {
DCHECK(block->mem_reuse());
_block_memory_usage -= block->allocated_bytes();
+ _scanner_memory_used_counter->set(_block_memory_usage);
// A free block is reused, so the memory usage should be decreased
// The caller of get_free_block will increase the memory usage
update_peak_memory_usage(-block->allocated_bytes());
@@ -187,6 +189,7 @@ void
ScannerContext::return_free_block(vectorized::BlockUPtr block) {
if (block->mem_reuse() && _block_memory_usage < _max_bytes_in_queue) {
size_t block_size_to_reuse = block->allocated_bytes();
_block_memory_usage += block_size_to_reuse;
+ _scanner_memory_used_counter->set(_block_memory_usage);
block->clear_column_data();
if (_free_blocks.enqueue(std::move(block))) {
update_peak_memory_usage(block_size_to_reuse);
diff --git a/be/src/vec/exec/scan/scanner_context.h
b/be/src/vec/exec/scan/scanner_context.h
index 1ebad17d418..6d042bc028f 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -225,6 +225,8 @@ protected:
const int _num_parallel_instances;
std::shared_ptr<RuntimeProfile> _scanner_profile;
RuntimeProfile::Counter* _scanner_sched_counter = nullptr;
+ // This counter refers to scan operator's local state
+ RuntimeProfile::Counter* _scanner_memory_used_counter = nullptr;
RuntimeProfile::Counter* _newly_create_free_blocks_num = nullptr;
RuntimeProfile::Counter* _scanner_wait_batch_timer = nullptr;
RuntimeProfile::Counter* _scanner_ctx_sched_time = nullptr;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
