This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0e651365ca [profile](scanner) add per scanner running time profile
(#15321)
0e651365ca is described below
commit 0e651365caf9ac8b90d77e7e77b3dd4b25926ae9
Author: yiguolei <[email protected]>
AuthorDate: Mon Dec 26 08:55:07 2022 +0800
[profile](scanner) add per scanner running time profile (#15321)
* [profile](scanner) add per scanner running time profile
Co-authored-by: yiguolei <[email protected]>
---
be/src/vec/exec/scan/scanner_context.cpp | 34 +++++++++++++++++++++++++++++++-
be/src/vec/exec/scan/scanner_context.h | 2 ++
be/src/vec/exec/scan/vscan_node.h | 1 -
be/src/vec/exec/scan/vscanner.cpp | 2 +-
be/src/vec/exec/scan/vscanner.h | 5 +++++
5 files changed, 41 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/exec/scan/scanner_context.cpp
b/be/src/vec/exec/scan/scanner_context.cpp
index 15b8131448..1940758907 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -74,6 +74,8 @@ Status ScannerContext::init() {
COUNTER_SET(_parent->_pre_alloc_free_blocks_num,
(int64_t)pre_alloc_block_count);
COUNTER_SET(_parent->_max_scanner_thread_num, (int64_t)_max_thread_num);
+ _parent->_runtime_profile->add_info_string("UseSpecificThreadToken",
+ thread_token == nullptr ?
"False" : "True");
return Status::OK();
}
@@ -174,6 +176,35 @@ bool ScannerContext::set_status_on_error(const Status&
status) {
Status ScannerContext::_close_and_clear_scanners() {
std::unique_lock<std::mutex> l(_scanners_lock);
+ if (_state->enable_profile()) {
+ std::stringstream scanner_statistics;
+ std::stringstream scanner_rows_read;
+ scanner_statistics << "[";
+ scanner_rows_read << "[";
+ for (auto finished_scanner_time : _finished_scanner_runtime) {
+ scanner_statistics << PrettyPrinter::print(finished_scanner_time,
TUnit::TIME_NS)
+ << ", ";
+ }
+ for (auto finished_scanner_rows : _finished_scanner_rows_read) {
+ scanner_rows_read << PrettyPrinter::print(finished_scanner_rows,
TUnit::UNIT) << ", ";
+ }
+ // Only unfinished scanners here
+ for (auto scanner : _scanners) {
+ // Scanners are in ObjPool in ScanNode,
+ // so no need to delete them here.
+ // Add per scanner running time before close them
+ scanner_statistics <<
PrettyPrinter::print(scanner->get_time_cost_ns(), TUnit::TIME_NS)
+ << ", ";
+ scanner_rows_read <<
PrettyPrinter::print(scanner->get_rows_read(), TUnit::UNIT)
+ << ", ";
+ }
+ scanner_statistics << "]";
+ scanner_rows_read << "]";
+ _parent->_scanner_profile->add_info_string("PerScannerRunningTime",
+ scanner_statistics.str());
+ _parent->_scanner_profile->add_info_string("PerScannerRowsRead",
scanner_rows_read.str());
+ }
+ // Only unfinished scanners here
for (auto scanner : _scanners) {
scanner->close(_state);
// Scanners are in ObjPool in ScanNode,
@@ -194,7 +225,6 @@ void ScannerContext::clear_and_join() {
break;
}
} while (false);
-
// Must wait all running scanners stop running.
// So that we can make sure to close all scanners.
_close_and_clear_scanners();
@@ -288,6 +318,8 @@ void
ScannerContext::get_next_batch_of_scanners(std::list<VScanner*>* current_ru
auto scanner = _scanners.front();
_scanners.pop_front();
if (scanner->need_to_close()) {
+
_finished_scanner_runtime.push_back(scanner->get_time_cost_ns());
+
_finished_scanner_rows_read.push_back(scanner->get_rows_read());
scanner->close(_state);
} else {
current_run->push_back(scanner);
diff --git a/be/src/vec/exec/scan/scanner_context.h
b/be/src/vec/exec/scan/scanner_context.h
index 41bd0b4d14..2a2d0ffcf1 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -221,6 +221,8 @@ protected:
// Not need to protect by lock, because only one scheduler thread will
access to it.
std::mutex _scanners_lock;
std::list<VScanner*> _scanners;
+ std::vector<int64_t> _finished_scanner_runtime;
+ std::vector<int64_t> _finished_scanner_rows_read;
int64_t _num_ctx_scheduling = 0;
int64_t _num_scanner_scheduling = 0;
diff --git a/be/src/vec/exec/scan/vscan_node.h
b/be/src/vec/exec/scan/vscan_node.h
index 3a91d91b91..6279330bee 100644
--- a/be/src/vec/exec/scan/vscan_node.h
+++ b/be/src/vec/exec/scan/vscan_node.h
@@ -259,7 +259,6 @@ protected:
RuntimeProfile::Counter* _scanner_ctx_sched_counter = nullptr;
RuntimeProfile::Counter* _scanner_wait_batch_timer = nullptr;
RuntimeProfile::Counter* _scanner_wait_worker_timer = nullptr;
-
// Num of pre allocated free blocks
RuntimeProfile::Counter* _pre_alloc_free_blocks_num = nullptr;
// Num of newly created free blocks when running query
diff --git a/be/src/vec/exec/scan/vscanner.cpp
b/be/src/vec/exec/scan/vscanner.cpp
index e9518fbe91..2009478fb6 100644
--- a/be/src/vec/exec/scan/vscanner.cpp
+++ b/be/src/vec/exec/scan/vscanner.cpp
@@ -35,7 +35,7 @@ VScanner::VScanner(RuntimeState* state, VScanNode* parent,
int64_t limit)
Status VScanner::get_block(RuntimeState* state, Block* block, bool* eof) {
// only empty block should be here
DCHECK(block->rows() == 0);
-
+ SCOPED_RAW_TIMER(&_per_scanner_timer);
int64_t raw_rows_threshold = raw_rows_read() +
config::doris_scanner_row_num;
if (!block->mem_reuse()) {
for (const auto slot_desc : _output_tuple_desc->slots()) {
diff --git a/be/src/vec/exec/scan/vscanner.h b/be/src/vec/exec/scan/vscanner.h
index 07ddf65298..e8d83f73e1 100644
--- a/be/src/vec/exec/scan/vscanner.h
+++ b/be/src/vec/exec/scan/vscanner.h
@@ -64,6 +64,10 @@ protected:
public:
VScanNode* get_parent() { return _parent; }
+ int64_t get_time_cost_ns() const { return _per_scanner_timer; }
+
+ int64_t get_rows_read() const { return _num_rows_read; }
+
Status try_append_late_arrival_runtime_filter();
// Call start_wait_worker_timer() when submit the scanner to the thread
pool.
@@ -185,6 +189,7 @@ protected:
bool _is_counted_down = false;
ScannerCounter _counter;
+ int64_t _per_scanner_timer = 0;
};
} // namespace doris::vectorized
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
