This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 631d1f43cf7 [Chore](profile) add some profile on ReaderInit #45556
(#47427)
631d1f43cf7 is described below
commit 631d1f43cf7ea665dc73dc3b4ff74182a55b01f7
Author: Pxl <[email protected]>
AuthorDate: Sun Jan 26 10:51:08 2025 +0800
[Chore](profile) add some profile on ReaderInit #45556 (#47427)
pick from #45556
---
be/src/olap/olap_common.h | 24 +++++++++
be/src/olap/rowset/beta_rowset_reader.cpp | 19 +++++---
be/src/olap/rowset/segment_v2/segment.cpp | 24 ++++++---
be/src/olap/rowset/segment_v2/segment.h | 6 ++-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 10 ++--
be/src/olap/tablet.cpp | 4 +-
be/src/olap/tablet_reader.cpp | 7 +++
be/src/pipeline/exec/olap_scan_operator.cpp | 41 ++++++++++++++++
be/src/pipeline/exec/olap_scan_operator.h | 27 ++++++++++
be/src/vec/exec/scan/new_olap_scanner.cpp | 43 ++++++++++++++++
be/src/vec/olap/block_reader.cpp | 57 +++++++++++++---------
be/src/vec/olap/vgeneric_iterators.cpp | 5 +-
12 files changed, 220 insertions(+), 47 deletions(-)
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 02e1b542680..77d5228de41 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -390,6 +390,30 @@ struct OlapReaderStatistics {
int64_t collect_iterator_merge_next_timer = 0;
int64_t collect_iterator_normal_next_timer = 0;
int64_t delete_bitmap_get_agg_ns = 0;
+
+ int64_t tablet_reader_init_timer_ns = 0;
+ int64_t tablet_reader_capture_rs_readers_timer_ns = 0;
+ int64_t tablet_reader_init_return_columns_timer_ns = 0;
+ int64_t tablet_reader_init_keys_param_timer_ns = 0;
+ int64_t tablet_reader_init_orderby_keys_param_timer_ns = 0;
+ int64_t tablet_reader_init_conditions_param_timer_ns = 0;
+ int64_t tablet_reader_init_delete_condition_param_timer_ns = 0;
+ int64_t block_reader_vcollect_iter_init_timer_ns = 0;
+ int64_t block_reader_rs_readers_init_timer_ns = 0;
+ int64_t block_reader_build_heap_init_timer_ns = 0;
+
+ int64_t rowset_reader_get_segment_iterators_timer_ns = 0;
+ int64_t rowset_reader_create_iterators_timer_ns = 0;
+ int64_t rowset_reader_init_iterators_timer_ns = 0;
+ int64_t rowset_reader_load_segments_timer_ns = 0;
+
+ int64_t segment_iterator_init_timer_ns = 0;
+ int64_t segment_iterator_init_return_column_iterators_timer_ns = 0;
+ int64_t segment_iterator_init_bitmap_index_iterators_timer_ns = 0;
+ int64_t segment_iterator_init_inverted_index_iterators_timer_ns = 0;
+
+ int64_t segment_create_column_readers_timer_ns = 0;
+ int64_t segment_load_index_timer_ns = 0;
};
using ColumnId = uint32_t;
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 13bf1f6d8c8..e13551e7322 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -78,7 +78,6 @@ bool BetaRowsetReader::update_profile(RuntimeProfile*
profile) {
Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext*
read_context,
std::vector<RowwiseIteratorUPtr>* out_iters,
bool use_cache) {
- RETURN_IF_ERROR(_rowset->load());
_read_context = read_context;
// The segment iterator is created with its own statistics,
// and the member variable '_stats' is initialized by
'_stats(&owned_stats)'.
@@ -92,6 +91,9 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
if (_read_context->stats != nullptr) {
_stats = _read_context->stats;
}
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_get_segment_iterators_timer_ns);
+
+ RETURN_IF_ERROR(_rowset->load());
// convert RowsetReaderContext to StorageReadOptions
_read_options.block_row_max = read_context->batch_size;
@@ -207,8 +209,11 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
// load segments
bool should_use_cache = use_cache || _read_context->reader_type ==
ReaderType::READER_QUERY;
SegmentCacheHandle segment_cache_handle;
- RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(_rowset,
&segment_cache_handle,
-
should_use_cache));
+ {
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_load_segments_timer_ns);
+ RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(_rowset,
&segment_cache_handle,
+
should_use_cache));
+ }
// create iterator for each segment
auto& segments = segment_cache_handle.get_segments();
@@ -234,6 +239,7 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
const bool use_lazy_init_iterators =
!is_merge_iterator && _read_context->reader_type ==
ReaderType::READER_QUERY;
for (int i = seg_start; i < seg_end; i++) {
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_create_iterators_timer_ns);
auto& seg_ptr = segments[i];
std::unique_ptr<RowwiseIterator> iter;
@@ -298,6 +304,8 @@ Status BetaRowsetReader::_init_iterator() {
std::vector<RowwiseIteratorUPtr> iterators;
RETURN_IF_ERROR(get_segment_iterators(_read_context, &iterators));
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_init_iterators_timer_ns);
+
if (_read_context->merged_rows == nullptr) {
_read_context->merged_rows = &_merged_rows;
}
@@ -333,8 +341,8 @@ Status BetaRowsetReader::_init_iterator() {
}
Status BetaRowsetReader::next_block(vectorized::Block* block) {
- SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RETURN_IF_ERROR(_init_iterator_once());
+ SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
if (_empty) {
return Status::Error<END_OF_FILE>("BetaRowsetReader is empty");
}
@@ -362,9 +370,8 @@ Status BetaRowsetReader::next_block(vectorized::Block*
block) {
}
Status BetaRowsetReader::next_block_view(vectorized::BlockView* block_view) {
- SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RETURN_IF_ERROR(_init_iterator_once());
-
+ SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RuntimeState* runtime_state = nullptr;
if (_read_context != nullptr) {
runtime_state = _read_context->runtime_state;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index dd77a14e150..9126a081d54 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -147,7 +147,7 @@ Status Segment::_open_inverted_index() {
Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions&
read_options,
std::unique_ptr<RowwiseIterator>* iter) {
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
read_options.stats->total_segment_number++;
// trying to prune the current segment by segment-level zone map
@@ -208,7 +208,11 @@ Status Segment::new_iterator(SchemaSPtr schema, const
StorageReadOptions& read_o
}
}
- RETURN_IF_ERROR(load_index());
+ {
+ SCOPED_RAW_TIMER(&read_options.stats->segment_load_index_timer_ns);
+ RETURN_IF_ERROR(load_index());
+ }
+
if (read_options.delete_condition_predicates->num_of_column_predicate() ==
0 &&
read_options.push_down_agg_type_opt != TPushAggOp::NONE &&
read_options.push_down_agg_type_opt != TPushAggOp::COUNT_ON_INDEX) {
@@ -418,7 +422,8 @@ vectorized::DataTypePtr Segment::get_data_type_of(const
ColumnIdentifier& identi
return nullptr;
}
-Status Segment::_create_column_readers_once() {
+Status Segment::_create_column_readers_once(OlapReaderStatistics* stats) {
+ SCOPED_RAW_TIMER(&stats->segment_create_column_readers_timer_ns);
return _create_column_readers_once_call.call([&] {
DCHECK(_footer_pb);
Defer defer([&]() { _footer_pb.reset(); });
@@ -642,7 +647,7 @@ Status Segment::new_column_iterator_with_path(const
TabletColumn& tablet_column,
Status Segment::new_column_iterator(const TabletColumn& tablet_column,
std::unique_ptr<ColumnIterator>* iter,
const StorageReadOptions* opt) {
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(opt->stats));
// init column iterator by path info
if (tablet_column.has_path_info() || tablet_column.is_variant_type()) {
@@ -670,8 +675,9 @@ Status Segment::new_column_iterator(const TabletColumn&
tablet_column,
return Status::OK();
}
-Status Segment::new_column_iterator(int32_t unique_id,
std::unique_ptr<ColumnIterator>* iter) {
- RETURN_IF_ERROR(_create_column_readers_once());
+Status Segment::new_column_iterator(int32_t unique_id, const
StorageReadOptions* opt,
+ std::unique_ptr<ColumnIterator>* iter) {
+ RETURN_IF_ERROR(_create_column_readers_once(opt->stats));
ColumnIterator* it;
RETURN_IF_ERROR(_column_readers.at(unique_id)->new_iterator(&it));
iter->reset(it);
@@ -699,8 +705,9 @@ ColumnReader* Segment::_get_column_reader(const
TabletColumn& col) {
}
Status Segment::new_bitmap_index_iterator(const TabletColumn& tablet_column,
+ const StorageReadOptions&
read_options,
std::unique_ptr<BitmapIndexIterator>* iter) {
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
ColumnReader* reader = _get_column_reader(tablet_column);
if (reader != nullptr && reader->has_bitmap_index()) {
BitmapIndexIterator* it;
@@ -715,7 +722,7 @@ Status Segment::new_inverted_index_iterator(const
TabletColumn& tablet_column,
const TabletIndex* index_meta,
const StorageReadOptions&
read_options,
std::unique_ptr<InvertedIndexIterator>* iter) {
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
ColumnReader* reader = _get_column_reader(tablet_column);
if (reader != nullptr && index_meta) {
if (_inverted_index_file_reader == nullptr) {
@@ -872,6 +879,7 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema&
schema, SlotDescripto
OlapReaderStatistics& stats,
std::unique_ptr<ColumnIterator>&
iterator_hint) {
StorageReadOptions storage_read_opt;
+ storage_read_opt.stats = &stats;
storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
segment_v2::ColumnIteratorOptions opt {
.use_page_cache = !config::disable_storage_page_cache,
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index 442dab3393e..b8e23629aca 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -104,9 +104,11 @@ public:
std::unique_ptr<ColumnIterator>* iter,
const StorageReadOptions* opt);
- Status new_column_iterator(int32_t unique_id,
std::unique_ptr<ColumnIterator>* iter);
+ Status new_column_iterator(int32_t unique_id, const StorageReadOptions*
opt,
+ std::unique_ptr<ColumnIterator>* iter);
Status new_bitmap_index_iterator(const TabletColumn& tablet_column,
+ const StorageReadOptions& read_options,
std::unique_ptr<BitmapIndexIterator>*
iter);
Status new_inverted_index_iterator(const TabletColumn& tablet_column,
@@ -220,7 +222,7 @@ private:
Status _open_inverted_index();
- Status _create_column_readers_once();
+ Status _create_column_readers_once(OlapReaderStatistics* stats);
private:
friend class SegmentIterator;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index e9812da465a..52e0ecd5d20 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -280,9 +280,10 @@ Status SegmentIterator::_init_impl(const
StorageReadOptions& opts) {
if (_inited) {
return Status::OK();
}
+ _opts = opts;
+ SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_timer_ns);
_inited = true;
_file_reader = _segment->_file_reader;
- _opts = opts;
_col_predicates.clear();
for (const auto& predicate : opts.column_predicates) {
@@ -1008,6 +1009,7 @@ bool
SegmentIterator::_check_all_conditions_passed_inverted_index_for_column(Col
}
Status SegmentIterator::_init_return_column_iterators() {
+
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_return_column_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
@@ -1050,19 +1052,21 @@ Status SegmentIterator::_init_return_column_iterators()
{
}
Status SegmentIterator::_init_bitmap_index_iterators() {
+
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_bitmap_index_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
for (auto cid : _schema->column_ids()) {
if (_bitmap_index_iterators[cid] == nullptr) {
-
RETURN_IF_ERROR(_segment->new_bitmap_index_iterator(_opts.tablet_schema->column(cid),
-
&_bitmap_index_iterators[cid]));
+ RETURN_IF_ERROR(_segment->new_bitmap_index_iterator(
+ _opts.tablet_schema->column(cid), _opts,
&_bitmap_index_iterators[cid]));
}
}
return Status::OK();
}
Status SegmentIterator::_init_inverted_index_iterators() {
+
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_inverted_index_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 948d8b4b484..bb6d8401a98 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2725,7 +2725,9 @@ Status Tablet::_get_segment_column_iterator(
rowset->rowset_id().to_string(),
segid));
}
segment_v2::SegmentSharedPtr segment = *it;
- RETURN_IF_ERROR(segment->new_column_iterator(target_column,
column_iterator, nullptr));
+ StorageReadOptions opts;
+ opts.stats = stats;
+ RETURN_IF_ERROR(segment->new_column_iterator(target_column,
column_iterator, &opts));
segment_v2::ColumnIteratorOptions opt {
.use_page_cache = !config::disable_storage_page_cache,
.file_reader = segment->file_reader().get(),
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 17e23fb5165..51b2384bc90 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -120,6 +120,7 @@ TabletReader::~TabletReader() {
}
Status TabletReader::init(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns);
_predicate_arena.reset(new vectorized::Arena());
Status res = _init_params(read_params);
@@ -159,6 +160,7 @@ bool TabletReader::_optimize_for_single_rowset(
}
Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_capture_rs_readers_timer_ns);
if (read_params.rs_splits.empty()) {
return Status::InternalError("fail to acquire data sources. tablet={}",
_tablet->tablet_id());
@@ -330,6 +332,7 @@ Status TabletReader::_init_params(const ReaderParams&
read_params) {
}
Status TabletReader::_init_return_columns(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns);
if (read_params.reader_type == ReaderType::READER_QUERY) {
_return_columns = read_params.return_columns;
_tablet_columns_convert_to_null_set =
read_params.tablet_columns_convert_to_null_set;
@@ -386,6 +389,7 @@ Status TabletReader::_init_return_columns(const
ReaderParams& read_params) {
}
Status TabletReader::_init_keys_param(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_keys_param_timer_ns);
if (read_params.start_key.empty()) {
return Status::OK();
}
@@ -460,6 +464,7 @@ Status TabletReader::_init_keys_param(const ReaderParams&
read_params) {
}
Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params)
{
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_orderby_keys_param_timer_ns);
// UNIQUE_KEYS will compare all keys as before
if (_tablet_schema->keys_type() == DUP_KEYS ||
(_tablet_schema->keys_type() == UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write())) {
@@ -486,6 +491,7 @@ Status TabletReader::_init_orderby_keys_param(const
ReaderParams& read_params) {
}
Status TabletReader::_init_conditions_param(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_conditions_param_timer_ns);
for (auto& condition : read_params.conditions) {
TCondition tmp_cond = condition;
RETURN_IF_ERROR(_tablet_schema->have_column(tmp_cond.column_name));
@@ -612,6 +618,7 @@ ColumnPredicate* TabletReader::_parse_to_predicate(const
FunctionFilter& functio
}
Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
+
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_delete_condition_param_timer_ns);
// If it's cumu and not allow do delete when cumu
if (read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION ||
(read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index b32f9e0457e..aa7413b4a16 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -149,6 +149,47 @@ Status OlapScanLocalState::_init_profile() {
_tablet_counter = ADD_COUNTER(_runtime_profile, "TabletNum", TUnit::UNIT);
_key_range_counter = ADD_COUNTER(_runtime_profile, "KeyRangesNum",
TUnit::UNIT);
_runtime_filter_info = ADD_LABEL_COUNTER_WITH_LEVEL(_runtime_profile,
"RuntimeFilterInfo", 1);
+
+ _tablet_reader_init_timer = ADD_TIMER(_scanner_profile,
"TabletReaderInitTimer");
+ _tablet_reader_capture_rs_readers_timer =
+ ADD_TIMER(_scanner_profile, "TabletReaderCaptureRsReadersTimer");
+ _tablet_reader_init_return_columns_timer =
+ ADD_TIMER(_scanner_profile, "TabletReaderInitReturnColumnsTimer");
+ _tablet_reader_init_keys_param_timer =
+ ADD_TIMER(_scanner_profile, "TabletReaderInitKeysParamTimer");
+ _tablet_reader_init_orderby_keys_param_timer =
+ ADD_TIMER(_scanner_profile,
"TabletReaderInitOrderbyKeysParamTimer");
+ _tablet_reader_init_conditions_param_timer =
+ ADD_TIMER(_scanner_profile,
"TabletReaderInitConditionsParamTimer");
+ _tablet_reader_init_delete_condition_param_timer =
+ ADD_TIMER(_scanner_profile,
"TabletReaderInitDeleteConditionParamTimer");
+ _block_reader_vcollect_iter_init_timer =
+ ADD_TIMER(_scanner_profile, "BlockReaderVcollectIterInitTimer");
+ _block_reader_rs_readers_init_timer =
+ ADD_TIMER(_scanner_profile, "BlockReaderRsReadersInitTimer");
+ _block_reader_build_heap_init_timer =
+ ADD_TIMER(_scanner_profile, "BlockReaderBuildHeapInitTimer");
+
+ _rowset_reader_get_segment_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"RowsetReaderGetSegmentIteratorsTimer");
+ _rowset_reader_create_iterators_timer =
+ ADD_TIMER(_scanner_profile, "RowsetReaderCreateIteratorsTimer");
+ _rowset_reader_init_iterators_timer =
+ ADD_TIMER(_scanner_profile, "RowsetReaderInitIteratorsTimer");
+ _rowset_reader_load_segments_timer =
+ ADD_TIMER(_scanner_profile, "RowsetReaderLoadSegmentsTimer");
+
+ _segment_iterator_init_timer = ADD_TIMER(_scanner_profile,
"SegmentIteratorInitTimer");
+ _segment_iterator_init_return_column_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"SegmentIteratorInitReturnColumnIteratorsTimer");
+ _segment_iterator_init_bitmap_index_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"SegmentIteratorInitBitmapIndexIteratorsTimer");
+ _segment_iterator_init_inverted_index_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"SegmentIteratorInitInvertedIndexIteratorsTimer");
+
+ _segment_create_column_readers_timer =
+ ADD_TIMER(_scanner_profile, "SegmentCreateColumnReadersTimer");
+ _segment_load_index_timer = ADD_TIMER(_scanner_profile,
"SegmentLoadIndexTimer");
return Status::OK();
}
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index e21af94cea0..de00cd4f37a 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -191,6 +191,33 @@ private:
RuntimeProfile::Counter* _runtime_filter_info = nullptr;
+ // timer about tablet reader
+ RuntimeProfile::Counter* _tablet_reader_init_timer = nullptr;
+ RuntimeProfile::Counter* _tablet_reader_capture_rs_readers_timer = nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_return_columns_timer =
nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_keys_param_timer = nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_orderby_keys_param_timer =
nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_conditions_param_timer =
nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_delete_condition_param_timer
= nullptr;
+
+ // timer about block reader
+ RuntimeProfile::Counter* _block_reader_vcollect_iter_init_timer = nullptr;
+ RuntimeProfile::Counter* _block_reader_rs_readers_init_timer = nullptr;
+ RuntimeProfile::Counter* _block_reader_build_heap_init_timer = nullptr;
+
+ RuntimeProfile::Counter* _rowset_reader_get_segment_iterators_timer =
nullptr;
+ RuntimeProfile::Counter* _rowset_reader_create_iterators_timer = nullptr;
+ RuntimeProfile::Counter* _rowset_reader_init_iterators_timer = nullptr;
+ RuntimeProfile::Counter* _rowset_reader_load_segments_timer = nullptr;
+
+ RuntimeProfile::Counter* _segment_iterator_init_timer = nullptr;
+ RuntimeProfile::Counter*
_segment_iterator_init_return_column_iterators_timer = nullptr;
+ RuntimeProfile::Counter*
_segment_iterator_init_bitmap_index_iterators_timer = nullptr;
+ RuntimeProfile::Counter*
_segment_iterator_init_inverted_index_iterators_timer = nullptr;
+
+ RuntimeProfile::Counter* _segment_create_column_readers_timer = nullptr;
+ RuntimeProfile::Counter* _segment_load_index_timer = nullptr;
+
std::mutex _profile_mtx;
};
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index dd4a12b021a..6ffb60f425b 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -656,10 +656,53 @@ void NewOlapScanner::_collect_profile_before_close() {
} else {
pipeline::OlapScanLocalState* local_state =
(pipeline::OlapScanLocalState*)_local_state;
INCR_COUNTER(local_state);
+
+ COUNTER_UPDATE(local_state->_tablet_reader_init_timer,
stats.tablet_reader_init_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_capture_rs_readers_timer,
+ stats.tablet_reader_capture_rs_readers_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_init_return_columns_timer,
+ stats.tablet_reader_init_return_columns_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_init_keys_param_timer,
+ stats.tablet_reader_init_keys_param_timer_ns);
+
COUNTER_UPDATE(local_state->_tablet_reader_init_orderby_keys_param_timer,
+ stats.tablet_reader_init_orderby_keys_param_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_init_conditions_param_timer,
+ stats.tablet_reader_init_conditions_param_timer_ns);
+
COUNTER_UPDATE(local_state->_tablet_reader_init_delete_condition_param_timer,
+
stats.tablet_reader_init_delete_condition_param_timer_ns);
+ COUNTER_UPDATE(local_state->_block_reader_vcollect_iter_init_timer,
+ stats.block_reader_vcollect_iter_init_timer_ns);
+ COUNTER_UPDATE(local_state->_block_reader_rs_readers_init_timer,
+ stats.block_reader_rs_readers_init_timer_ns);
+ COUNTER_UPDATE(local_state->_block_reader_build_heap_init_timer,
+ stats.block_reader_build_heap_init_timer_ns);
+
+ COUNTER_UPDATE(local_state->_rowset_reader_get_segment_iterators_timer,
+ stats.rowset_reader_get_segment_iterators_timer_ns);
+ COUNTER_UPDATE(local_state->_rowset_reader_create_iterators_timer,
+ stats.rowset_reader_create_iterators_timer_ns);
+ COUNTER_UPDATE(local_state->_rowset_reader_init_iterators_timer,
+ stats.rowset_reader_init_iterators_timer_ns);
+ COUNTER_UPDATE(local_state->_rowset_reader_load_segments_timer,
+ stats.rowset_reader_load_segments_timer_ns);
+
+ COUNTER_UPDATE(local_state->_segment_iterator_init_timer,
+ stats.segment_iterator_init_timer_ns);
+
COUNTER_UPDATE(local_state->_segment_iterator_init_return_column_iterators_timer,
+
stats.segment_iterator_init_return_column_iterators_timer_ns);
+
COUNTER_UPDATE(local_state->_segment_iterator_init_bitmap_index_iterators_timer,
+
stats.segment_iterator_init_bitmap_index_iterators_timer_ns);
+
COUNTER_UPDATE(local_state->_segment_iterator_init_inverted_index_iterators_timer,
+
stats.segment_iterator_init_inverted_index_iterators_timer_ns);
+
+ COUNTER_UPDATE(local_state->_segment_create_column_readers_timer,
+ stats.segment_create_column_readers_timer_ns);
+ COUNTER_UPDATE(local_state->_segment_load_index_timer,
stats.segment_load_index_timer_ns);
}
#undef INCR_COUNTER
#endif
+
// Update metrics
DorisMetrics::instance()->query_scan_bytes->increment(_scan_bytes);
DorisMetrics::instance()->query_scan_rows->increment(_scan_rows);
diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp
index 0df19961ee3..490cdb99e81 100644
--- a/be/src/vec/olap/block_reader.cpp
+++ b/be/src/vec/olap/block_reader.cpp
@@ -107,40 +107,49 @@ Status BlockReader::_init_collect_iter(const
ReaderParams& read_params) {
return res;
}
// check if rowsets are noneoverlapping
- _is_rowsets_overlapping = _rowsets_mono_asc_disjoint(read_params);
- _vcollect_iter.init(this, _is_rowsets_overlapping,
read_params.read_orderby_key,
- read_params.read_orderby_key_reverse);
+ {
+ SCOPED_RAW_TIMER(&_stats.block_reader_vcollect_iter_init_timer_ns);
+ _is_rowsets_overlapping = _rowsets_mono_asc_disjoint(read_params);
+ _vcollect_iter.init(this, _is_rowsets_overlapping,
read_params.read_orderby_key,
+ read_params.read_orderby_key_reverse);
+ }
std::vector<RowsetReaderSharedPtr> valid_rs_readers;
RuntimeState* runtime_state = read_params.runtime_state;
- for (int i = 0; i < read_params.rs_splits.size(); ++i) {
- if (runtime_state != nullptr && runtime_state->is_cancelled()) {
- return Status::Cancelled(runtime_state->cancel_reason());
- }
+ {
+ SCOPED_RAW_TIMER(&_stats.block_reader_rs_readers_init_timer_ns);
+ for (int i = 0; i < read_params.rs_splits.size(); ++i) {
+ if (runtime_state != nullptr && runtime_state->is_cancelled()) {
+ return Status::Cancelled(runtime_state->cancel_reason());
+ }
- auto& rs_split = read_params.rs_splits[i];
+ auto& rs_split = read_params.rs_splits[i];
- // _vcollect_iter.topn_next() will init rs_reader by itself
- if (!_vcollect_iter.use_topn_next()) {
- RETURN_IF_ERROR(rs_split.rs_reader->init(&_reader_context,
rs_split));
- }
+ // _vcollect_iter.topn_next() will init rs_reader by itself
+ if (!_vcollect_iter.use_topn_next()) {
+ RETURN_IF_ERROR(rs_split.rs_reader->init(&_reader_context,
rs_split));
+ }
- Status res = _vcollect_iter.add_child(rs_split);
- if (!res.ok() && !res.is<END_OF_FILE>()) {
- LOG(WARNING) << "failed to add child to iterator, err=" << res;
- return res;
- }
- if (res.ok()) {
- valid_rs_readers.push_back(rs_split.rs_reader);
+ Status res = _vcollect_iter.add_child(rs_split);
+ if (!res.ok() && !res.is<END_OF_FILE>()) {
+ LOG(WARNING) << "failed to add child to iterator, err=" << res;
+ return res;
+ }
+ if (res.ok()) {
+ valid_rs_readers.push_back(rs_split.rs_reader);
+ }
}
}
- RETURN_IF_ERROR(_vcollect_iter.build_heap(valid_rs_readers));
- // _vcollect_iter.topn_next() can not use current_row
- if (!_vcollect_iter.use_topn_next()) {
- auto status = _vcollect_iter.current_row(&_next_row);
- _eof = status.is<END_OF_FILE>();
+ {
+ SCOPED_RAW_TIMER(&_stats.block_reader_build_heap_init_timer_ns);
+ RETURN_IF_ERROR(_vcollect_iter.build_heap(valid_rs_readers));
+ // _vcollect_iter.topn_next() can not use current_row
+ if (!_vcollect_iter.use_topn_next()) {
+ auto status = _vcollect_iter.current_row(&_next_row);
+ _eof = status.is<END_OF_FILE>();
+ }
}
return Status::OK();
diff --git a/be/src/vec/olap/vgeneric_iterators.cpp
b/be/src/vec/olap/vgeneric_iterators.cpp
index d8a073fc11a..fe37abd08fa 100644
--- a/be/src/vec/olap/vgeneric_iterators.cpp
+++ b/be/src/vec/olap/vgeneric_iterators.cpp
@@ -50,9 +50,8 @@ Status VStatisticsIterator::init(const StorageReadOptions&
opts) {
auto cid = _schema.column_id(i);
auto unique_id = _schema.column(cid)->unique_id();
if (_column_iterators_map.count(unique_id) < 1) {
-
RETURN_IF_ERROR(_segment->new_column_iterator(opts.tablet_schema->column(cid),
-
&_column_iterators_map[unique_id],
- nullptr));
+ RETURN_IF_ERROR(_segment->new_column_iterator(
+ opts.tablet_schema->column(cid),
&_column_iterators_map[unique_id], &opts));
}
_column_iterators.push_back(_column_iterators_map[unique_id].get());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
