This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new facc2cef561 [Enchancement](profile) More accurate display of
predicates on storage layer (#48650)
facc2cef561 is described below
commit facc2cef5612b4faed41ea801119bab8933b162d
Author: Pxl <[email protected]>
AuthorDate: Thu Mar 6 11:07:04 2025 +0800
[Enchancement](profile) More accurate display of predicates on storage
layer (#48650)
### What problem does this PR solve?
before:
```
- PreEvaluatePredicates:
shared_predicate(unknow), column_id=0, opposite=false, can_ignore=false
```
after:
```
- PreEvaluatePredicates:
shared_predicate(passnull predicate for ComparisonPredicateBase(INT,
LE), column_id=0, opposite=false, can_ignore=false, column_id=0,
opposite=false, can_ignore=false), column_id=0, opposite=false,
can_ignore=false
```
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [x] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [x] Confirm the release note
- [x] Confirm test cases
- [x] Confirm document
- [x] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/olap/iterators.h | 2 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 5 ++---
be/src/olap/rowset/beta_rowset_reader.h | 2 +-
be/src/olap/rowset/rowset_reader.h | 2 +-
.../olap/rowset/segment_v2/lazy_init_segment_iterator.h | 5 ++---
be/src/olap/rowset/segment_v2/segment_iterator.h | 16 ++++++----------
be/src/olap/tablet_reader.h | 2 +-
be/src/pipeline/exec/scan_operator.cpp | 5 +++++
be/src/vec/exec/scan/new_olap_scanner.cpp | 5 +----
be/src/vec/exec/scan/new_olap_scanner.h | 3 ---
be/src/vec/olap/block_reader.h | 2 +-
be/src/vec/olap/vcollect_iterator.cpp | 6 ------
be/src/vec/olap/vcollect_iterator.h | 17 +++++++----------
be/src/vec/olap/vgeneric_iterators.cpp | 5 ++---
be/src/vec/olap/vgeneric_iterators.h | 5 ++---
15 files changed, 32 insertions(+), 50 deletions(-)
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index 1d3c2ddf6b6..6fda571eeda 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -182,7 +182,7 @@ public:
// merge sort in priority queue
virtual uint64_t data_id() const { return 0; }
- virtual bool update_profile(RuntimeProfile* profile) { return false; }
+ virtual void update_profile(RuntimeProfile* profile) {}
// return rows merged count by iterator
virtual uint64_t merged_rows() const { return 0; }
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 822916819fe..16fa386fdf6 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -68,11 +68,10 @@ RowsetReaderSharedPtr BetaRowsetReader::clone() {
return RowsetReaderSharedPtr(new BetaRowsetReader(_rowset));
}
-bool BetaRowsetReader::update_profile(RuntimeProfile* profile) {
+void BetaRowsetReader::update_profile(RuntimeProfile* profile) {
if (_iterator != nullptr) {
- return _iterator->update_profile(profile);
+ _iterator->update_profile(profile);
}
- return false;
}
Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext*
read_context,
diff --git a/be/src/olap/rowset/beta_rowset_reader.h
b/be/src/olap/rowset/beta_rowset_reader.h
index b191480f7c7..adf29ea1880 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -80,7 +80,7 @@ public:
return _iterator->current_block_row_locations(locations);
}
- bool update_profile(RuntimeProfile* profile) override;
+ void update_profile(RuntimeProfile* profile) override;
RowsetReaderSharedPtr clone() override;
diff --git a/be/src/olap/rowset/rowset_reader.h
b/be/src/olap/rowset/rowset_reader.h
index 6c637f47cc1..628d78bf0fa 100644
--- a/be/src/olap/rowset/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -83,7 +83,7 @@ public:
return Status::NotSupported("to be implemented");
}
- virtual bool update_profile(RuntimeProfile* profile) = 0;
+ virtual void update_profile(RuntimeProfile* profile) = 0;
virtual RowsetReaderSharedPtr clone() = 0;
diff --git a/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h
b/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h
index c31918d092c..90ba8f891b2 100644
--- a/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h
@@ -54,11 +54,10 @@ public:
return _inner_iterator->current_block_row_locations(locations);
}
- bool update_profile(RuntimeProfile* profile) override {
+ void update_profile(RuntimeProfile* profile) override {
if (_inner_iterator != nullptr) {
- return _inner_iterator->update_profile(profile);
+ _inner_iterator->update_profile(profile);
}
- return false;
}
private:
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h
b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 5b4c8f6d73d..e684ec372d7 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -138,18 +138,15 @@ public:
RowsetId rowset_id() const { return _segment->rowset_id(); }
int64_t tablet_id() const { return _tablet_id; }
- bool update_profile(RuntimeProfile* profile) override {
- bool updated = false;
- updated |= _update_profile(profile, _short_cir_eval_predicate,
"ShortCircuitPredicates");
- updated |= _update_profile(profile, _pre_eval_block_predicate,
"PreEvaluatePredicates");
+ void update_profile(RuntimeProfile* profile) override {
+ _update_profile(profile, _short_cir_eval_predicate,
"ShortCircuitPredicates");
+ _update_profile(profile, _pre_eval_block_predicate,
"PreEvaluatePredicates");
if (_opts.delete_condition_predicates != nullptr) {
std::set<const ColumnPredicate*> delete_predicate_set;
_opts.delete_condition_predicates->get_all_column_predicate(delete_predicate_set);
- updated |= _update_profile(profile, delete_predicate_set,
"DeleteConditionPredicates");
+ _update_profile(profile, delete_predicate_set,
"DeleteConditionPredicates");
}
-
- return updated;
}
std::vector<std::unique_ptr<InvertedIndexIterator>>&
inverted_index_iterators() {
@@ -165,17 +162,16 @@ private:
Status _next_batch_internal(vectorized::Block* block);
template <typename Container>
- bool _update_profile(RuntimeProfile* profile, const Container& predicates,
+ void _update_profile(RuntimeProfile* profile, const Container& predicates,
const std::string& title) {
if (predicates.empty()) {
- return false;
+ return;
}
std::string info;
for (auto pred : predicates) {
info += "\n" + pred->debug_string();
}
profile->add_info_string(title, info);
- return true;
}
[[nodiscard]] Status _lazy_init();
diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h
index dd9d39d9dec..e7d7d417def 100644
--- a/be/src/olap/tablet_reader.h
+++ b/be/src/olap/tablet_reader.h
@@ -230,7 +230,7 @@ public:
const OlapReaderStatistics& stats() const { return _stats; }
OlapReaderStatistics* mutable_stats() { return &_stats; }
- virtual bool update_profile(RuntimeProfile* profile) { return false; }
+ virtual void update_profile(RuntimeProfile* profile) {}
static Status init_reader_params_and_create_block(
TabletSharedPtr tablet, ReaderType reader_type,
const std::vector<RowsetSharedPtr>& input_rowsets,
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index ea8ee0e5cbe..ec283f46683 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1082,6 +1082,11 @@ Status ScanLocalState<Derived>::_init_profile() {
template <typename Derived>
Status ScanLocalState<Derived>::_get_topn_filters(RuntimeState* state) {
auto& p = _parent->cast<typename Derived::Parent>();
+ std::stringstream result;
+ std::copy(p.topn_filter_source_node_ids.begin(),
p.topn_filter_source_node_ids.end(),
+ std::ostream_iterator<int>(result, ","));
+ _runtime_profile->add_info_string("TopNFilterSourceNodeIds", result.str());
+
for (auto id : get_topn_filter_source_node_ids(state, false)) {
const auto& pred = state->get_query_ctx()->get_runtime_predicate(id);
vectorized::VExprSPtr topn_pred;
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index 15b034321e7..6bae4b3319e 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -503,9 +503,6 @@ Status NewOlapScanner::_get_block_impl(RuntimeState* state,
Block* block, bool*
// ATTN: Here we need to let the _get_block_impl method guarantee the
semantics of the interface,
// that is, eof can be set to true only when the returned block is empty.
RETURN_IF_ERROR(_tablet_reader->next_block_with_aggregation(block, eof));
- if (!_profile_updated) {
- _profile_updated = _tablet_reader->update_profile(_profile);
- }
if (block->rows() > 0) {
_tablet_reader_params.tablet->read_block_count.fetch_add(1,
std::memory_order_relaxed);
*eof = false;
@@ -518,7 +515,6 @@ Status NewOlapScanner::close(RuntimeState* state) {
if (_is_closed) {
return Status::OK();
}
-
RETURN_IF_ERROR(VScanner::close(state));
return Status::OK();
}
@@ -542,6 +538,7 @@ void NewOlapScanner::_collect_profile_before_close() {
return;
}
_has_updated_counter = true;
+ _tablet_reader->update_profile(_profile);
VScanner::_collect_profile_before_close();
diff --git a/be/src/vec/exec/scan/new_olap_scanner.h
b/be/src/vec/exec/scan/new_olap_scanner.h
index fd1246b120b..1f327376433 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.h
+++ b/be/src/vec/exec/scan/new_olap_scanner.h
@@ -99,9 +99,6 @@ private:
std::vector<uint32_t> _return_columns;
std::unordered_set<uint32_t> _tablet_columns_convert_to_null_set;
-
- // ========= profiles ==========
- bool _profile_updated = false;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/olap/block_reader.h b/be/src/vec/olap/block_reader.h
index f33fe743109..46beee3a454 100644
--- a/be/src/vec/olap/block_reader.h
+++ b/be/src/vec/olap/block_reader.h
@@ -51,7 +51,7 @@ public:
std::vector<RowLocation> current_block_row_locations() { return
_block_row_locations; }
- bool update_profile(RuntimeProfile* profile) override {
+ void update_profile(RuntimeProfile* profile) override {
return _vcollect_iter.update_profile(profile);
}
diff --git a/be/src/vec/olap/vcollect_iterator.cpp
b/be/src/vec/olap/vcollect_iterator.cpp
index 8c910656837..d2f7ab58672 100644
--- a/be/src/vec/olap/vcollect_iterator.cpp
+++ b/be/src/vec/olap/vcollect_iterator.cpp
@@ -734,12 +734,10 @@ Status
VCollectIterator::Level1Iterator::_merge_next(IteratorRowRef* ref) {
_heap->pop();
} else {
_ref.reset();
- _cur_child.reset();
return Status::Error<END_OF_FILE>("");
}
} else {
_ref.reset();
- _cur_child.reset();
LOG(WARNING) << "failed to get next from child, res=" << res;
return res;
}
@@ -770,11 +768,9 @@ Status
VCollectIterator::Level1Iterator::_normal_next(IteratorRowRef* ref) {
_children.pop_front();
return _normal_next(ref);
} else {
- _cur_child.reset();
return Status::Error<END_OF_FILE>("");
}
} else {
- _cur_child.reset();
LOG(WARNING) << "failed to get next from child, res=" << res;
return res;
}
@@ -877,10 +873,8 @@ Status
VCollectIterator::Level1Iterator::_normal_next(Block* block) {
if (LIKELY(res.ok())) {
return Status::OK();
} else if (res.is<END_OF_FILE>()) {
- _cur_child.reset();
return Status::Error<END_OF_FILE>("");
} else {
- _cur_child.reset();
LOG(WARNING) << "failed to get next from child, res=" << res;
return res;
}
diff --git a/be/src/vec/olap/vcollect_iterator.h
b/be/src/vec/olap/vcollect_iterator.h
index e83f56559f4..05adc997f0e 100644
--- a/be/src/vec/olap/vcollect_iterator.h
+++ b/be/src/vec/olap/vcollect_iterator.h
@@ -78,11 +78,10 @@ public:
return _inner_iter->current_block_row_locations(block_row_locations);
}
- bool update_profile(RuntimeProfile* profile) {
+ void update_profile(RuntimeProfile* profile) {
if (_inner_iter != nullptr) {
- return _inner_iter->update_profile(profile);
+ _inner_iter->update_profile(profile);
}
- return false;
}
inline bool use_topn_next() const { return _topn_limit > 0; }
@@ -148,7 +147,7 @@ private:
[[nodiscard]] virtual Status ensure_first_row_ref() = 0;
- virtual bool update_profile(RuntimeProfile* profile) = 0;
+ virtual void update_profile(RuntimeProfile* profile) = 0;
protected:
const TabletSchema& _schema;
@@ -209,11 +208,10 @@ private:
Status current_block_row_locations(std::vector<RowLocation>*
block_row_locations) override;
- bool update_profile(RuntimeProfile* profile) override {
+ void update_profile(RuntimeProfile* profile) override {
if (_rs_reader != nullptr) {
- return _rs_reader->update_profile(profile);
+ _rs_reader->update_profile(profile);
}
- return false;
}
Status refresh_current_row();
@@ -289,11 +287,10 @@ private:
~Level1Iterator() override;
- bool update_profile(RuntimeProfile* profile) override {
+ void update_profile(RuntimeProfile* profile) override {
if (_cur_child != nullptr) {
- return _cur_child->update_profile(profile);
+ _cur_child->update_profile(profile);
}
- return false;
}
void init_level0_iterators_for_union();
diff --git a/be/src/vec/olap/vgeneric_iterators.cpp
b/be/src/vec/olap/vgeneric_iterators.cpp
index bb107a5c1d6..66bf13d64d4 100644
--- a/be/src/vec/olap/vgeneric_iterators.cpp
+++ b/be/src/vec/olap/vgeneric_iterators.cpp
@@ -367,11 +367,10 @@ public:
Status current_block_row_locations(std::vector<RowLocation>* locations)
override;
- bool update_profile(RuntimeProfile* profile) override {
+ void update_profile(RuntimeProfile* profile) override {
if (_cur_iter != nullptr) {
- return _cur_iter->update_profile(profile);
+ _cur_iter->update_profile(profile);
}
- return false;
}
private:
diff --git a/be/src/vec/olap/vgeneric_iterators.h
b/be/src/vec/olap/vgeneric_iterators.h
index 89eb130348f..c61ba5e2881 100644
--- a/be/src/vec/olap/vgeneric_iterators.h
+++ b/be/src/vec/olap/vgeneric_iterators.h
@@ -209,11 +209,10 @@ public:
return Status::OK();
}
- bool update_profile(RuntimeProfile* profile) override {
+ void update_profile(RuntimeProfile* profile) override {
if (!_origin_iters.empty()) {
- return _origin_iters[0]->update_profile(profile);
+ _origin_iters[0]->update_profile(profile);
}
- return false;
}
private:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
