This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b5b90ef0968 [env](compiler) reduce template instantiation in
scan_operator to improve compile time (#61227)
b5b90ef0968 is described below
commit b5b90ef09683c57b5f3a374bd6df78f22672ca81
Author: Mryange <[email protected]>
AuthorDate: Fri Mar 13 14:15:35 2026 +0800
[env](compiler) reduce template instantiation in scan_operator to improve
compile time (#61227)
`scan_operator.cpp` was the 2nd slowest file to compile (~289s), caused
by template
instantiation blowup. Methods defined on `ScanLocalState<Derived>` were
instantiated
for each of the 6 `Derived` types (OlapScan, JDBCScan, FileScan, EsScan,
MetaScan,
GroupCommitScan). Specifically:
- 4 inner `PrimitiveType`-templated methods × 6 Derived × 21
PrimitiveTypes = **504 instantiations**
- 5 non-template normalize methods × 6 Derived = **30 extra
compilations**
None of these methods actually depend on `Derived` — they only use base
class members
or virtual dispatch.
## Solution
Move the methods that do not depend on `Derived` up to
`ScanLocalStateBase`:
**Inner PrimitiveType-template methods** (instantiation reduced from
6×21 → 21, ÷6):
- `_normalize_in_predicate<T>`
- `_normalize_binary_predicate<T>`
- `_normalize_is_null_predicate<T>`
- `_change_value_range<T, Func>`
**Non-template normalize methods** (each compiled 6× → 1×):
- `_eval_const_conjuncts`
- `_normalize_bloom_filter`
- `_normalize_topn_filter`
- `_normalize_bitmap_filter`
- `_normalize_function_filters`
**Accompanying changes:**
- Move `_should_push_down_*` virtual methods, `_eos`,
`_push_down_functions` members
to `ScanLocalStateBase` so the moved methods can call them without
`Derived`.
- Add `_max_pushdown_conditions_per_column` to base class, initialized
from parent
in `ScanLocalState<Derived>::init()`.
- `_normalize_topn_filter`: replace `p.node_id()` (via `Derived::Parent`
cast) with
`_parent->node_id()` (base class method).
before
```
288.82s be/src/exec/operator/scan_operator.cpp
```
now
```
[1/1] Compiling: be/src/exec/operator/scan_operator.cpp
→ 209.9s
```
---
be/src/exec/operator/scan_operator.cpp | 45 ++++------
be/src/exec/operator/scan_operator.h | 154 +++++++++++++++++----------------
2 files changed, 96 insertions(+), 103 deletions(-)
diff --git a/be/src/exec/operator/scan_operator.cpp
b/be/src/exec/operator/scan_operator.cpp
index 9282353604f..ae41f3303b0 100644
--- a/be/src/exec/operator/scan_operator.cpp
+++ b/be/src/exec/operator/scan_operator.cpp
@@ -141,6 +141,7 @@ Status ScanLocalState<Derived>::init(RuntimeState* state,
LocalStateInfo& info)
SCOPED_TIMER(exec_time_counter());
SCOPED_TIMER(_init_timer);
auto& p = _parent->cast<typename Derived::Parent>();
+ _max_pushdown_conditions_per_column =
p._max_pushdown_conditions_per_column;
RETURN_IF_ERROR(_helper.init(state, p.is_serial_operator(), p.node_id(),
p.operator_id(),
_filter_dependencies, p.get_name() +
"_FILTER_DEPENDENCY"));
RETURN_IF_ERROR(_init_profile());
@@ -454,8 +455,7 @@ Status
ScanLocalState<Derived>::_normalize_predicate(VExprContext* context, cons
return Status::OK();
}
-template <typename Derived>
-Status ScanLocalState<Derived>::_normalize_bloom_filter(
+Status ScanLocalStateBase::_normalize_bloom_filter(
VExprContext* expr_ctx, const VExprSPtr& root, SlotDescriptor* slot,
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
PushDownType* pdt) {
std::shared_ptr<ColumnPredicate> pred = nullptr;
@@ -483,8 +483,7 @@ Status ScanLocalState<Derived>::_normalize_bloom_filter(
return Status::OK();
}
-template <typename Derived>
-Status ScanLocalState<Derived>::_normalize_topn_filter(
+Status ScanLocalStateBase::_normalize_topn_filter(
VExprContext* expr_ctx, const VExprSPtr& root, SlotDescriptor* slot,
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
PushDownType* pdt) {
std::shared_ptr<ColumnPredicate> pred = nullptr;
@@ -500,18 +499,16 @@ Status ScanLocalState<Derived>::_normalize_topn_filter(
DCHECK(root->is_topn_filter());
*pdt = _should_push_down_topn_filter();
if (*pdt != PushDownType::UNACCEPTABLE) {
- auto& p = _parent->cast<typename Derived::Parent>();
auto& tmp = _state->get_query_ctx()->get_runtime_predicate(
assert_cast<VTopNPred*>(root.get())->source_node_id());
if (_push_down_topn(tmp)) {
- pred = tmp.get_predicate(p.node_id());
+ pred = tmp.get_predicate(_parent->node_id());
}
}
return Status::OK();
}
-template <typename Derived>
-Status ScanLocalState<Derived>::_normalize_bitmap_filter(
+Status ScanLocalStateBase::_normalize_bitmap_filter(
VExprContext* expr_ctx, const VExprSPtr& root, SlotDescriptor* slot,
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
PushDownType* pdt) {
std::shared_ptr<ColumnPredicate> pred = nullptr;
@@ -539,10 +536,8 @@ Status ScanLocalState<Derived>::_normalize_bitmap_filter(
return Status::OK();
}
-template <typename Derived>
-Status ScanLocalState<Derived>::_normalize_function_filters(VExprContext*
expr_ctx,
- SlotDescriptor*
slot,
- PushDownType* pdt)
{
+Status ScanLocalStateBase::_normalize_function_filters(VExprContext* expr_ctx,
SlotDescriptor* slot,
+ PushDownType* pdt) {
auto expr = expr_ctx->root()->is_rf_wrapper() ?
expr_ctx->root()->get_impl() : expr_ctx->root();
bool opposite = false;
VExpr* fn_expr = expr.get();
@@ -622,8 +617,7 @@ std::string ScanLocalState<Derived>::debug_string(int
indentation_level) const {
return fmt::to_string(debug_string_buffer);
}
-template <typename Derived>
-Status ScanLocalState<Derived>::_eval_const_conjuncts(VExprContext* expr_ctx,
PushDownType* pdt) {
+Status ScanLocalStateBase::_eval_const_conjuncts(VExprContext* expr_ctx,
PushDownType* pdt) {
auto vexpr =
expr_ctx->root()->is_rf_wrapper() ? expr_ctx->root()->get_impl() :
expr_ctx->root();
// Used to handle constant expressions, such as '1 = 1'
_eval_const_conjuncts does not handle cases like 'colA = 1'
@@ -671,9 +665,8 @@ Status
ScanLocalState<Derived>::_eval_const_conjuncts(VExprContext* expr_ctx, Pu
return Status::OK();
}
-template <typename Derived>
template <PrimitiveType T>
-Status ScanLocalState<Derived>::_normalize_in_predicate(
+Status ScanLocalStateBase::_normalize_in_predicate(
VExprContext* expr_ctx, const VExprSPtr& root, SlotDescriptor* slot,
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
ColumnValueRange<T>& range,
PushDownType* pdt) {
@@ -705,8 +698,7 @@ Status ScanLocalState<Derived>::_normalize_in_predicate(
auto is_in = false;
if (hybrid_set != nullptr) {
// runtime filter produce VDirectInPredicate
- if (hybrid_set->size() <=
- _parent->cast<typename
Derived::Parent>()._max_pushdown_conditions_per_column) {
+ if (hybrid_set->size() <=
static_cast<size_t>(_max_pushdown_conditions_per_column)) {
iter = hybrid_set->begin();
}
is_in = true;
@@ -784,9 +776,8 @@ Status ScanLocalState<Derived>::_normalize_in_predicate(
return Status::OK();
}
-template <typename Derived>
template <PrimitiveType T>
-Status ScanLocalState<Derived>::_normalize_binary_predicate(
+Status ScanLocalStateBase::_normalize_binary_predicate(
VExprContext* expr_ctx, const VExprSPtr& root, SlotDescriptor* slot,
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
ColumnValueRange<T>& range,
PushDownType* pdt) {
@@ -895,13 +886,12 @@ Status
ScanLocalState<Derived>::_normalize_binary_predicate(
return Status::OK();
}
-template <typename Derived>
template <PrimitiveType PrimitiveType, typename ChangeFixedValueRangeFunc>
-Status ScanLocalState<Derived>::_change_value_range(bool is_equal_op,
-
ColumnValueRange<PrimitiveType>& temp_range,
- const Field& value,
- const
ChangeFixedValueRangeFunc& func,
- const std::string&
fn_name) {
+Status ScanLocalStateBase::_change_value_range(bool is_equal_op,
+
ColumnValueRange<PrimitiveType>& temp_range,
+ const Field& value,
+ const
ChangeFixedValueRangeFunc& func,
+ const std::string& fn_name) {
if constexpr (PrimitiveType == TYPE_DATE) {
auto tmp_value = value.template get<TYPE_DATE>();
if (is_equal_op) {
@@ -937,9 +927,8 @@ Status ScanLocalState<Derived>::_change_value_range(bool
is_equal_op,
return Status::OK();
}
-template <typename Derived>
template <PrimitiveType T>
-Status ScanLocalState<Derived>::_normalize_is_null_predicate(
+Status ScanLocalStateBase::_normalize_is_null_predicate(
VExprContext* expr_ctx, const VExprSPtr& root, SlotDescriptor* slot,
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
ColumnValueRange<T>& range,
PushDownType* pdt) {
diff --git a/be/src/exec/operator/scan_operator.h
b/be/src/exec/operator/scan_operator.h
index a93a3ae3fa5..65faacca022 100644
--- a/be/src/exec/operator/scan_operator.h
+++ b/be/src/exec/operator/scan_operator.h
@@ -19,9 +19,11 @@
#include <cstdint>
#include <mutex>
+#include <set>
#include <string>
#include "common/status.h"
+#include "core/field.h"
#include "exec/common/util.hpp"
#include "exec/operator/operator.h"
#include "exec/pipeline/dependency.h"
@@ -128,6 +130,83 @@ protected:
RuntimeFilterConsumerHelper _helper;
// magic number as seed to generate hash value for condition cache
uint64_t _condition_cache_digest = 0;
+
+ // Moved from ScanLocalState<Derived> to avoid re-instantiation for each
Derived type.
+ std::atomic<bool> _eos = false;
+ int _max_pushdown_conditions_per_column = 1024;
+ // Save all function predicates which may be pushed down to data source.
+ std::vector<FunctionFilter> _push_down_functions;
+
+ // Virtual methods with default implementations; overridden by subclasses
when supported.
+ // Declared here so that the normalize methods below
(non-Derived-template) can call them.
+ virtual bool _push_down_topn(const RuntimePredicate& predicate) { return
false; }
+ virtual PushDownType _should_push_down_bloom_filter() const {
+ return PushDownType::UNACCEPTABLE;
+ }
+ virtual PushDownType _should_push_down_topn_filter() const {
+ return PushDownType::UNACCEPTABLE;
+ }
+ virtual PushDownType _should_push_down_bitmap_filter() const {
+ return PushDownType::UNACCEPTABLE;
+ }
+ virtual PushDownType _should_push_down_is_null_predicate(VectorizedFnCall*
fn_call) const {
+ return PushDownType::UNACCEPTABLE;
+ }
+ virtual PushDownType _should_push_down_in_predicate() const {
+ return PushDownType::UNACCEPTABLE;
+ }
+ virtual PushDownType _should_push_down_binary_predicate(
+ VectorizedFnCall* fn_call, VExprContext* expr_ctx, Field&
constant_val,
+ const std::set<std::string> fn_name) const {
+ return PushDownType::UNACCEPTABLE;
+ }
+ virtual Status _should_push_down_function_filter(VectorizedFnCall* fn_call,
+ VExprContext* expr_ctx,
+ StringRef* constant_str,
+ doris::FunctionContext**
fn_ctx,
+ PushDownType& pdt) {
+ pdt = PushDownType::UNACCEPTABLE;
+ return Status::OK();
+ }
+
+ // Non-templated normalize methods, moved here to avoid re-compilation per
Derived type.
+ Status _eval_const_conjuncts(VExprContext* expr_ctx, PushDownType* pdt);
+ Status _normalize_bloom_filter(VExprContext* expr_ctx, const VExprSPtr&
root,
+ SlotDescriptor* slot,
+
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
+ PushDownType* pdt);
+ Status _normalize_topn_filter(VExprContext* expr_ctx, const VExprSPtr&
root,
+ SlotDescriptor* slot,
+
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
+ PushDownType* pdt);
+ Status _normalize_bitmap_filter(VExprContext* expr_ctx, const VExprSPtr&
root,
+ SlotDescriptor* slot,
+
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
+ PushDownType* pdt);
+ Status _normalize_function_filters(VExprContext* expr_ctx, SlotDescriptor*
slot,
+ PushDownType* pdt);
+
+ // Inner PrimitiveType-template methods. Moved to base to avoid
N(Derived)×M(PrimitiveType)
+ // instantiation blowup: now instantiated M times total instead of N×M
times.
+ template <PrimitiveType T>
+ Status _normalize_in_predicate(VExprContext* expr_ctx, const VExprSPtr&
root,
+ SlotDescriptor* slot,
+
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
+ ColumnValueRange<T>& range, PushDownType*
pdt);
+ template <PrimitiveType T>
+ Status _normalize_binary_predicate(VExprContext* expr_ctx, const
VExprSPtr& root,
+ SlotDescriptor* slot,
+
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
+ ColumnValueRange<T>& range,
PushDownType* pdt);
+ template <PrimitiveType T>
+ Status _normalize_is_null_predicate(VExprContext* expr_ctx, const
VExprSPtr& root,
+ SlotDescriptor* slot,
+
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
+ ColumnValueRange<T>& range,
PushDownType* pdt);
+ template <PrimitiveType PrimitiveType, typename ChangeFixedValueRangeFunc>
+ Status _change_value_range(bool is_equal_op,
ColumnValueRange<PrimitiveType>& range,
+ const Field& value, const
ChangeFixedValueRangeFunc& func,
+ const std::string& fn_name);
};
template <typename LocalStateType>
@@ -202,37 +281,7 @@ protected:
virtual bool _should_push_down_common_expr() { return false; }
virtual bool _storage_no_merge() { return false; }
- virtual bool _push_down_topn(const RuntimePredicate& predicate) { return
false; }
virtual bool _is_key_column(const std::string& col_name) { return false; }
- virtual PushDownType _should_push_down_bloom_filter() const {
- return PushDownType::UNACCEPTABLE;
- }
- virtual PushDownType _should_push_down_topn_filter() const {
- return PushDownType::UNACCEPTABLE;
- }
- virtual PushDownType _should_push_down_bitmap_filter() const {
- return PushDownType::UNACCEPTABLE;
- }
- virtual PushDownType _should_push_down_is_null_predicate(VectorizedFnCall*
fn_call) const {
- return PushDownType::UNACCEPTABLE;
- }
- virtual PushDownType _should_push_down_in_predicate() const {
- return PushDownType::UNACCEPTABLE;
- }
- virtual PushDownType _should_push_down_binary_predicate(
- VectorizedFnCall* fn_call, VExprContext* expr_ctx, Field&
constant_val,
- const std::set<std::string> fn_name) const {
- return PushDownType::UNACCEPTABLE;
- }
-
- virtual Status _should_push_down_function_filter(VectorizedFnCall* fn_call,
- VExprContext* expr_ctx,
- StringRef* constant_str,
- doris::FunctionContext**
fn_ctx,
- PushDownType& pdt) {
- pdt = PushDownType::UNACCEPTABLE;
- return Status::OK();
- }
// Create a list of scanners.
// The number of scanners is related to the implementation of the data
source,
@@ -247,46 +296,6 @@ protected:
VExprSPtr& output_expr);
bool _is_predicate_acting_on_slot(const VExprSPtrs& children,
SlotDescriptor** slot_desc,
ColumnValueRangeType** range);
- Status _eval_const_conjuncts(VExprContext* expr_ctx, PushDownType* pdt);
-
- template <PrimitiveType T>
- Status _normalize_in_predicate(VExprContext* expr_ctx, const VExprSPtr&
root,
- SlotDescriptor* slot,
-
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
- ColumnValueRange<T>& range, PushDownType*
pdt);
- template <PrimitiveType T>
- Status _normalize_binary_predicate(VExprContext* expr_ctx, const
VExprSPtr& root,
- SlotDescriptor* slot,
-
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
- ColumnValueRange<T>& range,
PushDownType* pdt);
- Status _normalize_bloom_filter(VExprContext* expr_ctx, const VExprSPtr&
root,
- SlotDescriptor* slot,
-
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
- PushDownType* pdt);
- Status _normalize_topn_filter(VExprContext* expr_ctx, const VExprSPtr&
root,
- SlotDescriptor* slot,
-
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
- PushDownType* pdt);
-
- Status _normalize_bitmap_filter(VExprContext* expr_ctx, const VExprSPtr&
root,
- SlotDescriptor* slot,
-
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
- PushDownType* pdt);
-
- Status _normalize_function_filters(VExprContext* expr_ctx, SlotDescriptor*
slot,
- PushDownType* pdt);
-
- template <PrimitiveType T>
- Status _normalize_is_null_predicate(VExprContext* expr_ctx, const
VExprSPtr& root,
- SlotDescriptor* slot,
-
std::vector<std::shared_ptr<ColumnPredicate>>& predicates,
- ColumnValueRange<T>& range,
PushDownType* pdt);
-
- template <PrimitiveType PrimitiveType, typename ChangeFixedValueRangeFunc>
- Status _change_value_range(bool is_equal_op,
ColumnValueRange<PrimitiveType>& range,
- const Field& value, const
ChangeFixedValueRangeFunc& func,
- const std::string& fn_name);
-
Status _prepare_scanners();
// Submit the scanner to the thread pool and start execution
@@ -310,9 +319,6 @@ protected:
atomic_shared_ptr<ScannerContext> _scanner_ctx;
- // Save all function predicates which may be pushed down to data source.
- std::vector<FunctionFilter> _push_down_functions;
-
// colname -> cast dst type
std::map<std::string, DataTypePtr> _cast_types_for_variants;
@@ -322,8 +328,6 @@ protected:
phmap::flat_hash_map<int, std::vector<std::shared_ptr<ColumnPredicate>>>
_slot_id_to_predicates;
std::vector<std::shared_ptr<MutilColumnBlockPredicate>> _or_predicates;
- std::atomic<bool> _eos = false;
-
std::vector<std::shared_ptr<Dependency>> _filter_dependencies;
// ScanLocalState owns the ownership of scanner, scanner context only has
its weakptr
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
