This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 69d0094f62e [improvement](scan) Align scanner split to segments for
MATCH (#67915)
69d0094f62e is described below
commit 69d0094f62efa47cd877b12a914d9d4e0dc4d07f
Author: Chenyang Sun <[email protected]>
AuthorDate: Wed Oct 28 10:47:01 2026 +0800
[improvement](scan) Align scanner split to segments for MATCH (#67915)
### What problem does this PR solve?
An inverted index query is run over the whole segment and only then
intersected with the scanner's
row range, so its cost is priced per segment while
`ParallelScannerBuilder` splits by rows. A
segment shared by two scanners gets its posting lists walked twice.
An ann topn has the same property and already asks for one scanner per
segment. This PR reuses that
path for MATCH, so both of these now build one scanner per segment:
```sql
SELECT k1 FROM t ORDER BY l2_distance_approximate(embedding, [1.0, 2.0])
LIMIT 2; -- already did
SELECT k1 FROM t WHERE msg MATCH_PHRASE 'error timeout';
-- new
Cloud mode only. Queries without MATCH or ann topn are unchanged.
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [x] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR should
merge into -->
---------
Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
---
be/src/exec/operator/olap_scan_operator.cpp | 39 +++++++++++++++++-----
be/src/exec/operator/olap_scan_operator.h | 3 ++
.../analyzer/test_custom_analyzer1.groovy | 8 ++---
.../analyzer/test_ik_analyzer.groovy | 16 ++++-----
.../suites/inverted_index_p0/test_stopwords.groovy | 16 ++++-----
.../suites/search/test_search_dsl_syntax.groovy | 6 ++--
6 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/be/src/exec/operator/olap_scan_operator.cpp
b/be/src/exec/operator/olap_scan_operator.cpp
index a61fa641819..aecb90e02a8 100644
--- a/be/src/exec/operator/olap_scan_operator.cpp
+++ b/be/src/exec/operator/olap_scan_operator.cpp
@@ -477,6 +477,17 @@ static bool contains_expr_node_type(const VExprSPtr& expr,
TExprNodeType::type n
});
}
+// Find MATCH recursively; ones nested in AND / OR / NOT count too.
+static bool is_match_expr(const VExprSPtr& expr) {
+ DORIS_CHECK(expr != nullptr);
+ if (expr->node_type() == TExprNodeType::MATCH_PRED ||
+ expr->node_type() == TExprNodeType::SEARCH_EXPR ||
expr->can_push_down_to_index()) {
+ return true;
+ }
+ return std::ranges::any_of(expr->children(),
+ [](const auto& child) { return
is_match_expr(child); });
+}
+
static Status validate_residual_scan_conjuncts(RuntimeState* state,
TPushAggOp::type
push_down_agg_type,
const VExprContextSPtrs&
conjuncts) {
@@ -683,6 +694,24 @@ void OlapScanLocalState::_prune_rowsets_by_tso(const
TPaloScanRange& scan_range,
COUNTER_UPDATE(_segments_pruned_by_tso_counter, pruned_segments);
}
+// Give each segment a scanner of its own for queries like:
+// SELECT k1 FROM t ORDER BY l2_distance_approximate(embedding, [1.0, 2.0])
LIMIT 2
+// SELECT k1 FROM t WHERE msg MATCH_PHRASE 'error timeout'
+bool OlapScanLocalState::_use_scan_parallelism_by_per_segment() {
+ if (!state()->query_options().__isset.optimize_index_scan_parallelism ||
+ !state()->query_options().optimize_index_scan_parallelism) {
+ return false;
+ }
+ // TODO: Use optimize_index_scan_parallelism for ann range search in the
future.
+ // Currently, ann topn is enough
+ if (_ann_topn_runtime != nullptr) {
+ return true;
+ }
+ return config::is_cloud_mode() &&
+ std::ranges::any_of(_common_expr_ctxs_push_down,
+ [](const auto& ctx) { return
is_match_expr(ctx->root()); });
+}
+
Status OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
if (_scan_ranges.empty()) {
_eos = true;
@@ -835,15 +864,7 @@ Status
OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
scanner_builder.set_min_rows_per_scanner(min_rows_per_scanner);
// BM25 statistics must not depend on how a tablet is split into
scanners.
scanner_builder.set_share_tablet_collection_statistics(_score_runtime
!= nullptr);
- // If the session variable is set, force one scanner per segment.
- if (state()->query_options().__isset.optimize_index_scan_parallelism &&
- state()->query_options().optimize_index_scan_parallelism) {
- // TODO: Use optimize_index_scan_parallelism for ann range search
in the future.
- // Currently, ann topn is enough
- if (_ann_topn_runtime != nullptr) {
- scanner_builder.set_scan_parallelism_by_per_segment(true);
- }
- }
+
scanner_builder.set_scan_parallelism_by_per_segment(_use_scan_parallelism_by_per_segment());
RETURN_IF_ERROR(scanner_builder.build_scanners(*scanners));
for (auto& scanner : *scanners) {
diff --git a/be/src/exec/operator/olap_scan_operator.h
b/be/src/exec/operator/olap_scan_operator.h
index 0c593e66f7b..0c3c0099c18 100644
--- a/be/src/exec/operator/olap_scan_operator.h
+++ b/be/src/exec/operator/olap_scan_operator.h
@@ -137,6 +137,9 @@ private:
void _prune_rowsets_by_tso(const TPaloScanRange& scan_range,
TabletReadSource& read_source);
+ // Whether each segment should be scanned by a scanner of its own.
+ bool _use_scan_parallelism_by_per_segment();
+
Status _build_key_ranges_and_filters();
bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int32_t
bucket_seq,
diff --git
a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
index 1f40f3c42d4..7dbbb9f25bb 100644
---
a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
+++
b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
@@ -120,11 +120,11 @@ suite("test_custom_analyzer1", "p0") {
sql "sync"
sql """ set enable_segment_limit_pushdown = true; """
- qt_sql """ select * from test_custom_analyzer1 where ch match 'with';
"""
- qt_sql """ select * from test_custom_analyzer1 where ch match 'the';
"""
+ order_qt_sql """ select * from test_custom_analyzer1 where ch match
'with'; """
+ order_qt_sql """ select * from test_custom_analyzer1 where ch match
'the'; """
- qt_sql """ select * from test_custom_analyzer2 where ch match 'hm'; """
- qt_sql """ select * from test_custom_analyzer2 where ch match 'bg'; """
+ order_qt_sql """ select * from test_custom_analyzer2 where ch match
'hm'; """
+ order_qt_sql """ select * from test_custom_analyzer2 where ch match
'bg'; """
} finally {
}
}
diff --git
a/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
b/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
index e806a339fb0..250d8da7b87 100644
--- a/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
+++ b/regression-test/suites/inverted_index_p0/analyzer/test_ik_analyzer.groovy
@@ -70,17 +70,17 @@ suite("test_ik_analyzer", "p0") {
// Testing ik smart mode
println "Testing ik smart mode:"
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'北京'; """
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'计算机科学'; """
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'数据库管理系统'; """
- qt_sql """ select * from ${tableNameSmart} where content match_phrase
'中华人民共和国'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '北京'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '计算机科学'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '数据库管理系统'; """
+ order_qt_sql """ select * from ${tableNameSmart} where content
match_phrase '中华人民共和国'; """
// Testing ik max_word mode
println "Testing ik max_word mode:"
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '北京'; """
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '计算机科学'; """
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '数据库管理系统'; """
- qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '中华人民共和国'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '北京'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '计算机科学'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '数据库管理系统'; """
+ order_qt_sql """ select * from ${tableNameMaxWord} where content
match_phrase '中华人民共和国'; """
} finally {
sql "DROP TABLE IF EXISTS ${tableNameSmart}"
diff --git a/regression-test/suites/inverted_index_p0/test_stopwords.groovy
b/regression-test/suites/inverted_index_p0/test_stopwords.groovy
index 31e30cf6e48..2ff7dd3f556 100644
--- a/regression-test/suites/inverted_index_p0/test_stopwords.groovy
+++ b/regression-test/suites/inverted_index_p0/test_stopwords.groovy
@@ -44,16 +44,16 @@ suite("test_stopwords", "p0"){
sql "sync"
sql """ set enable_segment_limit_pushdown = true """
- qt_sql """ select * from ${indexTbName} where b match 'a'; """
- qt_sql """ select * from ${indexTbName} where b match 'are'; """
- qt_sql """ select * from ${indexTbName} where b match 'to'; """
+ order_qt_sql """ select * from ${indexTbName} where b match 'a'; """
+ order_qt_sql """ select * from ${indexTbName} where b match 'are'; """
+ order_qt_sql """ select * from ${indexTbName} where b match 'to'; """
- qt_sql """ select * from ${indexTbName} where c match 'a'; """
- qt_sql """ select * from ${indexTbName} where c match 'are'; """
- qt_sql """ select * from ${indexTbName} where c match 'to'; """
+ order_qt_sql """ select * from ${indexTbName} where c match 'a'; """
+ order_qt_sql """ select * from ${indexTbName} where c match 'are'; """
+ order_qt_sql """ select * from ${indexTbName} where c match 'to'; """
- qt_sql """ select * from ${indexTbName} where b match_phrase 'like a
box'; """
- qt_sql """ select * from ${indexTbName} where c match_phrase 'like a
box'; """
+ order_qt_sql """ select * from ${indexTbName} where b match_phrase
'like a box'; """
+ order_qt_sql """ select * from ${indexTbName} where c match_phrase
'like a box'; """
} finally {
//try_sql("DROP TABLE IF EXISTS ${testTable}")
diff --git a/regression-test/suites/search/test_search_dsl_syntax.groovy
b/regression-test/suites/search/test_search_dsl_syntax.groovy
index 5a4d9d4729a..7b1293ab876 100644
--- a/regression-test/suites/search/test_search_dsl_syntax.groovy
+++ b/regression-test/suites/search/test_search_dsl_syntax.groovy
@@ -92,16 +92,16 @@ suite("test_search_dsl_syntax", "p0") {
qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:Machine') ORDER BY id"
// DSL Syntax Test 2: Phrase queries with quotes
- qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:\"Machine Learning\"')"
+ qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:\"Machine Learning\"') ORDER BY id"
// DSL Syntax Test 3: Wildcard queries
qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:*Learning') ORDER BY id"
// DSL Syntax Test 4: Prefix queries
- qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:Data*')"
+ qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:Data*') ORDER BY id"
// DSL Syntax Test 5: Regular expression queries
- qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:/.*[Dd]ata.*/')"
+ qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('title:/.*[Dd]ata.*/') ORDER BY id"
// DSL Syntax Test 6: Boolean AND operator
qt_sql "SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, title
FROM ${tableName} WHERE search('category:Technology AND tags:AI') ORDER BY id"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
