This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2eb014caf0b [improvement](be) Skip reading zonemap always-true columns
for count on index (#65242)
2eb014caf0b is described below
commit 2eb014caf0b114a0d2664d72bab3809a929d0cc6
Author: Jerry Hu <[email protected]>
AuthorDate: Tue Jul 14 11:36:34 2026 +0800
[improvement](be) Skip reading zonemap always-true columns for count on
index (#65242)
### What problem does this PR solve?
Issue Number: None
Problem Summary:
COUNT_ON_INDEX can still read predicate columns even after segment zone
maps have proven all predicates on those columns are always true and
removed them before SegmentIterator initialization. This makes count
queries with range predicates over high-cardinality columns scan
unnecessary column data.
This PR records columns whose predicates are fully removed by segment
zone map pruning, then lets SegmentIterator reuse the existing
no-need-read path for non-output columns or COUNT_ON_INDEX columns. It
keeps the existing output-column protection and adds a common-expression
guard so expression materialization does not skip required data.
### Release note
None
### Check List (For Author)
- Test:
- Unit Test: `./run-be-ut.sh --run
--filter=SegmentIteratorNoNeedReadDataTest.*`
- Regression test: generated and verified
`fault_injection_p0/test_zonemap_always_true_count_on_index` on a local
single-node cluster
- Format: `build-support/clang-format.sh`;
`build-support/check-format.sh`
- Static check: attempted `build-support/run-clang-tidy.sh --build-dir
be/ut_build_ASAN`, but clang-tidy could not analyze this checkout
because the current toolchain reported `stddef.h` file not found and a
pre-existing `clang-tidy-nolint` issue in `be/src/core/types.h`
- Behavior changed: Yes. COUNT_ON_INDEX scans can skip data reads for
predicate columns proven always true by segment zone maps.
- Does this need documentation: No
---
be/src/storage/iterators.h | 4 ++
be/src/storage/segment/segment.cpp | 10 +++
be/src/storage/segment/segment_iterator.cpp | 24 ++++---
.../segment_iterator_no_need_read_data_test.cpp | 38 ++++++++++
.../test_zonemap_always_true_count_on_index.out | 7 ++
.../test_zonemap_always_true_count_on_index.groovy | 83 ++++++++++++++++++++++
6 files changed, 155 insertions(+), 11 deletions(-)
diff --git a/be/src/storage/iterators.h b/be/src/storage/iterators.h
index 3b6fa114be3..0f64daf1eba 100644
--- a/be/src/storage/iterators.h
+++ b/be/src/storage/iterators.h
@@ -107,6 +107,10 @@ public:
std::unordered_map<int32_t, std::vector<std::shared_ptr<const
ColumnPredicate>>>
del_predicates_for_zone_map;
TPushAggOp::type push_down_agg_type_opt = TPushAggOp::NONE;
+ // Non-key columns whose predicates were proven always true by the segment
zone map and then
+ // removed from column_predicates. SegmentIterator can skip reading these
non-output columns,
+ // or COUNT_ON_INDEX columns, because the column values no longer affect
filtering or counting.
+ std::set<uint32_t> zonemap_always_true_pred_cols;
// REQUIRED (null is not allowed)
OlapReaderStatistics* stats = nullptr;
diff --git a/be/src/storage/segment/segment.cpp
b/be/src/storage/segment/segment.cpp
index 054f21cec6f..99351502589 100644
--- a/be/src/storage/segment/segment.cpp
+++ b/be/src/storage/segment/segment.cpp
@@ -499,6 +499,16 @@ Status Segment::new_iterator(SchemaSPtr schema, const
StorageReadOptions& read_o
options_with_pruned_predicates.col_id_to_predicates[pred->column_id()]
->add_column_predicate(SingleColumnBlockPredicate::create_unique(pred));
}
+ for (const auto& pred : read_options.column_predicates) {
+ const auto pred_cid = pred->column_id();
+ // Key columns may still be required by key range seeks even
if the segment zone
+ // map proves their predicates always true. Only mark non-key
columns as safe for
+ // the no-need-read path.
+ if (!read_options.tablet_schema->column(pred_cid).is_key() &&
+
!options_with_pruned_predicates.col_id_to_predicates.contains(pred_cid)) {
+
options_with_pruned_predicates.zonemap_always_true_pred_cols.insert(pred_cid);
+ }
+ }
return iter->get()->init(options_with_pruned_predicates);
}
}
diff --git a/be/src/storage/segment/segment_iterator.cpp
b/be/src/storage/segment/segment_iterator.cpp
index f667efc0db1..038544e46d6 100644
--- a/be/src/storage/segment/segment_iterator.cpp
+++ b/be/src/storage/segment/segment_iterator.cpp
@@ -1454,13 +1454,6 @@ bool SegmentIterator::_need_read_data(ColumnId cid) {
// occurring, return true here that column data needs to be read
return true;
}
- // Check the following conditions:
- // 1. If the column represented by the unique ID is an inverted index
column (indicated by '_need_read_data_indices.count(unique_id) > 0 &&
!_need_read_data_indices[unique_id]')
- // and it's not marked for projection in '_output_columns'.
- // 2. Or, if the column is an inverted index column and it's marked for
projection in '_output_columns',
- // and the operation is a push down of the 'COUNT_ON_INDEX' aggregation
function.
- // If any of the above conditions are met, log a debug message indicating
that there's no need to read data for the indexed column.
- // Then, return false.
const auto& column = _opts.tablet_schema->column(cid);
// Different subcolumns may share the same parent_unique_id, so we choose
to abandon this optimization.
if (column.is_extracted_column() &&
@@ -1471,10 +1464,19 @@ bool SegmentIterator::_need_read_data(ColumnId cid) {
if (unique_id < 0) {
unique_id = column.parent_unique_id();
}
- if ((_need_read_data_indices.contains(cid) &&
!_need_read_data_indices[cid] &&
- !_output_columns.contains(unique_id)) ||
- (_need_read_data_indices.contains(cid) &&
!_need_read_data_indices[cid] &&
- _output_columns.count(unique_id) == 1 &&
+ // A column can skip data reads when its predicates have already been
fully resolved.
+ // zonemap_always_true_pred_cols is produced only for non-key columns
because key columns
+ // must remain readable for short-key range seeks.
+ const bool used_by_common_expr =
+ cid < _is_common_expr_column.size() && _is_common_expr_column[cid];
+ const bool zonemap_always_true_filter_column =
+ _opts.zonemap_always_true_pred_cols.contains(cid);
+ DCHECK(!zonemap_always_true_filter_column || !column.is_key());
+ const bool no_need_read_filter_column =
+ (_need_read_data_indices.contains(cid) &&
!_need_read_data_indices[cid]) ||
+ (zonemap_always_true_filter_column && !used_by_common_expr);
+ if ((no_need_read_filter_column && !_output_columns.contains(unique_id)) ||
+ (no_need_read_filter_column && _output_columns.count(unique_id) == 1 &&
_opts.push_down_agg_type_opt == TPushAggOp::COUNT_ON_INDEX)) {
VLOG_DEBUG << "SegmentIterator no need read data for column: "
<< _opts.tablet_schema->column_by_uid(unique_id).name();
diff --git
a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp
b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp
index 0bd73f93c97..dbe226a0608 100644
--- a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp
+++ b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp
@@ -61,4 +61,42 @@ TEST(SegmentIteratorNoNeedReadDataTest,
extracted_variant_count_on_index) {
EXPECT_TRUE(iter._need_read_data(subcol_cid));
}
+TEST(SegmentIteratorNoNeedReadDataTest, zonemap_always_true_predicate_column) {
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(KeysType::DUP_KEYS);
+ auto* key = schema_pb.add_column();
+ key->set_unique_id(1);
+ key->set_name("k");
+ key->set_type("INT");
+ key->set_is_key(true);
+ key->set_is_nullable(false);
+
+ auto* pred_col = schema_pb.add_column();
+ pred_col->set_unique_id(2);
+ pred_col->set_name("event_time");
+ pred_col->set_type("DATETIMEV2");
+ pred_col->set_is_key(false);
+ pred_col->set_is_nullable(false);
+
+ auto tablet_schema = std::make_shared<TabletSchema>();
+ tablet_schema->init_from_pb(schema_pb);
+
+ std::vector<ColumnId> read_column_ids(tablet_schema->num_columns());
+ for (uint32_t cid = 0; cid < read_column_ids.size(); ++cid) {
+ read_column_ids[cid] = cid;
+ }
+ auto read_schema = std::make_shared<Schema>(tablet_schema->columns(),
read_column_ids);
+ SegmentIterator iter(nullptr, read_schema);
+ iter._opts.tablet_schema = tablet_schema;
+ iter._opts.zonemap_always_true_pred_cols.emplace(1);
+
+ EXPECT_FALSE(iter._need_read_data(1));
+
+ iter._output_columns.emplace(2);
+ EXPECT_TRUE(iter._need_read_data(1));
+
+ iter._opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX;
+ EXPECT_FALSE(iter._need_read_data(1));
+}
+
} // namespace doris::segment_v2
diff --git
a/regression-test/data/fault_injection_p0/test_zonemap_always_true_count_on_index.out
b/regression-test/data/fault_injection_p0/test_zonemap_always_true_count_on_index.out
new file mode 100644
index 00000000000..ba9a96051e6
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_zonemap_always_true_count_on_index.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !count --
+2
+
+-- !count_with_key_range --
+2
+
diff --git
a/regression-test/suites/fault_injection_p0/test_zonemap_always_true_count_on_index.groovy
b/regression-test/suites/fault_injection_p0/test_zonemap_always_true_count_on_index.groovy
new file mode 100644
index 00000000000..dfb50609579
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_zonemap_always_true_count_on_index.groovy
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_zonemap_always_true_count_on_index", "p0, nonConcurrent") {
+ sql "DROP TABLE IF EXISTS test_zonemap_always_true_count_on_index"
+ sql "set enable_count_on_index_pushdown = true"
+ sql "set enable_no_need_read_data_opt = true"
+ sql "set experimental_enable_nereids_planner = true"
+ sql "set enable_fallback_to_original_planner = false"
+ sql "set inverted_index_skip_threshold = 0"
+
+ sql """
+ CREATE TABLE test_zonemap_always_true_count_on_index (
+ k INT,
+ app_id VARCHAR(32) NOT NULL,
+ event_time DATETIMEV2 NOT NULL,
+ v INT,
+ INDEX idx_app_id (`app_id`) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k, app_id)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ sql """
+ INSERT INTO test_zonemap_always_true_count_on_index VALUES
+ (1, 'app_a', '2026-03-28 00:10:00', 10),
+ (2, 'app_a', '2026-03-28 12:20:00', 20),
+ (3, 'app_b', '2026-03-28 23:30:00', 30);
+ """
+ sql "sync"
+
+ def countSql = """
+ SELECT COUNT(1) FROM test_zonemap_always_true_count_on_index
+ WHERE event_time >= '2026-03-28 00:00:00'
+ AND event_time < '2026-03-29 00:00:00'
+ AND app_id = 'app_a'
+ """
+
+ explain {
+ sql(countSql)
+ contains "pushAggOp=COUNT_ON_INDEX"
+ }
+
+ def countWithKeyRangeSql = """
+ SELECT COUNT(1) FROM test_zonemap_always_true_count_on_index
+ WHERE k >= 1
+ AND k < 4
+ AND app_id = 'app_a'
+ """
+
+ explain {
+ sql(countWithKeyRangeSql)
+ contains "pushAggOp=COUNT_ON_INDEX"
+ }
+
+ try {
+
GetDebugPoint().enableDebugPointForAllBEs("segment_iterator._read_columns_by_index",
+ [column_name: "event_time"])
+ qt_count countSql
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("segment_iterator._read_columns_by_index")
+ }
+
+ qt_count_with_key_range countWithKeyRangeSql
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
