This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9521f75b89f [fix](scan) Read all nested access paths before evaluating
TopN filters on pruned STRUCT columns (#68110)
9521f75b89f is described below
commit 9521f75b89f78ffb8672e501e0c7e8c449996382
Author: Jerry Hu <[email protected]>
AuthorDate: Sun Sep 20 09:45:09 2026 +0800
[fix](scan) Read all nested access paths before evaluating TopN filters on
pruned STRUCT columns (#68110)
### What problem does this PR solve?
Issue Number: None
Problem Summary:
With `enable_prune_nested_column = true`, the planner splits a STRUCT
column's
access paths into predicate paths (read before filtering) and lazy paths
(read
only for surviving rows). SegmentIterator relies on this split when a
common
expression references the column: it reads the column in the PREDICATE
phase,
evaluates the expression, and recovers the lazy nested fields
afterwards.
The TopN filter is a common expression that BE attaches to the scan at
runtime,
after the planner computed the access paths. When ORDER BY and WHERE
reference
different fields of the same STRUCT, for example
```sql
SELECT struct_element(s, 'a') FROM t
WHERE struct_element(s, 'b') IS NOT NULL
ORDER BY 1 NULLS LAST LIMIT 1;
```
the predicate paths only contain `s.b.NULL`, so `s.a` is a placeholder
(NULL)
while the TopN filter `struct_element(s, 'a') <= current_top` is
evaluated.
Every row of the later tablets is rejected and the global minimum is
lost,
silently returning a wrong LIMIT result.
This PR marks columns referenced by runtime-generated common expressions
(TopN filters and runtime filters) and keeps them in the NORMAL read
phase, so
all access paths are materialized before the expression is evaluated.
The lazy
split is still used for planner-visible expressions.
### Release note
None
### Check List (For Author)
- Test:
- Unit Test: `SegmentIteratorRuntimeCommonExprTest` covers the lazy
split
decision for planner expressions, TopN filters, and both on one column.
- Regression test: `topn_filter_nested_column_pruning` reproduces the
wrong
result with TIMESTAMPTZ/INT/DATE STRUCT fields, ASC/DESC, NULLS
FIRST/LAST,
selective predicates and multi-tablet scans.
- Behavior changed: No
- Does this need documentation: No
https://claude.ai/code/session_01E3gDAafAXZELs6AHVfD9UG
---
be/src/storage/segment/segment_iterator.cpp | 19 ++-
be/src/storage/segment/segment_iterator.h | 9 +-
.../segment_iterator_runtime_common_expr_test.cpp | 174 +++++++++++++++++++++
.../topn_filter_nested_column_pruning.out | 37 +++++
.../topn_filter_nested_column_pruning.groovy | 144 +++++++++++++++++
5 files changed, 377 insertions(+), 6 deletions(-)
diff --git a/be/src/storage/segment/segment_iterator.cpp
b/be/src/storage/segment/segment_iterator.cpp
index 99d53fd7f86..8a281ba1968 100644
--- a/be/src/storage/segment/segment_iterator.cpp
+++ b/be/src/storage/segment/segment_iterator.cpp
@@ -498,12 +498,13 @@ void SegmentIterator::_rebuild_scan_predicate_states() {
}
}
-void SegmentIterator::_mark_common_expr_states(const VExprSPtr& expr) {
+void SegmentIterator::_mark_common_expr_states(const VExprSPtr& expr, bool
runtime_generated) {
if (expr->is_slot_ref()) {
const auto ordinal =
cast_set<ColumnId>(assert_cast<const
VSlotRef*>(expr.get())->column_id());
DORIS_CHECK_LT(ordinal, _schema->num_block_columns());
_column_states[ordinal].has_common_expr = true;
+ _column_states[ordinal].has_runtime_common_expr |= runtime_generated;
_is_need_expr_eval = true;
return;
}
@@ -511,11 +512,11 @@ void SegmentIterator::_mark_common_expr_states(const
VExprSPtr& expr) {
const auto& virtual_expr =
assert_cast<const
VirtualSlotRef*>(expr.get())->get_virtual_column_expr();
DORIS_CHECK(virtual_expr != nullptr);
- _mark_common_expr_states(virtual_expr);
+ _mark_common_expr_states(virtual_expr, runtime_generated);
return;
}
for (const auto& child : expr->children()) {
- _mark_common_expr_states(child);
+ _mark_common_expr_states(child, runtime_generated);
}
}
@@ -2120,12 +2121,20 @@ Status
SegmentIterator::_vec_init_lazy_materialization() {
// Step2: extract columns that can execute expr context
if (!_common_expr_ctxs_push_down.empty()) {
for (const auto& expr_ctx : _common_expr_ctxs_push_down) {
- _mark_common_expr_states(expr_ctx->root());
+ const auto& root = expr_ctx->root();
+ // TopN filters and runtime filters are attached to the scan on BE
after the FE
+ // planner computed nested predicate access paths.
+ _mark_common_expr_states(root, root->is_topn_filter() ||
root->is_rf_wrapper());
}
if (_is_need_expr_eval) {
for (uint32_t cid = 0; cid < _schema->num_block_columns(); ++cid) {
const auto field_type = _schema->column(cid)->type();
- if (_column_states[cid].has_common_expr &&
_enable_prune_nested_column &&
+ // A runtime-generated common expression may read nested
fields outside the
+ // FE predicate access paths, e.g. ORDER BY s.a with WHERE on
s.b. Keep such
+ // a column in the NORMAL read phase so every access path is
materialized
+ // before the expression is evaluated.
+ if (_column_states[cid].has_common_expr &&
+ !_column_states[cid].has_runtime_common_expr &&
_enable_prune_nested_column &&
(field_type == FieldType::OLAP_FIELD_TYPE_STRUCT ||
field_type == FieldType::OLAP_FIELD_TYPE_ARRAY ||
field_type == FieldType::OLAP_FIELD_TYPE_MAP)) {
diff --git a/be/src/storage/segment/segment_iterator.h
b/be/src/storage/segment/segment_iterator.h
index 5602c6280ab..8645f296025 100644
--- a/be/src/storage/segment/segment_iterator.h
+++ b/be/src/storage/segment/segment_iterator.h
@@ -189,7 +189,7 @@ private:
void _init_column_states();
void _rebuild_scan_predicate_states();
- void _mark_common_expr_states(const VExprSPtr& expr);
+ void _mark_common_expr_states(const VExprSPtr& expr, bool
runtime_generated);
Status _vec_init_lazy_materialization();
uint32_t segment_id() const { return _segment->id(); }
@@ -341,6 +341,13 @@ private:
// predicates, then only residual predicates after index evaluation.
bool has_scan_pred = false;
bool has_common_expr = false;
+ // Set when a pushed-down common expression was generated on BE at
runtime
+ // (TopN filter or runtime filter) rather than by the FE planner. FE
computes
+ // predicate access paths only from planner-visible predicates, so
such an
+ // expression may touch nested fields that are not predicate paths.
The column
+ // must then read all of its access paths before filtering instead of
splitting
+ // lazy nested-column recovery.
+ bool has_runtime_common_expr = false;
// Index evaluation sets this to false when it fully supplies the
column result.
// _need_read_data() applies the remaining read constraints.
bool need_read_data = true;
diff --git
a/be/test/storage/segment/segment_iterator_runtime_common_expr_test.cpp
b/be/test/storage/segment/segment_iterator_runtime_common_expr_test.cpp
new file mode 100644
index 00000000000..30e33681649
--- /dev/null
+++ b/be/test/storage/segment/segment_iterator_runtime_common_expr_test.cpp
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/Exprs_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/primitive_type.h"
+#include "exprs/vexpr.h"
+#include "exprs/vexpr_context.h"
+#include "exprs/vslot_ref.h"
+#include "exprs/vtopn_pred.h"
+#include "storage/olap_common.h"
+#include "storage/schema.h"
+#include "storage/segment/column_reader.h"
+#include "storage/tablet/tablet_schema.h"
+
+// White-box access to SegmentIterator lazy materialization planning. This
mirrors the
+// existing segment_iterator_* white-box tests.
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wkeyword-macro"
+#endif
+#include "storage/segment/segment_iterator.h"
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+
+namespace doris::segment_v2 {
+namespace {
+
+// Stands in for a STRUCT column iterator whose predicate access paths cover
only one
+// nested field while another nested field is a lazy materialization target.
+class SplitStructColumnIterator final : public ColumnIterator {
+public:
+ Status seek_to_ordinal(ordinal_t ord) override { return Status::OK(); }
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ MutableColumnPtr& dst) override {
+ return Status::OK();
+ }
+
+ ordinal_t get_current_ordinal() const override { return 0; }
+
+ bool has_lazy_read_target() const override { return true; }
+};
+
+constexpr ColumnId kKeyOrdinal = 0;
+constexpr ColumnId kStructOrdinal = 1;
+
+ReadSchemaSPtr make_read_schema() {
+ auto key = std::make_shared<TabletColumn>();
+ key->set_unique_id(0);
+ key->set_name("k");
+ key->set_type(FieldType::OLAP_FIELD_TYPE_INT);
+ key->set_is_key(true);
+ key->set_is_nullable(false);
+
+ auto s = std::make_shared<TabletColumn>();
+ s->set_unique_id(1);
+ s->set_name("s");
+ s->set_type(FieldType::OLAP_FIELD_TYPE_STRUCT);
+ s->set_is_nullable(true);
+ for (const auto* name : {"a", "b"}) {
+ TabletColumn sub;
+ sub.set_name(name);
+ sub.set_type(FieldType::OLAP_FIELD_TYPE_INT);
+ sub.set_is_nullable(true);
+ s->add_sub_column(sub);
+ }
+
+ return std::make_shared<ReadSchema>(std::vector<TabletColumnPtr> {key, s});
+}
+
+VExprSPtr make_struct_slot_ref() {
+ return VSlotRef::create_shared(/*slot_id=*/1, /*column_id=*/kStructOrdinal,
+ /*column_uniq_id=*/1,
std::make_shared<DataTypeInt32>(), "s");
+}
+
+// A planner-visible common expression: its nested accesses are covered by the
FE
+// predicate access paths.
+VExprContextSPtr make_planner_expr_ctx() {
+ return VExprContext::create_shared(make_struct_slot_ref());
+}
+
+// A TopN filter is created on BE at runtime, after the FE computed access
paths.
+VExprContextSPtr make_topn_filter_ctx() {
+ TExprNode node;
+ node.__set_node_type(TExprNodeType::FUNCTION_CALL);
+ node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ node.__set_is_nullable(true);
+ auto topn_pred = VTopNPred::create_shared(node, /*source_node_id=*/0,
nullptr);
+ topn_pred->add_child(make_struct_slot_ref());
+ return VExprContext::create_shared(topn_pred);
+}
+
+} // namespace
+
+class SegmentIteratorRuntimeCommonExprTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ _read_schema = make_read_schema();
+ _iter = std::make_unique<SegmentIterator>(nullptr, _read_schema);
+ _iter->_opts.stats = &_stats;
+ _iter->_enable_prune_nested_column = true;
+
+ auto struct_iter = std::make_unique<SplitStructColumnIterator>();
+
struct_iter->set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE);
+ _iter->_column_iterators[kStructOrdinal] = std::move(struct_iter);
+ }
+
+ ReadSchemaSPtr _read_schema;
+ std::unique_ptr<SegmentIterator> _iter;
+ OlapReaderStatistics _stats;
+};
+
+TEST_F(SegmentIteratorRuntimeCommonExprTest,
plannerExprKeepsLazyNestedRecovery) {
+ _iter->_common_expr_ctxs_push_down = {make_planner_expr_ctx()};
+
+ auto st = _iter->_vec_init_lazy_materialization();
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ EXPECT_TRUE(_iter->_column_states[kStructOrdinal].has_common_expr);
+
EXPECT_FALSE(_iter->_column_states[kStructOrdinal].has_runtime_common_expr);
+ EXPECT_EQ(_iter->_lazy_pruned_ordinals, (std::vector<ColumnId>
{kStructOrdinal}));
+ EXPECT_EQ(_iter->_common_expr_ordinals, (std::vector<ColumnId>
{kStructOrdinal}));
+ EXPECT_EQ(_iter->_output_ordinals, (std::vector<ColumnId> {kKeyOrdinal}));
+}
+
+TEST_F(SegmentIteratorRuntimeCommonExprTest,
topnFilterDisablesLazyNestedRecovery) {
+ _iter->_common_expr_ctxs_push_down = {make_topn_filter_ctx()};
+
+ auto st = _iter->_vec_init_lazy_materialization();
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ EXPECT_TRUE(_iter->_column_states[kStructOrdinal].has_common_expr);
+ EXPECT_TRUE(_iter->_column_states[kStructOrdinal].has_runtime_common_expr);
+ EXPECT_FALSE(_iter->_column_states[kKeyOrdinal].has_runtime_common_expr);
+ // The TopN filter may read nested fields outside the predicate access
paths, so the
+ // struct must be fully materialized before the expression runs.
+ EXPECT_TRUE(_iter->_lazy_pruned_ordinals.empty());
+ EXPECT_EQ(_iter->_common_expr_ordinals, (std::vector<ColumnId>
{kStructOrdinal}));
+}
+
+TEST_F(SegmentIteratorRuntimeCommonExprTest,
topnFilterWinsOverPlannerExprOnSameColumn) {
+ _iter->_common_expr_ctxs_push_down = {make_planner_expr_ctx(),
make_topn_filter_ctx()};
+
+ auto st = _iter->_vec_init_lazy_materialization();
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ EXPECT_TRUE(_iter->_column_states[kStructOrdinal].has_runtime_common_expr);
+ EXPECT_TRUE(_iter->_lazy_pruned_ordinals.empty());
+}
+
+} // namespace doris::segment_v2
diff --git
a/regression-test/data/nereids_rules_p0/column_pruning/topn_filter_nested_column_pruning.out
b/regression-test/data/nereids_rules_p0/column_pruning/topn_filter_nested_column_pruning.out
new file mode 100644
index 00000000000..1b8669d4355
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/column_pruning/topn_filter_nested_column_pruning.out
@@ -0,0 +1,37 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !timestamptz_asc --
+2000-01-01 00:00:00+08:00
+
+-- !timestamptz_desc --
+2030-01-01 00:00:00+08:00
+
+-- !timestamptz_asc_limit2 --
+2000-01-01 00:00:00+08:00
+2023-03-12 14:59:59+08:00
+
+-- !int_asc --
+1
+
+-- !int_desc --
+20
+
+-- !int_nulls_first --
+1
+
+-- !int_selective --
+2 9
+
+-- !int_same_field --
+10
+
+-- !date_asc --
+2000-01-01
+
+-- !date_desc --
+2030-01-01
+
+-- !multi_struct --
+0 1 2000-01-01
+
+-- !timestamptz_asc_default_parallel --
+2000-01-01 00:00:00+08:00
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/topn_filter_nested_column_pruning.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/topn_filter_nested_column_pruning.groovy
new file mode 100644
index 00000000000..2d4698e348e
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/topn_filter_nested_column_pruning.groovy
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// The TopN filter is attached to the scan on BE at runtime, after the planner
computed
+// nested predicate access paths. When ORDER BY and WHERE touch different
fields of the
+// same STRUCT, the TopN filter must still see the ORDER BY field on every
tablet.
+suite("topn_filter_nested_column_pruning") {
+ sql "set enable_prune_nested_column = true"
+ sql "set topn_filter_ratio = 0.5"
+ sql "set parallel_pipeline_task_num = 1"
+ sql "set time_zone = '+08:00'"
+
+ sql "DROP TABLE IF EXISTS topn_filter_nested_prune_tbl"
+ sql """
+ CREATE TABLE topn_filter_nested_prune_tbl (
+ pk INT,
+ ts STRUCT<a: TIMESTAMPTZ(0), b: TIMESTAMPTZ(0)>,
+ si STRUCT<a: INT, b: INT>,
+ sd STRUCT<a: DATE, b: DATE>
+ )
+ DUPLICATE KEY(pk)
+ DISTRIBUTED BY HASH(pk) BUCKETS 2
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+ sql """
+ INSERT INTO topn_filter_nested_prune_tbl VALUES
+ (0, struct('2000-01-01 00:00:00+08:00', '2024-01-01 00:00:00+00:00'),
+ struct(1, 10), struct('2000-01-01', '2024-01-01')),
+ (1, struct('2023-03-12 01:59:59-05:00', '2023-11-05 01:30:00-05:00'),
+ struct(5, NULL), struct('2023-03-12', NULL)),
+ (2, struct('2023-11-05 01:30:00-04:00', '2024-12-31 00:00:00+00:00'),
+ struct(9, 30), struct('2023-11-05', '2024-12-31')),
+ (3, struct('2030-01-01 00:00:00+08:00', '2030-01-01 00:00:00+00:00'),
+ struct(20, 40), struct('2030-01-01', '2030-01-01'))
+ """
+ // The TopN filter is only generated when the planner has row count
statistics.
+ sql "ANALYZE TABLE topn_filter_nested_prune_tbl WITH SYNC"
+
+ explain {
+ sql """
+ SELECT struct_element(ts, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(ts, 'b') IS NOT NULL
+ ORDER BY 1 NULLS LAST LIMIT 1
+ """
+ contains("TOPN OPT:1")
+ contains("all access paths: [ts.a, ts.b.NULL]")
+ contains("predicate access paths: [ts.b.NULL]")
+ }
+
+ qt_timestamptz_asc """
+ SELECT struct_element(ts, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(ts, 'b') IS NOT NULL
+ ORDER BY 1 NULLS LAST LIMIT 1
+ """
+ qt_timestamptz_desc """
+ SELECT struct_element(ts, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(ts, 'b') IS NOT NULL
+ ORDER BY 1 DESC NULLS LAST LIMIT 1
+ """
+ qt_timestamptz_asc_limit2 """
+ SELECT struct_element(ts, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(ts, 'b') IS NOT NULL
+ ORDER BY 1 NULLS LAST LIMIT 2
+ """
+
+ qt_int_asc """
+ SELECT struct_element(si, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(si, 'b') IS NOT NULL
+ ORDER BY 1 NULLS LAST LIMIT 1
+ """
+ qt_int_desc """
+ SELECT struct_element(si, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(si, 'b') IS NOT NULL
+ ORDER BY 1 DESC NULLS LAST LIMIT 1
+ """
+ qt_int_nulls_first """
+ SELECT struct_element(si, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(si, 'b') IS NOT NULL
+ ORDER BY 1 NULLS FIRST LIMIT 1
+ """
+ // Selective predicate on the other field.
+ qt_int_selective """
+ SELECT pk, struct_element(si, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(si, 'b') > 15
+ ORDER BY 2 NULLS LAST LIMIT 1
+ """
+ // ORDER BY and WHERE on the same field.
+ qt_int_same_field """
+ SELECT struct_element(si, 'b')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(si, 'b') IS NOT NULL
+ ORDER BY 1 NULLS LAST LIMIT 1
+ """
+
+ qt_date_asc """
+ SELECT struct_element(sd, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(sd, 'b') IS NOT NULL
+ ORDER BY 1 NULLS LAST LIMIT 1
+ """
+ qt_date_desc """
+ SELECT struct_element(sd, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(sd, 'b') IS NOT NULL
+ ORDER BY 1 DESC NULLS LAST LIMIT 1
+ """
+
+ qt_multi_struct """
+ SELECT pk, struct_element(si, 'a'), struct_element(sd, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(si, 'b') IS NOT NULL AND struct_element(sd, 'b')
IS NOT NULL
+ ORDER BY 2, 3 NULLS LAST LIMIT 1
+ """
+
+ sql "set parallel_pipeline_task_num = 0"
+ qt_timestamptz_asc_default_parallel """
+ SELECT struct_element(ts, 'a')
+ FROM topn_filter_nested_prune_tbl
+ WHERE struct_element(ts, 'b') IS NOT NULL
+ ORDER BY 1 NULLS LAST LIMIT 1
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
