This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d518544d7a6 [fix](fe) Fix TopN lazy materialization for queries
ordered by an alias (#68019)
d518544d7a6 is described below
commit d518544d7a69066f0c6591a67a0ecad47f16eeb1
Author: minghong <[email protected]>
AuthorDate: Tue Sep 22 19:42:27 2026 +0800
[fix](fe) Fix TopN lazy materialization for queries ordered by an alias
(#68019)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary:
`SELECT lazy_col AS x, lazy_col AS y FROM t ORDER BY x LIMIT 1` failed
planning with
`A expression contains slot not from children`.
The TopN order key is the alias slot, so that alias has to be computed
below the TopN.
`MaterializeProbeVisitor` only protects the order key slot itself (an
order key slot is in
`TopN.getInputSlots()`) and never resolves an identity alias down to the
column the alias reads.
The probe of the other output (`lazy_col AS y`) therefore resolved to
the base column `lazy_col`
and classified it as lazily materialized, so `LazySlotPruning` removed
`lazy_col` from the scan while
`lazy_col AS x` below the TopN still read it. The final `Validator`
rejected the resulting plan and
the query returned an error. With `fe_debug=true` the failure was caught
inside `LazyMaterializeTopN`
instead, which silently skipped lazy materialization (the query
succeeded but lost the optimization).
Reproduction (master, `fe_debug=false`):
```sql
create table t(sort_col int, lazy_col int) duplicate key(sort_col)
distributed by hash(sort_col) buckets 1 properties('replication_num'='1');
select lazy_col as x, lazy_col as y from t order by x limit 1;
-- ERROR 1105: A expression contains slot not from children
-- Slot: lazy_col#1 Children Output:{0, 4}
-- Plan: PhysicalProject[lazy_col#1 AS x#2, __DORIS_GLOBAL_ROWID_COL__t#4]
-- +--PhysicalLazyMaterializeOlapScan[PhysicalOlapScan[t]]
```
Fix: `LazyMaterializeTopN` resolves the TopN order keys through the
identity alias chain of the
Projects under the TopN and adds the resolved slots (plus the
intermediate alias slots) to
`requiredMaterializedSlots`, so the probe rejects every lazy candidate
backed by a column an order
key reads. The resolution stops at set operations, which the probe never
materializes through
(lazy materialization is not supported through set operations today; if
that ever changes, order
keys have to be resolved per branch).
Effect: affected plans now either keep only the ordering column
materialized (other columns are
still fetched lazily) or skip lazy materialization, and the plan stays
valid. Plans that order by a
plain column are unchanged.
### Release note
TopN lazy materialization no longer builds an invalid plan (no more
`A expression contains slot not from children`) when a query orders by
an alias of a column.
The column that feeds the order key is materialized during the scan,
while other columns keep using
lazy materialization.
### Check List (For Author)
- Test
- [x] Regression test
(`regression-test/suites/query_p0/topn_lazy/order_by_alias`)
- [x] Unit Test (`TopnLazyMaterializeTest`, `LazyMaterializeTopNTest`)
- Behavior changed:
- [x] Yes. Queries that order by an alias of a projected column no
longer fail planning; the
ordering column is kept materialized instead of being pruned from the
scan.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../post/materialize/LazyMaterializeTopN.java | 118 ++++++++++++
.../post/materialize/LazySlotPruning.java | 8 +-
.../post/materialize/MaterializeProbeVisitor.java | 30 ++-
.../postprocess/TopnLazyMaterializeTest.java | 156 ++++++++++++++++
.../post/materialize/LazyMaterializeTopNTest.java | 30 +++
.../order_by_alias/topn_lazy_order_by_alias.out | 106 +++++++++++
.../order_by_alias/topn_lazy_order_by_alias.groovy | 202 +++++++++++++++++++++
7 files changed, 640 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
index 9acec57125a..11cd45d9115 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
@@ -36,16 +36,21 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
+import org.apache.doris.nereids.util.PlanUtils;
import org.apache.doris.qe.SessionVariable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -107,6 +112,7 @@ public class LazyMaterializeTopN extends PlanPostProcessor {
List<Slot> materializedSlots = new ArrayList<>();
Set<Slot> requiredMaterializedSlots = new HashSet<>();
collectProjectExprInputSlots(topN.child(), requiredMaterializedSlots);
+ collectRequiredAliasSources(topN, requiredMaterializedSlots);
/*
* requiredMaterializedSlots only records slots consumed by
Project/final-projection expressions inside the
@@ -119,6 +125,9 @@ public class LazyMaterializeTopN extends PlanPostProcessor {
* a is in Filter.getInputSlots(). Both return Optional.empty() and
are appended to materializedSlots below.
* Therefore an empty requiredMaterializedSlots set does not mean
every scan column can be delayed; it only
* means no extra Project/final-projection input must be forced
materialized by this local safety check.
+ *
+ * The probe only protects the slots the operators reference directly,
so the columns those slots read
+ * through identity aliases are protected by
collectRequiredAliasSources above.
*/
for (Slot slot : effectiveOutput) {
Optional<MaterializeSource> source =
computeMaterializeSource(topN, (SlotReference) slot,
@@ -292,6 +301,115 @@ public class LazyMaterializeTopN extends
PlanPostProcessor {
}
}
+ /**
+ * Keep the columns consumed below the TopN materialized during the scan,
resolved through identity
+ * aliases.
+ *
+ * <p>{@link MaterializeProbeVisitor} only protects the slot it is
tracing: a slot consumed by an
+ * operator on the way from the TopN down to the relation stops the probe,
but the columns an
+ * identity alias reads are never resolved. For
+ *
+ * <pre>
+ * OuterTopN(order by z)
+ * InnerTopN(order by x)
+ * Project(lazy_col AS x, lazy_col AS y, other_col AS z)
+ * OlapScan
+ * </pre>
+ *
+ * probing the outer output {@code y} resolves to the base column {@code
lazy_col}, so {@code lazy_col}
+ * is classified lazy and {@link LazySlotPruning} removes it from the
scan, while {@code lazy_col AS x}
+ * below the outer TopN is still read by the inner TopN. The plan then
references a slot its child no
+ * longer produces and the final {@link Validator} rejects it. The same
happens when an identity alias
+ * is consumed by a filter, a join condition or any other operator that
stays below the TopN.
+ *
+ * <p>Therefore every slot consumed by this TopN and by the operators
below it (its own order keys, the
+ * expressions of every descendant operator and the slots that are
required materialized already) is
+ * resolved through its identity alias chain. The only consumed slots that
are not required are the ones
+ * {@link MaterializeProbeVisitor#isIndexLazyFilter} keeps lazy: {@link
LazySlotPruning#visitPhysicalFilter}
+ * drops those from the scan's lazy slots for the same reason, so the scan
keeps producing them for the
+ * predicate and nothing below the TopN can be starved. Project
expressions are handled by
+ * {@link #collectProjectExprInputSlots}, which knows that a transparent
{@code Alias(Slot)} output may
+ * still be fetched lazily.
+ *
+ * <p>A set operation is a boundary: {@link MaterializeProbeVisitor} never
reports a lazy source for a
+ * slot produced by a set operation, and {@link #collectIdentityAliasMap}
stops at it, so the aliases
+ * below a set operation are neither resolved nor reachable. If lazy
materialization is ever extended
+ * through set operations, the consumed slots have to be resolved per set
operation branch instead.
+ */
+ private void collectRequiredAliasSources(PhysicalTopN<? extends Plan> topN,
+ Set<Slot> requiredMaterializedSlots) {
+ Map<Slot, Slot> aliasToChild = new HashMap<>();
+ collectIdentityAliasMap(topN.child(), aliasToChild);
+
+ Set<Slot> consumedSlots = new HashSet<>();
+ Set<Slot> indexLazySlots = new HashSet<>();
+ collectConsumedSlots(topN, consumedSlots, indexLazySlots);
+ consumedSlots.addAll(requiredMaterializedSlots);
+ for (Slot slot : consumedSlots) {
+ if (!indexLazySlots.contains(slot)) {
+ collectAliasChain(slot, aliasToChild,
requiredMaterializedSlots);
+ }
+ }
+ }
+
+ /** Collect the slots consumed by {@code plan} itself and by every
operator below it. */
+ private void collectConsumedSlots(Plan plan, Set<Slot> consumedSlots,
Set<Slot> indexLazySlots) {
+ if (plan instanceof PhysicalSetOperation) {
+ // Set operations are not materialized lazily, so nothing below
them can be lazy either.
+ return;
+ }
+ if (!(plan instanceof PhysicalProject)) {
+ // Project expressions are covered by
collectProjectExprInputSlots, which keeps the input of a
+ // transparent Alias(Slot) lazy because that alias output may
still be fetched later.
+ consumedSlots.addAll(plan.getInputSlots());
+ }
+ if (plan instanceof PhysicalFilter
+ &&
MaterializeProbeVisitor.isIndexLazyFilter((PhysicalFilter<?>) plan)) {
+ indexLazySlots.addAll(plan.getInputSlots());
+ }
+ if (plan instanceof PhysicalGenerate) {
+ // PhysicalGenerate.getExpressions() exposes the generators only,
while the lateral conjuncts
+ // are preserved by the implementation rule and executed by the
generate itself. Generator
+ // outputs are produced by the generate, never by the child,
exactly like LogicalGenerate.
+ PhysicalGenerate<?> generate = (PhysicalGenerate<?>) plan;
+ consumedSlots.addAll(Sets.difference(
+ PlanUtils.fastGetInputSlots(generate.getConjuncts()),
+ new HashSet<>(generate.getGeneratorOutput())));
+ }
+ for (Plan child : plan.children()) {
+ collectConsumedSlots(child, consumedSlots, indexLazySlots);
+ }
+ }
+
+ /** Collect {@code alias slot -> child slot} for every identity Alias of
the Projects under the TopN. */
+ private void collectIdentityAliasMap(Plan plan, Map<Slot, Slot>
aliasToChild) {
+ if (plan instanceof PhysicalSetOperation) {
+ // Set operations are not materialized lazily, so aliases below
them are never reached.
+ return;
+ }
+ if (plan instanceof PhysicalProject) {
+ for (NamedExpression project : ((PhysicalProject<?>)
plan).getProjects()) {
+ if (project instanceof Alias && project.child(0) instanceof
Slot) {
+ aliasToChild.putIfAbsent(project.toSlot(), (Slot)
project.child(0));
+ }
+ }
+ }
+ for (Plan child : plan.children()) {
+ collectIdentityAliasMap(child, aliasToChild);
+ }
+ }
+
+ /** Add a slot together with every slot of its alias chain, so the column
the chain ends at is protected. */
+ @VisibleForTesting
+ static void collectAliasChain(Slot slot, Map<Slot, Slot> aliasToChild,
Set<Slot> requiredMaterializedSlots) {
+ Set<Slot> visited = new HashSet<>();
+ Slot current = slot;
+ while (current != null && visited.add(current)) {
+ requiredMaterializedSlots.add(current);
+ current = aliasToChild.get(current);
+ }
+ }
+
private List<SlotReference> moveRowIdsToTail(List<Slot> slots,
Set<SlotReference> rowIds) {
List<SlotReference> reArrangedSlots = new ArrayList<>();
List<SlotReference> reArrangedRowIds = new ArrayList<>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java
index fce964758f1..edc11ee1261 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java
@@ -40,7 +40,6 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
-import org.apache.doris.qe.SessionVariable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
@@ -117,7 +116,7 @@ public class LazySlotPruning extends
DefaultPlanRewriter<LazySlotPruning.Context
@Override
public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter,
Context context) {
- if (SessionVariable.getTopNLazyMaterializationUsingIndex() &&
filter.child() instanceof PhysicalOlapScan) {
+ if (MaterializeProbeVisitor.isIndexLazyFilter(filter)) {
/*
materialization(materializedSlots=[a, b], lazy=[c])
->topn(b)
@@ -128,7 +127,10 @@ public class LazySlotPruning extends
DefaultPlanRewriter<LazySlotPruning.Context
->topn(b)
->project(rowid, b)
->filter(a=1, output=(rowid, a, b))
- ->materializeOlapScan(rowid, lazy=[a,c], T[a,b,c])
+ ->materializeOlapScan(rowid, lazy=[c], T[a,b,c])
+
+ The predicate slot a is dropped from the TopN tuple by the extra
project, but the scan keeps
+ producing it for the predicate: a is removed from the scan's lazy
slots below.
*/
List<Slot> lazySlotsToScan = new ArrayList<>();
boolean lazySlotsChanged = false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
index b112595dcef..5d98af97cc6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java
@@ -77,15 +77,31 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
}
+ /**
+ * Whether {@code filter} keeps its predicate slots lazy.
+ *
+ * <p>With {@code topn_lazy_materialization_using_index} the predicate of
a filter directly above an OLAP
+ * scan is evaluated while scanning, so its slots only have to be
re-materialized above the TopN instead
+ * of being carried through it. This is the single definition of that
shape:
+ * {@link MaterializeProbeVisitor#visitPhysicalFilter} reports those slots
as lazy sources,
+ * {@link LazySlotPruning#visitPhysicalFilter} keeps them out of the
scan's lazy slots (the scan still
+ * produces them for the predicate) and {@link LazyMaterializeTopN} must
not require them materialized.
+ */
+ static boolean isIndexLazyFilter(PhysicalFilter<? extends Plan> filter) {
+ if (!(filter.child() instanceof PhysicalOlapScan)) {
+ return false;
+ }
+ if (!SessionVariable.getTopNLazyMaterializationUsingIndex()) {
+ return false;
+ }
+ // Reject OLAP tables whose storage semantics cannot be reconstructed
from one row-id.
+ return supportOlapTopnLazyMaterialize(((PhysicalOlapScan)
filter.child()).getTable());
+ }
+
@Override
public Optional<MaterializeSource> visitPhysicalFilter(PhysicalFilter<?
extends Plan> filter,
ProbeContext
context) {
- if (SessionVariable.getTopNLazyMaterializationUsingIndex() &&
filter.child() instanceof PhysicalOlapScan) {
- // Reject OLAP tables whose storage semantics cannot be
reconstructed from one row-id.
- OlapTable table = ((PhysicalOlapScan) filter.child()).getTable();
- if (!supportOlapTopnLazyMaterialize(table)) {
- return Optional.empty();
- }
+ if (isIndexLazyFilter(filter)) {
if (filter.getInputSlots().contains(context.slot)) {
Relation relation = (Relation) filter.child();
return Optional.of(new MaterializeSource(
@@ -152,7 +168,7 @@ public class MaterializeProbeVisitor extends
DefaultPlanVisitor<Optional<Materia
* topn lazy materialization keeps only one row-id for each
relation.</li>
* </ul>
*/
- private boolean supportOlapTopnLazyMaterialize(OlapTable table) {
+ private static boolean supportOlapTopnLazyMaterialize(OlapTable table) {
if (KeysType.AGG_KEYS.equals(table.getKeysType())) {
return false;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
index ee0942c3071..010d3749758 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
@@ -142,6 +142,162 @@ public class TopnLazyMaterializeTest extends SSBTestBase {
materializeNodes.get(0).getLazyBaseColumnIndices());
}
+ @Test
+ public void testOrderByAliasKeepsItsSourceMaterialized() throws Exception {
+ this.createTable("create table lazy_materialize_order_by_alias_tbl("
+ + "sort_col int, lazy_col int, other_col int) "
+ + "duplicate key(sort_col) distributed by hash(sort_col)
buckets 1 "
+ + "properties('replication_num' = '1')");
+ // TestWithFeService enables feDebug, which makes LazyMaterializeTopN
validate its own result and
+ // silently fall back on an invalid plan. Turn it off so an invalid
plan fails the whole test.
+ boolean feDebug = connectContext.getSessionVariable().feDebug;
+ connectContext.getSessionVariable().feDebug = false;
+ try {
+ // The TopN sorts by an alias of lazy_col, so lazy_col must be
materialized for the sort.
+ // Otherwise LazySlotPruning removes it from the scan while
`lazy_col AS x` below the TopN
+ // still reads it, and the resulting plan references a slot its
child no longer produces.
+ PhysicalPlan plan = postProcess("select lazy_col as x, lazy_col as
y "
+ + "from lazy_materialize_order_by_alias_tbl order by x
limit 1");
+ Assertions.assertTrue(
+ plan.collectToList(node -> node instanceof
PhysicalLazyMaterialize).isEmpty(),
+ plan.treeString());
+
+ // Sorting by a bare column that is also aliased must keep that
column materialized.
+ plan = postProcess("select lazy_col, lazy_col as y "
+ + "from lazy_materialize_order_by_alias_tbl order by
lazy_col limit 1");
+ Assertions.assertTrue(
+ plan.collectToList(node -> node instanceof
PhysicalLazyMaterialize).isEmpty(),
+ plan.treeString());
+
+ // Only the column the order key reads is forced materialized,
other_col is still fetched lazily.
+ plan = postProcess("select lazy_col as x, other_col as y "
+ + "from lazy_materialize_order_by_alias_tbl order by x
limit 1");
+ List<PhysicalLazyMaterialize<? extends Plan>> materializeNodes =
plan.collectToList(
+ node -> node instanceof PhysicalLazyMaterialize);
+ Assertions.assertEquals(1, materializeNodes.size(),
plan.treeString());
+ Assertions.assertEquals(ImmutableList.of(ImmutableList.of(2)),
+ materializeNodes.get(0).getLazyBaseColumnIndices());
+
+ // Sorting by a column that is not aliased keeps every projected
column lazily fetched.
+ plan = postProcess("select lazy_col as x, other_col as y "
+ + "from lazy_materialize_order_by_alias_tbl order by
sort_col limit 1");
+ materializeNodes = plan.collectToList(node -> node instanceof
PhysicalLazyMaterialize);
+ Assertions.assertEquals(1, materializeNodes.size(),
plan.treeString());
+ List<Integer> lazyColumnIndexes = Lists.newArrayList();
+
materializeNodes.get(0).getLazyBaseColumnIndices().forEach(lazyColumnIndexes::addAll);
+ lazyColumnIndexes.sort(Integer::compareTo);
+ Assertions.assertEquals(ImmutableList.of(1, 2), lazyColumnIndexes);
+ } finally {
+ connectContext.getSessionVariable().feDebug = feDebug;
+ }
+ }
+
+ @Test
+ public void testNestedTopNKeepsAliasSourceMaterialized() throws Exception {
+ this.createTable("create table lazy_materialize_nested_topn_tbl("
+ + "sort_col int, lazy_col int, other_col int) "
+ + "duplicate key(sort_col) distributed by hash(sort_col)
buckets 1 "
+ + "properties('replication_num' = '1')");
+ boolean feDebug = connectContext.getSessionVariable().feDebug;
+ connectContext.getSessionVariable().feDebug = false;
+ try {
+ // Only the outer TopN is rewritten. Probing its output `y`
resolves to lazy_col, which the
+ // inner TopN still reads through `lazy_col AS x`, so lazy_col
must stay materialized.
+ PhysicalPlan plan = postProcess("select y from (select lazy_col as
x, lazy_col as y, other_col as z "
+ + "from lazy_materialize_nested_topn_tbl order by x limit
2) s order by z limit 1");
+ Assertions.assertTrue(
+ plan.collectToList(node -> node instanceof
PhysicalLazyMaterialize).isEmpty(),
+ plan.treeString());
+
+ // A column that nothing below the TopN reads may still be fetched
lazily through the TopNs.
+ plan = postProcess("select w from (select lazy_col as x, other_col
as w, lazy_col as y, sort_col "
+ + "from lazy_materialize_nested_topn_tbl order by x limit
2) s order by sort_col limit 1");
+ List<PhysicalLazyMaterialize<? extends Plan>> materializeNodes =
plan.collectToList(
+ node -> node instanceof PhysicalLazyMaterialize);
+ Assertions.assertEquals(1, materializeNodes.size(),
plan.treeString());
+ Assertions.assertEquals(ImmutableList.of(ImmutableList.of(2)),
+ materializeNodes.get(0).getLazyBaseColumnIndices());
+ } finally {
+ connectContext.getSessionVariable().feDebug = feDebug;
+ }
+ }
+
+ @Test
+ public void testIndexFilterPredicateSlotStaysLazy() throws Exception {
+ this.createTable("create table lazy_materialize_index_filter_tbl("
+ + "user_id bigint, username varchar(50), age int, addr
varchar(50)) "
+ + "duplicate key(user_id, username) distributed by
hash(user_id) buckets 1 "
+ + "properties('replication_num' = '1')");
+ boolean feDebug = connectContext.getSessionVariable().feDebug;
+ boolean usingIndex =
connectContext.getSessionVariable().topNLazyMaterializationUsingIndex;
+ connectContext.getSessionVariable().feDebug = false;
+ connectContext.getSessionVariable().topNLazyMaterializationUsingIndex
= true;
+ try {
+ // The filter is evaluated through the index, so its predicate
column is kept by the scan for
+ // the predicate and re-materialized lazily above the TopN. It
must stay lazy even though an
+ // operator below the TopN consumes it.
+ PhysicalPlan plan = postProcess("select * from
lazy_materialize_index_filter_tbl "
+ + "where user_id = 1 order by username limit 1");
+ List<PhysicalLazyMaterialize<? extends Plan>> materializeNodes =
plan.collectToList(
+ node -> node instanceof PhysicalLazyMaterialize);
+ Assertions.assertEquals(1, materializeNodes.size(),
plan.treeString());
+ List<Integer> lazyColumnIndexes = Lists.newArrayList();
+
materializeNodes.get(0).getLazyBaseColumnIndices().forEach(lazyColumnIndexes::addAll);
+ lazyColumnIndexes.sort(Integer::compareTo);
+ Assertions.assertEquals(ImmutableList.of(0, 2, 3),
lazyColumnIndexes, plan.treeString());
+ } finally {
+ connectContext.getSessionVariable().feDebug = feDebug;
+
connectContext.getSessionVariable().topNLazyMaterializationUsingIndex =
usingIndex;
+ }
+ }
+
+ @Test
+ public void testLateralGenerateConjunctKeepsItsAliasSourceMaterialized()
throws Exception {
+ this.createTable("create table lazy_materialize_lateral_tbl("
+ + "sort_col int, lazy_col int, other_col int, arr array<int>) "
+ + "duplicate key(sort_col) distributed by hash(sort_col)
buckets 1 "
+ + "properties('replication_num' = '1')");
+ boolean feDebug = connectContext.getSessionVariable().feDebug;
+ connectContext.getSessionVariable().feDebug = false;
+ try {
+ // The generate conjunct reads `x`, which is an alias of lazy_col.
PhysicalGenerate exposes
+ // only its generators through getInputSlots(), so the conjunct
inputs have to be resolved
+ // explicitly: otherwise lazy_col is pruned from the scan while
`lazy_col AS x` below the
+ // generate still reads it. Only other_col (read by nothing below
the TopN) stays lazy.
+ PhysicalPlan plan = postProcess("select s.y, s.w from (select
lazy_col as x, lazy_col as y, "
+ + "other_col as w, arr, sort_col from
lazy_materialize_lateral_tbl) s "
+ + "left join lateral unnest(s.arr) tt(tag) on tt.tag = s.x
"
+ + "order by s.sort_col limit 1");
+ List<PhysicalLazyMaterialize<? extends Plan>> materializeNodes =
plan.collectToList(
+ node -> node instanceof PhysicalLazyMaterialize);
+ Assertions.assertEquals(1, materializeNodes.size(),
plan.treeString());
+ Assertions.assertEquals(ImmutableList.of(ImmutableList.of(2)),
+ materializeNodes.get(0).getLazyBaseColumnIndices(),
plan.treeString());
+
+ // The conjunct reads the bare lazy_col, which the same Project
also aliases as `y`. Probing
+ // `y` resolves through that alias to lazy_col and no operator
below the TopN stops the probe
+ // for the bare slot, so lazy_col has to stay materialized for the
conjunct as well.
+ plan = postProcess("select s.y, s.w from (select lazy_col as y,
other_col as w, arr, sort_col, "
+ + "lazy_col from lazy_materialize_lateral_tbl) s "
+ + "left join lateral unnest(s.arr) tt(tag) on tt.tag =
s.lazy_col "
+ + "order by s.sort_col limit 1");
+ materializeNodes = plan.collectToList(node -> node instanceof
PhysicalLazyMaterialize);
+ Assertions.assertEquals(1, materializeNodes.size(),
plan.treeString());
+ Assertions.assertEquals(ImmutableList.of(ImmutableList.of(2)),
+ materializeNodes.get(0).getLazyBaseColumnIndices(),
plan.treeString());
+ } finally {
+ connectContext.getSessionVariable().feDebug = feDebug;
+ }
+ }
+
+ private PhysicalPlan postProcess(String sql) {
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .implement();
+ return new
PlanPostProcessors(checker.getCascadesContext()).process(checker.getPhysicalPlan());
+ }
+
@Test
public void testLightSchemaChangeFalse() throws Exception {
this.createTable("create table tm_lsc_false (k int, v int) duplicate
key(k) "
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopNTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopNTest.java
index d8763f27082..eb7ec84a039 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopNTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopNTest.java
@@ -30,8 +30,10 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public class LazyMaterializeTopNTest {
@@ -51,4 +53,32 @@ public class LazyMaterializeTopNTest {
Assertions.assertEquals(ImmutableList.of(aliasSlot),
requiredOutputSlots);
}
+
+ @Test
+ public void testCollectAliasChainKeepsTheColumnsOrderKeysRead() {
+ SlotReference baseSlot = new SlotReference("base",
IntegerType.INSTANCE);
+ Slot aliasSlot = new Alias(baseSlot, "alias").toSlot();
+ Slot aliasOfAliasSlot = new Alias(aliasSlot,
"alias_of_alias").toSlot();
+ Map<Slot, Slot> aliasToChild = ImmutableMap.of(aliasOfAliasSlot,
aliasSlot, aliasSlot, baseSlot);
+ Set<Slot> requiredMaterializedSlots = new HashSet<>();
+
+ LazyMaterializeTopN.collectAliasChain(aliasOfAliasSlot, aliasToChild,
requiredMaterializedSlots);
+
+ Assertions.assertEquals(
+ ImmutableSet.of(aliasOfAliasSlot, aliasSlot, baseSlot),
requiredMaterializedSlots);
+ }
+
+ @Test
+ public void testCollectAliasChainStopsOnCycle() {
+ // An alias chain must never keep the planner spinning, even if two
slots share one ExprId.
+ SlotReference baseSlot = new SlotReference("base",
IntegerType.INSTANCE);
+ Slot firstSlot = new Alias(baseSlot, "first").toSlot();
+ Slot secondSlot = new Alias(baseSlot, "second").toSlot();
+ Map<Slot, Slot> aliasToChild = ImmutableMap.of(firstSlot, secondSlot,
secondSlot, firstSlot);
+ Set<Slot> requiredMaterializedSlots = new HashSet<>();
+
+ LazyMaterializeTopN.collectAliasChain(firstSlot, aliasToChild,
requiredMaterializedSlots);
+
+ Assertions.assertEquals(ImmutableSet.of(firstSlot, secondSlot),
requiredMaterializedSlots);
+ }
}
diff --git
a/regression-test/data/query_p0/topn_lazy/order_by_alias/topn_lazy_order_by_alias.out
b/regression-test/data/query_p0/topn_lazy/order_by_alias/topn_lazy_order_by_alias.out
new file mode 100644
index 00000000000..8ad3b624152
--- /dev/null
+++
b/regression-test/data/query_p0/topn_lazy/order_by_alias/topn_lazy_order_by_alias.out
@@ -0,0 +1,106 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !repeated_alias_of_sort_key --
+10 10
+
+-- !repeated_alias_reversed --
+10 10
+
+-- !bare_column_and_alias --
+10 10
+
+-- !other_column_still_lazy --
+10 100
+
+-- !order_by_plain_column --
+10 100
+
+-- !using_index_repeated_alias --
+10 10
+
+-- !using_index_bare_column_and_alias --
+10 10
+
+-- !nested_topn_plan --
+PhysicalResultSink
+--PhysicalProject[s.y]
+----PhysicalTopN[GATHER_SORT]
+------PhysicalProject[s.y, s.z]
+--------PhysicalTopN[MERGE_SORT]
+----------PhysicalDistribute[DistributionSpecGather]
+------------PhysicalTopN[LOCAL_SORT]
+--------------PhysicalProject[topn_lazy_order_by_alias_tbl.lazy_col AS `x`,
topn_lazy_order_by_alias_tbl.lazy_col AS `y`,
topn_lazy_order_by_alias_tbl.other_col AS `z`]
+----------------PhysicalOlapScan[topn_lazy_order_by_alias_tbl]
+
+-- !nested_topn_result --
+10
+
+-- !nested_topn_still_lazy_plan --
+PhysicalResultSink
+--PhysicalProject[s.w]
+----PhysicalLazyMaterialize[materializedSlots:(s.sort_col) lazySlots:(s.w)]
+------PhysicalTopN[GATHER_SORT]
+--------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_tbl,
s.sort_col]
+----------PhysicalTopN[MERGE_SORT]
+------------PhysicalDistribute[DistributionSpecGather]
+--------------PhysicalTopN[LOCAL_SORT]
+----------------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_tbl,
topn_lazy_order_by_alias_tbl.lazy_col AS `x`,
topn_lazy_order_by_alias_tbl.sort_col]
+------------------PhysicalLazyMaterializeOlapScan[topn_lazy_order_by_alias_tbl
lazySlots:(topn_lazy_order_by_alias_tbl.other_col)]
+
+-- !index_mode_selective_lazy_plan --
+PhysicalResultSink
+--PhysicalProject[topn_lazy_order_by_alias_tbl.lazy_col,
topn_lazy_order_by_alias_tbl.other_col, x]
+----PhysicalLazyMaterialize[materializedSlots:(x,topn_lazy_order_by_alias_tbl.lazy_col)
lazySlots:(topn_lazy_order_by_alias_tbl.other_col)]
+------PhysicalTopN[MERGE_SORT]
+--------PhysicalDistribute[DistributionSpecGather]
+----------PhysicalTopN[LOCAL_SORT]
+------------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_tbl,
topn_lazy_order_by_alias_tbl.lazy_col, topn_lazy_order_by_alias_tbl.lazy_col
AS `x`]
+--------------filter((topn_lazy_order_by_alias_tbl.sort_col > 0))
+----------------PhysicalLazyMaterializeOlapScan[topn_lazy_order_by_alias_tbl
lazySlots:(topn_lazy_order_by_alias_tbl.other_col)]
+
+-- !index_mode_selective_lazy_result --
+10 10 100
+
+-- !index_mode_filter_predicate_lazy_plan --
+PhysicalResultSink
+--PhysicalProject[topn_lazy_order_by_alias_tbl.other_col,
topn_lazy_order_by_alias_tbl.sort_col, x]
+----PhysicalLazyMaterialize[materializedSlots:(x)
lazySlots:(topn_lazy_order_by_alias_tbl.other_col,topn_lazy_order_by_alias_tbl.sort_col)]
+------PhysicalTopN[MERGE_SORT]
+--------PhysicalDistribute[DistributionSpecGather]
+----------PhysicalTopN[LOCAL_SORT]
+------------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_tbl,
topn_lazy_order_by_alias_tbl.lazy_col AS `x`]
+--------------filter((topn_lazy_order_by_alias_tbl.sort_col > 0))
+----------------PhysicalLazyMaterializeOlapScan[topn_lazy_order_by_alias_tbl
lazySlots:(topn_lazy_order_by_alias_tbl.other_col)]
+
+-- !index_mode_filter_predicate_lazy_result --
+10 1 100
+
+-- !lateral_generate_conjunct_plan --
+PhysicalResultSink
+--PhysicalProject[s.w, s.y]
+----PhysicalLazyMaterialize[materializedSlots:(s.y,s.sort_col) lazySlots:(s.w)]
+------PhysicalTopN[MERGE_SORT]
+--------PhysicalDistribute[DistributionSpecGather]
+----------PhysicalTopN[LOCAL_SORT]
+------------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_arr_tbl,
s.sort_col, s.y]
+--------------PhysicalGenerate
+----------------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_arr_tbl,
s.arr, s.sort_col, topn_lazy_order_by_alias_arr_tbl.lazy_col AS `x`,
topn_lazy_order_by_alias_arr_tbl.lazy_col AS `y`]
+------------------PhysicalLazyMaterializeOlapScan[topn_lazy_order_by_alias_arr_tbl
lazySlots:(topn_lazy_order_by_alias_arr_tbl.other_col)]
+
+-- !lateral_generate_conjunct_result --
+10 100
+
+-- !lateral_generate_bare_conjunct_plan --
+PhysicalResultSink
+--PhysicalProject[s.w, s.y]
+----PhysicalLazyMaterialize[materializedSlots:(s.y,s.sort_col) lazySlots:(s.w)]
+------PhysicalTopN[MERGE_SORT]
+--------PhysicalDistribute[DistributionSpecGather]
+----------PhysicalTopN[LOCAL_SORT]
+------------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_arr_tbl,
s.sort_col, s.y]
+--------------PhysicalGenerate
+----------------PhysicalProject[regression_test_query_p0_topn_lazy_order_by_alias.__DORIS_GLOBAL_ROWID_COL__topn_lazy_order_by_alias_arr_tbl,
s.arr, s.lazy_col, s.sort_col, topn_lazy_order_by_alias_arr_tbl.lazy_col AS
`y`]
+------------------PhysicalLazyMaterializeOlapScan[topn_lazy_order_by_alias_arr_tbl
lazySlots:(topn_lazy_order_by_alias_arr_tbl.other_col)]
+
+-- !lateral_generate_bare_conjunct_result --
+10 100
+
diff --git
a/regression-test/suites/query_p0/topn_lazy/order_by_alias/topn_lazy_order_by_alias.groovy
b/regression-test/suites/query_p0/topn_lazy/order_by_alias/topn_lazy_order_by_alias.groovy
new file mode 100644
index 00000000000..8fefd563e59
--- /dev/null
+++
b/regression-test/suites/query_p0/topn_lazy/order_by_alias/topn_lazy_order_by_alias.groovy
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("topn_lazy_order_by_alias") {
+ // TopN lazy materialization only skips the rewrite when a plan validation
fails, and that
+ // validation only runs with fe_debug on. Pin it off so an invalid plan
surfaces as an error.
+ sql """ set fe_debug = false; """
+
+ sql """
+ drop table if exists topn_lazy_order_by_alias_tbl;
+ create table topn_lazy_order_by_alias_tbl (
+ sort_col int,
+ lazy_col int,
+ other_col int
+ ) duplicate key(sort_col)
+ distributed by hash(sort_col) buckets 1
+ properties('replication_num' = '1');
+ """
+ sql """
+ insert into topn_lazy_order_by_alias_tbl values
(3,30,300),(1,10,100),(2,20,200);
+ """
+
+ // Sorting by an alias of lazy_col: lazy_col feeds the sort key, so it
must stay materialized
+ // instead of being pruned from the scan while `lazy_col AS x` below the
TopN still reads it.
+ order_qt_repeated_alias_of_sort_key """
+ select lazy_col as x, lazy_col as y
+ from topn_lazy_order_by_alias_tbl order by x limit 1;
+ """
+
+ // Same shape ordered by the other alias.
+ order_qt_repeated_alias_reversed """
+ select lazy_col as x, lazy_col as y
+ from topn_lazy_order_by_alias_tbl order by y limit 1;
+ """
+
+ // A bare column plus an alias of it, sorted by the bare column.
+ order_qt_bare_column_and_alias """
+ select lazy_col, lazy_col as y
+ from topn_lazy_order_by_alias_tbl order by lazy_col limit 1;
+ """
+
+ // Only the column the order key reads is forced materialized; other_col
is still lazily fetched.
+ order_qt_other_column_still_lazy """
+ select lazy_col as x, other_col as y
+ from topn_lazy_order_by_alias_tbl order by x limit 1;
+ """
+
+ // Control: sorting by a column that is not aliased keeps both columns
lazily fetched.
+ order_qt_order_by_plain_column """
+ select lazy_col as x, other_col as y
+ from topn_lazy_order_by_alias_tbl order by sort_col limit 1;
+ """
+
+ // The same shapes through the inverted-index filter path.
+ sql """ set topn_lazy_materialization_using_index = true; """
+
+ order_qt_using_index_repeated_alias """
+ select lazy_col as x, lazy_col as y
+ from topn_lazy_order_by_alias_tbl where sort_col > 0 order by x limit
1;
+ """
+
+ order_qt_using_index_bare_column_and_alias """
+ select lazy_col, lazy_col as y
+ from topn_lazy_order_by_alias_tbl where sort_col > 0 order by lazy_col
limit 1;
+ """
+
+ sql """ set topn_lazy_materialization_using_index = false; """
+
+ // Only the outer TopN is rewritten. Probing its output `y` resolves to
lazy_col, which the inner TopN
+ // still reads through `lazy_col AS x`, so lazy_col has to stay
materialized. Plain execution hides the
+ // invalid plan behind StmtExecutor.queryRetry, so the plan shape is
asserted as well.
+ sql """ set detail_shape_nodes = 'PhysicalProject'; """
+
+ qt_nested_topn_plan """
+ explain shape plan
+ select y from (
+ select lazy_col as x, lazy_col as y, other_col as z
+ from topn_lazy_order_by_alias_tbl
+ order by x limit 2) s
+ order by z limit 1;
+ """
+
+ order_qt_nested_topn_result """
+ select y from (
+ select lazy_col as x, lazy_col as y, other_col as z
+ from topn_lazy_order_by_alias_tbl
+ order by x limit 2) s
+ order by z limit 1;
+ """
+
+ // Column `w` is read by nothing below the TopN, so it can still be
fetched lazily through both TopNs.
+ qt_nested_topn_still_lazy_plan """
+ explain shape plan
+ select w from (
+ select lazy_col as x, other_col as w, lazy_col as y, sort_col
+ from topn_lazy_order_by_alias_tbl
+ order by x limit 2) s
+ order by sort_col limit 1;
+ """
+
+ // Index mode: the bare lazy_col and its alias both feed the sort key, so
both stay materialized and
+ // only other_col is fetched lazily. Before the fix the bare lazy_col
could be pruned from the scan
+ // while `lazy_col AS x` below the TopN still read it.
+ sql """ set topn_lazy_materialization_using_index = true; """
+
+ qt_index_mode_selective_lazy_plan """
+ explain shape plan
+ select lazy_col as x, lazy_col, other_col
+ from topn_lazy_order_by_alias_tbl where sort_col > 0 order by x limit
1;
+ """
+
+ order_qt_index_mode_selective_lazy_result """
+ select lazy_col as x, lazy_col, other_col
+ from topn_lazy_order_by_alias_tbl where sort_col > 0 order by x limit
1;
+ """
+
+ // A direct filter predicate that no alias reads is evaluated through the
index and fetched by the
+ // scan for the predicate, so it stays lazy above the TopN while the alias
feeding the sort key keeps
+ // its base column materialized.
+ qt_index_mode_filter_predicate_lazy_plan """
+ explain shape plan
+ select lazy_col as x, sort_col, other_col
+ from topn_lazy_order_by_alias_tbl where sort_col > 0 order by x limit
1;
+ """
+
+ order_qt_index_mode_filter_predicate_lazy_result """
+ select lazy_col as x, sort_col, other_col
+ from topn_lazy_order_by_alias_tbl where sort_col > 0 order by x limit
1;
+ """
+
+ sql """ set topn_lazy_materialization_using_index = false; """
+
+ sql """
+ drop table if exists topn_lazy_order_by_alias_arr_tbl;
+ create table topn_lazy_order_by_alias_arr_tbl (
+ sort_col int,
+ lazy_col int,
+ other_col int,
+ arr array<int>
+ ) duplicate key(sort_col)
+ distributed by hash(sort_col) buckets 1
+ properties('replication_num' = '1');
+ """
+ sql """
+ insert into topn_lazy_order_by_alias_arr_tbl values
(3,30,300,[3]),(1,10,100,[1,10]),(2,20,200,[2,20]);
+ """
+
+ // The generate conjunct is evaluated by the generate itself, so the alias
it reads must keep its base
+ // column materialized: PhysicalGenerate exposes only its generators
through getInputSlots(), otherwise
+ // lazy_col is pruned from the scan while `lazy_col AS x` below the
generate still reads it. Only
+ // other_col, which nothing below the TopN reads, is fetched lazily.
+ qt_lateral_generate_conjunct_plan """
+ explain shape plan
+ select s.y, s.w from (
+ select lazy_col as x, lazy_col as y, other_col as w, arr, sort_col
+ from topn_lazy_order_by_alias_arr_tbl) s
+ left join lateral unnest(s.arr) tt(tag) on tt.tag = s.x
+ order by s.sort_col limit 1;
+ """
+
+ order_qt_lateral_generate_conjunct_result """
+ select s.y, s.w from (
+ select lazy_col as x, lazy_col as y, other_col as w, arr, sort_col
+ from topn_lazy_order_by_alias_arr_tbl) s
+ left join lateral unnest(s.arr) tt(tag) on tt.tag = s.x
+ order by s.sort_col limit 1;
+ """
+
+ // The conjunct reads the bare lazy_col, which the same Project also
aliases as `y`. Probing `y`
+ // resolves through that alias to lazy_col, and no operator below the TopN
stops the probe for the bare
+ // slot, so lazy_col has to stay materialized for the conjunct as well.
+ qt_lateral_generate_bare_conjunct_plan """
+ explain shape plan
+ select s.y, s.w from (
+ select lazy_col as y, other_col as w, arr, sort_col, lazy_col
+ from topn_lazy_order_by_alias_arr_tbl) s
+ left join lateral unnest(s.arr) tt(tag) on tt.tag = s.lazy_col
+ order by s.sort_col limit 1;
+ """
+
+ order_qt_lateral_generate_bare_conjunct_result """
+ select s.y, s.w from (
+ select lazy_col as y, other_col as w, arr, sort_col, lazy_col
+ from topn_lazy_order_by_alias_arr_tbl) s
+ left join lateral unnest(s.arr) tt(tag) on tt.tag = s.lazy_col
+ order by s.sort_col limit 1;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
