This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4fec15443ab [fix](topn) Skip TopN lazy materialization when
light_schema_change=false (#64441)
4fec15443ab is described below
commit 4fec15443ab977318f8be9fb69f3e2d2ccd53d49
Author: HappenLee <[email protected]>
AuthorDate: Wed Jun 17 12:29:24 2026 +0800
[fix](topn) Skip TopN lazy materialization when light_schema_change=false
(#64441)
Problem Summary:
TopN lazy materialization asks BE to rebuild the scan schema with
`GLOBAL_ROWID_COL` from `columns_desc`. For tables created with
`light_schema_change=false`, table columns use `col_unique_id=-1` and BE
skips that rebuild, so `GLOBAL_ROWID_COL` is not added and the scan can
fail with `field name is invalid`.
This PR disables TopN lazy materialization for OLAP tables whose base
schema disables light schema change, so those queries fall back to the
normal scan path instead of hitting the BE schema mismatch. It also adds
FE unit coverage for the `light_schema_change=false` case.
---
.../post/materialize/LazyMaterializeTopN.java | 11 ++++
.../postprocess/TopnLazyMaterializeTest.java | 70 ++++++++++++++++++++++
2 files changed, 81 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
index aa55eab77ad..1b5f262bc88 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java
@@ -162,6 +162,17 @@ public class LazyMaterializeTopN extends PlanPostProcessor
{
HashSet<SlotReference> rowIdSet = new HashSet<>();
StatementContext threadStatementContext =
StatementScopeIdGenerator.getStatementContext();
for (Relation relation : relationToLazySlotMap.keySet()) {
+ // TopN lazy materialization relies on BE adding a
GLOBAL_ROWID_COL to the
+ // tablet schema. When light_schema_change=false, the table
columns have
+ // col_unique_id=-1, which causes BE to skip the schema rebuild
from
+ // columns_desc, so the GLOBAL_ROWID_COL is never added and the
scan fails.
+ if (relation instanceof CatalogRelation
+ && ((CatalogRelation) relation).getTable() instanceof
OlapTable
+ && !((OlapTable) ((CatalogRelation)
relation).getTable()).getEnableLightSchemaChange()) {
+ LOG.debug("Skip TopN lazy materialization for table {} with
light_schema_change=false",
+ ((CatalogRelation) relation).getTable().getName());
+ return topN;
+ }
if (relation instanceof CatalogRelation) {
CatalogRelation catalogRelation = (CatalogRelation) relation;
Column rowIdCol = new Column(Column.GLOBAL_ROWID_COL +
catalogRelation.getTable().getName(),
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
index ea384d35bd2..cb5045ad76d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java
@@ -17,16 +17,20 @@
package org.apache.doris.nereids.postprocess;
+import org.apache.doris.analysis.ColumnAccessPath;
import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.nereids.datasets.ssb.SSBTestBase;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.processor.post.PlanPostProcessors;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.planner.MaterializationNode;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanFragment;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -78,4 +82,70 @@ public class TopnLazyMaterializeTest extends SSBTestBase {
Assertions.assertEquals(1, slots.size());
Assertions.assertEquals("k2", slots.get(0).getColumn().getName());
}
+
+ @Test
+ public void testNestedColumnAccessPathInLazyMaterialize() throws Exception
{
+ this.createTables("create table lazy_materialize_struct_tbl("
+ + "id bigint, "
+ + "user_profile struct<"
+ + "personal:struct<name:varchar(100)>,"
+ + "professional:struct<skills:array<varchar(100)>>>) "
+ + "duplicate key(id) distributed by hash(id) buckets 1 "
+ + "properties('replication_num' = '1')");
+ String sql = "select struct_element(struct_element(user_profile,
'professional'), 'skills') "
+ + "from lazy_materialize_struct_tbl order by id limit 10";
+
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .implement();
+ PhysicalPlan plan = checker.getPhysicalPlan();
+ plan = new
PlanPostProcessors(checker.getCascadesContext()).process(plan);
+ PlanTranslatorContext translatorContext = new
PlanTranslatorContext(checker.getCascadesContext());
+ PlanFragment fragment = new
PhysicalPlanTranslator(translatorContext).translatePlan(plan);
+
+ List<MaterializationNode> materializationNodes = Lists.newArrayList();
+ fragment.getPlanRoot().collect(MaterializationNode.class,
materializationNodes);
+ Assertions.assertEquals(1, materializationNodes.size());
+ TupleDescriptor materializeTupleDesc = translatorContext.getTupleDesc(
+ materializationNodes.get(0).getTupleIds().get(0));
+ SlotDescriptor userProfileSlot =
materializeTupleDesc.getSlots().stream()
+ .filter(slot ->
"user_profile".equals(slot.getColumn().getName()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("lazy user_profile slot
not found"));
+ Assertions.assertTrue(userProfileSlot.getAllAccessPaths().contains(
+ ColumnAccessPath.data(ImmutableList.of("user_profile",
"professional", "skills"))));
+ }
+
+ @Test
+ public void testLightSchemaChangeFalse() throws Exception {
+ this.createTable("create table tm_lsc_false (k int, v int) duplicate
key(k) "
+ + "distributed by hash(k) buckets 1 "
+ + "properties('replication_num' = '1', 'light_schema_change' =
'false')");
+ String sql = "select * from tm_lsc_false order by k limit 1";
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .implement();
+ PhysicalPlan plan = checker.getPhysicalPlan();
+ plan = new
PlanPostProcessors(checker.getCascadesContext()).process(plan);
+ PlanTranslatorContext translatorContext = new
PlanTranslatorContext(checker.getCascadesContext());
+ PlanFragment fragment = new
PhysicalPlanTranslator(translatorContext).translatePlan(plan);
+
+ // TopN lazy materialization should be skipped for
light_schema_change=false,
+ // so no MaterializationNode should be created.
+ List<MaterializationNode> materializationNodes = Lists.newArrayList();
+ fragment.getPlanRoot().collect(MaterializationNode.class,
materializationNodes);
+ Assertions.assertTrue(materializationNodes.isEmpty(),
+ "TopN lazy materialization should be skipped for
light_schema_change=false");
+
+ // All columns should be in the scan output (no lazy pruned columns).
+ List<OlapScanNode> scanNodes = Lists.newArrayList();
+ fragment.getPlanRoot().collect(OlapScanNode.class, scanNodes);
+ Assertions.assertEquals(1, scanNodes.size());
+ List<SlotDescriptor> slots =
scanNodes.get(0).getTupleDesc().getSlots();
+ Assertions.assertEquals(2, slots.size());
+ Assertions.assertTrue(slots.stream().anyMatch(s ->
s.getColumn().getName().equals("k")));
+ Assertions.assertTrue(slots.stream().anyMatch(s ->
s.getColumn().getName().equals("v")));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
