This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 82d7f28bffd [opt](mtmv) ensure rewritten plan output order correct
even project been eliminated (#31870)
82d7f28bffd is described below
commit 82d7f28bffdde22635704c404c5c59898d7e29fc
Author: morrySnow <[email protected]>
AuthorDate: Fri Mar 8 10:54:57 2024 +0800
[opt](mtmv) ensure rewritten plan output order correct even project been
eliminated (#31870)
---
.../main/java/org/apache/doris/mtmv/MTMVCache.java | 27 ++++-------
.../mv/AbstractMaterializedViewRule.java | 52 +++++++++++-----------
.../mv/InitMaterializationContextHook.java | 2 +-
.../exploration/mv/MaterializationContext.java | 19 +++-----
4 files changed, 43 insertions(+), 57 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
index ec64f8f5f2b..94076563ee1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -30,9 +29,6 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
-import java.util.List;
-import java.util.stream.Collectors;
-
/**
* The cache for materialized view cache
*/
@@ -40,20 +36,20 @@ public class MTMVCache {
// the materialized view plan which should be optimized by the same rules
to query
private final Plan logicalPlan;
- // this should be shuttle expression with lineage
- private final List<NamedExpression> mvOutputExpressions;
+ // for stable output order, we should use original plan
+ private final Plan originalPlan;
- public MTMVCache(Plan logicalPlan, List<NamedExpression>
mvOutputExpressions) {
+ public MTMVCache(Plan logicalPlan, Plan originalPlan) {
this.logicalPlan = logicalPlan;
- this.mvOutputExpressions = mvOutputExpressions;
+ this.originalPlan = originalPlan;
}
public Plan getLogicalPlan() {
return logicalPlan;
}
- public List<NamedExpression> getMvOutputExpressions() {
- return mvOutputExpressions;
+ public Plan getOriginalPlan() {
+ return originalPlan;
}
public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
@@ -66,15 +62,10 @@ public class MTMVCache {
if (mvSqlStatementContext.getConnectContext().getStatementContext() ==
null) {
mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext);
}
- Plan mvRewrittenPlan =
- planner.plan(unboundMvPlan, PhysicalProperties.ANY,
ExplainLevel.REWRITTEN_PLAN);
+ Plan mvRewrittenPlan = planner.plan(unboundMvPlan,
PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
+ // TODO: should use visitor or a new rule to remove result sink node
Plan mvPlan = mvRewrittenPlan instanceof LogicalResultSink
? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() :
mvRewrittenPlan;
- // use rewritten plan output expression currently, if expression
rewrite fail,
- // consider to use the analyzed plan for output expressions only
- List<NamedExpression> mvOutputExpressions = mvPlan.getOutput().stream()
- .map(NamedExpression.class::cast)
- .collect(Collectors.toList());
- return new MTMVCache(mvPlan, mvOutputExpressions);
+ return new MTMVCache(mvPlan, mvRewrittenPlan);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index a11b9101045..e001359574e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -35,6 +35,7 @@ import
org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
@@ -55,6 +56,7 @@ import org.apache.doris.nereids.util.TypeUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
@@ -259,27 +261,32 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
* Rewrite by rules and try to make output is the same after optimize by
rules
*/
protected Plan rewriteByRules(CascadesContext cascadesContext, Plan
rewrittenPlan, Plan originPlan) {
+ List<Slot> originOutputs = originPlan.getOutput();
+ if (originOutputs.size() != rewrittenPlan.getOutput().size()) {
+ return null;
+ }
+ Map<Slot, ExprId> originSlotToRewrittenExprId = Maps.newHashMap();
+ for (int i = 0; i < originOutputs.size(); i++) {
+ originSlotToRewrittenExprId.put(originOutputs.get(i),
rewrittenPlan.getOutput().get(i).getExprId());
+ }
// run rbo job on mv rewritten plan
CascadesContext rewrittenPlanContext = CascadesContext.initContext(
cascadesContext.getStatementContext(), rewrittenPlan,
cascadesContext.getCurrentJobContext().getRequiredProperties());
Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute();
rewrittenPlan = rewrittenPlanContext.getRewritePlan();
- List<Slot> originPlanOutput = originPlan.getOutput();
- List<Slot> rewrittenPlanOutput = rewrittenPlan.getOutput();
- if (originPlanOutput.size() != rewrittenPlanOutput.size()) {
- return null;
- }
- List<NamedExpression> expressions = new ArrayList<>();
- // should add project above rewritten plan if top plan is not project,
if aggregate above will nu
- if (!isOutputValid(originPlan, rewrittenPlan)) {
- for (int i = 0; i < originPlanOutput.size(); i++) {
- expressions.add(((NamedExpression)
normalizeExpression(originPlanOutput.get(i),
- rewrittenPlanOutput.get(i))));
- }
- return new LogicalProject<>(expressions, rewrittenPlan, false);
+
+ // for get right nullable after rewritten, we need this map
+ Map<ExprId, Slot> exprIdToNewRewrittenSlot = Maps.newHashMap();
+ for (Slot slot : rewrittenPlan.getOutput()) {
+ exprIdToNewRewrittenSlot.put(slot.getExprId(), slot);
}
- return rewrittenPlan;
+
+ // normalize nullable
+ ImmutableList<NamedExpression> convertNullable = originOutputs.stream()
+ .map(s -> normalizeExpression(s,
exprIdToNewRewrittenSlot.get(originSlotToRewrittenExprId.get(s))))
+ .collect(ImmutableList.toImmutableList());
+ return new LogicalProject<>(convertNullable, rewrittenPlan);
}
/**
@@ -398,22 +405,17 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
* Normalize expression with query, keep the consistency of exprId and
nullable props with
* query
*/
- protected Expression normalizeExpression(Expression sourceExpression,
Expression replacedExpression) {
- if (sourceExpression instanceof NamedExpression
- && replacedExpression.nullable() !=
sourceExpression.nullable()) {
+ private NamedExpression normalizeExpression(
+ NamedExpression sourceExpression, NamedExpression
replacedExpression) {
+ Expression innerExpression = replacedExpression;
+ if (replacedExpression.nullable() != sourceExpression.nullable()) {
// if enable join eliminate, query maybe inner join and mv maybe
outer join.
// If the slot is at null generate side, the nullable maybe
different between query and view
// So need to force to consistent.
- replacedExpression = sourceExpression.nullable()
+ innerExpression = sourceExpression.nullable()
? new Nullable(replacedExpression) : new
NonNullable(replacedExpression);
}
- if (sourceExpression instanceof NamedExpression
- && !sourceExpression.equals(replacedExpression)) {
- NamedExpression sourceNamedExpression = (NamedExpression)
sourceExpression;
- replacedExpression = new Alias(sourceNamedExpression.getExprId(),
replacedExpression,
- sourceNamedExpression.getName());
- }
- return replacedExpression;
+ return new Alias(sourceExpression.getExprId(), innerExpression,
sourceExpression.getName());
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 8046639d1ba..ab8f4d69d45 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -95,7 +95,7 @@ public class InitMaterializationContextHook implements
PlannerHook {
// todo should force keep consistency to mv sql plan output
Plan projectScan = new LogicalProject<Plan>(mvProjects, mvScan);
cascadesContext.addMaterializationContext(
-
MaterializationContext.fromMaterializedView(materializedView, projectScan,
cascadesContext));
+
MaterializationContext.fromMaterializedView(materializedView, projectScan));
});
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index 0414f7007f3..e64e1290954 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -22,7 +22,6 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.mtmv.MTMVCache;
-import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.trees.plans.ObjectId;
@@ -47,7 +46,8 @@ import java.util.stream.Collectors;
public class MaterializationContext {
private static final Logger LOG =
LogManager.getLogger(MaterializationContext.class);
- private MTMV mtmv;
+
+ private final MTMV mtmv;
// Should use stmt id generator in query context
private final Plan mvScanPlan;
private final List<Table> baseTables;
@@ -68,11 +68,7 @@ public class MaterializationContext {
/**
* MaterializationContext, this contains necessary info for query
rewriting by mv
*/
- public MaterializationContext(MTMV mtmv,
- Plan mvScanPlan,
- CascadesContext cascadesContext,
- List<Table> baseTables,
- List<Table> baseViews) {
+ public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table>
baseTables, List<Table> baseViews) {
this.mtmv = mtmv;
this.mvScanPlan = mvScanPlan;
this.baseTables = baseTables;
@@ -91,8 +87,8 @@ public class MaterializationContext {
// mv output expression shuttle, this will be used to expression
rewrite
this.mvExprToMvScanExprMapping = ExpressionMapping.generate(
ExpressionUtils.shuttleExpressionWithLineage(
- mtmvCache.getMvOutputExpressions(),
- mtmvCache.getLogicalPlan()),
+ mtmvCache.getOriginalPlan().getOutput(),
+ mtmvCache.getOriginalPlan()),
mvScanPlan.getExpressions());
// copy the plan from cache, which the plan in cache may change
this.mvPlan = mtmvCache.getLogicalPlan();
@@ -229,13 +225,10 @@ public class MaterializationContext {
/**
* MaterializationContext fromMaterializedView
*/
- public static MaterializationContext fromMaterializedView(MTMV
materializedView,
- Plan mvScanPlan,
- CascadesContext cascadesContext) {
+ public static MaterializationContext fromMaterializedView(MTMV
materializedView, Plan mvScanPlan) {
return new MaterializationContext(
materializedView,
mvScanPlan,
- cascadesContext,
ImmutableList.of(),
ImmutableList.of());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
