This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6cde7bc8ad [feature](Nereids) just reserve logical expression in memo
after do dphyp (#20843)
6cde7bc8ad is described below
commit 6cde7bc8ad1146564f60f77edefba4ac71f34f1f
Author: jakevin <[email protected]>
AuthorDate: Fri Jun 16 18:12:39 2023 +0800
[feature](Nereids) just reserve logical expression in memo after do dphyp
(#20843)
After DPHyp, clear all physicalExpression and other,
just keep logicalExpression as original plan as input of cascades optimize.
---
.../org/apache/doris/nereids/StatementContext.java | 9 +++++++
.../jobs/cascades/OptimizeGroupExpressionJob.java | 7 ++++-
.../doris/nereids/jobs/executor/Optimizer.java | 30 +++++++++++++---------
.../hypergraph/receiver/PlanReceiver.java | 7 +----
.../java/org/apache/doris/nereids/memo/Group.java | 9 +++++++
.../apache/doris/nereids/memo/GroupExpression.java | 4 +++
.../java/org/apache/doris/nereids/memo/Memo.java | 17 ++++++++++++
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +-
8 files changed, 65 insertions(+), 20 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 0d997021bd..a6e73b32fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -47,6 +47,7 @@ public class StatementContext {
private int maxNAryInnerJoin = 0;
private boolean isDpHyp = false;
+ private boolean isOtherJoinReorder = false;
private final IdGenerator<ExprId> exprIdGenerator =
ExprId.createGenerator();
@@ -108,6 +109,14 @@ public class StatementContext {
isDpHyp = dpHyp;
}
+ public boolean isOtherJoinReorder() {
+ return isOtherJoinReorder;
+ }
+
+ public void setOtherJoinReorder(boolean otherJoinReorder) {
+ isOtherJoinReorder = otherJoinReorder;
+ }
+
public ExprId getNextExprId() {
return exprIdGenerator.getNextId();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
index c9fba86d78..1c3247de91 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
@@ -61,12 +61,17 @@ public class OptimizeGroupExpressionJob extends Job {
||
context.getCascadesContext().getMemo().getGroupExpressionsSize() >
context.getCascadesContext()
.getConnectContext().getSessionVariable().memoMaxGroupExpressionSize;
boolean isDpHyp =
context.getCascadesContext().getStatementContext().isDpHyp();
+ boolean isOtherJoinReorder =
context.getCascadesContext().getStatementContext().isOtherJoinReorder();
boolean isEnableBushyTree =
context.getCascadesContext().getConnectContext().getSessionVariable()
.isEnableBushyTree();
if (isDisableJoinReorder) {
return Collections.emptyList();
} else if (isDpHyp) {
- return getRuleSet().getDPHypReorderRules();
+ if (isOtherJoinReorder) {
+ return getRuleSet().getDPHypReorderRules();
+ } else {
+ return Collections.emptyList();
+ }
} else if (isEnableBushyTree) {
return getRuleSet().getBushyTreeJoinReorder();
} else if
(context.getCascadesContext().getStatementContext().getMaxNAryInnerJoin() <= 5)
{
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Optimizer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Optimizer.java
index 00cca76a76..286336d526 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Optimizer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Optimizer.java
@@ -25,6 +25,7 @@ import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.minidump.MinidumpUtils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -51,25 +52,23 @@ public class Optimizer {
// init memo
cascadesContext.toMemo();
// stats derive
- cascadesContext.pushJob(
- new
DeriveStatsJob(cascadesContext.getMemo().getRoot().getLogicalExpression(),
- cascadesContext.getCurrentJobContext()));
+ cascadesContext.pushJob(new
DeriveStatsJob(cascadesContext.getMemo().getRoot().getLogicalExpression(),
+ cascadesContext.getCurrentJobContext()));
cascadesContext.getJobScheduler().executeJobPool(cascadesContext);
serializeStatUsed(cascadesContext.getConnectContext());
- // optimize
+ // DPHyp optimize
StatementContext statementContext =
cascadesContext.getStatementContext();
- boolean isDpHyp =
cascadesContext.getConnectContext().getSessionVariable().enableDPHypOptimizer
- || statementContext.getMaxNAryInnerJoin() >
statementContext.getConnectContext()
- .getSessionVariable().getMaxTableCountUseCascadesJoinReorder();
+ boolean isDpHyp = getSessionVariable().enableDPHypOptimizer ||
statementContext.getMaxNAryInnerJoin()
+ >
getSessionVariable().getMaxTableCountUseCascadesJoinReorder();
cascadesContext.getStatementContext().setDpHyp(isDpHyp);
- if
(!statementContext.getConnectContext().getSessionVariable().isDisableJoinReorder()
&& isDpHyp) {
+ cascadesContext.getStatementContext().setOtherJoinReorder(false);
+ if (!getSessionVariable().isDisableJoinReorder() && isDpHyp) {
dpHypOptimize();
}
- cascadesContext.pushJob(new OptimizeGroupJob(
- cascadesContext.getMemo().getRoot(),
- cascadesContext.getCurrentJobContext())
- );
+ // Cascades optimize
+ cascadesContext.pushJob(
+ new OptimizeGroupJob(cascadesContext.getMemo().getRoot(),
cascadesContext.getCurrentJobContext()));
cascadesContext.getJobScheduler().executeJobPool(cascadesContext);
}
@@ -79,6 +78,9 @@ public class Optimizer {
// Due to EnsureProjectOnTopJoin, root group can't be Join Group, so
DPHyp doesn't change the root group
cascadesContext.pushJob(new JoinOrderJob(root,
cascadesContext.getCurrentJobContext()));
cascadesContext.getJobScheduler().executeJobPool(cascadesContext);
+ // after DPHyp just keep logical expression
+ cascadesContext.getMemo().removePhysicalExpression();
+ cascadesContext.getStatementContext().setOtherJoinReorder(true);
}
private void serializeStatUsed(ConnectContext connectContext) {
@@ -95,4 +97,8 @@ public class Optimizer {
cascadesContext.getConnectContext().getTotalHistogramMap());
jsonObj.put("Histogram", histogramArray);
}
+
+ private SessionVariable getSessionVariable() {
+ return cascadesContext.getConnectContext().getSessionVariable();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
index 9436c10977..aa9d6f787b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
@@ -29,7 +29,6 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -326,11 +325,7 @@ public class PlanReceiver implements AbstractReceiver {
} else {
throw new RuntimeException("DPhyp can only handle join and
project operator");
}
- // shadow all join order rule
- CopyInResult copyInResult =
jobContext.getCascadesContext().getMemo().copyIn(logicalPlan, root, false);
- for (Rule rule :
jobContext.getCascadesContext().getRuleSet().getImplementationRules()) {
- copyInResult.correspondingExpression.setApplied(rule);
- }
+ jobContext.getCascadesContext().getMemo().copyIn(logicalPlan,
root, false);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
index 5bc32cb64f..3987047371 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
@@ -28,6 +28,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.util.TreeStringUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
@@ -180,6 +181,10 @@ public class Group {
return move;
}
+ public void clearLowestCostPlans() {
+ lowestCostPlans.clear();
+ }
+
public double getCostLowerBound() {
return -1D;
}
@@ -291,6 +296,10 @@ public class Group {
return parentExpressions.size();
}
+ public void removeParentPhysicalExpressions() {
+ parentExpressions.entrySet().removeIf(entry ->
entry.getKey().getPlan() instanceof PhysicalPlan);
+ }
+
/**
* move the ownerGroup to target group.
*
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
index b921c80000..1757cd7dac 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
@@ -168,6 +168,10 @@ public class GroupExpression {
toGroupExpression.ruleMasks.or(ruleMasks);
}
+ public void clearApplied() {
+ ruleMasks.clear();
+ }
+
public boolean isStatDerived() {
return statDerived;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index e10c39c323..d39b92d360 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -114,6 +114,23 @@ public class Memo {
return groupExpressions.size();
}
+ /** just keep LogicalExpression in Memo. */
+ public void removePhysicalExpression() {
+ groupExpressions.entrySet().removeIf(entry ->
entry.getValue().getPlan() instanceof PhysicalPlan);
+
+ for (Group group : groups.values()) {
+ group.clearPhysicalExpressions();
+ group.clearLowestCostPlans();
+ group.removeParentPhysicalExpressions();
+ group.setExplored(false);
+ }
+
+ // logical groupExpression reset ruleMask
+ groupExpressions.values().stream()
+ .filter(groupExpression -> groupExpression.getPlan()
instanceof LogicalPlan)
+ .forEach(GroupExpression::clearApplied);
+ }
+
private Plan skipProject(Plan plan, Group targetGroup) {
// Some top project can't be eliminated
if (plan instanceof LogicalProject && ((LogicalProject<?>)
plan).canEliminate()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 2307f0b5c5..8e0a4c07e0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -202,7 +202,7 @@ public class RuleSet {
.build();
public static final List<Rule> DPHYP_REORDER_RULES =
ImmutableList.<Rule>builder()
- .add(JoinCommute.NON_INNER.build())
+ .add(JoinCommute.BUSHY.build())
.addAll(OTHER_REORDER_RULES)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
