This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8e15388074 [fix](Nereids): use CBO rule instead of using rewrite rule.
(#18256)
8e15388074 is described below
commit 8e1538807430ad3be26e2e5f6c79765698c5e04c
Author: jakevin <[email protected]>
AuthorDate: Fri Mar 31 11:23:26 2023 +0800
[fix](Nereids): use CBO rule instead of using rewrite rule. (#18256)
---
.../org/apache/doris/nereids/NereidsPlanner.java | 7 ++--
.../org/apache/doris/nereids/StatementContext.java | 10 ++++++
.../jobs/cascades/OptimizeGroupExpressionJob.java | 6 +++-
.../hypergraph/receiver/PlanReceiver.java | 3 --
.../org/apache/doris/nereids/rules/RuleSet.java | 21 ++++++------
.../rules/exploration/MergeProjectsCBO.java | 34 +++++++++++++++++++
.../PushdownFilterThroughProjectCBO.java | 38 +++++++++++++++++++++
.../rules/rewrite/logical/MergeProjects.java | 39 +++++++++++-----------
.../logical/PushdownFilterThroughProject.java | 24 +++++++------
.../trees/plans/logical/LogicalProject.java | 7 +---
10 files changed, 136 insertions(+), 53 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index fdb15a3907..fb5ca5afcd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -40,7 +40,6 @@ import
org.apache.doris.nereids.processor.post.PlanPostProcessors;
import org.apache.doris.nereids.processor.pre.PlanPreprocessors;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
@@ -55,6 +54,7 @@ import org.apache.doris.planner.ScanNode;
import org.apache.doris.qe.ConnectContext;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.context.Context;
@@ -261,11 +261,12 @@ public class NereidsPlanner extends Planner {
if (root.isJoinGroup()) {
// If the root group is join group, DPHyp can change the root
group.
// To keep the root group is not changed, we add a project
operator above join
- List<Slot> outputs =
root.getLogicalExpression().getPlan().getOutput();
- LogicalPlan plan = new LogicalProject(outputs,
root.getLogicalExpression().getPlan());
+ List<NamedExpression> outputs =
ImmutableList.copyOf(root.getLogicalExpression().getPlan().getOutput());
+ LogicalPlan plan = new LogicalProject<>(outputs,
root.getLogicalExpression().getPlan());
CopyInResult copyInResult = cascadesContext.getMemo().copyIn(plan,
null, false);
root = copyInResult.correspondingExpression.getOwnerGroup();
}
+ cascadesContext.getStatementContext().setDpHyp(true);
cascadesContext.pushJob(new JoinOrderJob(root,
cascadesContext.getCurrentJobContext()));
cascadesContext.getJobScheduler().executeJobPool(cascadesContext);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index c1ed58c307..7a764941db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -45,6 +45,8 @@ public class StatementContext {
private int maxNAryInnerJoin = 0;
+ private boolean isDpHyp = false;
+
private final IdGenerator<ExprId> exprIdGenerator =
ExprId.createGenerator();
private final IdGenerator<ObjectId> objectIdGenerator =
ObjectId.createGenerator();
@@ -93,6 +95,14 @@ public class StatementContext {
return maxNAryInnerJoin;
}
+ public boolean isDpHyp() {
+ return isDpHyp;
+ }
+
+ public void setDpHyp(boolean dpHyp) {
+ isDpHyp = dpHyp;
+ }
+
public StatementBase getParsedStatement() {
return parsedStatement;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
index 7492a01f01..28bb48da31 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.Rule;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -57,10 +58,13 @@ public class OptimizeGroupExpressionJob extends Job {
private List<Rule> getExplorationRules() {
boolean isDisableJoinReorder =
context.getCascadesContext().getConnectContext().getSessionVariable()
.isDisableJoinReorder();
+ boolean isDpHyp =
context.getCascadesContext().getStatementContext().isDpHyp();
boolean isEnableBushyTree =
context.getCascadesContext().getConnectContext().getSessionVariable()
.isEnableBushyTree();
if (isDisableJoinReorder) {
- return getRuleSet().getExplorationRulesWithoutReorder();
+ return Collections.emptyList();
+ } else if (isDpHyp) {
+ return getRuleSet().getOtherReorderRules();
} else if (isEnableBushyTree) {
return getRuleSet().getBushyTreeJoinReorder();
} else if
(context.getCascadesContext().getStatementContext().getMaxNAryInnerJoin() <= 5)
{
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
index 6a6828f5b1..e7f9dea653 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
@@ -347,9 +347,6 @@ public class PlanReceiver implements AbstractReceiver {
}
// shadow all join order rule
CopyInResult copyInResult =
jobContext.getCascadesContext().getMemo().copyIn(logicalPlan, root, false);
- for (Rule rule :
jobContext.getCascadesContext().getRuleSet().getJoinOrderRule()) {
- copyInResult.correspondingExpression.setApplied(rule);
- }
for (Rule rule :
jobContext.getCascadesContext().getRuleSet().getImplementationRules()) {
copyInResult.correspondingExpression.setApplied(rule);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index ee07d069a4..2917996e27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.rules;
+import org.apache.doris.nereids.rules.exploration.MergeProjectsCBO;
+import
org.apache.doris.nereids.rules.exploration.PushdownFilterThroughProjectCBO;
import org.apache.doris.nereids.rules.exploration.join.InnerJoinLAsscom;
import org.apache.doris.nereids.rules.exploration.join.InnerJoinLAsscomProject;
import org.apache.doris.nereids.rules.exploration.join.InnerJoinLeftAssociate;
@@ -83,8 +85,8 @@ import java.util.List;
*/
public class RuleSet {
public static final List<Rule> EXPLORATION_RULES = planRuleFactories()
- .add(new PushdownFilterThroughProject())
- .add(new MergeProjects())
+ .add(new PushdownFilterThroughProjectCBO())
+ .add(new MergeProjectsCBO())
.build();
public static final List<Rule> OTHER_REORDER_RULES = planRuleFactories()
@@ -158,6 +160,13 @@ public class RuleSet {
.add(JoinExchangeBothProject.INSTANCE)
.build();
+ public List<Rule> getOtherReorderRules() {
+ List<Rule> rules = new ArrayList<>();
+ rules.addAll(OTHER_REORDER_RULES);
+ rules.addAll(EXPLORATION_RULES);
+ return rules;
+ }
+
public List<Rule> getZigZagTreeJoinReorder() {
List<Rule> rules = new ArrayList<>();
rules.addAll(ZIG_ZAG_TREE_JOIN_REORDER);
@@ -174,14 +183,6 @@ public class RuleSet {
return rules;
}
- public List<Rule> getExplorationRulesWithoutReorder() {
- return new ArrayList<>(EXPLORATION_RULES);
- }
-
- public List<Rule> getJoinOrderRule() {
- return BUSHY_TREE_JOIN_REORDER;
- }
-
public List<Rule> getImplementationRules() {
return IMPLEMENTATION_RULES;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/MergeProjectsCBO.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/MergeProjectsCBO.java
new file mode 100644
index 0000000000..6263e4f159
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/MergeProjectsCBO.java
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects;
+
+/**
+ * this rule aims to merge projects.
+ */
+public class MergeProjectsCBO extends OneExplorationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalProject(logicalProject())
+ .then(project -> MergeProjects.mergeProjects(project))
+ .toRule(RuleType.MERGE_PROJECTS);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/PushdownFilterThroughProjectCBO.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/PushdownFilterThroughProjectCBO.java
new file mode 100644
index 0000000000..76e05516dd
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/PushdownFilterThroughProjectCBO.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import
org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject;
+
+/**
+ * Push down filter through project.
+ * input:
+ * filter(a>2, b=0) -> project(c+d as a, e as b)
+ * output:
+ * project(c+d as a, e as b) -> filter(c+d>2, e=0).
+ */
+public class PushdownFilterThroughProjectCBO extends OneExplorationRuleFactory
{
+ @Override
+ public Rule build() {
+ return logicalFilter(logicalProject())
+
.then(PushdownFilterThroughProject::pushdownFilterThroughProject)
+ .toRule(RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java
index 427a86be63..49209b8a1e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java
@@ -21,35 +21,34 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import java.util.List;
/**
- * this rule aims to merge consecutive filters.
- * For example:
- * logical plan tree:
- * project(a)
- * |
- * project(a,b)
- * |
- * project(a, b, c)
- * |
- * scan
- * transformed to:
- * project(a)
- * |
- * scan
+ * this rule aims to merge consecutive project. For example:
+ * <pre>
+ * project(a)
+ * |
+ * project(a,b) -> project(a)
+ * |
+ * project(a, b, c)
+ * </pre>
*/
public class MergeProjects extends OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalProject(logicalProject()).then(project -> {
- LogicalProject childProject = project.child();
- List<NamedExpression> projectExpressions =
project.mergeProjections(childProject);
- LogicalProject newProject = childProject.canEliminate() ? project
: childProject;
- return newProject.withProjectsAndChild(projectExpressions,
childProject.child(0));
- }).toRule(RuleType.MERGE_PROJECTS);
+ return logicalProject(logicalProject())
+ .then(project -> mergeProjects(project))
+ .toRule(RuleType.MERGE_PROJECTS);
+ }
+
+ public static Plan mergeProjects(LogicalProject project) {
+ LogicalProject childProject = (LogicalProject) project.child();
+ List<NamedExpression> projectExpressions =
project.mergeProjections(childProject);
+ LogicalProject newProject = childProject.canEliminate() ? project :
childProject;
+ return newProject.withProjectsAndChild(projectExpressions,
childProject.child(0));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
index 12cae4772a..c7fbf05e78 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
@@ -42,16 +42,8 @@ public class PushdownFilterThroughProject implements
RewriteRuleFactory {
public List<Rule> buildRules() {
return ImmutableList.of(
RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT.build(
- logicalFilter(logicalProject()).then(filter -> {
- LogicalProject<Plan> project = filter.child();
- return project.withProjectsAndChild(
- project.getProjects(),
- new LogicalFilter<>(
-
ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()),
- project.child()
- )
- );
- })
+ logicalFilter(logicalProject())
+
.then(PushdownFilterThroughProject::pushdownFilterThroughProject)
),
// filter(project(limit)) will change to filter(limit(project)) by
PushdownProjectThroughLimit,
// then we should change filter(limit(project)) to
project(filter(limit))
@@ -71,4 +63,16 @@ public class PushdownFilterThroughProject implements
RewriteRuleFactory {
)
);
}
+
+ /** pushdown Filter through project */
+ public static Plan pushdownFilterThroughProject(LogicalFilter filter) {
+ LogicalProject<Plan> project = (LogicalProject) filter.child();
+ return project.withProjectsAndChild(
+ project.getProjects(),
+ new LogicalFilter<>(
+ ExpressionUtils.replace(filter.getConjuncts(),
project.getAliasToProducer()),
+ project.child()
+ )
+ );
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
index 8debace279..e707528581 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
@@ -78,12 +78,7 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends
LogicalUnary<CHILD_
this(projects, excepts, canEliminate, Optional.empty(),
Optional.empty(), child, isDistinct);
}
- /**
- * Constructor for LogicalProject.
- *
- * @param projects project list
- */
- public LogicalProject(List<NamedExpression> projects,
List<NamedExpression> excepts, boolean canEliminate,
+ private LogicalProject(List<NamedExpression> projects,
List<NamedExpression> excepts, boolean canEliminate,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
CHILD_TYPE child, boolean isDistinct) {
super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties,
child);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
