This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2c897b82ad [enhance](Nereids) Pushdown Project Through OuterJoin.
(#21730)
2c897b82ad is described below
commit 2c897b82adf7f6a412cf3cdfaffcc27920584759
Author: jakevin <[email protected]>
AuthorDate: Fri Jul 14 11:46:29 2023 +0800
[enhance](Nereids) Pushdown Project Through OuterJoin. (#21730)
PushdownJoinOtherCondition will pushdown expression in condition into
project, it will block JoinReorder, so we need to pushdown project to help
JoinReorder
---
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +--
.../org/apache/doris/nereids/rules/RuleType.java | 2 +-
...a => PushdownProjectThroughInnerOuterJoin.java} | 37 ++++++++++++++++++----
... PushdownProjectThroughInnerOuterJoinTest.java} | 10 +++---
4 files changed, 38 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 6eb68e3ec5..98e42eb311 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -35,7 +35,7 @@ import
org.apache.doris.nereids.rules.exploration.join.OuterJoinAssoc;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinAssocProject;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinLAsscom;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinLAsscomProject;
-import
org.apache.doris.nereids.rules.exploration.join.PushdownProjectThroughInnerJoin;
+import
org.apache.doris.nereids.rules.exploration.join.PushdownProjectThroughInnerOuterJoin;
import
org.apache.doris.nereids.rules.exploration.join.PushdownProjectThroughSemiJoin;
import
org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose;
import
org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
@@ -111,7 +111,7 @@ public class RuleSet {
.add(SemiJoinSemiJoinTransposeProject.INSTANCE)
.add(LogicalJoinSemiJoinTranspose.INSTANCE)
.add(LogicalJoinSemiJoinTransposeProject.INSTANCE)
- .add(PushdownProjectThroughInnerJoin.INSTANCE)
+ .add(PushdownProjectThroughInnerOuterJoin.INSTANCE)
.add(PushdownProjectThroughSemiJoin.INSTANCE)
.add(TransposeAggSemiJoin.INSTANCE)
.add(TransposeAggSemiJoinProject.INSTANCE)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 9553e86343..c2ac8567cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -271,7 +271,7 @@ public enum RuleType {
TRANSPOSE_LOGICAL_AGG_SEMI_JOIN_PROJECT(RuleTypeClass.EXPLORATION),
TRANSPOSE_LOGICAL_JOIN_UNION(RuleTypeClass.EXPLORATION),
PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN(RuleTypeClass.EXPLORATION),
- PUSH_DOWN_PROJECT_THROUGH_INNER_JOIN(RuleTypeClass.EXPLORATION),
+ PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN(RuleTypeClass.EXPLORATION),
EAGER_COUNT(RuleTypeClass.EXPLORATION),
EAGER_GROUP_BY(RuleTypeClass.EXPLORATION),
EAGER_GROUP_BY_COUNT(RuleTypeClass.EXPLORATION),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoin.java
similarity index 79%
rename from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoin.java
rename to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoin.java
index bd9d146426..6d01078cdc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoin.java
@@ -33,7 +33,9 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -48,13 +50,15 @@ import java.util.stream.Collectors;
* A B A B
* </pre>
*/
-public class PushdownProjectThroughInnerJoin implements ExplorationRuleFactory
{
- public static final PushdownProjectThroughInnerJoin INSTANCE = new
PushdownProjectThroughInnerJoin();
+public class PushdownProjectThroughInnerOuterJoin implements
ExplorationRuleFactory {
+ public static final PushdownProjectThroughInnerOuterJoin INSTANCE = new
PushdownProjectThroughInnerOuterJoin();
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalJoin(logicalProject(innerLogicalJoin()), group())
+ logicalJoin(logicalProject(logicalJoin()), group())
+ .when(j ->
j.left().child().getJoinType().isOuterJoin() || j.left().child().getJoinType()
+ .isInnerJoin())
// Just pushdown project with non-column expr like
(t.id + 1)
.whenNot(j -> j.left().isAllSlots())
.whenNot(j -> j.left().child().hasJoinHint())
@@ -65,8 +69,10 @@ public class PushdownProjectThroughInnerJoin implements
ExplorationRuleFactory {
return null;
}
return topJoin.withChildren(newLeft,
topJoin.right());
-
}).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_JOIN),
- logicalJoin(group(), logicalProject(innerLogicalJoin()))
+
}).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN),
+ logicalJoin(group(), logicalProject(logicalJoin()))
+ .when(j ->
j.right().child().getJoinType().isOuterJoin() || j.right().child().getJoinType()
+ .isInnerJoin())
// Just pushdown project with non-column expr like
(t.id + 1)
.whenNot(j -> j.right().isAllSlots())
.whenNot(j -> j.right().child().hasJoinHint())
@@ -77,7 +83,7 @@ public class PushdownProjectThroughInnerJoin implements
ExplorationRuleFactory {
return null;
}
return topJoin.withChildren(topJoin.left(),
newRight);
-
}).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_JOIN)
+
}).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN)
);
}
@@ -97,7 +103,24 @@ public class PushdownProjectThroughInnerJoin implements
ExplorationRuleFactory {
List<NamedExpression> aProjects = new ArrayList<>();
List<NamedExpression> bProjects = new ArrayList<>();
- for (NamedExpression namedExpression : project.getProjects()) {
+ List<NamedExpression> projects;
+ if (join.getJoinType().isInnerJoin()) {
+ projects = project.getProjects();
+ } else {
+ Map<Slot, Slot> childrenSlots = new HashMap<>();
+ join.left().getOutputSet().forEach(slot -> childrenSlots.put(slot,
slot));
+ join.right().getOutputSet().forEach(slot ->
childrenSlots.put(slot, slot));
+ join.getOutputSet().forEach(slot -> {
+ if (childrenSlots.containsKey(slot)) {
+ childrenSlots.put(slot, childrenSlots.get(slot));
+ }
+ });
+
+ projects = project.getProjects().stream().map(expr ->
expr.rewriteUp(e ->
+ e instanceof Slot ? childrenSlots.get((Slot) e) : e
+ )).map(e -> (NamedExpression) e).collect(Collectors.toList());
+ }
+ for (NamedExpression namedExpression : projects) {
Set<ExprId> usedExprIds = namedExpression.getInputSlotExprIds();
if (aOutputExprIdSet.containsAll(usedExprIds)) {
aProjects.add(namedExpression);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoinTest.java
similarity index 94%
rename from
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoinTest.java
rename to
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoinTest.java
index d739039676..19be848332 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoinTest.java
@@ -37,7 +37,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
-class PushdownProjectThroughInnerJoinTest implements MemoPatternMatchSupported
{
+class PushdownProjectThroughInnerOuterJoinTest implements
MemoPatternMatchSupported {
private final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
private final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
private final LogicalOlapScan scan3 =
PlanConstructor.newLogicalOlapScan(2, "t3", 0);
@@ -59,7 +59,7 @@ class PushdownProjectThroughInnerJoinTest implements
MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
-
.applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+
.applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
@@ -90,7 +90,7 @@ class PushdownProjectThroughInnerJoinTest implements
MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
-
.applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+
.applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
@@ -121,7 +121,7 @@ class PushdownProjectThroughInnerJoinTest implements
MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
-
.applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+
.applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
@@ -158,7 +158,7 @@ class PushdownProjectThroughInnerJoinTest implements
MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
-
.applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+
.applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.checkMemo(memo -> Assertions.assertEquals(1,
memo.getRoot().getLogicalExpressions().size()));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
