This is an automated email from the ASF dual-hosted git repository. HappenLee pushed a commit to branch branch-4.2_tpc in repository https://gitbox.apache.org/repos/asf/doris.git
commit d718c83bddaecfebd318eabc3ca314f8c3b51961 Author: HappenLee <[email protected]> AuthorDate: Fri Oct 23 17:42:40 2026 +0800 Revert "branch:4.2: [improvement](cascades) Fuse project pushdown into join reorder #67541 (#68289)" This reverts commit cf9361be41e01e778df510ba042a5071239de5fe. --- .../org/apache/doris/nereids/rules/RuleSet.java | 4 + .../exploration/join/InnerJoinLAsscomProject.java | 17 +- .../join/InnerJoinLeftAssociateProject.java | 17 +- .../join/InnerJoinRightAssociateProject.java | 17 +- .../exploration/join/JoinExchangeBothProject.java | 25 +- .../join/LogicalJoinSemiJoinTransposeProject.java | 33 +- .../exploration/join/OuterJoinAssocProject.java | 21 +- .../exploration/join/OuterJoinLAsscomProject.java | 24 +- .../exploration/join/ProjectJoinReorderHelper.java | 197 ------------ .../join/PushDownProjectThroughInnerOuterJoin.java | 94 +++++- .../join/PushDownProjectThroughSemiJoin.java | 41 ++- .../join/SemiJoinSemiJoinTransposeProject.java | 17 +- .../ProjectAwareJoinReorderComplexProjectTest.java | 354 --------------------- .../join/ProjectJoinReorderHelperTest.java | 190 ----------- .../nereids_p0/repeat/test_repeat_output_slot.out | 1 - .../tpcds_sf100/noStatsRfPrune/query2.out | 10 +- .../tpcds_sf100/noStatsRfPrune/query59.out | 20 +- .../tpcds_sf100/no_stats_shape/query2.out | 10 +- .../tpcds_sf100/no_stats_shape/query59.out | 22 +- 19 files changed, 225 insertions(+), 889 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index af630378f8c..cedd031e1c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.rules.exploration.join.JoinExchangeBothProject; import org.apache.doris.nereids.rules.exploration.join.LogicalJoinSemiJoinTransposeProject; import org.apache.doris.nereids.rules.exploration.join.OuterJoinAssocProject; import org.apache.doris.nereids.rules.exploration.join.OuterJoinLAsscomProject; +import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughInnerOuterJoin; +import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughSemiJoin; import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateOnNoneAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateRule; @@ -154,6 +156,8 @@ public class RuleSet { .add(OuterJoinLAsscomProject.INSTANCE) .add(SemiJoinSemiJoinTransposeProject.INSTANCE) .add(LogicalJoinSemiJoinTransposeProject.INSTANCE) + .add(PushDownProjectThroughInnerOuterJoin.INSTANCE) + .add(PushDownProjectThroughSemiJoin.INSTANCE) .add(TransposeAggSemiJoinProject.INSTANCE) .addAll(new PushDownTopNThroughJoin().buildRules()) .addAll(new PushDownLimitDistinctThroughJoin().buildRules()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java index 89e9552a72d..9ff7f01b829 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java @@ -32,7 +32,6 @@ import org.apache.doris.nereids.util.Utils; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -63,20 +62,16 @@ public class InnerJoinLAsscomProject extends OneExplorationRuleFactory { innerLogicalJoin(logicalProject(innerLogicalJoin()), group()) .when(topJoin -> checkReorder(topJoin, topJoin.left().child(), enableLeftZigZag)) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) + .when(join -> join.left().isAllSlots())) .then(topProject -> { LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); - if (!normalizedProject.isPresent()) { - return null; - } /* ********** init ********** */ - LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child(); - Plan a = bottomJoin.left(); - Plan b = bottomJoin.right(); - Plan c = topJoin.right(); + LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child(); + GroupPlan a = bottomJoin.left(); + GroupPlan b = bottomJoin.right(); + GroupPlan c = topJoin.right(); Set<ExprId> bExprIdSet = b.getOutputExprIdSet(); /* ********** split Conjuncts ********** */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java index 7deeb550705..e836eac3692 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateProject.java @@ -31,7 +31,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; /** @@ -51,19 +50,15 @@ public class InnerJoinLeftAssociateProject extends OneExplorationRuleFactory { public Rule build() { return logicalProject(innerLogicalJoin(group(), logicalProject(innerLogicalJoin())) .when(topJoin -> checkReorder(topJoin)) - .whenNot(join -> join.hasDistributeHint() || join.right().child().hasDistributeHint())) + .whenNot(join -> join.hasDistributeHint() || join.right().child().hasDistributeHint()) + .when(join -> join.right().isAllSlots())) .then(topProject -> { LogicalJoin<GroupPlan, LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.right()); - if (!normalizedProject.isPresent()) { - return null; - } - LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child(); - Plan a = topJoin.left(); - Plan b = bottomJoin.left(); - Plan c = bottomJoin.right(); + LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.right().child(); + GroupPlan a = topJoin.left(); + GroupPlan b = bottomJoin.left(); + GroupPlan c = bottomJoin.right(); Set<ExprId> cExprIdSet = c.getOutputExprIdSet(); // Split condition diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java index 6c76465444a..841963e9a7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateProject.java @@ -31,7 +31,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; /** @@ -49,19 +48,15 @@ public class InnerJoinRightAssociateProject extends OneExplorationRuleFactory { public Rule build() { return logicalProject(innerLogicalJoin(logicalProject(innerLogicalJoin()), group()) .when(topJoin -> checkReorder(topJoin)) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) + .when(join -> join.left().isAllSlots())) .then(topProject -> { LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); - if (!normalizedProject.isPresent()) { - return null; - } - LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child(); - Plan a = bottomJoin.left(); - Plan b = bottomJoin.right(); - Plan c = topJoin.right(); + LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child(); + GroupPlan a = bottomJoin.left(); + GroupPlan b = bottomJoin.right(); + GroupPlan c = topJoin.right(); Set<ExprId> aExprIdSet = a.getOutputExprIdSet(); // Split condition diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java index 939bc2daac9..bc79a715c0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeBothProject.java @@ -37,7 +37,6 @@ import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; /** @@ -57,24 +56,18 @@ public class JoinExchangeBothProject extends OneExplorationRuleFactory { public Rule build() { return logicalProject(innerLogicalJoin(logicalProject(innerLogicalJoin()), logicalProject(innerLogicalJoin())) .when(JoinExchangeBothProject::checkReorder) + .when(join -> join.left().isAllSlots() && join.right().isAllSlots()) .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint() || join.right().child().hasDistributeHint())) .then(topProject -> { LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedLeftProject = ProjectJoinReorderHelper.normalize(topJoin.left()); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedRightProject = ProjectJoinReorderHelper.normalize(topJoin.right()); - if (!normalizedLeftProject.isPresent() || !normalizedRightProject.isPresent()) { - return null; - } - LogicalJoin<Plan, Plan> leftJoin = normalizedLeftProject.get().child(); - LogicalJoin<Plan, Plan> rightJoin = normalizedRightProject.get().child(); - Plan a = leftJoin.left(); - Plan b = leftJoin.right(); - Plan c = rightJoin.left(); - Plan d = rightJoin.right(); + LogicalJoin<GroupPlan, GroupPlan> leftJoin = topJoin.left().child(); + LogicalJoin<GroupPlan, GroupPlan> rightJoin = topJoin.right().child(); + GroupPlan a = leftJoin.left(); + GroupPlan b = leftJoin.right(); + GroupPlan c = rightJoin.left(); + GroupPlan d = rightJoin.right(); Set<ExprId> acOutputExprIdSet = JoinUtils.getJoinOutputExprIdSet(a, c); Set<ExprId> bdOutputExprIdSet = JoinUtils.getJoinOutputExprIdSet(b, d); @@ -99,10 +92,10 @@ public class JoinExchangeBothProject extends OneExplorationRuleFactory { return null; } - LogicalJoin<Plan, Plan> newLeftJoin = new LogicalJoin<>(JoinType.INNER_JOIN, + LogicalJoin<GroupPlan, GroupPlan> newLeftJoin = new LogicalJoin<>(JoinType.INNER_JOIN, newLeftJoinHashJoinConjuncts, newLeftJoinOtherJoinConjuncts, new DistributeHint(DistributeType.NONE), a, c, null); - LogicalJoin<Plan, Plan> newRightJoin = new LogicalJoin<>(JoinType.INNER_JOIN, + LogicalJoin<GroupPlan, GroupPlan> newRightJoin = new LogicalJoin<>(JoinType.INNER_JOIN, newRightJoinHashJoinConjuncts, newRightJoinOtherJoinConjuncts, new DistributeHint(DistributeType.NONE), b, d, null); Set<ExprId> topUsedExprIds = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java index a3f6477d3cb..0531c6e54ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/LogicalJoinSemiJoinTransposeProject.java @@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableList; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; /** @@ -50,22 +49,18 @@ public class LogicalJoinSemiJoinTransposeProject implements ExplorationRuleFacto && (topJoin.getJoinType().isInnerJoin() || topJoin.getJoinType().isLeftOuterJoin()))) .whenNot(topJoin -> topJoin.hasDistributeHint() - || topJoin.left().child().hasDistributeHint())) + || topJoin.left().child().hasDistributeHint()) + .when(join -> join.left().isAllSlots())) .then(topProject -> { LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); - if (!normalizedProject.isPresent()) { - return null; - } - LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child(); + LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child(); if (!JoinUtils.checkReorderPrecondition(topJoin, bottomJoin)) { return null; } - Plan a = bottomJoin.left(); - Plan b = bottomJoin.right(); - Plan c = topJoin.right(); + GroupPlan a = bottomJoin.left(); + GroupPlan b = bottomJoin.right(); + GroupPlan c = topJoin.right(); Set<ExprId> topUsedExprIds = new HashSet<>(); topProject.getProjects().forEach(expr -> topUsedExprIds.addAll(expr.getInputSlotExprIds())); @@ -88,22 +83,18 @@ public class LogicalJoinSemiJoinTransposeProject implements ExplorationRuleFacto && (topJoin.getJoinType().isInnerJoin() || topJoin.getJoinType().isRightOuterJoin()))) .whenNot(topJoin -> topJoin.hasDistributeHint() - || topJoin.right().child().hasDistributeHint())) + || topJoin.right().child().hasDistributeHint()) + .when(join -> join.right().isAllSlots())) .then(topProject -> { LogicalJoin<GroupPlan, LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.right()); - if (!normalizedProject.isPresent()) { - return null; - } - LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child(); + LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.right().child(); if (!JoinUtils.checkReorderPrecondition(topJoin, bottomJoin)) { return null; } - Plan a = topJoin.left(); - Plan b = bottomJoin.left(); - Plan c = bottomJoin.right(); + GroupPlan a = topJoin.left(); + GroupPlan b = bottomJoin.left(); + GroupPlan c = bottomJoin.right(); Set<ExprId> topUsedExprIds = new HashSet<>(); topProject.getProjects().forEach(expr -> topUsedExprIds.addAll(expr.getInputSlotExprIds())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java index 6ad9cf6a861..0dcb3b8d344 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java @@ -38,7 +38,6 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableSet; import java.util.HashSet; -import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -67,25 +66,19 @@ public class OuterJoinAssocProject extends OneExplorationRuleFactory { .when(join -> VALID_TYPE_PAIR_SET.contains( Pair.of(join.left().child().getJoinType(), join.getJoinType()))) .when(topJoin -> OuterJoinLAsscomProject.checkReorder(topJoin, topJoin.left().child())) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) + .when(join -> checkCondition(join, join.left().child().left().getOutputSet())) + .when(join -> join.left().isAllSlots())) .thenApply(ctx -> { LogicalProject<LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan>> topProject = ctx.root; LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); - if (!normalizedProject.isPresent()) { - return null; - } /* ********** init ********** */ - LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child(); - Plan a = bottomJoin.left(); - Plan b = bottomJoin.right(); - Plan c = topJoin.right(); - if (!checkCondition(topJoin, a.getOutputSet())) { - return null; - } + LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child(); + GroupPlan a = bottomJoin.left(); + GroupPlan b = bottomJoin.right(); + GroupPlan c = topJoin.right(); /* * Paper `On the Correct and Complete Enumeration of the Core Search Space`. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java index af9fc9855af..ac9787bed48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java @@ -35,7 +35,6 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableSet; import java.util.HashSet; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -68,23 +67,18 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { .when(join -> OuterJoinLAsscomProject.VALID_TYPE_PAIR_SET.contains( Pair.of(join.left().child().getJoinType(), join.getJoinType()))) .when(topJoin -> OuterJoinLAsscomProject.checkReorder(topJoin, topJoin.left().child())) - .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint())) + .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) + .when(topJoin -> OuterJoinLAsscomProject.checkCondition(topJoin, + topJoin.left().child().right().getOutputExprIdSet())) + .when(join -> join.left().isAllSlots())) .then(topProject -> { LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topJoin = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topJoin.left()); - if (!normalizedProject.isPresent()) { - return null; - } /* ********** init ********** */ - LogicalJoin<Plan, Plan> bottomJoin = normalizedProject.get().child(); - Plan a = bottomJoin.left(); - Plan b = bottomJoin.right(); - Plan c = topJoin.right(); - if (!OuterJoinLAsscomProject.checkCondition(topJoin, b.getOutputExprIdSet())) { - return null; - } + LogicalJoin<GroupPlan, GroupPlan> bottomJoin = topJoin.left().child(); + GroupPlan a = bottomJoin.left(); + GroupPlan b = bottomJoin.right(); + GroupPlan c = topJoin.right(); /* ********** new Plan ********** */ LogicalJoin newBottomJoin = topJoin.withChildrenNoContext(a, c, null); @@ -134,7 +128,7 @@ public class OuterJoinLAsscomProject extends OneExplorationRuleFactory { * check join reorder masks. */ public static boolean checkReorder(LogicalJoin<? extends Plan, GroupPlan> topJoin, - LogicalJoin<? extends Plan, ? extends Plan> bottomJoin) { + LogicalJoin<GroupPlan, GroupPlan> bottomJoin) { // hasCommute will cause to lack of OuterJoinAssocRule:Left return !topJoin.getJoinReorderContext().hasLAsscom() && !topJoin.getJoinReorderContext().hasLeftAssociate() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelper.java deleted file mode 100644 index 223b271d0ac..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelper.java +++ /dev/null @@ -1,197 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.join; - -import org.apache.doris.nereids.rules.exploration.CBOUtils; -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.JoinType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** Normalize a Project over Join only when a Project-aware join reorder rule produces an alternative. */ -final class ProjectJoinReorderHelper { - private ProjectJoinReorderHelper() { - } - - /** - * Keep a slot-only Project unchanged, or push single-side complex expressions below its Join. - * - * <p>The returned Project is slot-only and has the same output as the input Project. Empty means that - * the Project cannot be moved without changing semantics.</p> - */ - static Optional<LogicalProject<LogicalJoin<Plan, Plan>>> normalize(LogicalProject<?> project) { - if (project.isAllSlots()) { - return Optional.of(castProject(project)); - } - - LogicalJoin<Plan, Plan> join = childJoin(project); - JoinType joinType = join.getJoinType(); - if (joinType.isLeftSemiOrAntiJoin()) { - if (join.isMarkJoin() || projectBothJoinSides(project)) { - return Optional.empty(); - } - return Optional.of(pushDownLeftSemiProject(project, join)); - } - if (joinType.isInnerJoin() || joinType.isOuterJoin() - || joinType.isAsofInnerJoin() || joinType.isAsofOuterJoin()) { - return pushDownInnerOuterProject(project, join); - } - return Optional.empty(); - } - - private static boolean projectBothJoinSides(LogicalProject<?> project) { - LogicalJoin<Plan, Plan> join = childJoin(project); - Set<Slot> projectOutput = project.getOutputSet(); - boolean containLeft = join.left().getOutput().stream().anyMatch(projectOutput::contains); - boolean containRight = join.right().getOutput().stream().anyMatch(projectOutput::contains); - return containLeft && containRight; - } - - private static LogicalProject<LogicalJoin<Plan, Plan>> pushDownLeftSemiProject( - LogicalProject<?> project, LogicalJoin<Plan, Plan> join) { - Set<Slot> conditionLeftSlots = CBOUtils.joinChildConditionSlots(join, true); - List<NamedExpression> newProjects = new ArrayList<>(project.getProjects()); - Set<Slot> projectUsedSlots = project.getProjects().stream() - .map(NamedExpression::toSlot) - .collect(Collectors.toSet()); - conditionLeftSlots.stream() - .filter(slot -> !projectUsedSlots.contains(slot)) - .forEach(newProjects::add); - - Plan newLeft = new LogicalProject<>(newProjects, join.left()); - LogicalJoin<Plan, Plan> newJoin = join.withChildren(ImmutableList.of(newLeft, join.right())); - return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); - } - - private static Optional<LogicalProject<LogicalJoin<Plan, Plan>>> pushDownInnerOuterProject( - LogicalProject<?> project, LogicalJoin<Plan, Plan> join) { - Set<ExprId> leftOutputExprIds = join.left().getOutputExprIdSet(); - Set<ExprId> rightOutputExprIds = join.right().getOutputExprIdSet(); - - boolean containsHyperEdge = project.getProjects().stream().anyMatch(expression -> { - Set<ExprId> inputExprIds = expression.getInputSlotExprIds(); - return !leftOutputExprIds.containsAll(inputExprIds) - && !rightOutputExprIds.containsAll(inputExprIds); - }); - if (containsHyperEdge) { - return Optional.empty(); - } - - List<NamedExpression> projects = adjustProjectsNullable(project, join); - List<NamedExpression> leftProjects = new ArrayList<>(); - List<NamedExpression> rightProjects = new ArrayList<>(); - for (NamedExpression expression : projects) { - if (leftOutputExprIds.containsAll(expression.getInputSlotExprIds())) { - leftProjects.add(expression); - } else { - rightProjects.add(expression); - } - } - - boolean leftContainsComplexExpression = leftProjects.stream() - .anyMatch(expression -> !(expression instanceof Slot)); - boolean rightContainsComplexExpression = rightProjects.stream() - .anyMatch(expression -> !(expression instanceof Slot)); - // JoinCommute supplies the orientation in which a movable complex expression is on the left. - if (!leftContainsComplexExpression) { - return Optional.empty(); - } - JoinType joinType = join.getJoinType(); - boolean rightSideNullable = joinType.isLeftOuterJoin() - || joinType.isAsofLeftOuterJoin() || joinType.isFullOuterJoin(); - boolean leftSideNullable = joinType.isRightOuterJoin() - || joinType.isAsofRightOuterJoin() || joinType.isFullOuterJoin(); - if ((rightSideNullable && rightContainsComplexExpression) - || (leftSideNullable && leftContainsComplexExpression)) { - return Optional.empty(); - } - - Builder<NamedExpression> newLeftProjects = ImmutableList.<NamedExpression>builder() - .addAll(leftProjects); - Set<Slot> leftConditionSlots = CBOUtils.joinChildConditionSlots(join, true); - Set<Slot> leftProjectSlots = leftProjects.stream() - .map(NamedExpression::toSlot) - .collect(Collectors.toSet()); - leftConditionSlots.stream() - .filter(slot -> !leftProjectSlots.contains(slot)) - .forEach(newLeftProjects::add); - Plan newLeft = new LogicalProject<>(newLeftProjects.build(), join.left()); - - Plan newRight = join.right(); - if (rightContainsComplexExpression) { - Builder<NamedExpression> newRightProjects = ImmutableList.<NamedExpression>builder() - .addAll(rightProjects); - Set<Slot> rightConditionSlots = CBOUtils.joinChildConditionSlots(join, false); - Set<Slot> rightProjectSlots = rightProjects.stream() - .map(NamedExpression::toSlot) - .collect(Collectors.toSet()); - rightConditionSlots.stream() - .filter(slot -> !rightProjectSlots.contains(slot)) - .forEach(newRightProjects::add); - newRight = new LogicalProject<>(newRightProjects.build(), join.right()); - } - - LogicalJoin<Plan, Plan> newJoin = join.withChildren(ImmutableList.of(newLeft, newRight)); - return Optional.of(new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin)); - } - - private static List<NamedExpression> adjustProjectsNullable( - LogicalProject<?> project, LogicalJoin<Plan, Plan> join) { - if (join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()) { - return project.getProjects(); - } - - Map<Slot, Slot> childSlots = new HashMap<>(); - join.left().getOutputSet().forEach(slot -> childSlots.put(slot, slot)); - join.right().getOutputSet().forEach(slot -> childSlots.put(slot, slot)); - join.getOutputSet().forEach(slot -> { - if (childSlots.containsKey(slot)) { - childSlots.put(slot, childSlots.get(slot)); - } - }); - return project.getProjects().stream() - .map(expression -> expression.rewriteUp(child -> - child instanceof Slot ? childSlots.get((Slot) child) : child)) - .map(NamedExpression.class::cast) - .collect(Collectors.toList()); - } - - @SuppressWarnings("unchecked") - private static LogicalJoin<Plan, Plan> childJoin(LogicalProject<?> project) { - return (LogicalJoin<Plan, Plan>) project.child(); - } - - @SuppressWarnings("unchecked") - private static LogicalProject<LogicalJoin<Plan, Plan>> castProject(LogicalProject<?> project) { - return (LogicalProject<LogicalJoin<Plan, Plan>>) project; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java index 8a9c44015d5..905af662a2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java @@ -19,15 +19,25 @@ package org.apache.doris.nereids.rules.exploration.join; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.CBOUtils; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /** * Rule for pushdown project through inner/outer join @@ -56,7 +66,7 @@ public class PushDownProjectThroughInnerOuterJoin implements ExplorationRuleFact .whenNot(j -> j.left().child().hasDistributeHint()) .then(topJoin -> { LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> project = topJoin.left(); - Plan newLeft = ProjectJoinReorderHelper.normalize(project).orElse(null); + Plan newLeft = pushdownProject(project); if (newLeft == null) { return null; } @@ -72,7 +82,7 @@ public class PushDownProjectThroughInnerOuterJoin implements ExplorationRuleFact .whenNot(j -> j.right().child().hasDistributeHint()) .then(topJoin -> { LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> project = topJoin.right(); - Plan newRight = ProjectJoinReorderHelper.normalize(project).orElse(null); + Plan newRight = pushdownProject(project); if (newRight == null) { return null; } @@ -80,4 +90,84 @@ public class PushDownProjectThroughInnerOuterJoin implements ExplorationRuleFact }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN_RIGHT) ); } + + private Plan pushdownProject(LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> project) { + LogicalJoin<GroupPlan, GroupPlan> join = project.child(); + Set<ExprId> aOutputExprIdSet = join.left().getOutputExprIdSet(); + Set<ExprId> bOutputExprIdSet = join.right().getOutputExprIdSet(); + + // reject hyper edge in Project. + if (!project.getProjects().stream().allMatch(expr -> { + Set<ExprId> inputSlotExprIds = expr.getInputSlotExprIds(); + return aOutputExprIdSet.containsAll(inputSlotExprIds) + || bOutputExprIdSet.containsAll(inputSlotExprIds); + })) { + return null; + } + + List<NamedExpression> aProjects = new ArrayList<>(); + List<NamedExpression> bProjects = new ArrayList<>(); + List<NamedExpression> projects; + if (join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()) { + projects = project.getProjects(); + } else { + Map<Slot, Slot> childrenSlots = new HashMap<>(); + join.left().getOutputSet().forEach(slot -> childrenSlots.put(slot, slot)); + join.right().getOutputSet().forEach(slot -> childrenSlots.put(slot, slot)); + join.getOutputSet().forEach(slot -> { + if (childrenSlots.containsKey(slot)) { + childrenSlots.put(slot, childrenSlots.get(slot)); + } + }); + + projects = project.getProjects().stream().map(expr -> expr.rewriteUp(e -> + e instanceof Slot ? childrenSlots.get((Slot) e) : e + )).map(e -> (NamedExpression) e).collect(Collectors.toList()); + } + for (NamedExpression namedExpression : projects) { + Set<ExprId> usedExprIds = namedExpression.getInputSlotExprIds(); + if (aOutputExprIdSet.containsAll(usedExprIds)) { + aProjects.add(namedExpression); + } else { + bProjects.add(namedExpression); + } + } + + boolean leftContains = aProjects.stream().anyMatch(e -> !(e instanceof Slot)); + boolean rightContains = bProjects.stream().anyMatch(e -> !(e instanceof Slot)); + // due to JoinCommute, we don't need to consider just right contains. + if (!leftContains) { + return null; + } + // we could not push nullable side project + if (((join.getJoinType().isLeftOuterJoin() || join.getJoinType().isAsofLeftOuterJoin() + || join.getJoinType().isFullOuterJoin()) && rightContains) + || ((join.getJoinType().isRightOuterJoin() || join.getJoinType().isAsofRightOuterJoin() + || join.getJoinType().isFullOuterJoin()) && leftContains)) { + return null; + } + + Builder<NamedExpression> newAProject = ImmutableList.<NamedExpression>builder().addAll(aProjects); + Set<Slot> aConditionSlots = CBOUtils.joinChildConditionSlots(join, true); + Set<Slot> aProjectSlots = aProjects.stream().map(NamedExpression::toSlot) + .collect(Collectors.toSet()); + aConditionSlots.stream().filter(slot -> !aProjectSlots.contains(slot)).forEach(newAProject::add); + Plan newLeft = new LogicalProject<>(newAProject.build(), join.left()); + + if (!rightContains) { + Plan newJoin = join.withChildren(newLeft, join.right()); + return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); + } + + Builder<NamedExpression> newBProject = ImmutableList.<NamedExpression>builder().addAll(bProjects); + Set<Slot> bConditionSlots = CBOUtils.joinChildConditionSlots(join, false); + Set<Slot> bProjectSlots = bProjects.stream().map(NamedExpression::toSlot) + .collect(Collectors.toSet()); + bConditionSlots.stream().filter(slot -> !bProjectSlots.contains(slot)).forEach(newBProject::add); + Plan newRight = new LogicalProject<>(newBProject.build(), join.right()); + + Plan newJoin = join.withChildren(newLeft, newRight); + return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java index 74d8befe3fa..11efc82fc21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughSemiJoin.java @@ -19,7 +19,10 @@ package org.apache.doris.nereids.rules.exploration.join; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.CBOUtils; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -27,7 +30,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * Rule for pushdown project through left-semi/anti join @@ -56,10 +62,10 @@ public class PushDownProjectThroughSemiJoin implements ExplorationRuleFactory { .whenNot(j -> j.left().child().hasDistributeHint()) .then(topJoin -> { LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> project = topJoin.left(); - Plan newLeft = ProjectJoinReorderHelper.normalize(project).orElse(null); - if (newLeft == null) { + if (projectBothJoinSide(project)) { return null; } + Plan newLeft = pushdownProject(project); return topJoin.withChildren(newLeft, topJoin.right()); }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_LEFT), @@ -70,12 +76,39 @@ public class PushDownProjectThroughSemiJoin implements ExplorationRuleFactory { .whenNot(j -> j.right().child().hasDistributeHint()) .then(topJoin -> { LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> project = topJoin.right(); - Plan newRight = ProjectJoinReorderHelper.normalize(project).orElse(null); - if (newRight == null) { + if (projectBothJoinSide(project)) { return null; } + Plan newRight = pushdownProject(project); return topJoin.withChildren(topJoin.left(), newRight); }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_RIGHT) ); } + + private boolean projectBothJoinSide(LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> project) { + // if project contains both side of join, it can't be pushed. + // such as: + // Project(l, null as r) + // ------ L(l) left anti join R(r) + LogicalJoin<?, ?> join = project.child(); + Set<Slot> projectOutput = project.getOutputSet(); + boolean containLeft = join.left().getOutput().stream().anyMatch(projectOutput::contains); + boolean containRight = join.right().getOutput().stream().anyMatch(projectOutput::contains); + return containRight && containLeft; + } + + private Plan pushdownProject(LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> project) { + LogicalJoin<GroupPlan, GroupPlan> join = project.child(); + Set<Slot> conditionLeftSlots = CBOUtils.joinChildConditionSlots(join, true); + + List<NamedExpression> newProject = new ArrayList<>(project.getProjects()); + Set<Slot> projectUsedSlots = project.getProjects().stream().map(NamedExpression::toSlot) + .collect(Collectors.toSet()); + conditionLeftSlots.stream().filter(slot -> !projectUsedSlots.contains(slot)) + .forEach(newProject::add); + Plan newLeft = new LogicalProject<>(newProject, join.left()); + + Plan newJoin = join.withChildren(newLeft, join.right()); + return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), newJoin); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java index 18c79ce6b20..8d1d67af5c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java @@ -33,7 +33,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableSet; import java.util.HashSet; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -69,6 +68,7 @@ public class SemiJoinSemiJoinTransposeProject extends OneExplorationRuleFactory .when(this::typeChecker) .when(topSemi -> InnerJoinLAsscomProject.checkReorder(topSemi, topSemi.left().child(), false)) .whenNot(join -> join.hasDistributeHint() || join.left().child().hasDistributeHint()) + .when(join -> join.left().isAllSlots()) // the transpose swaps the bottom semi join to the top, so the mark slot // produced by the bottom mark join would be produced by the new top semi // join. if the top semi join references the mark slot in its conjuncts, @@ -80,16 +80,11 @@ public class SemiJoinSemiJoinTransposeProject extends OneExplorationRuleFactory .then(topProject -> { LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>, GroupPlan> topSemi = topProject.child(); - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> - normalizedProject = ProjectJoinReorderHelper.normalize(topSemi.left()); - if (!normalizedProject.isPresent()) { - return null; - } - LogicalJoin<Plan, Plan> bottomSemi = normalizedProject.get().child(); - LogicalProject<LogicalJoin<Plan, Plan>> abProject = normalizedProject.get(); - Plan a = bottomSemi.left(); - Plan b = bottomSemi.right(); - Plan c = topSemi.right(); + LogicalJoin<GroupPlan, GroupPlan> bottomSemi = topSemi.left().child(); + LogicalProject<LogicalJoin<GroupPlan, GroupPlan>> abProject = topSemi.left(); + GroupPlan a = bottomSemi.left(); + GroupPlan b = bottomSemi.right(); + GroupPlan c = topSemi.right(); Set<ExprId> aOutputExprIdSet = a.getOutputExprIdSet(); // if bottom semi join is mark join, we need remove the mark join slot creating by bottom semi join // from the project list before swapping the bottom semi to top semi diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectAwareJoinReorderComplexProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectAwareJoinReorderComplexProjectTest.java deleted file mode 100644 index e124f9e3b93..00000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectAwareJoinReorderComplexProjectTest.java +++ /dev/null @@ -1,354 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.join; - -import org.apache.doris.common.Pair; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.rewrite.AddProjectForJoin; -import org.apache.doris.nereids.rules.rewrite.MergeProjectable; -import org.apache.doris.nereids.trees.expressions.Add; -import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; -import org.apache.doris.nereids.trees.plans.JoinType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.LogicalPlanBuilder; -import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PlanChecker; -import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -class ProjectAwareJoinReorderComplexProjectTest { - - @Test - void completeClassicOptimizerReordersPlanAfterAddProjectForJoin() { - LogicalPlan barePlan = buildBarePlan(4); - List<String> expectedOutput = outputSignature(barePlan); - ConnectContext connectContext = MemoTestUtils.createConnectContext(); - SessionVariable sessionVariable = connectContext.getSessionVariable(); - sessionVariable.enableDPHypOptimizer = false; - sessionVariable.setMaxTableCountUseCascadesJoinReorder(64); - sessionVariable.joinReorderTimeLimit = 600_000; - - PlanChecker checker = PlanChecker.from(connectContext, barePlan) - .customRewrite(new AddProjectForJoin()) - .applyTopDown(new MergeProjectable()); - Plan rewrittenPlan = checker.getCascadesContext().getRewritePlan(); - - long joinCount = rewrittenPlan.collect(plan -> plan instanceof LogicalJoin).size(); - long projectOnJoinCount = rewrittenPlan.collect(plan -> plan instanceof LogicalProject - && plan.child(0) instanceof LogicalJoin).size(); - Assertions.assertEquals(3, joinCount); - Assertions.assertEquals(joinCount, projectOnJoinCount); - Assertions.assertEquals(expectedOutput, outputSignature(rewrittenPlan)); - - checker.optimize(); - Assertions.assertEquals(expectedOutput, outputSignature(checker.getBestPlanTree())); - Set<String> joinTopologies = checker.getAllPlan().stream() - .map(ProjectAwareJoinReorderComplexProjectTest::joinTopology) - .filter(topology -> topology.startsWith("(")) - .collect(Collectors.toSet()); - Assertions.assertTrue(joinTopologies.size() > 1, joinTopologies::toString); - } - - @Test - void innerLAsscomMovesComplexExpressionWithItsInput() { - LogicalOlapScan a = scan(0, "a"); - LogicalOlapScan b = scan(1, "b"); - LogicalOlapScan c = scan(2, "c"); - Alias alias = alias(a, "a_alias"); - LogicalPlan ab = project(join(a, b, JoinType.INNER_JOIN, 0, 0), - alias, a.getOutput().get(1), b.getOutput().get(0), b.getOutput().get(1)); - LogicalPlan original = projectAll(join(ab, c, JoinType.INNER_JOIN, 0, 0)); - - Plan reordered = applyAndFind(InnerJoinLAsscomProject.INSTANCE.build(), original, - plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("c")) - && hasJoin(plan, JoinType.INNER_JOIN, names("a", "c"), names("b"))); - - assertSemanticsPreserved(original, reordered); - assertAliasPlacedOnScan(reordered, alias, "a"); - } - - @Test - void innerAssociatesInBothDirectionsWithComplexExpression() { - LogicalOlapScan a = scan(0, "a"); - LogicalOlapScan b = scan(1, "b"); - LogicalOlapScan c = scan(2, "c"); - Alias rightAssociateAlias = alias(a, "a_alias"); - LogicalPlan ab = project(join(a, b, JoinType.INNER_JOIN, 0, 0), - rightAssociateAlias, a.getOutput().get(1), b.getOutput().get(0), b.getOutput().get(1)); - LogicalPlan rightAssociateOriginal = projectAll(join(ab, c, JoinType.INNER_JOIN, 2, 0)); - - Plan rightAssociated = applyAndFind(InnerJoinRightAssociateProject.INSTANCE.build(), - rightAssociateOriginal, - plan -> hasJoin(plan, JoinType.INNER_JOIN, names("b"), names("c")) - && hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("b", "c"))); - assertSemanticsPreserved(rightAssociateOriginal, rightAssociated); - assertAliasPlacedOnScan(rightAssociated, rightAssociateAlias, "a"); - - Alias leftAssociateAlias = alias(b, "b_alias"); - LogicalPlan bc = project(join(b, c, JoinType.INNER_JOIN, 0, 0), - leftAssociateAlias, b.getOutput().get(0), b.getOutput().get(1), - c.getOutput().get(0), c.getOutput().get(1)); - LogicalPlan leftAssociateOriginal = projectAll(join(a, bc, JoinType.INNER_JOIN, 0, 1)); - - Plan leftAssociated = applyAndFind(InnerJoinLeftAssociateProject.INSTANCE.build(), - leftAssociateOriginal, - plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("b")) - && hasJoin(plan, JoinType.INNER_JOIN, names("a", "b"), names("c"))); - assertSemanticsPreserved(leftAssociateOriginal, leftAssociated); - assertAliasPlacedOnScan(leftAssociated, leftAssociateAlias, "b"); - } - - @Test - void exchangePushesIndependentComplexExpressionsOnBothBranches() { - LogicalOlapScan a = scan(0, "a"); - LogicalOlapScan b = scan(1, "b"); - LogicalOlapScan c = scan(2, "c"); - LogicalOlapScan d = scan(3, "d"); - Alias aAlias = alias(a, "a_alias"); - Alias cAlias = alias(c, "c_alias"); - LogicalPlan ab = project(join(a, b, JoinType.INNER_JOIN, 0, 0), - aAlias, a.getOutput().get(0), a.getOutput().get(1), - b.getOutput().get(0), b.getOutput().get(1)); - LogicalPlan cd = project(join(c, d, JoinType.INNER_JOIN, 0, 0), - cAlias, c.getOutput().get(0), c.getOutput().get(1), - d.getOutput().get(0), d.getOutput().get(1)); - LogicalPlan original = projectAll(new LogicalPlanBuilder(ab) - .join(cd, JoinType.INNER_JOIN, ImmutableList.of(Pair.of(1, 1), Pair.of(3, 3))) - .build()); - - Plan reordered = applyAndFind(JoinExchangeBothProject.INSTANCE.build(), original, - plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("c")) - && hasJoin(plan, JoinType.INNER_JOIN, names("b"), names("d"))); - - assertSemanticsPreserved(original, reordered); - assertAliasPlacedOnScan(reordered, aAlias, "a"); - assertAliasPlacedOnScan(reordered, cAlias, "c"); - } - - @Test - void transposeLogicalJoinAndSemiJoinWithComplexExpression() { - LogicalOlapScan a = scan(0, "a"); - LogicalOlapScan b = scan(1, "b"); - LogicalOlapScan c = scan(2, "c"); - Alias alias = alias(a, "a_alias"); - LogicalPlan semi = project(join(a, b, JoinType.LEFT_SEMI_JOIN, 0, 0), - alias, a.getOutput().get(0), a.getOutput().get(1)); - LogicalPlan original = projectAll(join(semi, c, JoinType.INNER_JOIN, 0, 0)); - - Plan reordered = applyAndFind( - LogicalJoinSemiJoinTransposeProject.INSTANCE.buildRules().get(0), original, - plan -> hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("c")) - && hasJoin(plan, JoinType.LEFT_SEMI_JOIN, names("a", "c"), names("b"))); - - assertSemanticsPreserved(original, reordered); - assertAliasPlacedOnScan(reordered, alias, "a"); - } - - @Test - void transposeTwoSemiJoinsWithComplexExpression() { - LogicalOlapScan a = scan(0, "a"); - LogicalOlapScan b = scan(1, "b"); - LogicalOlapScan c = scan(2, "c"); - Alias alias = alias(a, "a_alias"); - LogicalPlan bottomSemi = project(join(a, b, JoinType.LEFT_SEMI_JOIN, 0, 0), - alias, a.getOutput().get(0), a.getOutput().get(1)); - LogicalPlan original = projectAll(join(bottomSemi, c, JoinType.LEFT_SEMI_JOIN, 0, 0)); - - Plan reordered = applyAndFind(SemiJoinSemiJoinTransposeProject.INSTANCE.build(), original, - plan -> hasJoin(plan, JoinType.LEFT_SEMI_JOIN, names("a"), names("c")) - && hasJoin(plan, JoinType.LEFT_SEMI_JOIN, names("a", "c"), names("b"))); - - assertSemanticsPreserved(original, reordered); - assertAliasPlacedOnScan(reordered, alias, "a"); - } - - @Test - void outerJoinReordersKeepComplexExpressionOnNonNullableInput() { - LogicalOlapScan a = scan(0, "a"); - LogicalOlapScan b = scan(1, "b"); - LogicalOlapScan c = scan(2, "c"); - Alias lAsscomAlias = alias(a, "a_lasscom_alias"); - LogicalPlan abForLAsscom = project(join(a, b, JoinType.INNER_JOIN, 0, 0), - lAsscomAlias, a.getOutput().get(0), a.getOutput().get(1), - b.getOutput().get(0), b.getOutput().get(1)); - LogicalPlan lAsscomOriginal = projectAll(join(abForLAsscom, c, JoinType.LEFT_OUTER_JOIN, 0, 0)); - - Plan lAsscom = applyAndFind(OuterJoinLAsscomProject.INSTANCE.build(), lAsscomOriginal, - plan -> hasJoin(plan, JoinType.LEFT_OUTER_JOIN, names("a"), names("c")) - && hasJoin(plan, JoinType.INNER_JOIN, names("a", "c"), names("b"))); - assertSemanticsPreserved(lAsscomOriginal, lAsscom); - assertAliasPlacedOnScan(lAsscom, lAsscomAlias, "a"); - - Alias assocAlias = alias(a, "a_assoc_alias"); - LogicalPlan abForAssoc = project(join(a, b, JoinType.INNER_JOIN, 0, 0), - assocAlias, a.getOutput().get(0), a.getOutput().get(1), - b.getOutput().get(0), b.getOutput().get(1)); - LogicalPlan assocOriginal = projectAll(join(abForAssoc, c, JoinType.LEFT_OUTER_JOIN, 3, 0)); - - Plan assoc = applyAndFind(OuterJoinAssocProject.INSTANCE.build(), assocOriginal, - plan -> hasJoin(plan, JoinType.LEFT_OUTER_JOIN, names("b"), names("c")) - && hasJoin(plan, JoinType.INNER_JOIN, names("a"), names("b", "c"))); - assertSemanticsPreserved(assocOriginal, assoc); - assertAliasPlacedOnScan(assoc, assocAlias, "a"); - } - - private static Plan applyAndFind(Rule rule, LogicalPlan original, Predicate<Plan> predicate) { - List<Plan> plans = PlanChecker.from(MemoTestUtils.createConnectContext(), original) - .applyExploration(rule) - .getAllPlan(); - return plans.stream() - .filter(predicate) - .findFirst() - .orElseThrow(() -> new AssertionError("Expected reordered alternative, found " + plans.size())); - } - - private static void assertSemanticsPreserved(Plan original, Plan reordered) { - Assertions.assertEquals(outputSignature(original), outputSignature(reordered)); - Assertions.assertEquals(conditionSignatures(original), conditionSignatures(reordered)); - } - - private static void assertAliasPlacedOnScan(Plan plan, Alias alias, String scanName) { - Assertions.assertTrue(anyPlan(plan, candidate -> candidate instanceof LogicalProject - && ((LogicalProject<?>) candidate).getProjects().contains(alias) - && scanNames(candidate.child(0)).equals(names(scanName)))); - } - - private static boolean hasJoin(Plan plan, JoinType type, Set<String> left, Set<String> right) { - return anyPlan(plan, candidate -> candidate instanceof LogicalJoin - && ((LogicalJoin<?, ?>) candidate).getJoinType() == type - && scanNames(candidate.child(0)).equals(left) - && scanNames(candidate.child(1)).equals(right)); - } - - private static boolean anyPlan(Plan plan, Predicate<Plan> predicate) { - if (predicate.test(plan)) { - return true; - } - for (Plan child : plan.children()) { - if (anyPlan(child, predicate)) { - return true; - } - } - return false; - } - - private static Set<String> scanNames(Plan plan) { - ImmutableSet.Builder<String> names = ImmutableSet.builder(); - collectScanNames(plan, names); - return names.build(); - } - - private static void collectScanNames(Plan plan, ImmutableSet.Builder<String> names) { - if (plan instanceof LogicalOlapScan) { - names.add(((LogicalOlapScan) plan).getTable().getName()); - } - plan.children().forEach(child -> collectScanNames(child, names)); - } - - private static List<String> conditionSignatures(Plan plan) { - List<String> signatures = new ArrayList<>(); - collectConditionSignatures(plan, signatures); - Collections.sort(signatures); - return signatures; - } - - private static void collectConditionSignatures(Plan plan, List<String> signatures) { - if (plan instanceof LogicalJoin) { - LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan; - join.getHashJoinConjuncts().stream().map(Expression::toSql).forEach(signatures::add); - join.getOtherJoinConjuncts().stream().map(Expression::toSql).forEach(signatures::add); - join.getMarkJoinConjuncts().stream().map(Expression::toSql).forEach(signatures::add); - } - plan.children().forEach(child -> collectConditionSignatures(child, signatures)); - } - - private static List<String> outputSignature(Plan plan) { - return plan.getOutput().stream() - .map(slot -> slot.getExprId() + ":" + slot.getName() + ":" + slot.nullable()) - .collect(Collectors.toList()); - } - - private static LogicalPlan buildBarePlan(int tableCount) { - LogicalPlan plan = scan(0, "t0"); - for (int i = 1; i < tableCount; i++) { - LogicalOlapScan right = scan(i, "t" + i); - plan = join(plan, right, JoinType.INNER_JOIN, 2 * (i - 1), 0); - } - return plan; - } - - private static String joinTopology(Plan plan) { - if (plan instanceof LogicalProject) { - return joinTopology(plan.child(0)); - } - if (plan instanceof LogicalOlapScan) { - return ((LogicalOlapScan) plan).getTable().getName(); - } - if (plan instanceof LogicalJoin) { - return "(" + joinTopology(plan.child(0)) + "," + joinTopology(plan.child(1)) + ")"; - } - return plan.getClass().getSimpleName(); - } - - private static LogicalOlapScan scan(int id, String name) { - return PlanConstructor.newLogicalOlapScan(id, name, 0); - } - - private static Alias alias(LogicalOlapScan scan, String name) { - return new Alias(new Add(scan.getOutput().get(0), new IntegerLiteral(1)), name); - } - - private static LogicalPlan join(LogicalPlan left, LogicalPlan right, JoinType type, - int leftIndex, int rightIndex) { - return new LogicalPlanBuilder(left) - .join(right, type, Pair.of(leftIndex, rightIndex)) - .build(); - } - - private static LogicalPlan project(LogicalPlan child, NamedExpression... projects) { - return new LogicalProject<>(ImmutableList.copyOf(projects), child); - } - - private static LogicalPlan projectAll(LogicalPlan child) { - return new LogicalProject<>(ImmutableList.copyOf(child.getOutput()), child); - } - - private static Set<String> names(String... names) { - return ImmutableSet.copyOf(names); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelperTest.java deleted file mode 100644 index 1431074fd8d..00000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/ProjectJoinReorderHelperTest.java +++ /dev/null @@ -1,190 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.join; - -import org.apache.doris.common.Pair; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Add; -import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.functions.scalar.Random; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; -import org.apache.doris.nereids.trees.plans.JoinType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.LogicalPlanBuilder; -import org.apache.doris.nereids.util.PlanConstructor; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -class ProjectJoinReorderHelperTest { - private final LogicalOlapScan left = PlanConstructor.newLogicalOlapScan(0, "left", 0); - private final LogicalOlapScan right = PlanConstructor.newLogicalOlapScan(1, "right", 0); - - @Test - void keepSlotOnlyProjectUnchanged() { - LogicalJoin<LogicalPlan, LogicalPlan> join = join(JoinType.INNER_JOIN); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> project - = new LogicalProject<>(ImmutableList.copyOf(join.getOutput()), join); - - Optional<LogicalProject<LogicalJoin<Plan, Plan>>> normalized - = ProjectJoinReorderHelper.normalize(project); - - Assertions.assertTrue(normalized.isPresent()); - Assertions.assertSame(project, normalized.get()); - } - - @Test - void removeStandalonePushDownRulesFromClassicReorder() { - List<RuleType> classicRuleTypes = RuleSet.OTHER_REORDER_RULES.stream() - .map(Rule::getRuleType) - .collect(Collectors.toList()); - - Assertions.assertFalse(classicRuleTypes.contains( - RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN_LEFT)); - Assertions.assertFalse(classicRuleTypes.contains( - RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN_RIGHT)); - Assertions.assertFalse(classicRuleTypes.contains( - RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_LEFT)); - Assertions.assertFalse(classicRuleTypes.contains( - RuleType.PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN_RIGHT)); - } - - @Test - void pushComplexExpressionsToBothInputsAndRestoreConditionSlots() { - LogicalJoin<LogicalPlan, LogicalPlan> join = join(JoinType.INNER_JOIN); - Alias leftAlias = new Alias(new Add(left.getOutput().get(1), new IntegerLiteral(1)), "left_alias"); - Alias rightAlias = new Alias(new Add(right.getOutput().get(1), new IntegerLiteral(2)), "right_alias"); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> project - = new LogicalProject<>(ImmutableList.of(leftAlias, rightAlias), join); - - LogicalProject<LogicalJoin<Plan, Plan>> normalized - = ProjectJoinReorderHelper.normalize(project).orElseThrow(AssertionError::new); - LogicalJoin<? extends Plan, ? extends Plan> normalizedJoin = normalized.child(); - - Assertions.assertTrue(normalized.getProjects().stream().allMatch(Slot.class::isInstance)); - Assertions.assertEquals(outputSignature(project), outputSignature(normalized)); - Assertions.assertEquals(join.getHashJoinConjuncts(), normalizedJoin.getHashJoinConjuncts()); - assertInputProject(normalizedJoin.left(), leftAlias, left.getOutput().get(0)); - assertInputProject(normalizedJoin.right(), rightAlias, right.getOutput().get(0)); - } - - @Test - void rejectHyperEdgeProject() { - LogicalJoin<LogicalPlan, LogicalPlan> join = join(JoinType.INNER_JOIN); - Alias hyperEdge = new Alias(new Add(left.getOutput().get(0), right.getOutput().get(0)), "both"); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> project - = new LogicalProject<>(ImmutableList.of(hyperEdge), join); - - Assertions.assertFalse(ProjectJoinReorderHelper.normalize(project).isPresent()); - } - - @Test - void respectOuterJoinNullableSides() { - Alias leftAlias = new Alias(new Add(left.getOutput().get(1), new IntegerLiteral(1)), "left_alias"); - Alias rightAlias = new Alias(new Add(right.getOutput().get(1), new IntegerLiteral(1)), "right_alias"); - - LogicalJoin<LogicalPlan, LogicalPlan> leftOuter = join(JoinType.LEFT_OUTER_JOIN); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> leftOnly - = new LogicalProject<>(ImmutableList.of(leftAlias, leftOuter.getOutput().get(2)), leftOuter); - LogicalProject<LogicalJoin<Plan, Plan>> normalized - = ProjectJoinReorderHelper.normalize(leftOnly).orElseThrow(AssertionError::new); - Assertions.assertEquals(outputSignature(leftOnly), outputSignature(normalized)); - Assertions.assertFalse(normalized.child().right() instanceof LogicalProject); - - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> leftAndNullableRight - = new LogicalProject<>(ImmutableList.of(leftAlias, rightAlias), leftOuter); - Assertions.assertFalse(ProjectJoinReorderHelper.normalize(leftAndNullableRight).isPresent()); - - LogicalJoin<LogicalPlan, LogicalPlan> rightOuter = join(JoinType.RIGHT_OUTER_JOIN); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> nullableLeft - = new LogicalProject<>(ImmutableList.of(leftAlias), rightOuter); - Assertions.assertFalse(ProjectJoinReorderHelper.normalize(nullableLeft).isPresent()); - - LogicalJoin<LogicalPlan, LogicalPlan> fullOuter = join(JoinType.FULL_OUTER_JOIN); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> fullOuterProject - = new LogicalProject<>(ImmutableList.of(leftAlias, rightAlias), fullOuter); - Assertions.assertFalse(ProjectJoinReorderHelper.normalize(fullOuterProject).isPresent()); - } - - @Test - void pushLeftSemiProjectAndRejectMarkJoin() { - LogicalJoin<LogicalPlan, LogicalPlan> semiJoin = join(JoinType.LEFT_SEMI_JOIN); - Alias alias = new Alias(new Add(left.getOutput().get(1), new IntegerLiteral(1)), "semi_alias"); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> project - = new LogicalProject<>(ImmutableList.of(alias), semiJoin); - - LogicalProject<LogicalJoin<Plan, Plan>> normalized - = ProjectJoinReorderHelper.normalize(project).orElseThrow(AssertionError::new); - assertInputProject(normalized.child().left(), alias, left.getOutput().get(0)); - Assertions.assertEquals(outputSignature(project), outputSignature(normalized)); - - LogicalJoin<LogicalPlan, LogicalPlan> markJoin = (LogicalJoin<LogicalPlan, LogicalPlan>) - new LogicalPlanBuilder(left) - .markJoin(right, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0)) - .build(); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> markProject - = new LogicalProject<>(ImmutableList.of(alias), markJoin); - Assertions.assertFalse(ProjectJoinReorderHelper.normalize(markProject).isPresent()); - } - - @Test - void preserveExistingLiteralAndVolatilePlacementBehavior() { - LogicalJoin<LogicalPlan, LogicalPlan> join = join(JoinType.INNER_JOIN); - Alias literal = new Alias(new IntegerLiteral(1), "literal_alias"); - Alias random = new Alias(new Random(), "random_alias"); - LogicalProject<LogicalJoin<LogicalPlan, LogicalPlan>> project - = new LogicalProject<>(ImmutableList.of(literal, random), join); - - LogicalProject<LogicalJoin<Plan, Plan>> normalized - = ProjectJoinReorderHelper.normalize(project).orElseThrow(AssertionError::new); - - assertInputProject(normalized.child().left(), literal, random, left.getOutput().get(0)); - Assertions.assertEquals(outputSignature(project), outputSignature(normalized)); - } - - private LogicalJoin<LogicalPlan, LogicalPlan> join(JoinType joinType) { - return (LogicalJoin<LogicalPlan, LogicalPlan>) new LogicalPlanBuilder(left) - .join(right, joinType, Pair.of(0, 0)) - .build(); - } - - private static void assertInputProject(Plan input, NamedExpression... expectedProjects) { - Assertions.assertInstanceOf(LogicalProject.class, input); - LogicalProject<?> project = (LogicalProject<?>) input; - Assertions.assertEquals(ImmutableList.copyOf(expectedProjects), project.getProjects()); - } - - private static List<String> outputSignature(Plan plan) { - return plan.getOutput().stream() - .map(slot -> slot.getExprId() + ":" + slot.getName() + ":" + slot.nullable()) - .collect(Collectors.toList()); - } -} diff --git a/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out index f8ab9595435..8339e9252c2 100644 --- a/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out +++ b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out @@ -64,4 +64,3 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) 2020-01-04T00:00 ALL 1 6 \N \N b 2020-01-04T00:00 ALL 1 6 \N \N b 2020-01-04T00:00 ALL 1 7 \N \N \N - diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out index 3be5c07a3f9..19e82920b92 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query2.out @@ -23,16 +23,16 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF2 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF1 d_week_seq->[d_week_seq] ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF1 RF2 ----------------------PhysicalProject -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 +------------------------filter((date_dim.d_year = 1999)) +--------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject ---------------------filter((date_dim.d_year = 1999)) -----------------------PhysicalOlapScan[date_dim] +--------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 --------------PhysicalProject ----------------filter((date_dim.d_year = 1998)) ------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out index 417a1251638..28a4a118915 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query59.out @@ -19,24 +19,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF6 d_week_seq->[d_week_seq] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() +------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52)) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() build RFs:RF4 s_store_id2->[s_store_id];RF5 expr_(cast(d_week_seq2 as BIGINT) - 52)->[cast(d_week_seq as BIGINT)] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF5 RF6 +------------------------PhysicalProject +--------------------------PhysicalOlapScan[store] apply RFs: RF4 +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 52)] ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF3 ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject --------------------------filter((d.d_month_seq <= 1219) and (d.d_month_seq >= 1208)) ----------------------------PhysicalOlapScan[date_dim(d)] ---------------------PhysicalProject -----------------------PhysicalOlapScan[store] ----------------PhysicalProject ------------------filter((d.d_month_seq <= 1207) and (d.d_month_seq >= 1196)) --------------------PhysicalOlapScan[date_dim(d)] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out index fbbd4cd5abc..5ed4762bd44 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query2.out @@ -23,16 +23,16 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF2 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 53))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 53)] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((date_dim.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF1 d_week_seq->[d_week_seq] ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF1 RF2 ----------------------PhysicalProject -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 +------------------------filter((date_dim.d_year = 1999)) +--------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject ---------------------filter((date_dim.d_year = 1999)) -----------------------PhysicalOlapScan[date_dim] +--------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF3 --------------PhysicalProject ----------------filter((date_dim.d_year = 1998)) ------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out index 920d8670505..25713c879dc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query59.out @@ -19,24 +19,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq1)) otherCondition=() build RFs:RF6 d_week_seq->[d_week_seq] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() build RFs:RF4 s_store_id1->[s_store_id];RF5 s_store_sk->[ss_store_sk] +------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52)) and (y.s_store_id1 = x.s_store_id2)) otherCondition=() build RFs:RF4 s_store_id2->[s_store_id];RF5 expr_(cast(d_week_seq2 as BIGINT) - 52)->[cast(d_week_seq as BIGINT)] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF3 d_week_seq->[d_week_seq] +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk] +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF5 RF6 +------------------------PhysicalProject +--------------------------PhysicalOlapScan[store] apply RFs: RF4 +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_week_seq = d_week_seq2)) otherCondition=() build RFs:RF2 d_week_seq->[d_week_seq] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((wss.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(d_week_seq1 as BIGINT) = expr_(cast(d_week_seq2 as BIGINT) - 52))) otherCondition=() build RFs:RF1 expr_cast(d_week_seq1 as BIGINT)->[(cast(d_week_seq as BIGINT) - 52)] ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 RF3 ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5 RF6 +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[store] apply RFs: RF4 +------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject --------------------------filter((d.d_month_seq <= 1219) and (d.d_month_seq >= 1208)) ----------------------------PhysicalOlapScan[date_dim(d)] ---------------------PhysicalProject -----------------------PhysicalOlapScan[store] ----------------PhysicalProject ------------------filter((d.d_month_seq <= 1207) and (d.d_month_seq >= 1196)) --------------------PhysicalOlapScan[date_dim(d)] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
