This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0-beta in repository https://gitbox.apache.org/repos/asf/doris.git
commit cbfaec4c9c5da0cb7431d3fbf9be5083ddb92ee0 Author: morrySnow <[email protected]> AuthorDate: Fri Jun 9 23:41:00 2023 +0800 [minor](Nereids) remove some invasive code of minidump in cascades framework (#20606) --- .../java/org/apache/doris/nereids/cost/Cost.java | 14 +++++---- .../apache/doris/nereids/cost/CostCalculator.java | 16 ++++------- .../org/apache/doris/nereids/cost/CostWeight.java | 9 ++---- .../nereids/jobs/cascades/CostAndEnforcerJob.java | 33 +++++++++------------- .../jobs/joinorder/hypergraph/GraphSimplifier.java | 4 +-- .../apache/doris/nereids/memo/GroupExpression.java | 2 +- .../org/apache/doris/nereids/memo/MemoTest.java | 2 +- 7 files changed, 35 insertions(+), 45 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java index c0ec0b1956..18dfbad5e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/Cost.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.cost; +import org.apache.doris.qe.ConnectContext; + /** * Cost encapsulate the real cost with double type. * We do this because we want to customize the operation of adding child cost @@ -28,8 +30,8 @@ public interface Cost { /** * This is for calculating the cost in simplifier */ - static Cost withRowCount(double rowCount, boolean enableNewCostModel) { - if (enableNewCostModel) { + static Cost withRowCount(double rowCount) { + if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) { return new CostV2(0, rowCount, 0); } return new CostV1(rowCount); @@ -38,15 +40,15 @@ public interface Cost { /** * return zero cost */ - static Cost zero(boolean enableNewCostModel) { - if (enableNewCostModel) { + static Cost zero() { + if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) { return CostV2.zero(); } return CostV1.zero(); } - static Cost infinite(boolean enableNewCostModel) { - if (enableNewCostModel) { + static Cost infinite() { + if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) { return CostV2.infinite(); } return CostV1.infinite(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java index b1e1570171..7d6125494f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpecReplicated; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.qe.ConnectContext; import java.util.List; @@ -32,11 +33,6 @@ import java.util.List; @Developing //TODO: memory cost and network cost should be estimated by byte size. public class CostCalculator { - private static boolean enableNewCostModel = false; - - public static void setEnableNewCostModel(boolean enableNewCostModel) { - CostCalculator.enableNewCostModel = enableNewCostModel; - } /** * Calculate cost for groupExpression @@ -47,7 +43,7 @@ public class CostCalculator { && childrenProperties.get(1).getDistributionSpec() instanceof DistributionSpecReplicated) { planContext.setBroadcastJoin(); } - if (enableNewCostModel) { + if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) { CostModelV2 costModelV2 = new CostModelV2(); return groupExpression.getPlan().accept(costModelV2, planContext); } else { @@ -60,7 +56,7 @@ public class CostCalculator { * Calculate cost without groupExpression */ public static Cost calculateCost(Plan plan, PlanContext planContext) { - if (enableNewCostModel) { + if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) { CostModelV2 costModel = new CostModelV2(); return plan.accept(costModel, planContext); } else { @@ -70,9 +66,9 @@ public class CostCalculator { } public static Cost addChildCost(Plan plan, Cost planCost, Cost childCost, int index) { - if (!enableNewCostModel) { - return CostModelV1.addChildCost(plan, planCost, childCost, index); + if (ConnectContext.get().getSessionVariable().getEnableNewCostModel()) { + return CostModelV2.addChildCost(plan, planCost, childCost, index); } - return CostModelV2.addChildCost(plan, planCost, childCost, index); + return CostModelV1.addChildCost(plan, planCost, childCost, index); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java index 5cd576ac28..a0bb8c7a87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostWeight.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.cost; +import org.apache.doris.qe.ConnectContext; + import com.google.common.base.Preconditions; /** @@ -37,7 +39,6 @@ public class CostWeight { static final double NETWORK_WEIGHT = 1.5; static final double DELAY = 0.5; - static double nereidsCboPenaltyFactor = 0.7; final double cpuWeight; final double memoryWeight; final double networkWeight; @@ -69,7 +70,7 @@ public class CostWeight { public static CostWeight get() { return new CostWeight(CPU_WEIGHT, MEMORY_WEIGHT, NETWORK_WEIGHT, - nereidsCboPenaltyFactor); + ConnectContext.get().getSessionVariable().getNereidsCboPenaltyFactor()); } //TODO: add it in session variable @@ -80,8 +81,4 @@ public class CostWeight { public double weightSum(double cpuCost, double ioCost, double netCost) { return cpuCost * cpuWeight + ioCost * ioWeight + netCost * networkWeight; } - - public static void setNereidsCboPenaltyFactor(double nereidsCboPenaltyFactor) { - CostWeight.nereidsCboPenaltyFactor = nereidsCboPenaltyFactor; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java index ff82addb25..b673bd6e87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.jobs.cascades; import org.apache.doris.common.Pair; import org.apache.doris.nereids.cost.Cost; import org.apache.doris.nereids.cost.CostCalculator; -import org.apache.doris.nereids.cost.CostWeight; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; @@ -47,6 +46,7 @@ import java.util.Optional; * Inspired by NoisePage and ORCA-Paper. */ public class CostAndEnforcerJob extends Job implements Cloneable { + private static final Logger LOG = LogManager.getLogger(CostAndEnforcerJob.class); // GroupExpression to optimize @@ -110,10 +110,6 @@ public class CostAndEnforcerJob extends Job implements Cloneable { */ @Override public void execute() { - CostCalculator.setEnableNewCostModel( - context.getCascadesContext().getConnectContext().getSessionVariable().getEnableNewCostModel()); - CostWeight.setNereidsCboPenaltyFactor( - context.getCascadesContext().getConnectContext().getSessionVariable().getNereidsCboPenaltyFactor()); if (groupExpression.isUnused()) { return; } @@ -121,10 +117,8 @@ public class CostAndEnforcerJob extends Job implements Cloneable { countJobExecutionTimesOfGroupExpressions(groupExpression); // Do init logic of root plan/groupExpr of `subplan`, only run once per task. if (curChildIndex == -1) { - curNodeCost = Cost.zero( - context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump()); - curTotalCost = Cost.zero( - context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump()); + curNodeCost = Cost.zero(); + curTotalCost = Cost.zero(); curChildIndex = 0; // List<request property to children> // [ child item: [leftProperties, rightProperties]] @@ -196,8 +190,7 @@ public class CostAndEnforcerJob extends Job implements Cloneable { lowestCostExpr.getCostValueByProperties(requestChildProperty), curChildIndex); if (curTotalCost.getValue() > context.getCostUpperBound()) { - curTotalCost = Cost.infinite( - context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump()); + curTotalCost = Cost.infinite(); } // the request child properties will be covered by the output properties // that corresponding to the request properties. so if we run a costAndEnforceJob of the same @@ -248,14 +241,18 @@ public class CostAndEnforcerJob extends Job implements Cloneable { // if we come here, mean that we have some error in stats calculator and should fix it. return false; } + StatsCalculator statsCalculator = StatsCalculator.estimate(groupExpression, context.getCascadesContext().getConnectContext().getSessionVariable().getForbidUnknownColStats(), context.getCascadesContext().getConnectContext().getTotalColumnStatisticMap(), context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump()); - context.getCascadesContext().getConnectContext().getTotalColumnStatisticMap() - .putAll(statsCalculator.getTotalColumnStatisticMap()); - context.getCascadesContext().getConnectContext().getTotalHistogramMap() - .putAll(statsCalculator.getTotalHistogramMap()); + if (!context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump() + && context.getCascadesContext().getConnectContext().getSessionVariable().isEnableMinidump()) { + context.getCascadesContext().getConnectContext().getTotalColumnStatisticMap() + .putAll(statsCalculator.getTotalColumnStatisticMap()); + context.getCascadesContext().getConnectContext().getTotalHistogramMap() + .putAll(statsCalculator.getTotalHistogramMap()); + } // recompute cost after adjusting property curNodeCost = CostCalculator.calculateCost(groupExpression, requestChildrenProperties); @@ -328,10 +325,8 @@ public class CostAndEnforcerJob extends Job implements Cloneable { lowestCostChildren.clear(); prevChildIndex = -1; curChildIndex = 0; - curTotalCost = Cost.zero( - context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump()); - curNodeCost = Cost.zero( - context.getCascadesContext().getConnectContext().getSessionVariable().isPlayNereidsDump()); + curTotalCost = Cost.zero(); + curNodeCost = Cost.zero(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java index adee7f662e..efe9896dd2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java @@ -82,7 +82,7 @@ public class GraphSimplifier { } for (Node node : graph.getNodes()) { cacheStats.put(node.getNodeMap(), node.getGroup().getStatistics()); - cacheCost.put(node.getNodeMap(), Cost.withRowCount(node.getRowCount(), false)); + cacheCost.put(node.getNodeMap(), Cost.withRowCount(node.getRowCount())); } circleDetector = new CircleDetector(edgeSize); @@ -427,7 +427,7 @@ public class GraphSimplifier { Statistics leftStats, Statistics rightStats) { LogicalJoin join = edge.getJoin(); PlanContext planContext = new PlanContext(stats, leftStats, rightStats); - Cost cost = Cost.zero(false); + Cost cost; if (JoinUtils.shouldNestedLoopJoin(join)) { PhysicalNestedLoopJoin nestedLoopJoin = new PhysicalNestedLoopJoin<>( join.getJoinType(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java index 43f54ca494..b921c80000 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java @@ -77,7 +77,7 @@ public class GroupExpression { // After mergeGroup(), source Group was cleaned up, but it may be in the Job Stack. So use this to mark and skip it. private boolean isUnused = false; - private ObjectId id = StatementScopeIdGenerator.newObjectId(); + private final ObjectId id = StatementScopeIdGenerator.newObjectId(); public GroupExpression(Plan plan) { this(plan, Lists.newArrayList()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java index 776b68fb10..43b1c48761 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java @@ -95,7 +95,7 @@ class MemoTest implements MemoPatternMatchSupported { FakePlan fakePlan = new FakePlan(); GroupExpression srcParentExpression = new GroupExpression(fakePlan, Lists.newArrayList(srcGroup)); Group srcParentGroup = new Group(new GroupId(0), srcParentExpression, new LogicalProperties(ArrayList::new)); - srcParentGroup.setBestPlan(srcParentExpression, Cost.zero(false), PhysicalProperties.ANY); + srcParentGroup.setBestPlan(srcParentExpression, Cost.zero(), PhysicalProperties.ANY); GroupExpression dstParentExpression = new GroupExpression(fakePlan, Lists.newArrayList(dstGroup)); Group dstParentGroup = new Group(new GroupId(1), dstParentExpression, new LogicalProperties(ArrayList::new)); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
