This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch opt_memtable_speed
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/opt_memtable_speed by this
push:
new a186907b13 [opt](Nereids): enable distinct pushdown (#23196)
a186907b13 is described below
commit a186907b1329c8277937be2f1a22517bb3eadfd5
Author: xzj7019 <[email protected]>
AuthorDate: Sat Aug 19 12:11:47 2023 +0800
[opt](Nereids): enable distinct pushdown (#23196)
* [opt](Nereids): enable distinct pushdown
* [opt](Nereids): enable distinct pushdown
* [opt](Nereids): enable distinct pushdown
---------
Co-authored-by: zhongjian.xzj
<[email protected]>
---
.../apache/doris/nereids/jobs/executor/Rewriter.java | 7 ++++---
.../nereids/rules/expression/rules/CaseWhenToIf.java | 5 +++++
.../rules/rewrite/PushdownDistinctThroughJoin.java | 11 +++++++++++
.../java/org/apache/doris/nereids/util/PlanUtils.java | 9 +++++++++
.../main/java/org/apache/doris/qe/SessionVariable.java | 17 +++++++++++++++++
5 files changed, 46 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 626c6094be..6f20208dac 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -83,6 +83,7 @@ import
org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin;
import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation;
import org.apache.doris.nereids.rules.rewrite.PushProjectThroughUnion;
+import org.apache.doris.nereids.rules.rewrite.PushdownDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughProject;
import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushdownLimit;
@@ -251,9 +252,9 @@ public class Rewriter extends AbstractBatchJobExecutor {
topDown(new BuildAggForUnion())
),
- // topic("Distinct",
- //
costBased(custom(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN,
PushdownDistinctThroughJoin::new))
- // ),
+ topic("Distinct",
+ custom(RuleType.PUSHDOWN_DISTINCT_THROUGH_JOIN,
PushdownDistinctThroughJoin::new)
+ ),
topic("Limit optimization",
topDown(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToIf.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToIf.java
index 6372338406..08a07bbb6b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToIf.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/CaseWhenToIf.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.qe.ConnectContext;
/**
* Rewrite rule to convert CASE WHEN to IF.
@@ -37,6 +38,10 @@ public class CaseWhenToIf extends
AbstractExpressionRewriteRule {
@Override
public Expression visitCaseWhen(CaseWhen caseWhen,
ExpressionRewriteContext context) {
Expression expr = caseWhen;
+ if (ConnectContext.get() == null ||
ConnectContext.get().getSessionVariable() == null
+ ||
!ConnectContext.get().getSessionVariable().isEnableCaseWhenTransformation()) {
+ return expr;
+ }
if (caseWhen.getWhenClauses().size() == 1) {
WhenClause whenClause = caseWhen.getWhenClauses().get(0);
Expression operand = whenClause.getOperand();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoin.java
index 7cfc9f35aa..c7bdafea9c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoin.java
@@ -19,11 +19,14 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
+import org.apache.doris.nereids.util.PlanUtils;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
@@ -40,7 +43,12 @@ public class PushdownDistinctThroughJoin extends
DefaultPlanRewriter<JobContext>
@Override
public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> agg,
JobContext context) {
+ if (ConnectContext.get() == null ||
ConnectContext.get().getSessionVariable() == null
+ ||
!ConnectContext.get().getSessionVariable().isEnableDistinctPushDown()) {
+ return agg;
+ }
agg = visitChildren(this, agg, context);
+
if (agg.hasPushed() || !agg.isDistinct() || isLeaf(agg.child())) {
return agg;
}
@@ -80,6 +88,9 @@ public class PushdownDistinctThroughJoin extends
DefaultPlanRewriter<JobContext>
}
private Plan withDistinct(Plan plan) {
+ if (PlanUtils.skipProjectFilterLimit(plan) instanceof Relation) {
+ return plan;
+ }
return new LogicalAggregate<>(ImmutableList.copyOf(plan.getOutput()),
true, true, plan);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
index 5ded1dab5e..17034c15e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
@@ -24,6 +24,7 @@ import
org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -96,4 +97,12 @@ public class PlanUtils {
}
}).collect(ImmutableList.toImmutableList());
}
+
+ public static Plan skipProjectFilterLimit(Plan plan) {
+ if (plan instanceof LogicalProject && ((LogicalProject<?>)
plan).isAllSlots()
+ || plan instanceof LogicalFilter || plan instanceof
LogicalLimit) {
+ return plan.child(0);
+ }
+ return plan;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index eb5f1434e6..57a2eb1f2d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -232,6 +232,9 @@ public class SessionVariable implements Serializable,
Writable {
public static final String DISABLE_NEREIDS_RULES = "disable_nereids_rules";
public static final String ENABLE_NEW_COST_MODEL = "enable_new_cost_model";
public static final String ENABLE_FALLBACK_TO_ORIGINAL_PLANNER =
"enable_fallback_to_original_planner";
+ public static final String ENABLE_DISTINCT_PUSHDOWN =
"enable_distinct_pushdown";
+
+ public static final String ENABLE_CASEWHEN_TRANSFORMATION =
"enable_casewhen_transformation";
public static final String ENABLE_NEREIDS_TIMEOUT =
"enable_nereids_timeout";
public static final String FORBID_UNKNOWN_COLUMN_STATS =
"forbid_unknown_col_stats";
@@ -908,6 +911,12 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
needForward = true)
public boolean enableFallbackToOriginalPlanner = true;
+ @VariableMgr.VarAttr(name = ENABLE_DISTINCT_PUSHDOWN)
+ public boolean enableDistinctPushDown = false;
+
+ @VariableMgr.VarAttr(name = ENABLE_CASEWHEN_TRANSFORMATION)
+ public boolean enableCaseWhenTransformation = true;
+
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true)
public boolean enableNereidsTimeout = true;
@@ -2457,6 +2466,14 @@ public class SessionVariable implements Serializable,
Writable {
new
SetVar(SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER, new
StringLiteral("true")));
}
+ public boolean isEnableDistinctPushDown() {
+ return enableDistinctPushDown;
+ }
+
+ public boolean isEnableCaseWhenTransformation() {
+ return enableCaseWhenTransformation;
+ }
+
public void disableNereidsPlannerOnce() throws DdlException {
if (!enableNereidsPlanner) {
return;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
