This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 17ca75f8340 [chore](Nereids): add eager aggregate into rules (#27505)
17ca75f8340 is described below
commit 17ca75f83400fdf66153de7946b39946de1cacc3
Author: jakevin <[email protected]>
AuthorDate: Fri Nov 24 10:06:04 2023 +0800
[chore](Nereids): add eager aggregate into rules (#27505)
Add `Eager Aggregate` rules into Rewrite rules.
---
.../doris/nereids/jobs/executor/Rewriter.java | 18 ++++++-----
.../rewrite/PushDownCountThroughJoinTest.java | 36 ++++------------------
.../rewrite/PushDownDistinctThroughJoinTest.java | 18 ++++-------
.../rewrite/PushDownMinMaxThroughJoinTest.java | 36 ++++------------------
.../rules/rewrite/PushDownSumThroughJoinTest.java | 24 ++++-----------
.../apache/doris/utframe/TestWithFeService.java | 11 +++++++
6 files changed, 45 insertions(+), 98 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 15e08d052e9..183bfc3c91a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -91,9 +91,12 @@ import
org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
+import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject;
import org.apache.doris.nereids.rules.rewrite.PushDownLimit;
import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownMinMaxThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownSumThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin;
@@ -272,14 +275,13 @@ public class Rewriter extends AbstractBatchJobExecutor {
topDown(new BuildAggForUnion())
),
- // topic("Eager aggregation",
- // topDown(
- // new PushDownSumThroughJoin(),
- // new PushDownMinMaxThroughJoin(),
- // new PushDownCountThroughJoin()
- // ),
- // custom(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN,
PushDownDistinctThroughJoin::new)
- // ),
+ topic("Eager aggregation",
+ topDown(
+ new PushDownSumThroughJoin(),
+ new PushDownMinMaxThroughJoin(),
+ new PushDownCountThroughJoin()
+ )
+ ),
topic("Limit optimization",
// TODO: the logical plan should not contains any phase
information,
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java
index 21eebe1b375..34ccfe70f70 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java
@@ -42,15 +42,15 @@ import java.util.Set;
class PushDownCountThroughJoinTest implements MemoPatternMatchSupported {
private static final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
private static final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
+ private MockUp<SessionVariable> mockUp = new MockUp<SessionVariable>() {
+ @Mock
+ public Set<Integer> getEnableNereidsRules() {
+ return
ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type());
+ }
+ };
@Test
void testSingleCount() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type());
- }
- };
Alias count = new Count(scan1.getOutput().get(0)).alias("count");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -71,12 +71,6 @@ class PushDownCountThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testMultiCount() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type());
- }
- };
Alias leftCnt1 = new Count(scan1.getOutput().get(0)).alias("leftCnt1");
Alias leftCnt2 = new Count(scan1.getOutput().get(1)).alias("leftCnt2");
Alias rightCnt1 = new
Count(scan2.getOutput().get(1)).alias("rightCnt1");
@@ -100,12 +94,6 @@ class PushDownCountThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testSingleCountStar() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type());
- }
- };
Alias count = new Count().alias("countStar");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -126,12 +114,6 @@ class PushDownCountThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testSingleCountStarEmptyGroupBy() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type());
- }
- };
Alias count = new Count().alias("countStar");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -153,12 +135,6 @@ class PushDownCountThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testBothSideCountAndCountStar() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type());
- }
- };
Alias leftCnt = new Count(scan1.getOutput().get(0)).alias("leftCnt");
Alias rightCnt = new Count(scan2.getOutput().get(0)).alias("rightCnt");
Alias countStar = new Count().alias("countStar");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java
index aac1bf2c7d4..1ce91d1c7c3 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java
@@ -42,15 +42,15 @@ class PushDownDistinctThroughJoinTest implements
MemoPatternMatchSupported {
private static final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
private static final LogicalOlapScan scan3 =
PlanConstructor.newLogicalOlapScan(2, "t3", 0);
private static final LogicalOlapScan scan4 =
PlanConstructor.newLogicalOlapScan(3, "t4", 0);
+ private MockUp<SessionVariable> mockUp = new MockUp<SessionVariable>() {
+ @Mock
+ public Set<Integer> getEnableNereidsRules() {
+ return
ImmutableSet.of(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN.type());
+ }
+ };
@Test
void testPushdownJoin() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN.type());
- }
- };
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
.join(scan3, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -73,12 +73,6 @@ class PushDownDistinctThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testPushdownProjectJoin() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN.type());
- }
- };
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
.project(ImmutableList.of(0, 2))
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java
index dafef59100a..cf28954a47c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java
@@ -45,15 +45,15 @@ class PushDownMinMaxThroughJoinTest implements
MemoPatternMatchSupported {
private static final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
private static final LogicalOlapScan scan3 =
PlanConstructor.newLogicalOlapScan(2, "t3", 0);
private static final LogicalOlapScan scan4 =
PlanConstructor.newLogicalOlapScan(3, "t4", 0);
+ private MockUp<SessionVariable> mockUp = new MockUp<SessionVariable>() {
+ @Mock
+ public Set<Integer> getEnableNereidsRules() {
+ return
ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type());
+ }
+ };
@Test
void testSingleJoin() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type());
- }
- };
Alias min = new Min(scan1.getOutput().get(0)).alias("min");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -74,12 +74,6 @@ class PushDownMinMaxThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testMultiJoin() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type());
- }
- };
Alias min = new Min(scan1.getOutput().get(0)).alias("min");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -113,12 +107,6 @@ class PushDownMinMaxThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testAggNotOutputGroupBy() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type());
- }
- };
// agg don't output group by
Alias min = new Min(scan1.getOutput().get(0)).alias("min");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
@@ -146,12 +134,6 @@ class PushDownMinMaxThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testBothSideSingleJoin() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type());
- }
- };
Alias min = new Min(scan1.getOutput().get(1)).alias("min");
Alias max = new Max(scan2.getOutput().get(1)).alias("max");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
@@ -174,12 +156,6 @@ class PushDownMinMaxThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testBothSide() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type());
- }
- };
Alias min = new Min(scan1.getOutput().get(1)).alias("min");
Alias max = new Max(scan3.getOutput().get(1)).alias("max");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java
index 5828d2319f2..088372b0d76 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java
@@ -42,15 +42,15 @@ import java.util.Set;
class PushDownSumThroughJoinTest implements MemoPatternMatchSupported {
private static final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
private static final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
+ private MockUp<SessionVariable> mockUp = new MockUp<SessionVariable>() {
+ @Mock
+ public Set<Integer> getEnableNereidsRules() {
+ return ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type());
+ }
+ };
@Test
void testSingleJoinLeftSum() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type());
- }
- };
Alias sum = new Sum(scan1.getOutput().get(1)).alias("sum");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -71,12 +71,6 @@ class PushDownSumThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testSingleJoinRightSum() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type());
- }
- };
Alias sum = new Sum(scan2.getOutput().get(1)).alias("sum");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
@@ -97,12 +91,6 @@ class PushDownSumThroughJoinTest implements
MemoPatternMatchSupported {
@Test
void testAggNotOutputGroupBy() {
- new MockUp<SessionVariable>() {
- @Mock
- public Set<Integer> getEnableNereidsRules() {
- return
ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type());
- }
- };
// agg don't output group by
Alias sum = new Sum(scan1.getOutput().get(1)).alias("sum");
LogicalPlan plan = new LogicalPlanBuilder(scan1)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 911155df7a2..946e159529d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -69,6 +69,7 @@ import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.QueryState;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.ShowExecutor;
import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.qe.StmtExecutor;
@@ -84,9 +85,12 @@ import
org.apache.doris.utframe.MockedFrontend.NotInitException;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import mockit.Mock;
+import mockit.MockUp;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
@@ -129,6 +133,13 @@ public abstract class TestWithFeService {
protected static final String DEFAULT_CLUSTER_PREFIX = "default_cluster:";
+ private static final MockUp<SessionVariable> mockUp = new
MockUp<SessionVariable>() {
+ @Mock
+ public Set<Integer> getEnableNereidsRules() {
+ return ImmutableSet.of();
+ }
+ };
+
@BeforeAll
public final void beforeAll() throws Exception {
FeConstants.enableInternalSchemaDb = false;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
