This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8c2a721873 [opt](nereids)push down filter through window #23935
8c2a721873 is described below
commit 8c2a7218737d718aa7354eafd219bcf97cf5d82a
Author: minghong <[email protected]>
AuthorDate: Sat Sep 9 20:53:31 2023 +0800
[opt](nereids)push down filter through window #23935
select rank() over (partition by A, B) as r, sum(x) over(A, C) as s from T;
A is a common partition key for all windowExpressions, that is A is
intersection of {A,B} and {A, C}
we could push filter A=1 through this window, since A is a common Partition
key:
select * from (select a, row_number() over (partition by a) from win) T
where a=1;
origin plan:
----filter((T.a = 1))
----------PhysicalWindow
------------PhysicalQuickSort
--------------PhysicalProject
------------------PhysicalOlapScan[win]
transformed to
----PhysicalWindow
------PhysicalQuickSort
--------PhysicalProject
----------filter((T.a = 1))
------------PhysicalOlapScan[win]
But C=1 can not be pushed through window.
---
.../doris/nereids/jobs/executor/Rewriter.java | 4 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 7 +-
...dow.java => CreatePartitionTopNFromWindow.java} | 5 +-
.../rules/rewrite/PushdownFilterThroughWindow.java | 138 +++++----------------
.../nereids/trees/plans/logical/LogicalWindow.java | 33 +++++
...va => GeneratePartitionTopnFromWindowTest.java} | 6 +-
...st.java => PushdowFilterThroughWindowTest.java} | 78 ++++--------
.../push_filter_through_window.out | 46 +++++++
.../push_filter_through_window.groovy | 55 ++++++++
9 files changed, 205 insertions(+), 167 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 986947c262..de8e92eff3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -49,6 +49,7 @@ import org.apache.doris.nereids.rules.rewrite.ColumnPruning;
import org.apache.doris.nereids.rules.rewrite.ConvertInnerOrCrossJoin;
import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite;
import org.apache.doris.nereids.rules.rewrite.CountLiteralToCountStar;
+import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow;
import org.apache.doris.nereids.rules.rewrite.DeferMaterializeTopNResult;
import org.apache.doris.nereids.rules.rewrite.EliminateAggregate;
import org.apache.doris.nereids.rules.rewrite.EliminateDedupJoinCondition;
@@ -87,7 +88,6 @@ import
org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin;
import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation;
import org.apache.doris.nereids.rules.rewrite.PushProjectThroughUnion;
import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughProject;
-import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushdownLimit;
import org.apache.doris.nereids.rules.rewrite.PushdownTopNThroughWindow;
import org.apache.doris.nereids.rules.rewrite.ReorderJoin;
@@ -277,7 +277,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
new SplitLimit(),
new PushdownLimit(),
new PushdownTopNThroughWindow(),
- new PushdownFilterThroughWindow()
+ new CreatePartitionTopNFromWindow()
)
),
// TODO: these rules should be implementation rules, and generate
alternative physical plans.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index bba58ffa5b..6d2df7046c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -74,6 +74,7 @@ import
org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN;
import
org.apache.doris.nereids.rules.implementation.LogicalUnionToPhysicalUnion;
import
org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow;
import org.apache.doris.nereids.rules.rewrite.ConvertOuterJoinToAntiJoin;
+import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow;
import org.apache.doris.nereids.rules.rewrite.EliminateOuterJoin;
import org.apache.doris.nereids.rules.rewrite.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.MergeGenerates;
@@ -121,6 +122,7 @@ public class RuleSet {
.build();
public static final List<RuleFactory> PUSH_DOWN_FILTERS = ImmutableList.of(
+ new CreatePartitionTopNFromWindow(),
new PushdownFilterThroughProject(),
new PushdownFilterThroughSort(),
new PushdownJoinOtherCondition(),
@@ -129,7 +131,6 @@ public class RuleSet {
new PushdownFilterThroughAggregation(),
new PushdownFilterThroughRepeat(),
new PushdownFilterThroughSetOperation(),
- new PushdownFilterThroughWindow(),
new PushdownProjectThroughLimit(),
new EliminateOuterJoin(),
new ConvertOuterJoinToAntiJoin(),
@@ -137,7 +138,9 @@ public class RuleSet {
new MergeFilters(),
new MergeGenerates(),
new MergeLimits(),
- new PushdownAliasThroughJoin());
+ new PushdownAliasThroughJoin(),
+ new PushdownFilterThroughWindow()
+ );
public static final List<Rule> IMPLEMENTATION_RULES = planRuleFactories()
.add(new LogicalCTEProducerToPhysicalCTEProducer())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CreatePartitionTopNFromWindow.java
similarity index 98%
copy from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java
copy to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CreatePartitionTopNFromWindow.java
index 0addaeac0a..1a4ae3ef1b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CreatePartitionTopNFromWindow.java
@@ -44,7 +44,7 @@ import java.util.Set;
import java.util.function.Predicate;
/**
- * Push down the 'filter' into the 'window'.
+ * Push down the 'partitionTopN' into the 'window'.
* It will convert the filter condition to the 'limit value' and push down
below the 'window'.
* But there are some restrictions, the details are explained below.
* For example:
@@ -74,8 +74,7 @@ import java.util.function.Predicate;
* any_node
*/
-public class PushdownFilterThroughWindow extends OneRewriteRuleFactory {
-
+public class CreatePartitionTopNFromWindow extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalWindow()).thenApply(ctx -> {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java
index 0addaeac0a..1902ff5030 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java
@@ -19,45 +19,22 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.expressions.BinaryOperator;
-import org.apache.doris.nereids.trees.expressions.EqualTo;
-import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.LessThan;
-import org.apache.doris.nereids.trees.expressions.LessThanEqual;
-import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
-import org.apache.doris.nereids.trees.expressions.WindowExpression;
-import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
-import java.util.List;
-import java.util.Optional;
import java.util.Set;
-import java.util.function.Predicate;
/**
- * Push down the 'filter' into the 'window'.
- * It will convert the filter condition to the 'limit value' and push down
below the 'window'.
- * But there are some restrictions, the details are explained below.
- * For example:
- * 'SELECT * FROM (
- * SELECT *, ROW_NUMBER() OVER (ORDER BY b) AS row_number
- * FROM t
- * ) AS tt WHERE row_number <= 100;'
- * The filter 'row_number <= 100' can be pushed down into the window operator.
- * The following will demonstrate how the plan changes:
+ * Push down the 'filter' into the 'window' if filter key is window partition
key.
* Logical plan tree:
* any_node
* |
- * filter (row_number <= 100)
+ * filter (a <= 100)
* |
* window (PARTITION BY a ORDER BY b)
* |
@@ -65,11 +42,9 @@ import java.util.function.Predicate;
* transformed to:
* any_node
* |
- * filter (row_number <= 100)
- * |
* window (PARTITION BY a ORDER BY b)
* |
- * partition_topn(PARTITION BY: a, ORDER BY b, Partition Limit:
100)
+ * filter (a <= 100)
* |
* any_node
*/
@@ -81,88 +56,43 @@ public class PushdownFilterThroughWindow extends
OneRewriteRuleFactory {
return logicalFilter(logicalWindow()).thenApply(ctx -> {
LogicalFilter<LogicalWindow<Plan>> filter = ctx.root;
LogicalWindow<Plan> window = filter.child();
-
- // We have already done such optimization rule, so just ignore it.
- if (window.child(0) instanceof LogicalPartitionTopN) {
- return filter;
- }
-
- List<NamedExpression> windowExprs = window.getWindowExpressions();
- if (windowExprs.size() != 1) {
- return filter;
- }
- NamedExpression windowExpr = windowExprs.get(0);
- if (windowExpr.children().size() != 1 || !(windowExpr.child(0)
instanceof WindowExpression)) {
- return filter;
- }
-
- // Check the filter conditions. Now, we currently only support
simple conditions of the form
- // 'column </ <=/ = constant'. We will extract some related
conjuncts and do some check.
- Set<Expression> conjuncts = filter.getConjuncts();
- Set<Expression> relatedConjuncts =
extractRelatedConjuncts(conjuncts, windowExpr.getExprId());
-
- boolean hasPartitionLimit = false;
- long partitionLimit = Long.MAX_VALUE;
-
- for (Expression conjunct : relatedConjuncts) {
- Preconditions.checkArgument(conjunct instanceof
BinaryOperator);
- BinaryOperator op = (BinaryOperator) conjunct;
- Expression leftChild = op.children().get(0);
- Expression rightChild = op.children().get(1);
-
- Preconditions.checkArgument(leftChild instanceof SlotReference
- && rightChild instanceof IntegerLikeLiteral);
-
- long limitVal = ((IntegerLikeLiteral)
rightChild).getLongValue();
- // Adjust the value for 'limitVal' based on the comparison
operators.
- if (conjunct instanceof LessThan) {
- limitVal--;
- }
- if (limitVal < 0) {
- return new
LogicalEmptyRelation(ctx.statementContext.getNextRelationId(),
filter.getOutput());
+ // now we only handle single slot used as partition key
+ // for example:
+ // select * from (select T.*, rank() over(partition by c2+c3 order
by c4) rn from T) abc where c2=1;
+ // c2=1 cannot be pushed down.
+ Set<SlotReference> commonPartitionKeys =
window.getCommonPartitionKeyFromWindowExpressions();
+ Set<Expression> bottomConjuncts = Sets.newHashSet();
+ Set<Expression> upperConjuncts = Sets.newHashSet();
+ for (Expression expr : filter.getConjuncts()) {
+ boolean pushed = false;
+ for (Expression partitionKey : commonPartitionKeys) {
+ // partitionKey is a single slot reference,
+ // we want to push expressions which have only one input
slot, and the input slot is used as
+ // partition key in all windowExpressions.
+ if
(partitionKey.getInputSlots().containsAll(expr.getInputSlots())) {
+ bottomConjuncts.add(expr);
+ pushed = true;
+ break;
+ }
}
- if (hasPartitionLimit) {
- partitionLimit = Math.min(partitionLimit, limitVal);
- } else {
- partitionLimit = limitVal;
- hasPartitionLimit = true;
+ if (!pushed) {
+ upperConjuncts.add(expr);
}
}
-
- if (!hasPartitionLimit) {
- return filter;
+ if (bottomConjuncts.isEmpty()) {
+ return null;
}
- Optional<Plan> newWindow =
window.pushPartitionLimitThroughWindow(partitionLimit, false);
- if (!newWindow.isPresent()) {
- return filter;
+ LogicalFilter<Plan> bottomFilter = new
LogicalFilter<>(bottomConjuncts, window.child());
+ window = (LogicalWindow<Plan>) window.withChildren(bottomFilter);
+ if (upperConjuncts.isEmpty()) {
+ return window;
+ } else {
+ LogicalFilter<Plan> upperFilter = (LogicalFilter<Plan>) filter
+ .withConjuncts(upperConjuncts).withChildren(window);
+ return upperFilter;
}
- return filter.withChildren(newWindow.get());
}).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_WINDOW);
}
- private Set<Expression> extractRelatedConjuncts(Set<Expression> conjuncts,
ExprId slotRefID) {
- Predicate<Expression> condition = conjunct -> {
- if (!(conjunct instanceof BinaryOperator)) {
- return false;
- }
- BinaryOperator op = (BinaryOperator) conjunct;
- Expression leftChild = op.children().get(0);
- Expression rightChild = op.children().get(1);
-
- if (!(conjunct instanceof LessThan || conjunct instanceof
LessThanEqual || conjunct instanceof EqualTo)) {
- return false;
- }
-
- // TODO: Now, we only support the column on the left side.
- if (!(leftChild instanceof SlotReference) || !(rightChild
instanceof IntegerLikeLiteral)) {
- return false;
- }
- return ((SlotReference) leftChild).getExprId() == slotRefID;
- };
-
- return conjuncts.stream()
- .filter(condition)
- .collect(ImmutableSet.toImmutableSet());
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
index fcfe9906d6..67e1819779 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
@@ -19,9 +19,11 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
import org.apache.doris.nereids.trees.expressions.WindowFrame;
import org.apache.doris.nereids.trees.expressions.functions.window.DenseRank;
@@ -36,10 +38,14 @@ import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
/**
* logical node to deal with window functions;
@@ -224,4 +230,31 @@ public class LogicalWindow<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
return Optional.ofNullable(window);
}
+
+ /**
+ *
+ * select rank() over (partition by A, B) as r, sum(x) over(A, C) as s
from T;
+ * A is a common partition key for all windowExpressions.
+ * for a common Partition key A, we could push filter A=1 through this
window.
+ */
+ public Set<SlotReference> getCommonPartitionKeyFromWindowExpressions() {
+ ImmutableSet.Builder<SlotReference> commonPartitionKeySet =
ImmutableSet.builder();
+ Map<Expression, Integer> partitionKeyCount = Maps.newHashMap();
+ for (Expression expr : windowExpressions) {
+ if (expr instanceof Alias && expr.child(0) instanceof
WindowExpression) {
+ WindowExpression winExpr = (WindowExpression) expr.child(0);
+ for (Expression partitionKey : winExpr.getPartitionKeys()) {
+ int count = partitionKeyCount.getOrDefault(partitionKey,
0);
+ partitionKeyCount.put(partitionKey, count + 1);
+ }
+ }
+ }
+ int winExprCount = windowExpressions.size();
+ for (Map.Entry<Expression, Integer> entry :
partitionKeyCount.entrySet()) {
+ if (entry.getValue() == winExprCount && entry.getKey() instanceof
SlotReference) {
+ commonPartitionKeySet.add((SlotReference) entry.getKey());
+ }
+ }
+ return commonPartitionKeySet.build();
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/GeneratePartitionTopnFromWindowTest.java
similarity index 96%
copy from
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java
copy to
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/GeneratePartitionTopnFromWindowTest.java
index a53e8c6a05..e5f7efd38d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/GeneratePartitionTopnFromWindowTest.java
@@ -46,7 +46,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
-public class PushdownFilterThroughWindowTest implements
MemoPatternMatchSupported {
+public class GeneratePartitionTopnFromWindowTest implements
MemoPatternMatchSupported {
private final LogicalOlapScan scan = new
LogicalOlapScan(StatementScopeIdGenerator.newRelationId(),
PlanConstructor.student,
ImmutableList.of(""));
@@ -72,7 +72,7 @@ public class PushdownFilterThroughWindowTest implements
MemoPatternMatchSupporte
* scan(student)
*/
@Test
- public void pushDownFilterThroughWindowTest() {
+ public void testGeneratePartitionTopnFromWindow() {
ConnectContext context = MemoTestUtils.createConnectContext();
context.getSessionVariable().setEnablePartitionTopN(true);
NamedExpression gender = scan.getOutput().get(1).toSlot();
@@ -96,7 +96,7 @@ public class PushdownFilterThroughWindowTest implements
MemoPatternMatchSupporte
.build();
PlanChecker.from(context, plan)
- .applyTopDown(new PushdownFilterThroughWindow())
+ .applyTopDown(new CreatePartitionTopNFromWindow())
.matches(
logicalProject(
logicalFilter(
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdowFilterThroughWindowTest.java
similarity index 53%
rename from
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java
rename to
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdowFilterThroughWindowTest.java
index a53e8c6a05..6c8239c9a9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdowFilterThroughWindowTest.java
@@ -17,18 +17,15 @@
package org.apache.doris.nereids.rules.rewrite;
-import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.OrderExpression;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
import org.apache.doris.nereids.trees.expressions.WindowFrame;
import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
-import org.apache.doris.nereids.trees.plans.WindowFuncType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
@@ -40,81 +37,56 @@ import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import org.junit.jupiter.api.Test;
import java.util.List;
-public class PushdownFilterThroughWindowTest implements
MemoPatternMatchSupported {
+public class PushdowFilterThroughWindowTest implements
MemoPatternMatchSupported {
private final LogicalOlapScan scan = new
LogicalOlapScan(StatementScopeIdGenerator.newRelationId(),
PlanConstructor.student,
ImmutableList.of(""));
- /*-
- * origin plan:
- * project
- * |
- * filter row_number <= 100
- * |
- * window(ROW_NUMBER() as row_number PARTITION BY gender
ORDER BY age)
- * |
- * scan(student)
- *
- * transformed plan:
- * project
- * |
- * filter row_number <= 100
- * |
- * window(ROW_NUMBER() as row_number PARTITION BY gender
ORDER BY age)
- * |
- * partitionTopN(row_number(), partition by gender, order by age,
hasGlobalLimit: false, partitionLimit: 100)
- * |
- * scan(student)
- */
@Test
public void pushDownFilterThroughWindowTest() {
ConnectContext context = MemoTestUtils.createConnectContext();
- context.getSessionVariable().setEnablePartitionTopN(true);
- NamedExpression gender = scan.getOutput().get(1).toSlot();
NamedExpression age = scan.getOutput().get(3).toSlot();
-
- List<Expression> partitionKeyList = ImmutableList.of(gender);
- List<OrderExpression> orderKeyList = ImmutableList.of(new
OrderExpression(
- new OrderKey(age, true, true)));
+ List<Expression> partitionKeyList = ImmutableList.of(age);
WindowFrame windowFrame = new
WindowFrame(WindowFrame.FrameUnitsType.ROWS,
WindowFrame.FrameBoundary.newPrecedingBoundary(),
WindowFrame.FrameBoundary.newCurrentRowBoundary());
- WindowExpression window1 = new WindowExpression(new RowNumber(),
partitionKeyList, orderKeyList, windowFrame);
+ WindowExpression window1 = new WindowExpression(new RowNumber(),
partitionKeyList,
+ Lists.newArrayList(), windowFrame);
Alias windowAlias1 = new Alias(window1, window1.toSql());
List<NamedExpression> expressions = Lists.newArrayList(windowAlias1);
LogicalWindow<LogicalOlapScan> window = new
LogicalWindow<>(expressions, scan);
- Expression filterPredicate = new
LessThanEqual(window.getOutput().get(4).toSlot(), Literal.of(100));
+ Expression filterPredicate = new EqualTo(age, Literal.of(100));
LogicalPlan plan = new LogicalPlanBuilder(window)
.filter(filterPredicate)
.project(ImmutableList.of(0))
.build();
-
PlanChecker.from(context, plan)
.applyTopDown(new PushdownFilterThroughWindow())
.matches(
- logicalProject(
- logicalFilter(
- logicalWindow(
- logicalPartitionTopN(
- logicalOlapScan()
- ).when(logicalPartitionTopN -> {
- WindowFuncType funName =
logicalPartitionTopN.getFunction();
- List<Expression> partitionKeys =
logicalPartitionTopN.getPartitionKeys();
- List<OrderExpression> orderKeys =
logicalPartitionTopN.getOrderKeys();
- boolean hasGlobalLimit =
logicalPartitionTopN.hasGlobalLimit();
- long partitionLimit =
logicalPartitionTopN.getPartitionLimit();
- return funName ==
WindowFuncType.ROW_NUMBER && partitionKeys.equals(partitionKeyList)
- && orderKeys.equals(orderKeyList) &&
!hasGlobalLimit && partitionLimit == 100;
- })
- )
- ).when(filter ->
filter.getConjuncts().equals(ImmutableSet.of(filterPredicate)))
- )
+ logicalProject(
+ logicalWindow(
+ logicalFilter(
+ logicalOlapScan()
+ ).when(filter -> {
+ if (filter.getConjuncts().size()
!= 1) {
+ return false;
+ }
+ Expression conj =
filter.getConjuncts().iterator().next();
+ if (!(conj instanceof EqualTo)) {
+ return false;
+ }
+ EqualTo eq = (EqualTo) conj;
+ return eq.left().equals(age);
+
+ })
+ )
+ )
);
}
}
+
diff --git
a/regression-test/data/nereids_syntax_p0/push_filter_through_window.out
b/regression-test/data/nereids_syntax_p0/push_filter_through_window.out
new file mode 100644
index 0000000000..8d0356b5f1
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/push_filter_through_window.out
@@ -0,0 +1,46 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !1 --
+1 1
+1 2
+
+-- !shape_1 --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalWindow
+------PhysicalQuickSort
+--------PhysicalProject
+----------filter((T.a = 1))
+------------PhysicalOlapScan[push_filter_through_window_tbl]
+
+-- !2 --
+1 1
+1 1
+
+-- !shape_2 --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalProject
+------PhysicalWindow
+--------PhysicalQuickSort
+----------filter((T.a = 1))
+------------PhysicalOlapScan[push_filter_through_window_tbl]
+
+-- !4 --
+PhysicalResultSink
+--PhysicalDistribute
+----filter((T.b = 2))
+------PhysicalWindow
+--------PhysicalQuickSort
+----------PhysicalOlapScan[push_filter_through_window_tbl]
+
+-- !5 --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalProject
+------filter((T.b = 2))
+--------PhysicalWindow
+----------PhysicalQuickSort
+------------PhysicalDistribute
+--------------PhysicalProject
+----------------PhysicalOlapScan[push_filter_through_window_tbl]
+
diff --git
a/regression-test/suites/nereids_syntax_p0/push_filter_through_window.groovy
b/regression-test/suites/nereids_syntax_p0/push_filter_through_window.groovy
new file mode 100644
index 0000000000..8c10c04539
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/push_filter_through_window.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("push_filter_through_window") {
+ sql """set enable_nereids_planner=true"""
+ sql """
+ DROP TABLE IF EXISTS push_filter_through_window_tbl
+ """
+ sql """
+ CREATE TABLE `push_filter_through_window_tbl` (
+ `a` int(11) NULL,
+ `b` int NULL
+ ) ENGINE=OLAP
+ duplicate KEY(`a`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`a`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "is_being_synced" = "false",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false"
+ );
+ """
+ sql """
+ insert into push_filter_through_window_tbl values (1, 2), (1, 3), (2,
4), (2, 5);
+ """
+
+ qt_1 """select * from (select a, row_number() over (partition by a) from
push_filter_through_window_tbl) T where a=1;"""
+ qt_shape_1 """explain shape plan select * from (select a, row_number()
over (partition by a) from push_filter_through_window_tbl) T where a=1;"""
+ qt_2 """select * from (select a, row_number() over (partition by b, a)
from push_filter_through_window_tbl) T where a=1;"""
+ qt_shape_2 """explain shape plan select * from (select a, row_number()
over (partition by b, a) from push_filter_through_window_tbl) T where a=1;"""
+
+ // TODO open qt_3 after fix bug: cannot choose best plan when there are
more than one windowExpression
+ // qt_3 """ explain shape plan select * from (select a, row_number() over
(partition by b, a) as r1, row_number() over (partition by b) as r2 from
push_filter_through_window_tbl) T where a=1;"""
+
+ qt_4 """explain shape plan select * from (select a, b, row_number() over
(partition by a) from push_filter_through_window_tbl) T where b=2;"""
+ qt_5 """explain shape plan select * from (select a, b, row_number() over
(partition by a+b) from push_filter_through_window_tbl) T where b=2;"""
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
