This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 3738d0fddc7 branch-2.1: [fix](nereids) fix push down non-foldable
filter through project #47989 (#48084)
3738d0fddc7 is described below
commit 3738d0fddc7d1a85a918948c696a9a040bfd438a
Author: yujun <[email protected]>
AuthorDate: Fri Feb 21 10:09:03 2025 +0800
branch-2.1: [fix](nereids) fix push down non-foldable filter through
project #47989 (#48084)
cherry pick from #47989
---
.../post/PushDownFilterThroughProject.java | 13 +++++-
.../doris/nereids/processor/post/Validator.java | 7 ---
.../rewrite/PushDownFilterThroughProject.java | 46 ++++++++++++--------
.../expressions/functions/ExpressionTrait.java | 9 +++-
.../PushDownFilterThroughProjectTest.java | 41 ++++++++++++++++++
.../rewrite/PushDowFilterThroughProjectTest.java | 48 ++++++++++++++++++++-
.../filter_push_down/push_filter_through.out | Bin 10141 -> 10077 bytes
7 files changed, 136 insertions(+), 28 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushDownFilterThroughProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushDownFilterThroughProject.java
index 864e817dc1f..4807d66b320 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushDownFilterThroughProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushDownFilterThroughProject.java
@@ -18,12 +18,17 @@
package org.apache.doris.nereids.processor.post;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.util.ExpressionUtils;
+import java.util.Map;
+import java.util.Objects;
+
/**
* merge consecutive projects
*/
@@ -37,9 +42,13 @@ public class PushDownFilterThroughProject extends
PlanPostProcessor {
}
PhysicalProject<? extends Plan> project = (PhysicalProject<? extends
Plan>) child;
+ Map<Slot, Expression> childAlias = project.getAliasToProducer();
+ if
(filter.getInputSlots().stream().map(childAlias::get).filter(Objects::nonNull)
+ .anyMatch(Expression::containsNonfoldable)) {
+ return filter;
+ }
PhysicalFilter<? extends Plan> newFilter =
filter.withConjunctsAndChild(
- ExpressionUtils.replace(filter.getConjuncts(),
project.getAliasToProducer()),
- project.child());
+ ExpressionUtils.replace(filter.getConjuncts(), childAlias),
project.child());
return ((AbstractPhysicalPlan)
project.withChildren(newFilter.accept(this, context)))
.copyStatsAndGroupIdFrom(project);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java
index 8ff3a43bf79..00102b78dfe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java
@@ -59,13 +59,6 @@ public class Validator extends PlanPostProcessor {
Preconditions.checkArgument(!filter.getConjuncts().isEmpty()
&& filter.getPredicate() != BooleanLiteral.TRUE, "Filter
predicate can't be empty or true");
- Plan child = filter.child();
- // Forbidden filter-project, we must make filter-project ->
project-filter.
- if (child instanceof PhysicalProject) {
- throw new AnalysisException(
- "Nereids generate a filter-project plan, but backend not
support:\n" + filter.treeString());
- }
-
return visit(filter, context);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java
index 5842beaf3d6..bb1cf3245cc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java
@@ -33,6 +33,8 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
import java.util.Set;
/**
@@ -67,6 +69,7 @@ public class PushDownFilterThroughProject implements
RewriteRuleFactory {
private static Plan
pushDownFilterThroughProject(LogicalFilter<LogicalProject<Plan>> filter) {
LogicalProject<? extends Plan> project = filter.child();
Set<Slot> childOutputs = project.getOutputSet();
+ Map<Slot, Expression> childAlias = project.getAliasToProducer();
// we need run this rule before subquery unnesting
// therefore the conjuncts may contain slots from outer query
// we should only push down conjuncts without any outer query's slot,
@@ -74,16 +77,15 @@ public class PushDownFilterThroughProject implements
RewriteRuleFactory {
// splitConjuncts.first -> conjuncts having outer query slots which
should NOT be pushed down
// splitConjuncts.second -> conjuncts without any outer query slots
which should be pushed down
Pair<Set<Expression>, Set<Expression>> splitConjuncts =
- splitConjunctsByChildOutput(filter.getConjuncts(),
childOutputs);
- if (splitConjuncts.second.isEmpty()) {
- // all conjuncts contain outer query's slots, no conjunct can be
pushed down
- // just return unchanged plan
+ splitConjunctsByChildOutput(filter.getConjuncts(),
childOutputs, childAlias);
+ Set<Expression> remainPredicates = splitConjuncts.first;
+ Set<Expression> pushDownPredicates = splitConjuncts.second;
+ if (pushDownPredicates.isEmpty()) {
return null;
}
- project = (LogicalProject<? extends Plan>) project.withChildren(new
LogicalFilter<>(
- ExpressionUtils.replace(splitConjuncts.second,
project.getAliasToProducer()),
- project.child()));
- return PlanUtils.filterOrSelf(splitConjuncts.first, project);
+ project = (LogicalProject<? extends Plan>) project.withChildren(
+ new LogicalFilter<>(pushDownPredicates, project.child()));
+ return PlanUtils.filterOrSelf(remainPredicates, project);
}
private static Plan pushDownFilterThroughLimitProject(
@@ -91,32 +93,42 @@ public class PushDownFilterThroughProject implements
RewriteRuleFactory {
LogicalLimit<LogicalProject<Plan>> limit = filter.child();
LogicalProject<Plan> project = limit.child();
Set<Slot> childOutputs = project.getOutputSet();
+ Map<Slot, Expression> childAlias = project.getAliasToProducer();
// split the conjuncts by child's output
Pair<Set<Expression>, Set<Expression>> splitConjuncts =
- splitConjunctsByChildOutput(filter.getConjuncts(),
childOutputs);
- if (splitConjuncts.second.isEmpty()) {
+ splitConjunctsByChildOutput(filter.getConjuncts(),
childOutputs, childAlias);
+ Set<Expression> remainPredicates = splitConjuncts.first;
+ Set<Expression> pushDownPredicates = splitConjuncts.second;
+ if (pushDownPredicates.isEmpty()) {
return null;
}
project = project.withProjectsAndChild(project.getProjects(),
- new LogicalFilter<>(
- ExpressionUtils.replace(splitConjuncts.second,
- project.getAliasToProducer()),
- limit.withChildren(project.child())));
- return PlanUtils.filterOrSelf(splitConjuncts.first, project);
+ new LogicalFilter<>(pushDownPredicates,
limit.withChildren(project.child())));
+ return PlanUtils.filterOrSelf(remainPredicates, project);
}
private static Pair<Set<Expression>, Set<Expression>>
splitConjunctsByChildOutput(
- Set<Expression> conjuncts, Set<Slot> childOutputs) {
+ Set<Expression> conjuncts, Set<Slot> childOutputs, Map<Slot,
Expression> childAlias) {
Set<Expression> pushDownPredicates = Sets.newLinkedHashSet();
Set<Expression> remainPredicates = Sets.newLinkedHashSet();
for (Expression conjunct : conjuncts) {
Set<Slot> conjunctSlots = conjunct.getInputSlots();
- if (childOutputs.containsAll(conjunctSlots)) {
+ // If filter contains non-foldable expression, it can push down,
for example:
+ // `filter(a + random(1, 10) > 1) -> project(a)` => `project(a) ->
filter(a + random(1, 10) > 1)`.
+ // If filter slot is alias and its expression contains
non-foldable expression, it can't push down, example:
+ // `filter(a > 1) -> project(b + random(1, 10) as a)`, if push
down filter, it got
+ // `project(b + random(1, 10) as a) -> filter(b + random(1, 10) >
1)`, it contains two distinct RANDOM.
+ if (childOutputs.containsAll(conjunctSlots)
+ &&
conjunctSlots.stream().map(childAlias::get).filter(Objects::nonNull)
+ .noneMatch(Expression::containsNonfoldable)) {
pushDownPredicates.add(conjunct);
} else {
remainPredicates.add(conjunct);
}
}
+ if (!pushDownPredicates.isEmpty()) {
+ pushDownPredicates = ExpressionUtils.replace(pushDownPredicates,
childAlias);
+ }
return Pair.of(remainPredicates, pushDownPredicates);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
index daaab359e84..6e33f2a5e11 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
@@ -77,6 +77,13 @@ public interface ExpressionTrait extends
TreeNode<Expression> {
return true;
}
+ /**
+ * Identify the expression is containing non-foldable expr or not
+ */
+ default boolean containsNonfoldable() {
+ return anyMatch(expr -> !((ExpressionTrait) expr).foldable());
+ }
+
/**
* Identify the expression itself is deterministic or not, default true
*/
@@ -85,7 +92,7 @@ public interface ExpressionTrait extends TreeNode<Expression>
{
}
/**
- * Identify the expression is containing deterministic expr or not
+ * Identify the expression is containing non-deterministic expr or not
*/
default boolean containsNondeterministic() {
return anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java
index 08ccc56f9ba..fd434504b2e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java
@@ -23,12 +23,15 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.processor.post.PushDownFilterThroughProject;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Random;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
import org.apache.doris.nereids.trees.plans.RelationId;
@@ -111,4 +114,42 @@ public class PushDownFilterThroughProjectTest {
((PhysicalFilter<?>)
newPlan.child(0).child(0)).getExpressions();
Assertions.assertEquals(newFilterConjuncts.get(0).child(0), a);
}
+
+ @Test
+ public void testNotPushFilterWithNonfoldable(@Injectable LogicalProperties
placeHolder,
+ @Injectable CascadesContext ctx) {
+ OlapTable t1 = PlanConstructor.newOlapTable(0, "t1", 0,
KeysType.DUP_KEYS);
+ List<String> qualifier = new ArrayList<>();
+ qualifier.add("test");
+ List<Slot> t1Output = new ArrayList<>();
+ SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+ SlotReference b = new SlotReference("b", IntegerType.INSTANCE);
+ SlotReference c = new SlotReference("c", IntegerType.INSTANCE);
+ t1Output.add(a);
+ t1Output.add(b);
+ t1Output.add(c);
+ LogicalProperties t1Properties = new LogicalProperties(() -> t1Output,
() -> FunctionalDependencies.EMPTY_FUNC_DEPS);
+ PhysicalOlapScan scan = new
PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1,
+ qualifier, 0L, Collections.emptyList(),
Collections.emptyList(), null,
+ PreAggStatus.on(), ImmutableList.of(), Optional.empty(),
t1Properties,
+ Optional.empty());
+ Alias x = new Alias(a, "x");
+ List<NamedExpression> projList3 = Lists.newArrayList(x, b, c);
+ PhysicalProject proj3 = new PhysicalProject(projList3, placeHolder,
scan);
+ Alias y = new Alias(
+ new Add(x.toSlot(), new Random(new BigIntLiteral(1L), new
BigIntLiteral(10L))),
+ "y");
+ Alias z = new Alias(b, "z");
+ List<NamedExpression> projList2 = Lists.newArrayList(y, z, c);
+ PhysicalProject proj2 = new PhysicalProject(projList2, placeHolder,
proj3);
+ Set<Expression> conjuncts = Sets.newHashSet();
+ conjuncts.add(new EqualTo(y.toSlot(), Literal.of(0)));
+ PhysicalFilter filter = new PhysicalFilter(conjuncts,
proj2.getLogicalProperties(), proj2);
+
+ PushDownFilterThroughProject processor = new
PushDownFilterThroughProject();
+ PhysicalPlan newPlan = (PhysicalPlan) filter.accept(processor, ctx);
+ Assertions.assertTrue(newPlan instanceof PhysicalFilter);
+ Assertions.assertTrue(newPlan.child(0) instanceof PhysicalProject);
+ Assertions.assertTrue(newPlan.child(0).child(0) instanceof
PhysicalProject);
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDowFilterThroughProjectTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDowFilterThroughProjectTest.java
index a737acbf016..080c0a33522 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDowFilterThroughProjectTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDowFilterThroughProjectTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.properties.OrderKey;
+import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -25,9 +26,12 @@ import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.OrderExpression;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Random;
import org.apache.doris.nereids.trees.expressions.functions.window.Rank;
+import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.LogicalPlanBuilder;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
@@ -36,6 +40,7 @@ import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import org.junit.jupiter.api.Test;
class PushDowFilterThroughProjectTest implements MemoPatternMatchSupported {
@@ -70,7 +75,7 @@ class PushDowFilterThroughProjectTest implements
MemoPatternMatchSupported {
Alias alias = new Alias(new ExprId(1), window, "window");
ConnectContext context = MemoTestUtils.createConnectContext();
- // filter -> limit -> project(windows)
+ // filter -> project(windows)
LogicalPlan plan = new LogicalPlanBuilder(scan)
.projectExprs(ImmutableList.of(alias))
.filter(new IsNull(alias.toSlot()))
@@ -92,6 +97,47 @@ class PushDowFilterThroughProjectTest implements
MemoPatternMatchSupported {
.matches(logicalFilter(logicalLimit(logicalProject())));
}
+ @Test
+ void notPushDownFilterThroughNonfoldable() {
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ Alias foldableAlias = new Alias(new ExprId(1),
scan.getOutput().get(0), "a");
+ Alias nonfoldableAlias = new Alias(new ExprId(2),
+ new Random(new BigIntLiteral(1L), new BigIntLiteral(10L)),
"b");
+ Expression nonfoldableAdd = new Add(foldableAlias.toSlot(),
+ new Random(new BigIntLiteral(1L), new BigIntLiteral(2L)));
+ Expression condition = ExpressionUtils.and(Lists.newArrayList(new
IsNull(foldableAlias.toSlot()),
+ new IsNull(nonfoldableAlias.toSlot()), new
IsNull(nonfoldableAdd)));
+ LogicalPlan plan = new LogicalPlanBuilder(scan)
+ .projectExprs(ImmutableList.of(foldableAlias,
nonfoldableAlias))
+ .filter(condition)
+ .build();
+
+ PlanChecker.from(context, plan)
+ .applyTopDown(new PushDownFilterThroughProject())
+ .matches(
+ logicalFilter(
+ logicalProject(
+ logicalFilter().when(f ->
f.getPredicate().toSql().equals(
+ "(id IS NULL AND (id +
random(1, 2)) IS NULL)"))))
+ .when(f -> f.getPredicate().toSql().equals("b
IS NULL")));
+
+ plan = new LogicalPlanBuilder(scan)
+ .projectExprs(ImmutableList.of(foldableAlias,
nonfoldableAlias))
+ .limit(1)
+ .filter(condition)
+ .build();
+
+ PlanChecker.from(context, plan)
+ .applyTopDown(new PushDownFilterThroughProject())
+ .matches(
+ logicalFilter(
+ logicalProject(
+ logicalFilter(logicalLimit())
+ .when(f ->
f.getPredicate().toSql().equals(
+ "(id IS NULL AND (id +
random(1, 2)) IS NULL)"))))
+ .when(f -> f.getPredicate().toSql().equals("b
IS NULL")));
+ }
+
@Test
void pushDownFilterThroughLimit() {
ConnectContext context = MemoTestUtils.createConnectContext();
diff --git
a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
index 27ad895837c..50e35770b63 100644
Binary files
a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
and
b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
