This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 969002bc532 [feature](Nereids): add is null in predicate and put or
expansion rule in rewriter (#28348) (#28690)
969002bc532 is described below
commit 969002bc532b18e38d5ba1044a9a714162d4dec7
Author: 谢健 <[email protected]>
AuthorDate: Wed Dec 20 12:20:59 2023 +0800
[feature](Nereids): add is null in predicate and put or expansion rule in
rewriter (#28348) (#28690)
---
.../org/apache/doris/nereids/jobs/executor/Rewriter.java | 3 +++
.../java/org/apache/doris/nereids/rules/RuleSet.java | 2 --
.../rules/{exploration => rewrite}/OrExpansion.java | 15 ++++++++++-----
.../org/apache/doris/nereids/stats/FilterEstimation.java | 2 +-
regression-test/data/nereids_p0/union/or_expansion.out | 15 ++++++++++++++-
.../suites/nereids_p0/union/or_expansion.groovy | 16 +++++++++++-----
6 files changed, 39 insertions(+), 14 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index fcbbdee6ee6..24495ccc620 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -78,6 +78,7 @@ import
org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion;
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.NormalizeSort;
+import org.apache.doris.nereids.rules.rewrite.OrExpansion;
import org.apache.doris.nereids.rules.rewrite.PruneEmptyPartition;
import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition;
import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition;
@@ -377,6 +378,8 @@ public class Rewriter extends AbstractBatchJobExecutor {
topic("rewrite cte sub-tree",
custom(RuleType.REWRITE_CTE_CHILDREN, () -> new
RewriteCteChildren(jobs))
),
+ topic("or expansion",
+ topDown(new OrExpansion())),
topic("whole plan check",
custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new)
)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index dfb00b08b84..b7a43d66984 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -18,7 +18,6 @@
package org.apache.doris.nereids.rules;
import org.apache.doris.nereids.rules.exploration.MergeProjectsCBO;
-import org.apache.doris.nereids.rules.exploration.OrExpansion;
import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoin;
import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoinProject;
import org.apache.doris.nereids.rules.exploration.join.InnerJoinLAsscom;
@@ -119,7 +118,6 @@ public class RuleSet {
.add(PushdownProjectThroughSemiJoin.INSTANCE)
.add(TransposeAggSemiJoin.INSTANCE)
.add(TransposeAggSemiJoinProject.INSTANCE)
- .add(OrExpansion.INSTANCE)
.build();
public static final List<RuleFactory> PUSH_DOWN_FILTERS = ImmutableList.of(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
similarity index 95%
rename from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java
rename to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
index 9019565142b..fd7e7e844ad 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
@@ -15,15 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.nereids.rules.exploration;
+package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import
org.apache.doris.nereids.rules.rewrite.PushdownExpressionsInHashCondition;
+import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -72,7 +73,7 @@ public class OrExpansion extends OneExplorationRuleFactory {
@Override
public Rule build() {
- return logicalJoin().when(JoinUtils::shouldNestedLoopJoin)
+ return logicalJoin(any(), any()).when(JoinUtils::shouldNestedLoopJoin)
.when(join -> supportJoinType.contains(join.getJoinType())
&&
ConnectContext.get().getSessionVariable().getEnablePipelineEngine())
.thenApply(ctx -> {
@@ -82,7 +83,7 @@ public class OrExpansion extends OneExplorationRuleFactory {
//1. Try to split or conditions
Pair<List<Expression>, List<Expression>>
hashOtherConditions = splitOrCondition(join);
- if (hashOtherConditions == null) {
+ if (hashOtherConditions == null ||
hashOtherConditions.first.size() <= 1) {
return join;
}
@@ -221,7 +222,11 @@ public class OrExpansion extends OneExplorationRuleFactory
{
LogicalCTEProducer<? extends Plan> rightProducer) {
List<Expression> disjunctions = hashOtherConditions.first;
List<Expression> otherConditions = hashOtherConditions.second;
- List<Expression> notExprs =
disjunctions.stream().map(Not::new).collect(Collectors.toList());
+ // For null values, equalTo and not equalTo both return false
+ // To avoid it, we always return true when there is null
+ List<Expression> notExprs = disjunctions.stream()
+ .map(e -> ExpressionUtils.or(new Not(e), new IsNull(e)))
+ .collect(ImmutableList.toImmutableList());
List<Plan> joins = Lists.newArrayList();
for (int hashCondIdx = 0; hashCondIdx < disjunctions.size();
hashCondIdx++) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index 0a7d6b70c39..9c04b59ce6d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -367,7 +367,7 @@ public class FilterEstimation extends
ExpressionVisitor<Statistics, EstimationCo
// 4. not A like XXX
colBuilder.setNumNulls(0);
Preconditions.checkArgument(
- child instanceof EqualTo
+ child instanceof EqualPredicate
|| child instanceof InPredicate
|| child instanceof IsNull
|| child instanceof Like,
diff --git a/regression-test/data/nereids_p0/union/or_expansion.out
b/regression-test/data/nereids_p0/union/or_expansion.out
index dd82dbbcab7..4779bf2ace1 100644
--- a/regression-test/data/nereids_p0/union/or_expansion.out
+++ b/regression-test/data/nereids_p0/union/or_expansion.out
@@ -1,9 +1,13 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !order_ij --
+\N \N
+1 \N
+1 1
+1 1
-- !order_laj --
+\N
0
-1
2
3
4
@@ -14,6 +18,8 @@
9
-- !order_loj --
+\N \N
+\N \N
0 \N
1 \N
2 \N
@@ -24,8 +30,13 @@
7 \N
8 \N
9 \N
+1 1
+1 1
-- !order_foj --
+\N \N
+\N \N
+\N \N
0 \N
1 \N
2 \N
@@ -36,6 +47,8 @@
7 \N
8 \N
9 \N
+1 1
+1 1
\N 20
\N 21
\N 22
diff --git a/regression-test/suites/nereids_p0/union/or_expansion.groovy
b/regression-test/suites/nereids_p0/union/or_expansion.groovy
index 9139ea65c26..2d68098cd2d 100644
--- a/regression-test/suites/nereids_p0/union/or_expansion.groovy
+++ b/regression-test/suites/nereids_p0/union/or_expansion.groovy
@@ -45,11 +45,6 @@ suite("or_expansion") {
)
"""
- for (int i = 0; i < 10; i++) {
- sql "insert into oe1 values(${i}, ${i})"
- sql "insert into oe2 values(${i+20}, ${i+20})"
- }
-
sql """
alter table oe1 modify column k0 set stats ('row_count'='1000',
'ndv'='1000', 'min_value'='1', 'max_value'='1000', 'avg_size'='1000',
'max_size'='1000' )
"""
@@ -99,6 +94,17 @@ suite("or_expansion") {
contains "VHASH JOIN"
}
+ for (int i = 0; i < 10; i++) {
+ sql "insert into oe1 values(${i}, ${i})"
+ sql "insert into oe2 values(${i+20}, ${i+20})"
+ }
+ sql "insert into oe1 values(null, 1)"
+ sql "insert into oe1 values(1, null)"
+ sql "insert into oe1 values(null, null)"
+ sql "insert into oe2 values(null, 1)"
+ sql "insert into oe2 values(1, null)"
+ sql "insert into oe2 values(null, null)"
+
qt_order_ij """
select oe1.k0, oe2.k0
from oe1 inner join oe2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
