This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a043679ec8f [fix](nereids)check if correlated filter exists before
converting apply to join (#47985)
a043679ec8f is described below
commit a043679ec8f51ffc21344826d9c04dc46e9f7c67
Author: starocean999 <[email protected]>
AuthorDate: Wed Feb 19 17:30:56 2025 +0800
[fix](nereids)check if correlated filter exists before converting apply to
join (#47985)
example sql:
`SELECT *
FROM
t1
WHERE
t1.x IN (SELECT x FROM t2 WHERE t1.y is NULL AND t1.y is NOT NULL);`
the correlated filter `t1.y is NULL AND t1.y is NOT NULL` will be
replaced as `FALSE` by SimplifyConflictCompound rule. So we need check
if correlated filter exists before converting apply to join
Related PR: (https://github.com/apache/doris/pull/47385)
---
.../nereids/rules/rewrite/ExistsApplyToJoin.java | 5 +-
.../doris/nereids/rules/rewrite/InApplyToJoin.java | 6 ++-
.../nereids/rules/rewrite/ScalarApplyToJoin.java | 5 +-
.../subquery/test_correlated_filter_removed.groovy | 54 ++++++++++++++++++++++
4 files changed, 67 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java
index 003aa102860..930e4c467df 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java
@@ -81,7 +81,10 @@ public class ExistsApplyToJoin extends OneRewriteRuleFactory
{
@Override
public Rule build() {
return logicalApply().when(LogicalApply::isExist).then(apply -> {
- if (apply.isCorrelated()) {
+ // apply.isCorrelated() only check if correlated slot exits
+ // but correlation filter may be eliminated by
SimplifyConflictCompound rule
+ // so we need check both correlated slot and correlation filter
exists before creating LogicalJoin node
+ if (apply.isCorrelated() &&
apply.getCorrelationFilter().isPresent()) {
return correlatedToJoin(apply);
} else {
return unCorrelatedToJoin(apply);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java
index fdf6efb8167..67276e92bb3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java
@@ -122,7 +122,11 @@ public class InApplyToJoin extends OneRewriteRuleFactory {
new DistributeHint(DistributeType.NONE),
apply.getMarkJoinSlotReference(),
apply.children(), null);
} else {
- if (apply.isCorrelated()) {
+ // apply.isCorrelated() only check if correlated slot exits
+ // but correlation filter may be eliminated by
SimplifyConflictCompound rule
+ // so we need check both correlated slot and correlation
filter exists
+ // before creating LogicalJoin node
+ if (apply.isCorrelated() &&
apply.getCorrelationFilter().isPresent()) {
if (inSubquery.isNot()) {
predicate = ExpressionUtils.and(ExpressionUtils.or(new
EqualTo(left, right),
new IsNull(left), new IsNull(right)),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java
index a7b2b9a2045..5cb11914b66 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java
@@ -45,7 +45,10 @@ public class ScalarApplyToJoin extends OneRewriteRuleFactory
{
@Override
public Rule build() {
return logicalApply().when(LogicalApply::isScalar).then(apply -> {
- if (apply.isCorrelated()) {
+ // apply.isCorrelated() only check if correlated slot exits
+ // but correlation filter may be eliminated by
SimplifyConflictCompound rule
+ // so we need check both correlated slot and correlation filter
exists before creating LogicalJoin node
+ if (apply.isCorrelated() &&
apply.getCorrelationFilter().isPresent()) {
return correlatedToJoin(apply);
} else {
return unCorrelatedToJoin(apply);
diff --git
a/regression-test/suites/nereids_p0/subquery/test_correlated_filter_removed.groovy
b/regression-test/suites/nereids_p0/subquery/test_correlated_filter_removed.groovy
new file mode 100644
index 00000000000..2a3b5124048
--- /dev/null
+++
b/regression-test/suites/nereids_p0/subquery/test_correlated_filter_removed.groovy
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_correlated_filter_removed") {
+ multi_sql """
+ drop table if exists
table_6_undef_partitions2_keys3_properties4_distributed_by5;
+
+ create table
table_6_undef_partitions2_keys3_properties4_distributed_by5 (
+ col_int_undef_signed int/*agg_type_placeholder*/ ,
+ col_varchar_10__undef_signed varchar(10)/*agg_type_placeholder*/ ,
+ pk int/*agg_type_placeholder*/
+ ) engine=olap
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+
+ insert into
table_6_undef_partitions2_keys3_properties4_distributed_by5(pk,col_int_undef_signed,col_varchar_10__undef_signed)
values
(0,null,'think'),(1,null,''),(2,2,''),(3,null,'r'),(4,null,null),(5,8,'here');
+
+ drop table if exists
table_100_undef_partitions2_keys3_properties4_distributed_by5;
+
+ create table
table_100_undef_partitions2_keys3_properties4_distributed_by5 (
+ col_int_undef_signed int/*agg_type_placeholder*/ ,
+ col_varchar_10__undef_signed varchar(10)/*agg_type_placeholder*/ ,
+ pk int/*agg_type_placeholder*/
+ ) engine=olap
+ distributed by hash(pk) buckets 10
+ properties("replication_num" = "1");
+
+ insert into
table_100_undef_partitions2_keys3_properties4_distributed_by5(pk,col_int_undef_signed,col_varchar_10__undef_signed)
values
(0,null,''),(1,null,''),(2,null,''),(3,0,null),(4,7,null),(5,9,'d'),(6,9,null),(7,null,null),(8,null,null),(9,null,''),(10,null,'are'),(11,null,'were'),(12,2,''),(13,null,'one'),(14,null,'ok'),(15,null,'your'),(16,null,''),(17,null,null),(18,4,''),(19,null,null),(20,null,null),(21,null,null),(22,3,''),(23,null,null),(24,8,''),(25,2,'I''m'),(26,nul
[...]
+ """
+
+ sql """
+ SELECT *
+ FROM table_6_undef_partitions2_keys3_properties4_distributed_by5 AS t1
+ WHERE t1.`pk` + 2 IN
+ (SELECT 1
+ FROM table_100_undef_partitions2_keys3_properties4_distributed_by5
AS t2
+ WHERE t1.col_int_undef_signed is NULL
+ AND t1.col_int_undef_signed is NOT NULL) ;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
