This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f2cfc87aca1 [fix](nereids) temporary partition is selected only if
user manually specified (#27893)
f2cfc87aca1 is described below
commit f2cfc87aca19af0b3895b2d3f388ba2d4f8ec995
Author: minghong <[email protected]>
AuthorDate: Mon Dec 4 09:44:27 2023 +0800
[fix](nereids) temporary partition is selected only if user manually
specified (#27893)
q1: "select * from ut_p temporary partitions(tp1) where val > 0"
in q1, temporary partition tp1 is scaned
q2: "select * from ut_p where val > 0"
in q2, temporary partition tp1 is not scaned.
---
.../rules/expression/rules/PartitionPruner.java | 10 -------
.../rules/rewrite/PruneOlapScanPartition.java | 18 ++++++++----
.../data/nereids_syntax_p0/select_partition.out | 2 +-
.../nereids_syntax_p0/select_partition.groovy | 34 +++++++++++++++++++---
4 files changed, 43 insertions(+), 21 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index a0336295476..ae3a5add083 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -18,7 +18,6 @@
package org.apache.doris.nereids.rules.expression.rules;
import org.apache.doris.catalog.ListPartitionItem;
-import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.nereids.CascadesContext;
@@ -98,15 +97,6 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
.collect(ImmutableList.toImmutableList());
}
- /**
- * prune partition with `partitionInfo` as parameter.
- */
- public static List<Long> prune(List<Slot> partitionSlots, Expression
partitionPredicate,
- PartitionInfo partitionInfo, CascadesContext cascadesContext,
PartitionTableType partitionTableType) {
- return prune(partitionSlots, partitionPredicate,
partitionInfo.getAllPartitions(), cascadesContext,
- partitionTableType);
- }
-
/**
* prune partition with `idToPartitions` as parameter.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
index db183d66389..e9e59f1c113 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionInfo;
+import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
@@ -31,7 +32,6 @@ import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
-import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
@@ -66,15 +66,21 @@ public class PruneOlapScanPartition extends
OneRewriteRuleFactory {
.stream()
.map(column ->
scanOutput.get(column.getName().toLowerCase()))
.collect(Collectors.toList());
+ List<Long> manuallySpecifiedPartitions =
scan.getManuallySpecifiedPartitions();
+ Map<Long, PartitionItem> idToPartitions;
+ if (manuallySpecifiedPartitions.isEmpty()) {
+ idToPartitions = partitionInfo.getIdToItem(false);
+ } else {
+ Map<Long, PartitionItem> allPartitions =
partitionInfo.getAllPartitions();
+ idToPartitions = allPartitions.keySet().stream()
+ .filter(id -> manuallySpecifiedPartitions.contains(id))
+ .collect(Collectors.toMap(Function.identity(), id ->
allPartitions.get(id)));
+ }
List<Long> prunedPartitions = new
ArrayList<>(PartitionPruner.prune(
- partitionSlots, filter.getPredicate(), partitionInfo,
ctx.cascadesContext,
+ partitionSlots, filter.getPredicate(), idToPartitions,
ctx.cascadesContext,
PartitionTableType.OLAP));
- List<Long> manuallySpecifiedPartitions =
scan.getManuallySpecifiedPartitions();
- if (!CollectionUtils.isEmpty(manuallySpecifiedPartitions)) {
- prunedPartitions.retainAll(manuallySpecifiedPartitions);
- }
if (prunedPartitions.isEmpty()) {
return new LogicalEmptyRelation(
ConnectContext.get().getStatementContext().getNextRelationId(),
diff --git a/regression-test/data/nereids_syntax_p0/select_partition.out
b/regression-test/data/nereids_syntax_p0/select_partition.out
index e49bb9ea3d9..6ca642b594a 100644
--- a/regression-test/data/nereids_syntax_p0/select_partition.out
+++ b/regression-test/data/nereids_syntax_p0/select_partition.out
@@ -25,5 +25,5 @@
1 aaa aaa
-- !sql --
-16 1234 t
+6 1234 t
diff --git a/regression-test/suites/nereids_syntax_p0/select_partition.groovy
b/regression-test/suites/nereids_syntax_p0/select_partition.groovy
index 8b438b228c2..f5697a6f3b4 100644
--- a/regression-test/suites/nereids_syntax_p0/select_partition.groovy
+++ b/regression-test/suites/nereids_syntax_p0/select_partition.groovy
@@ -89,6 +89,7 @@ suite("query_on_specific_partition") {
SELECT * FROM test_iot PARTITION p1;
"""
+// temporary partition test
sql """
DROP TABLE IF EXISTS ut_p;
"""
@@ -110,14 +111,39 @@ suite("query_on_specific_partition") {
);
"""
- sql """ALTER TABLE ut_p ADD TEMPORARY PARTITION tp1 VALUES [("15"),
("20"));"""
+ sql """ALTER TABLE ut_p ADD TEMPORARY PARTITION tp1 VALUES [("5"),
("7"));"""
- sql "INSERT INTO ut_p TEMPORARY PARTITION(tp1) values(16,1234, 't');"
+ sql "INSERT INTO ut_p TEMPORARY PARTITION(tp1) values(6,1234, 't');"
+ sql "INSERT INTO ut_p values(6,1234, 't');"
+ sql "INSERT INTO ut_p values(3,1234, 't');"
+ sql "set enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
qt_sql """select * from ut_p temporary partitions(tp1);"""
-
-
+ explain {
+ sql "select * from ut_p temporary partitions(tp1);"
+ contains "partitions=1/2 (tp1)"
+ }
+
+ explain {
+ sql "select * from ut_p temporary partitions(tp1) where val > 0"
+ contains "partitions=1/2 (tp1)"
+ }
+
+ explain {
+ sql "select * from ut_p temporary partitions(tp1) where val > 0"
+ contains "partitions=1/2 (tp1)"
+ }
+
+ explain {
+ sql "select * from ut_p partitions(p2) where val > 0"
+ contains "partitions=1/2 (p2)"
+ }
+
+ explain {
+ sql "select * from ut_p temporary partitions(tp1) where id = 8"
+ contains "VEMPTYSET"
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
