This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 7b3357477db branch-2.1: [fix](nereids)Solve the problem of pruning
wrong partitions in multi-column partition pruning (#43658)
7b3357477db is described below
commit 7b3357477db591467ec89eb8fc29355e9a6e4e02
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Nov 12 19:14:51 2024 +0800
branch-2.1: [fix](nereids)Solve the problem of pruning wrong partitions in
multi-column partition pruning (#43658)
Cherry-picked from #43332
Co-authored-by: feiniaofeiafei
<[email protected]>
---
.../rules/OneRangePartitionEvaluator.java | 27 +++++++++++++---------
.../partition_prune/test_date_trunc_prune.groovy | 25 ++++++++++++++++++++
2 files changed, 41 insertions(+), 11 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index c3345c34693..84a037171f3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -93,7 +93,6 @@ public class OneRangePartitionEvaluator
private final List<List<Expression>> inputs;
private final Map<Expression, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
- private final Map<Expression, ColumnRange> rangeMap = new HashMap<>();
/** OneRangePartitionEvaluator */
public OneRangePartitionEvaluator(long partitionId, List<Slot>
partitionSlots,
@@ -175,8 +174,8 @@ public class OneRangePartitionEvaluator
@Override
public Expression evaluate(Expression expression, Map<Slot,
PartitionSlotInput> currentInputs) {
Map<Expression, ColumnRange> defaultColumnRanges =
currentInputs.values().iterator().next().columnRanges;
- rangeMap.putAll(defaultColumnRanges);
- EvaluateRangeResult result = expression.accept(this, new
EvaluateRangeInput(currentInputs));
+ Map<Expression, ColumnRange> rangeMap = new
HashMap<>(defaultColumnRanges);
+ EvaluateRangeResult result = expression.accept(this, new
EvaluateRangeInput(currentInputs, rangeMap));
return result.result;
}
@@ -397,6 +396,7 @@ public class OneRangePartitionEvaluator
public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
result = mergeRanges(result.result, result.childrenResult.get(0),
result.childrenResult.get(1),
+ context.rangeMap,
(leftRange, rightRange) -> leftRange.intersect(rightRange));
result = returnFalseIfExistEmptyRange(result);
@@ -425,6 +425,7 @@ public class OneRangePartitionEvaluator
result.childrenResult);
}
result = mergeRanges(result.result, result.childrenResult.get(0),
result.childrenResult.get(1),
+ context.rangeMap,
(leftRange, rightRange) -> leftRange.union(rightRange));
return returnFalseIfExistEmptyRange(result);
}
@@ -437,8 +438,8 @@ public class OneRangePartitionEvaluator
for (Map.Entry<Expression, ColumnRange> entry :
result.childrenResult.get(0).columnRanges.entrySet()) {
Expression expr = entry.getKey();
ColumnRange childRange = entry.getValue();
- ColumnRange partitionRange = rangeMap.containsKey(expr)
- ? rangeMap.get(expr) : ColumnRange.all();
+ ColumnRange partitionRange = context.rangeMap.containsKey(expr)
+ ? context.rangeMap.get(expr) : ColumnRange.all();
newRanges.put(expr,
partitionRange.intersect(childRange.complete()));
}
result = new EvaluateRangeResult(result.result, newRanges,
result.childrenResult);
@@ -562,6 +563,7 @@ public class OneRangePartitionEvaluator
private EvaluateRangeResult mergeRanges(
Expression originResult, EvaluateRangeResult left,
EvaluateRangeResult right,
+ Map<Expression, ColumnRange> rangeMap,
BiFunction<ColumnRange, ColumnRange, ColumnRange> mergeFunction) {
Map<Expression, ColumnRange> leftRanges = left.columnRanges;
@@ -623,7 +625,7 @@ public class OneRangePartitionEvaluator
if (partitionSlotContainsNull.containsKey(dateTruncChild)) {
partitionSlotContainsNull.put(dateTrunc, true);
}
- return computeMonotonicFunctionRange(result);
+ return computeMonotonicFunctionRange(result, context.rangeMap);
}
@Override
@@ -636,7 +638,7 @@ public class OneRangePartitionEvaluator
if (partitionSlotContainsNull.containsKey(dateChild)) {
partitionSlotContainsNull.put(date, true);
}
- return computeMonotonicFunctionRange(result);
+ return computeMonotonicFunctionRange(result, context.rangeMap);
}
@Override
@@ -649,7 +651,7 @@ public class OneRangePartitionEvaluator
if (partitionSlotContainsNull.containsKey(converTzChild)) {
partitionSlotContainsNull.put(convertTz, true);
}
- return computeMonotonicFunctionRange(result);
+ return computeMonotonicFunctionRange(result, context.rangeMap);
}
private boolean isPartitionSlot(Slot slot) {
@@ -681,10 +683,12 @@ public class OneRangePartitionEvaluator
/** EvaluateRangeInput */
public static class EvaluateRangeInput {
- private Map<Slot, PartitionSlotInput> slotToInput;
+ private final Map<Slot, PartitionSlotInput> slotToInput;
+ private final Map<Expression, ColumnRange> rangeMap;
- public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput) {
+ public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput,
Map<Expression, ColumnRange> rangeMap) {
this.slotToInput = slotToInput;
+ this.rangeMap = rangeMap;
}
}
@@ -816,7 +820,8 @@ public class OneRangePartitionEvaluator
return onePartitionInputs;
}
- private EvaluateRangeResult
computeMonotonicFunctionRange(EvaluateRangeResult result) {
+ private EvaluateRangeResult
computeMonotonicFunctionRange(EvaluateRangeResult result,
+ Map<Expression, ColumnRange> rangeMap) {
Monotonic func = (Monotonic) result.result;
if (rangeMap.containsKey(func)) {
return new EvaluateRangeResult((Expression) func,
ImmutableMap.of((Expression) func,
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
index 4ad3afbb9db..c5eb3855247 100644
---
a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
@@ -334,4 +334,29 @@ suite("test_date_trunc_prune") {
( NOT ( ( table1 . `col_int_undef_signed` != table1 .
`col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' )
AND
table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY field1
ORDER BY field1 LIMIT 1000;
"""
+
+ // test multi column partition table and predicate with date_trunc
+ sql "drop table if exists t_multi_column_partition"
+ sql """
+ create table t_multi_column_partition(a int, dt datetime, v int) partition
by range(a, dt)
+ (
+ partition p0 values [(0,'2024-01-01 00:00:00'), (10,'2024-01-10
00:00:00')),
+ partition p10 values [(10,'2024-01-10 00:00:00'), (20,'2024-01-20
00:00:00')),
+ partition p20 values [(20,'2024-01-20 00:00:00'), (30,'2024-01-31
00:00:00')),
+ partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10
00:00:00')),
+ partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20
00:00:00'))
+ )
+ distributed by hash(a) properties("replication_num"="1");
+ """
+ sql """
+ insert into t_multi_column_partition values(0,'2024-01-01
00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2),
+ (10,'2024-01-10 00:00:00',3),(10,'2024-01-11
00:00:00',200),(12,'2021-01-01 00:00:00',2),
+ (25,'2024-01-10 00:00:00',3),(20,'2024-01-11
00:00:00',200),(30,'2021-01-01 00:00:00',2),
+ (40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2),(10,'2024-01-9
00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000),
+ (2,'2023-01-10 01:00:00',1000)
+ """
+ explain {
+ sql """select * from t_multi_column_partition where a=2 and
date_trunc(dt,'day') <'2024-01-1 00:00:00';"""
+ contains ("partitions=1/5 (p0)")
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
