This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7b715fe93a6882ea457312232351b004d035fe3b Author: morrySnow <[email protected]> AuthorDate: Wed Sep 27 18:41:56 2023 +0800 [fix](Nereids) could not prune datev1 partition column (#24959) because storage engine could not process date comparison predicates. we convert it to datetime comparison predicates. however, partition prunner could not process cast(slot) cp literal. so, we convert back in partition pruner to let it work well. TODO: move convert date to datetime in translate stage and only convert predicates for storage engine. --- .../rules/HiveDefaultPartitionEvaluator.java | 4 +- .../rules/OneRangePartitionEvaluator.java | 18 ++++---- .../rules/expression/rules/PartitionPruner.java | 48 ++++++++++++++++++++-- .../trees/expressions/literal/DateTimeLiteral.java | 4 ++ 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java index 04785e1db14..f50cbe3cf33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java @@ -32,8 +32,8 @@ import java.util.Map; * For any partition predicate, the evaluate() will always return true. */ public class HiveDefaultPartitionEvaluator implements OnePartitionEvaluator { - private long id; - private List<Slot> partitionSlots; + private final long id; + private final List<Slot> partitionSlots; public HiveDefaultPartitionEvaluator(long id, List<Slot> partitionSlots) { this.id = id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index f9857646852..13ecddd150a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -76,15 +76,15 @@ public class OneRangePartitionEvaluator extends ExpressionVisitor<EvaluateRangeResult, EvaluateRangeInput> implements OnePartitionEvaluator { private final long partitionId; - private List<Slot> partitionSlots; - private RangePartitionItem partitionItem; - private ExpressionRewriteContext expressionRewriteContext; - private List<PartitionSlotType> partitionSlotTypes; - private List<Literal> lowers; - private List<Literal> uppers; - private List<List<Expression>> inputs; - private Map<Slot, Boolean> partitionSlotContainsNull; - private Map<Slot, PartitionSlotType> slotToType; + private final List<Slot> partitionSlots; + private final RangePartitionItem partitionItem; + private final ExpressionRewriteContext expressionRewriteContext; + private final List<PartitionSlotType> partitionSlotTypes; + private final List<Literal> lowers; + private final List<Literal> uppers; + private final List<List<Expression>> inputs; + private final Map<Slot, Boolean> partitionSlotContainsNull; + private final Map<Slot, PartitionSlotType> slotToType; /** OneRangePartitionEvaluator */ public OneRangePartitionEvaluator(long partitionId, List<Slot> partitionSlots, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index 8479c27776a..6c932f558b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -22,9 +22,16 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.types.DateTimeType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -36,9 +43,9 @@ import java.util.Objects; /** * PartitionPruner */ -public class PartitionPruner { - private List<OnePartitionEvaluator> partitions; - private Expression partitionPredicate; +public class PartitionPruner extends DefaultExpressionRewriter<Void> { + private final List<OnePartitionEvaluator> partitions; + private final Expression partitionPredicate; /** Different type of table may have different partition prune behavior. */ public enum PartitionTableType { @@ -48,7 +55,40 @@ public class PartitionPruner { private PartitionPruner(List<OnePartitionEvaluator> partitions, Expression partitionPredicate) { this.partitions = Objects.requireNonNull(partitions, "partitions cannot be null"); - this.partitionPredicate = Objects.requireNonNull(partitionPredicate, "partitionPredicate cannot be null"); + this.partitionPredicate = Objects.requireNonNull(partitionPredicate.accept(this, null), + "partitionPredicate cannot be null"); + } + + @Override + public Expression visitComparisonPredicate(ComparisonPredicate cp, Void context) { + // Date cp Date is not supported in BE storage engine. So cast to DateTime in SimplifyComparisonPredicate + // for easy process partition prune, we convert back to date compare date here + // see more info in SimplifyComparisonPredicate + Expression left = cp.left(); + Expression right = cp.right(); + if (left.getDataType() != DateTimeType.INSTANCE || right.getDataType() != DateTimeType.INSTANCE) { + return cp; + } + if (!(left instanceof DateTimeLiteral) && !(right instanceof DateTimeLiteral)) { + return cp; + } + if (left instanceof DateTimeLiteral && ((DateTimeLiteral) left).isMidnight() + && right instanceof Cast + && ((Cast) right).child() instanceof SlotReference + && ((Cast) right).child().getDataType().isDateType()) { + DateTimeLiteral dt = (DateTimeLiteral) left; + Cast cast = (Cast) right; + return cp.withChildren(new DateLiteral(dt.getYear(), dt.getMonth(), dt.getDay()), cast.child()); + } else if (right instanceof DateTimeLiteral && ((DateTimeLiteral) right).isMidnight() + && left instanceof Cast + && ((Cast) left).child() instanceof SlotReference + && ((Cast) left).child().getDataType().isDateType()) { + DateTimeLiteral dt = (DateTimeLiteral) right; + Cast cast = (Cast) left; + return cp.withChildren(cast.child(), new DateLiteral(dt.getYear(), dt.getMonth(), dt.getDay())); + } else { + return cp; + } } public List<Long> prune() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 2db3dacb49b..dd4c4bacabc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -147,6 +147,10 @@ public class DateTimeLiteral extends DateLiteral { this.day = day; } + public boolean isMidnight() { + return hour == 0 && minute == 0 && second == 0 && microSecond == 0; + } + @Override protected void init(String s) throws AnalysisException { try { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
