This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b9c68020aa5 [fix](partition) Preserve lexicographic range partition
boundaries (#67888)
b9c68020aa5 is described below
commit b9c68020aa51cd603dde200adfeb44f33ce4cb07
Author: morrySnow <[email protected]>
AuthorDate: Sun Sep 20 17:13:10 2026 +0800
[fix](partition) Preserve lexicographic range partition boundaries (#67888)
### What problem does this PR solve?
Problem Summary:
Problem: Composite RANGE partition pruning can discard partitions
containing matching tuples when suffix columns appear outside
independently projected column bounds even though the full tuple is
lexicographically within the partition.
Root cause: Expanded and unexpanded pruning paths modeled composite
tuple bounds independently. Input range construction made a non-terminal
upper column exclusive, while predicate refinement kept scanning after
the first decisive OTHER column, allowing later suffixes to overwrite
it. Expanded equality folding could also remove the range entry needed
by refinement.
Reproduction: For a partition `[(1, 10, 100), (100, 20, 200))`, tuples
such as `(1, 11, 50)`, `(100, 19, 250)`, and `(100, 20, 199)` are
lexicographically valid. With different
`partition_pruning_expand_threshold` values, the old logic could produce
a zero-partition scan or empty result.
Fix: Share explicit lower and upper lexicographic bound state across
input-range construction and predicate refinement. Constrain only the
first unresolved column, keep suffixes unbounded after a prefix
diverges, use inclusive upper bounds for non-terminal columns and an
exclusive bound only on the final column, stop at the first decisive
OTHER column, and conservatively reuse the input range when a folded
equality removes an expanded entry.
Tests: Extend `PartitionPrunerTest` with three- and four-column
lower/upper boundaries, first/middle/final divergence,
expanded/unexpanded paths, and suffix witnesses. Add an end-to-end
regression that asserts selected partitions and query results for
expanded and unexpanded paths.
### Release note
Fix incorrect pruning for composite RANGE partitions at lexicographic
boundary transitions.
---
.../rules/OneRangePartitionEvaluator.java | 341 ++++++++++++++++-----
.../nereids/rules/rewrite/PartitionPrunerTest.java | 199 ++++++++++++
.../test_lexicographic_range_partition.out | 21 ++
.../test_lexicographic_range_partition.groovy | 93 ++++++
4 files changed, 585 insertions(+), 69 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index f1e7d5dcf31..d5c6f7debdf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -23,7 +23,6 @@ import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import
org.apache.doris.nereids.rules.expression.rules.OneRangePartitionEvaluator.EvaluateRangeInput;
import
org.apache.doris.nereids.rules.expression.rules.OneRangePartitionEvaluator.EvaluateRangeResult;
@@ -52,9 +51,9 @@ import
org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.qe.SessionVariable;
import com.google.common.base.Preconditions;
-import com.google.common.collect.BoundType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
@@ -92,8 +91,26 @@ public class OneRangePartitionEvaluator<K>
// whether the Expression in partition range may be null.
private final Map<Expression, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
+ // An impossible expander/evaluator state makes pruning unsafe. Production
keeps the partition;
+ // fe_debug throws at the point where the broken invariant is observed.
+ private boolean disablePruning;
- /** OneRangePartitionEvaluator */
+ /**
+ * Create an evaluator for one RANGE partition.
+ *
+ * <p>The constructor converts the tuple endpoints to Nereids literals,
classifies every
+ * partition coordinate as {@code CONST}, {@code RANGE}, or {@code OTHER},
and expands the
+ * enumerable prefix up to {@code expandThreshold}. It also records
whether each coordinate can
+ * contain NULL so expression evaluation can fold null-sensitive
predicates safely. An unknown
+ * coordinate type violates the expander/evaluator contract: fe_debug
reports it immediately,
+ * while production marks this evaluator as non-prunable.
+ *
+ * @param partitionIdent identifier returned when this partition must be
scanned
+ * @param partitionSlots partition-key slots in lexicographic order
+ * @param partitionItem inclusive-lower/exclusive-upper RANGE partition
definition
+ * @param cascadesContext planner context used by expression rewrite rules
+ * @param expandThreshold maximum number of enumerable values expanded
from the partition range
+ */
public OneRangePartitionEvaluator(K partitionIdent, List<Slot>
partitionSlots,
RangePartitionItem partitionItem, CascadesContext cascadesContext,
int expandThreshold) {
this.partitionIdent = partitionIdent;
@@ -139,7 +156,10 @@ public class OneRangePartitionEvaluator<K>
maybeNull = true;
break;
default:
- throw new AnalysisException("Unknown partition slot
type: " + partitionSlotType);
+ disablePruningOnInvalidState(
+ "Unknown partition slot type while deriving
nullability: " + partitionSlotType);
+ maybeNull = true;
+ break;
}
partitionSlotContainsNull.put(slot, maybeNull);
}
@@ -170,12 +190,28 @@ public class OneRangePartitionEvaluator<K>
}
}
+ /**
+ * Evaluate a partition predicate under one projected input row.
+ *
+ * <p>The projected ranges seed expression evaluation and are then
narrowed by the predicate.
+ * If input construction or range refinement discovers an invalid internal
state, returning any
+ * folded result could incorrectly remove data. In production this method
therefore returns the
+ * original predicate, which {@link PartitionPruner} treats as unknown and
keeps the partition.
+ * In fe_debug the invalid state is thrown before this fallback is reached.
+ *
+ * @param expression partition predicate to simplify
+ * @param currentInputs replacement expression and projected range for
every partition slot
+ * @return the simplified predicate, or {@code expression} when pruning
has been disabled
+ */
@Override
public Expression evaluate(Expression expression, Map<Slot,
PartitionSlotInput> currentInputs) {
+ if (disablePruning) {
+ return expression;
+ }
Map<Expression, ColumnRange> defaultColumnRanges =
currentInputs.values().iterator().next().columnRanges;
Map<Expression, ColumnRange> rangeMap = new
HashMap<>(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new
EvaluateRangeInput(currentInputs, rangeMap));
- return result.result;
+ return disablePruning ? expression : result.result;
}
@Override
@@ -407,6 +443,19 @@ public class OneRangePartitionEvaluator<K>
return result;
}
+ /**
+ * Evaluate a conjunction and refine the independent column ranges with
the tuple boundary.
+ *
+ * <p>The normal child evaluation first intersects the ranges contributed
by every conjunct.
+ * A composite RANGE partition needs one additional pass: a suffix column
is constrained by a
+ * lower or upper endpoint only while every preceding coordinate is still
equal to that endpoint.
+ * Applying this pass after merging the children also recovers prefix
coordinates whose equality
+ * expression was folded to a literal and therefore disappeared from the
child range map.
+ *
+ * @param and conjunction being evaluated
+ * @param context replacement values and projected ranges for the current
expanded partition input
+ * @return the folded conjunction together with its lexicographically
refined column ranges
+ */
@Override
public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
@@ -424,8 +473,8 @@ public class OneRangePartitionEvaluator<K>
}
// shrink range and prune the other type: if previous column is
literal and equals to the bound
- andResult = determinateRangeOfOtherType(andResult, lowers, true);
- andResult = determinateRangeOfOtherType(andResult, uppers, false);
+ andResult = determinateRangeOfOtherType(andResult, lowers, true,
context.rangeMap);
+ andResult = determinateRangeOfOtherType(andResult, uppers, false,
context.rangeMap);
return andResult;
}
@@ -526,60 +575,83 @@ public class OneRangePartitionEvaluator<K>
}
}
+ /**
+ * Refine the first unresolved suffix column against one lexicographic
partition endpoint.
+ *
+ * <p>Partition slot types have the shape {@code CONST*, RANGE, OTHER*}.
The scan proceeds from
+ * left to right and keeps an endpoint active only while the observed
singleton values equal its
+ * prefix. For the first unresolved {@code OTHER} coordinate, the active
lower endpoint contributes
+ * {@code >= bound}; an active upper endpoint contributes {@code <=
bound}, except that the final
+ * partition column uses {@code < bound} because RANGE partitions are
upper-exclusive. The method
+ * returns immediately after that coordinate because once it may differ
from the endpoint, later
+ * coordinates are lexicographically unconstrained.
+ *
+ * <p>Constant folding can remove an expanded {@code RANGE} slot from
{@code context.columnRanges}.
+ * For that slot only, the method reads the singleton from {@code
defaultColumnRanges} so the
+ * omitted equal-prefix coordinate can still participate in prefix
comparison. A missing
+ * {@code OTHER} slot is different: its default range describes only the
partition boundary, not
+ * the predicate. Adding it to the predicate result would let {@code NOT}
complement a synthetic
+ * range and could incorrectly prune the partition, so a missing {@code
OTHER} stops refinement.
+ *
+ * @param context result of merging all conjunct ranges
+ * @param partitionBound lower or upper endpoint of the composite partition
+ * @param isLowerBound whether {@code partitionBound} is the inclusive
lower endpoint
+ * @param defaultColumnRanges projected ranges for the current expanded
partition input
+ * @return {@code context} refined at the first decisive suffix
coordinate, or an equivalent
+ * false result if the refined range is empty
+ */
private EvaluateRangeResult determinateRangeOfOtherType(
- EvaluateRangeResult context, List<Literal> partitionBound, boolean
isLowerBound) {
+ EvaluateRangeResult context, List<Literal> partitionBound, boolean
isLowerBound,
+ Map<Expression, ColumnRange> defaultColumnRanges) {
if (context.result instanceof Literal) {
return context;
}
- Slot qualifiedSlot = null;
- ColumnRange qualifiedRange = null;
+ LexicographicBoundState boundState = new LexicographicBoundState(
+ partitionBound, isLowerBound, partitionSlots.size());
for (int i = 0; i < partitionSlotTypes.size(); i++) {
PartitionSlotType partitionSlotType = partitionSlotTypes.get(i);
Slot slot = partitionSlots.get(i);
- if (!context.columnRanges.containsKey(slot)) {
- return context;
- }
+ ColumnRange columnRange = context.columnRanges.get(slot);
switch (partitionSlotType) {
case CONST: continue;
case RANGE:
- ColumnRange columnRange = context.columnRanges.get(slot);
- if (!columnRange.isSingleton()
- ||
!columnRange.getLowerBound().getValue().equals(partitionBound.get(i))) {
+ // Expanded RANGE literals can disappear after constant
folding. Recover only this
+ // equal-prefix coordinate from the partition projection.
+ if (columnRange == null) {
+ columnRange = defaultColumnRanges.get(slot);
+ }
+ if (columnRange == null || !columnRange.isSingleton()) {
+ return context;
+ }
+
boundState.observeLiteral(columnRange.getLowerBound().getValue(), i);
+ if (!boundState.hasEqualPrefix()) {
return context;
}
continue;
case OTHER:
- columnRange = context.columnRanges.get(slot);
+ // Do not expose a default-only partition range to the
predicate tree. In
+ // particular, visitNot() must never complement a range
that the predicate did
+ // not contribute.
+ if (columnRange == null) {
+ return context;
+ }
if (columnRange.isSingleton()
- &&
columnRange.getLowerBound().getValue().equals(partitionBound.get(i))) {
+ &&
columnRange.getLowerBound().getValue().equals(partitionBound.get(i))
+ && i + 1 < partitionSlots.size()) {
continue;
}
-
- qualifiedSlot = slot;
- if (isLowerBound) {
- qualifiedRange =
ColumnRange.atLeast(partitionBound.get(i));
- } else {
- qualifiedRange = i + 1 == partitionSlots.size()
- ? ColumnRange.lessThen(partitionBound.get(i))
- : ColumnRange.atMost(partitionBound.get(i));
- }
- break;
+ ColumnRange newRange =
boundState.constrainFirstUnresolvedColumn(columnRange, i);
+ Map<Expression, ColumnRange> newRanges = replaceExprRange(
+ context.columnRanges, slot, newRange);
+ return newRange.isEmptyRange()
+ ? new EvaluateRangeResult(BooleanLiteral.FALSE,
newRanges, context.childrenResult)
+ : new EvaluateRangeResult(context.result,
newRanges, context.childrenResult);
default:
- throw new AnalysisException("Unknown partition slot type:
" + partitionSlotType);
- }
- }
-
- if (qualifiedSlot != null) {
- ColumnRange origin = context.columnRanges.get(qualifiedSlot);
- ColumnRange newRange = origin.intersect(qualifiedRange);
-
- Map<Expression, ColumnRange> newRanges =
replaceExprRange(context.columnRanges, qualifiedSlot, newRange);
-
- if (newRange.isEmptyRange()) {
- return new EvaluateRangeResult(BooleanLiteral.FALSE,
newRanges, context.childrenResult);
- } else {
- return new EvaluateRangeResult(context.result, newRanges,
context.childrenResult);
+ disablePruningOnInvalidState(
+ "Unknown partition slot type while refining a
lexicographic bound: "
+ + partitionSlotType);
+ return context;
}
}
return context;
@@ -745,52 +817,69 @@ public class OneRangePartitionEvaluator<K>
return ImmutableList.of(slotToInputs);
}
+ /**
+ * Build evaluator inputs for every expanded representation of this
composite RANGE partition.
+ *
+ * <p>Range expansion replaces enumerable coordinates with literals and
leaves unexpanded
+ * coordinates as slots. Separate lower- and upper-bound states track
whether the literal prefix
+ * of each generated input still equals the corresponding endpoint. Only
the first unresolved
+ * coordinate while a state is active receives that endpoint's constraint;
after it can diverge,
+ * all suffix coordinates remain unbounded. This preserves tuple ordering
instead of incorrectly
+ * treating each partition column as an independent interval.
+ *
+ * <p>The returned {@link PartitionSlotInput}s all contain the complete
projected range map for
+ * their generated input. Expression evaluation can therefore recover
ranges for slots that were
+ * replaced by literals and removed by constant folding.
+ *
+ * <p>A {@code CONST} coordinate must have been expanded to a literal. If
it is not, the input is
+ * structurally invalid: fe_debug throws, while production makes the
coordinate unbounded and
+ * disables pruning for the whole evaluator so planning can continue
without dropping data.
+ *
+ * @return one slot-to-input map for each Cartesian-product row produced
by range expansion
+ */
private List<Map<Slot, PartitionSlotInput>>
commonComputeOnePartitionInputs() {
List<Map<Slot, PartitionSlotInput>> onePartitionInputs =
Lists.newArrayListWithCapacity(inputs.size());
for (List<Expression> input : inputs) {
- boolean previousIsLowerBoundLiteral = true;
- boolean previousIsUpperBoundLiteral = true;
+ LexicographicBoundState lowerState = new LexicographicBoundState(
+ lowers, true, partitionSlots.size());
+ LexicographicBoundState upperState = new LexicographicBoundState(
+ uppers, false, partitionSlots.size());
Builder<Slot, PartitionSlotInput> slotToInputs =
ImmutableMap.builderWithExpectedSize(16);
for (int i = 0; i < partitionSlots.size(); ++i) {
Slot partitionSlot = partitionSlots.get(i);
// partitionSlot will be replaced to this expression
Expression expression = input.get(i);
- ColumnRange slotRange = null;
+ ColumnRange slotRange;
PartitionSlotType partitionSlotType =
partitionSlotTypes.get(i);
if (expression instanceof Literal) {
// const or expanded range
slotRange = ColumnRange.singleton((Literal) expression);
- if (!expression.equals(lowers.get(i))) {
- previousIsLowerBoundLiteral = false;
- }
- if (!expression.equals(uppers.get(i))) {
- previousIsUpperBoundLiteral = false;
- }
+ lowerState.observeLiteral(expression, i);
+ upperState.observeLiteral(expression, i);
} else {
- // un expanded range
+ // The first unresolved column carries every still-active
lexicographic bound.
+ // Once that column can diverge, every suffix column must
remain unbounded.
switch (partitionSlotType) {
case RANGE:
- boolean isLastPartitionColumn = i + 1 ==
partitionSlots.size();
- BoundType rightBoundType = isLastPartitionColumn
- ? BoundType.OPEN : BoundType.CLOSED;
- slotRange = ColumnRange.range(
- lowers.get(i), BoundType.CLOSED,
uppers.get(i), rightBoundType);
- break;
case OTHER:
- if (previousIsLowerBoundLiteral) {
- slotRange = ColumnRange.atLeast(lowers.get(i));
- } else if (previousIsUpperBoundLiteral) {
- slotRange =
ColumnRange.lessThen(uppers.get(i));
- } else {
- // unknown range
- slotRange = ColumnRange.all();
- }
+ slotRange =
lowerState.constrainFirstUnresolvedColumn(ColumnRange.all(), i);
+ slotRange =
upperState.constrainFirstUnresolvedColumn(slotRange, i);
+ break;
+ case CONST:
+ disablePruningOnInvalidState("CONST partition
input must be a literal: slot="
+ + partitionSlot + ", expression=" +
expression);
+ slotRange = ColumnRange.all();
+ lowerState.diverge();
+ upperState.diverge();
break;
default:
- throw new AnalysisException("Unknown partition
slot type: " + partitionSlotType);
+ disablePruningOnInvalidState(
+ "Unknown partition slot type while
building evaluator inputs: "
+ + partitionSlotType);
+ slotRange = ColumnRange.all();
+ lowerState.diverge();
+ upperState.diverge();
}
- previousIsLowerBoundLiteral = false;
- previousIsUpperBoundLiteral = false;
}
ImmutableMap<Expression, ColumnRange> slotToRange =
ImmutableMap.of(partitionSlot, slotRange);
slotToInputs.put(partitionSlot, new
PartitionSlotInput(expression, slotToRange));
@@ -802,6 +891,120 @@ public class OneRangePartitionEvaluator<K>
return onePartitionInputs;
}
+ /**
+ * Handle a state that violates the contract between {@link
PartitionRangeExpander} and this evaluator.
+ *
+ * <p>Tests and debugging sessions set {@code fe_debug=true}, so they fail
immediately and expose
+ * the broken invariant. Production planning must remain available: after
logging the problem,
+ * evaluation returns the original predicate for this partition, which
conservatively keeps it.
+ *
+ * @param message description of the invalid state
+ */
+ private void disablePruningOnInvalidState(String message) {
+ SessionVariable.throwAnalysisExceptionWhenFeDebug(message);
+ disablePruning = true;
+ }
+
+ /** Describes whether the coordinates already consumed are still equal to
an endpoint prefix. */
+ private enum BoundPrefixState {
+ /** No consumed coordinate differs from the tracked lower or upper
endpoint. */
+ EQUAL_PREFIX,
+
+ /** An earlier coordinate can differ, so this endpoint cannot
constrain any later coordinate. */
+ DIVERGED
+ }
+
+ /**
+ * Tracks one lower or upper endpoint while projected ranges are built or
refined.
+ *
+ * <p>Lexicographic comparison makes an endpoint relevant only as long as
all preceding
+ * coordinates equal its prefix. For example, with lower endpoint {@code
(1, 10, 100)}, an input
+ * whose first coordinate is {@code 1} must constrain the first unresolved
coordinate to
+ * {@code >= 10}. If the first coordinate is {@code 2}, the lower endpoint
is already satisfied
+ * and neither the second nor third coordinate receives a lower constraint.
+ *
+ * <p>The state is monotonic: it starts at {@link
BoundPrefixState#EQUAL_PREFIX} and can transition
+ * to {@link BoundPrefixState#DIVERGED} only once.
+ */
+ private static class LexicographicBoundState {
+ private final List<Literal> bound;
+ private final boolean lowerBound;
+ private final int columnCount;
+ private BoundPrefixState prefixState = BoundPrefixState.EQUAL_PREFIX;
+
+ /**
+ * Create state for one endpoint of a composite partition.
+ *
+ * @param bound endpoint values in partition-column order
+ * @param lowerBound true for the inclusive lower endpoint, false for
the exclusive upper endpoint
+ * @param columnCount total number of partition columns, used to
identify the final coordinate
+ */
+ private LexicographicBoundState(List<Literal> bound, boolean
lowerBound, int columnCount) {
+ this.bound = bound;
+ this.lowerBound = lowerBound;
+ this.columnCount = columnCount;
+ }
+
+ /**
+ * Return whether every coordinate observed so far is equal to this
endpoint's prefix.
+ *
+ * @return true while this endpoint may still constrain the next
unresolved coordinate
+ */
+ private boolean hasEqualPrefix() {
+ return prefixState == BoundPrefixState.EQUAL_PREFIX;
+ }
+
+ /**
+ * Consume a literal coordinate and deactivate this endpoint if the
literal differs from it.
+ *
+ * <p>Once a coordinate differs, later values cannot make the tuple
equal to the endpoint
+ * again, so an already-diverged state is intentionally left unchanged.
+ *
+ * @param literal literal selected for the current partition coordinate
+ * @param index zero-based partition-column index of {@code literal}
+ */
+ private void observeLiteral(Expression literal, int index) {
+ if (hasEqualPrefix() && !literal.equals(bound.get(index))) {
+ diverge();
+ }
+ }
+
+ /**
+ * Intersect an unresolved coordinate with this endpoint when its
prefix is still equal.
+ *
+ * <p>The lower endpoint is inclusive on every coordinate. The upper
endpoint is inclusive on
+ * non-terminal coordinates because equality there leaves later
coordinates to decide tuple
+ * membership; only the final coordinate is exclusive. Consuming an
unresolved coordinate
+ * always deactivates the endpoint so no suffix coordinate is
independently constrained.
+ *
+ * @param origin range already inferred for the unresolved coordinate
+ * @param index zero-based partition-column index of that coordinate
+ * @return {@code origin} intersected with the active endpoint, or
unchanged if an earlier
+ * coordinate has already diverged
+ */
+ private ColumnRange constrainFirstUnresolvedColumn(ColumnRange origin,
int index) {
+ if (!hasEqualPrefix()) {
+ return origin;
+ }
+ diverge();
+ Literal boundary = bound.get(index);
+ ColumnRange boundaryRange;
+ if (lowerBound) {
+ boundaryRange = ColumnRange.atLeast(boundary);
+ } else if (index + 1 == columnCount) {
+ boundaryRange = ColumnRange.lessThen(boundary);
+ } else {
+ boundaryRange = ColumnRange.atMost(boundary);
+ }
+ return origin.intersect(boundaryRange);
+ }
+
+ /** Mark this endpoint as satisfied or violated by an earlier decisive
coordinate. */
+ private void diverge() {
+ prefixState = BoundPrefixState.DIVERGED;
+ }
+ }
+
public EvaluateRangeResult visitMonotonic(Expression monotonic,
EvaluateRangeInput context) {
EvaluateRangeResult rangeResult = evaluateChildrenThenThis(monotonic,
context);
if (!rangeResult.result.getClass().equals(monotonic.getClass())) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
index 3fe07a49f9c..f26c4701ce7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
@@ -23,11 +23,13 @@ import org.apache.doris.catalog.ListPartitionItem;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionKey;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import
org.apache.doris.nereids.rules.expression.rules.OneListPartitionEvaluator;
import org.apache.doris.nereids.rules.expression.rules.OnePartitionEvaluator;
+import
org.apache.doris.nereids.rules.expression.rules.OneRangePartitionEvaluator;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionPruneResult;
import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType;
@@ -35,24 +37,31 @@ import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Range;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -64,6 +73,7 @@ public class PartitionPrunerTest extends TestWithFeService {
private final SlotReference slotA = new SlotReference("a",
IntegerType.INSTANCE);
private final SlotReference slotB = new SlotReference("b",
IntegerType.INSTANCE);
private final SlotReference slotC = new SlotReference("c",
IntegerType.INSTANCE);
+ private final SlotReference slotD = new SlotReference("d",
IntegerType.INSTANCE);
@Override
protected void runBeforeAll() throws Exception {
@@ -344,6 +354,195 @@ public class PartitionPrunerTest extends
TestWithFeService {
Assertions.assertTrue(result.hasPartitionPredicate);
}
+ /**
+ * Verify lower/upper boundary transitions for a three-column partition in
both evaluator paths.
+ *
+ * <p>The low threshold leaves the RANGE coordinate unexpanded, while the
high threshold expands
+ * it into literals. The witness tuples cover divergence in a suffix
coordinate, an exactly equal
+ * lower endpoint, and tuples at or beyond the exclusive upper endpoint.
+ */
+ @Test
+ public void testThreeColumnLexicographicRangeBoundaries()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT),
+ new Column("c", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB, slotC);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10, 100}, new int[] {100, 20, 200});
+
+ for (int expandThreshold : new int[] {1, 200}) {
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 11, 50);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 100, 19, 250);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 100, 20, 199);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 10, 100);
+
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 1, 10, 99);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 100, 20, 200);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 100, 20, 250);
+ }
+ }
+
+ /**
+ * Verify that only the first unresolved coordinate receives a
lexicographic endpoint constraint.
+ *
+ * <p>Four columns exercise divergence at the first, middle, and final
coordinates. Extreme suffix
+ * values demonstrate that once an earlier coordinate differs, later
columns are intentionally
+ * unbounded; exact-prefix cases verify inclusive lower and exclusive
upper semantics at the end.
+ */
+ @Test
+ public void testFourColumnLexicographicRangeBoundaries()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT),
+ new Column("c", PrimitiveType.INT),
+ new Column("d", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB, slotC, slotD);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10, 100, 1000}, new int[] {4, 20, 200,
2000});
+
+ for (int expandThreshold : new int[] {1, 10}) {
+ // Once an earlier column diverges, all suffix columns are
unbounded.
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 2, -1000, -1000, -1000);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 3, 9999, 9999, 9999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 11, -1, -1);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 4, 19, 9999, 9999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 10, 101, -1);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 4, 20, 199, 9999);
+
+ // An equal prefix leaves the final column to decide the inclusive
lower and exclusive upper bounds.
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 1, 10, 100, 1000);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
false, 4, 20, 200, 1999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 1, 10, 100, 999);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 4, 20, 200, 2000);
+ assertRangeTuplePruned(partitionItem, slots, expandThreshold,
true, 4, 20, 200, 2001);
+ }
+ }
+
+ /**
+ * A range copied only from the partition projection must not become part
of a predicate result.
+ *
+ * <p>This is the reduced tree produced by the Date-IN partition-pruning
rewrite for
+ * {@code NOT(date(c) IN (day2, day3))}. For expanded {@code a = 1}, the
predicate constrains only
+ * {@code c}; injecting the default {@code b >= 10} range would let {@code
NOT} complement it and
+ * incorrectly turn the entire partition into false.
+ */
+ @Test
+ public void testNotDoesNotComplementDefaultOnlySuffixRange()
+ throws AnalysisException, InvocationTargetException,
IllegalAccessException {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT),
+ new Column("c", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB, slotC);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10, 1}, new int[] {2, 20, 4});
+ Expression predicate = new Not(new Or(
+ new And(new GreaterThanEqual(slotC, Literal.of(2)), new
LessThan(slotC, Literal.of(3))),
+ new And(new GreaterThanEqual(slotC, Literal.of(3)), new
LessThan(slotC, Literal.of(4)))));
+ OneRangePartitionEvaluator<String> evaluator = new
OneRangePartitionEvaluator<>(
+ "p1", slots, partitionItem, cascadesContext, 200);
+
+ Pair<Boolean, Boolean> result =
+ (Pair<Boolean, Boolean>) canBePrunedOutMethod.invoke(null,
predicate, evaluator);
+
+ Assertions.assertFalse(result.first);
+ }
+
+ /**
+ * Invalid expander/evaluator states fail in fe_debug but conservatively
keep the partition in production.
+ */
+ @Test
+ public void testInvalidConstInputUsesFeDebugPolicy() throws Exception {
+ List<Column> columns = ImmutableList.of(
+ new Column("a", PrimitiveType.INT),
+ new Column("b", PrimitiveType.INT));
+ List<Slot> slots = ImmutableList.of(slotA, slotB);
+ RangePartitionItem partitionItem = createRangePartitionItem(
+ columns, new int[] {1, 10}, new int[] {1, 20});
+ Expression predicate = new EqualTo(slotA, Literal.of(2));
+ boolean oldFeDebug = connectContext.getSessionVariable().feDebug;
+ try {
+ connectContext.getSessionVariable().feDebug = false;
+ OneRangePartitionEvaluator<String> productionEvaluator =
+ createEvaluatorWithNonLiteralConstInput(partitionItem,
slots);
+ Pair<Boolean, Boolean> productionResult = (Pair<Boolean, Boolean>)
canBePrunedOutMethod.invoke(
+ null, predicate, productionEvaluator);
+ Assertions.assertFalse(productionResult.first);
+
+ connectContext.getSessionVariable().feDebug = true;
+ OneRangePartitionEvaluator<String> debugEvaluator =
+ createEvaluatorWithNonLiteralConstInput(partitionItem,
slots);
+ InvocationTargetException exception =
Assertions.assertThrows(InvocationTargetException.class,
+ () -> canBePrunedOutMethod.invoke(null, predicate,
debugEvaluator));
+ Assertions.assertInstanceOf(
+
org.apache.doris.nereids.exceptions.AnalysisException.class,
exception.getCause());
+ } finally {
+ connectContext.getSessionVariable().feDebug = oldFeDebug;
+ }
+ }
+
+ private OneRangePartitionEvaluator<String>
createEvaluatorWithNonLiteralConstInput(
+ RangePartitionItem partitionItem, List<Slot> slots) throws
ReflectiveOperationException {
+ OneRangePartitionEvaluator<String> evaluator = new
OneRangePartitionEvaluator<>(
+ "p1", slots, partitionItem, cascadesContext, 200);
+ Field inputsField =
OneRangePartitionEvaluator.class.getDeclaredField("inputs");
+ inputsField.setAccessible(true);
+ inputsField.set(evaluator, ImmutableList.of(ImmutableList.of(slotA,
slotB)));
+ return evaluator;
+ }
+
+ /**
+ * Evaluate an equality predicate for one tuple and assert whether the
partition is pruned.
+ *
+ * @param partitionItem composite RANGE partition under test
+ * @param slots partition slots in tuple order
+ * @param expandThreshold threshold selecting expanded or unexpanded
evaluator inputs
+ * @param expectedPruned expected result from {@code
PartitionPruner.canBePrunedOut}
+ * @param values tuple values used to build one equality per partition slot
+ */
+ private void assertRangeTuplePruned(RangePartitionItem partitionItem,
List<Slot> slots,
+ int expandThreshold, boolean expectedPruned, int... values)
+ throws InvocationTargetException, IllegalAccessException {
+ ImmutableList.Builder<Expression> equalities =
ImmutableList.builderWithExpectedSize(values.length);
+ for (int i = 0; i < values.length; i++) {
+ equalities.add(new EqualTo(slots.get(i), Literal.of(values[i])));
+ }
+ Expression predicate = ExpressionUtils.and(equalities.build());
+ OneRangePartitionEvaluator<String> evaluator = new
OneRangePartitionEvaluator<>(
+ "p1", slots, partitionItem, cascadesContext, expandThreshold);
+ Pair<Boolean, Boolean> result =
+ (Pair<Boolean, Boolean>) canBePrunedOutMethod.invoke(null,
predicate, evaluator);
+ Assertions.assertEquals(expectedPruned, result.first,
+ "tuple=" + Arrays.toString(values) + ", expandThreshold=" +
expandThreshold);
+ }
+
+ /**
+ * Construct a closed-open composite RANGE partition from integer tuple
endpoints.
+ *
+ * @param columns partition columns in key order
+ * @param lowerValues inclusive lower endpoint values
+ * @param upperValues exclusive upper endpoint values
+ * @return partition item representing {@code [lowerValues, upperValues)}
+ * @throws AnalysisException if an endpoint cannot be converted to a typed
partition key
+ */
+ private RangePartitionItem createRangePartitionItem(
+ List<Column> columns, int[] lowerValues, int[] upperValues) throws
AnalysisException {
+ ImmutableList.Builder<PartitionValue> lower =
ImmutableList.builderWithExpectedSize(lowerValues.length);
+ ImmutableList.Builder<PartitionValue> upper =
ImmutableList.builderWithExpectedSize(upperValues.length);
+ for (int value : lowerValues) {
+ lower.add(new PartitionValue(Integer.toString(value)));
+ }
+ for (int value : upperValues) {
+ upper.add(new PartitionValue(Integer.toString(value)));
+ }
+ PartitionKey lowerKey = PartitionKey.createPartitionKey(lower.build(),
columns);
+ PartitionKey upperKey = PartitionKey.createPartitionKey(upper.build(),
columns);
+ return new RangePartitionItem(Range.closedOpen(lowerKey, upperKey));
+ }
+
private ListPartitionItem createListPartitionItem(String... values) throws
AnalysisException {
ImmutableList.Builder<PartitionKey> partitionKeys =
ImmutableList.builder();
for (String value : values) {
diff --git
a/regression-test/data/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.out
b/regression-test/data/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.out
new file mode 100644
index 00000000000..4cf8fb2485d
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !expanded_lower_suffix --
+1
+
+-- !expanded_upper_suffix --
+2
+
+-- !expanded_upper_equal_prefix --
+3
+
+-- !unexpanded_lower_suffix --
+1
+
+-- !unexpanded_upper_suffix --
+2
+
+-- !unexpanded_upper_equal_prefix --
+3
+
+-- !negated_multi_date_in --
+42
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.groovy
new file mode 100644
index 00000000000..91c2a36ad8c
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_lexicographic_range_partition.groovy
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lexicographic_range_partition") {
+ sql "DROP TABLE IF EXISTS lexicographic_range_partition"
+ sql """
+ CREATE TABLE lexicographic_range_partition (
+ k1 INT NOT NULL,
+ k2 INT NOT NULL,
+ k3 INT NOT NULL,
+ v INT NOT NULL
+ )
+ DUPLICATE KEY(k1, k2, k3)
+ PARTITION BY RANGE(k1, k2, k3) (
+ PARTITION p_target VALUES [("1", "10", "100"), ("100", "20",
"200"))
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO lexicographic_range_partition VALUES
+ (1, 11, 50, 1),
+ (100, 19, 250, 2),
+ (100, 20, 199, 3),
+ (1, 10, 100, 4)
+ """
+
+ // Verify the optimizer retains the only partition before checking the
query result. A wrong
+ // empty result alone is ambiguous here because it could also come from
execution or test data.
+ def assertTargetPartition = { String query ->
+ explain {
+ sql query
+ contains "partitions=1/1 (p_target)"
+ }
+ }
+
+ sql "SET partition_pruning_expand_threshold=200"
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=1 AND k2=11 AND k3=50")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=19 AND k3=250")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=20 AND k3=199")
+ qt_expanded_lower_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=1 AND k2=11 AND k3=50"
+ qt_expanded_upper_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=100 AND k2=19 AND k3=250"
+ qt_expanded_upper_equal_prefix "SELECT v FROM
lexicographic_range_partition WHERE k1=100 AND k2=20 AND k3=199"
+
+ sql "SET partition_pruning_expand_threshold=1"
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=1 AND k2=11 AND k3=50")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=19 AND k3=250")
+ assertTargetPartition("SELECT v FROM lexicographic_range_partition WHERE
k1=100 AND k2=20 AND k3=199")
+ qt_unexpanded_lower_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=1 AND k2=11 AND k3=50"
+ qt_unexpanded_upper_suffix "SELECT v FROM lexicographic_range_partition
WHERE k1=100 AND k2=19 AND k3=250"
+ qt_unexpanded_upper_equal_prefix "SELECT v FROM
lexicographic_range_partition WHERE k1=100 AND k2=20 AND k3=199"
+
+ sql "DROP TABLE IF EXISTS lexicographic_range_not_date_in"
+ sql """
+ CREATE TABLE lexicographic_range_not_date_in (
+ k1 INT NOT NULL,
+ k2 INT NOT NULL,
+ k3 DATE NOT NULL,
+ v INT NOT NULL
+ )
+ DUPLICATE KEY(k1, k2, k3)
+ PARTITION BY RANGE(k1, k2, k3) (
+ PARTITION p_target VALUES [("1", "10", "2020-01-01"), ("2", "20",
"2020-01-04"))
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql "INSERT INTO lexicographic_range_not_date_in VALUES (1, 11,
'2020-01-01', 42)"
+
+ // The Date-IN rewrite creates NOT(OR(day ranges)). Default-only k2 ranges
must not leak into
+ // that predicate tree, otherwise NOT turns the valid partition into an
empty set.
+ sql "SET partition_pruning_expand_threshold=200"
+ def negatedDateInQuery = """
+ SELECT v FROM lexicographic_range_not_date_in
+ WHERE NOT(DATE(k3) IN ('2020-01-02', '2020-01-03'))
+ """
+ assertTargetPartition(negatedDateInQuery)
+ qt_negated_multi_date_in negatedDateInQuery
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
