This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 6c8ccecbd1e branch-4.1: [improvement](repeat) Optimize shuffle key
selection for Repeat decomposition #66532 (#66568)
6c8ccecbd1e is described below
commit 6c8ccecbd1eeb2a51109e8989a05dec1e265d506
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Aug 7 14:23:09 2026 +0800
branch-4.1: [improvement](repeat) Optimize shuffle key selection for Repeat
decomposition #66532 (#66568)
Cherry-picked from #66532
Co-authored-by: feiniaofeiafei <[email protected]>
---
.../rules/rewrite/DecomposeRepeatWithPreAggregation.java | 13 ++++++++++---
.../org/apache/doris/statistics/util/StatisticsUtil.java | 7 ++++---
.../rewrite/DecomposeRepeatWithPreAggregationTest.java | 2 +-
3 files changed, 15 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
index 4f6fa967f5f..ef40c53e75f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java
@@ -97,6 +97,7 @@ public class DecomposeRepeatWithPreAggregation extends
DefaultPlanRewriter<Disti
private static final Set<Class<? extends AggregateFunction>>
SUPPORT_AGG_FUNCTIONS =
ImmutableSet.of(Sum.class, Sum0.class, Min.class, Max.class,
AnyValue.class, Count.class);
private static final int DECOMPOSE_REPEAT_THRESHOLD = 3;
+ private static final int BALANCE_MULTIPLIER = 128;
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
@@ -546,11 +547,10 @@ public class DecomposeRepeatWithPreAggregation extends
DefaultPlanRewriter<Disti
}
for (Expression candidate : candidates) {
ColumnStatistic columnStatistic =
inputStats.findColumnStatistics(candidate);
- if (columnStatistic == null || columnStatistic.isUnKnown() ||
columnStatistic.hotValues == null) {
+ if (columnStatistic == null || columnStatistic.isUnKnown()) {
continue;
}
- if (StatisticsUtil.isBalanced(columnStatistic, totalInstanceNum,
- ShuffleKeyPruneUtils.shuffleKeyHotValueThreshold,
inputStats.getRowCount())) {
+ if (isBalanced(columnStatistic, totalInstanceNum,
inputStats.getRowCount())) {
return Optional.of(candidate);
}
}
@@ -620,4 +620,11 @@ public class DecomposeRepeatWithPreAggregation extends
DefaultPlanRewriter<Disti
return repeat.withNormalizedExpr(replacedNewGroupingSets,
replacedRepeatOutputs,
repeat.getGroupingId().get(), child);
}
+
+ private static boolean isBalanced(ColumnStatistic columnStatistic, int
instanceNum, double rowCount) {
+ double ndv = columnStatistic.ndv;
+ return ndv > instanceNum * BALANCE_MULTIPLIER
+ && !StatisticsUtil.hasSignificantHotValues(columnStatistic,
+ ShuffleKeyPruneUtils.shuffleKeyHotValueThreshold, rowCount,
false);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index c59992082ad..0bc6ea8a699 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -1402,10 +1402,11 @@ public class StatisticsUtil {
* Used by shuffle key prune and skew detection rules.
* Returns false when hotValues is null (not collected) or empty
(collected but no hot values).
*/
- public static boolean hasSignificantHotValues(ColumnStatistic
columnStatistic, double minRatio, double rowCount) {
+ public static boolean hasSignificantHotValues(ColumnStatistic
columnStatistic, double minRatio, double rowCount,
+ boolean strictWhenHotValueUnknow) {
Map<Literal, Float> hotValues = columnStatistic.getHotValues();
if (hotValues == null) {
- return true;
+ return strictWhenHotValueUnknow;
}
return columnStatistic.numNulls / rowCount > minRatio
|| hotValues.values().stream().anyMatch(ratio -> ratio >=
minRatio);
@@ -1415,6 +1416,6 @@ public class StatisticsUtil {
double rowCount) {
double ndv = columnStatistic.ndv;
return ndv > instanceNum *
AggregateUtils.NDV_INSTANCE_BALANCE_MULTIPLIER
- && !hasSignificantHotValues(columnStatistic, minRatio,
rowCount);
+ && !hasSignificantHotValues(columnStatistic, minRatio,
rowCount, true);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java
index ef58076e061..e548999bb53 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java
@@ -631,7 +631,7 @@ public class DecomposeRepeatWithPreAggregationTest extends
TestWithFeService imp
Optional<Expression> chosen2 = (Optional<Expression>) method.invoke(
rule, groupingSets, -1, candidates, stats, 50);
Assertions.assertTrue(chosen2.isPresent());
- Assertions.assertEquals(b, chosen2.get());
+ Assertions.assertEquals(c, chosen2.get());
// inputStats null -> chooseByNdv returns empty for every group ->
empty
@SuppressWarnings("unchecked")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
