This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9fc6513ac05 (fix)(eager-agg) null-to-non-null safety check and remove
redundant normalize calls in eager aggregation (#64820)
9fc6513ac05 is described below
commit 9fc6513ac056076b1876d3c810ce9940f368c5de
Author: minghong <[email protected]>
AuthorDate: Mon Jul 6 18:57:12 2026 +0800
(fix)(eager-agg) null-to-non-null safety check and remove redundant
normalize calls in eager aggregation (#64820)
### What problem does this PR solve?
1. generalize null-to-non-null safety check via NullToNonNullFunction
marker interface. (covering
CaseWhen、If、Coalesce、Nvl、NullOrEmpty、NotNullOrEmpty、 nullSafeEqual)
2. remove redundant normalize calls in eager aggregation
---
.../nereids/rules/analysis/NormalizeAggregate.java | 2 +-
.../rewrite/eageraggregation/EagerAggRewriter.java | 54 ++---
.../eageraggregation/PushDownAggContext.java | 27 ++-
.../eageraggregation/PushDownAggregation.java | 34 +--
.../doris/nereids/trees/expressions/CaseWhen.java | 2 +-
.../doris/nereids/trees/expressions/IsFalse.java | 3 +-
.../doris/nereids/trees/expressions/IsNull.java | 2 +-
.../doris/nereids/trees/expressions/IsTrue.java | 3 +-
.../nereids/trees/expressions/NullSafeEqual.java | 2 +-
.../trees/expressions/NullToNonNullFunction.java | 57 +++++
.../expressions/functions/scalar/Coalesce.java | 3 +-
.../trees/expressions/functions/scalar/If.java | 3 +-
.../functions/scalar/Ipv4StringToNumOrDefault.java | 3 +-
.../functions/scalar/Ipv6StringToNumOrDefault.java | 3 +-
.../expressions/functions/scalar/NonNullable.java | 4 +-
.../functions/scalar/NotNullOrEmpty.java | 3 +-
.../expressions/functions/scalar/NullOrEmpty.java | 3 +-
.../trees/expressions/functions/scalar/Nvl.java | 3 +-
.../functions/scalar/ToIpv4OrDefault.java | 3 +-
.../functions/scalar/ToIpv6OrDefault.java | 3 +-
.../nereids/trees/plans/logical/LogicalUnion.java | 2 +-
.../eageraggregation/EagerAggRewriterTest.java | 239 ++++++++++++++++++++-
22 files changed, 386 insertions(+), 72 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
index 83a2332290f..7b5e88060de 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java
@@ -156,7 +156,7 @@ public class NormalizeAggregate implements
RewriteRuleFactory, NormalizeToSlot {
* c. alias with window-agg
*/
@SuppressWarnings("checkstyle:UnusedLocalVariable")
- public LogicalPlan normalizeAgg(LogicalAggregate<Plan> aggregate,
Optional<LogicalHaving<?>> having,
+ private LogicalPlan normalizeAgg(LogicalAggregate<Plan> aggregate,
Optional<LogicalHaving<?>> having,
CascadesContext ctx) {
// Push down exprs:
// collect group by exprs
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
index 5311cb96383..55ee5780607 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
@@ -18,18 +18,17 @@
package org.apache.doris.nereids.rules.rewrite.eageraggregation;
import org.apache.doris.common.Pair;
-import org.apache.doris.nereids.rules.analysis.NormalizeAggregate;
import org.apache.doris.nereids.rules.rewrite.StatsDerive;
import
org.apache.doris.nereids.rules.rewrite.eageraggregation.EagerAggHints.Action;
import org.apache.doris.nereids.stats.ExpressionEstimation;
import org.apache.doris.nereids.trees.expressions.Alias;
-import org.apache.doris.nereids.trees.expressions.CaseWhen;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.Multiply;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
@@ -223,7 +222,7 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
return Pair.of(false, false);
}
if (deduplicateOnly) {
- return adjustPushSideForCaseWhen(join, context, toLeft, toRight);
+ return adjustPushSideForNullToNonNull(join, context, toLeft,
toRight);
}
if (toLeft && toRight) {
return join.getJoinType().isInnerOrCrossJoin()
@@ -231,7 +230,7 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
: Pair.of(false, false);
}
// one-side push down
- Pair<Boolean, Boolean> pushSide = adjustPushSideForCaseWhen(join,
context, toLeft, toRight);
+ Pair<Boolean, Boolean> pushSide = adjustPushSideForNullToNonNull(join,
context, toLeft, toRight);
if (!pushSide.first && !pushSide.second) {
return pushSide;
}
@@ -251,13 +250,14 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
&& hasAggNeedJoinMultiplicityRecovery(oppositeAggFunctions);
}
- private Pair<Boolean, Boolean> adjustPushSideForCaseWhen(
+ private Pair<Boolean, Boolean> adjustPushSideForNullToNonNull(
LogicalJoin<? extends Plan, ? extends Plan> join,
PushDownAggContext context,
boolean toLeft, boolean toRight) {
- // Do not push aggregation to the nullable side of outer joins when
agg function contains case-when.
- // CaseWhen expressions may produce non-null values from null-padded
rows (e.g., WHEN col IS NULL THEN -54),
+ // Do not push aggregation to the nullable side of outer joins when
agg function contains
+ // a NullToNonNullFunction (e.g. COALESCE, NVL, IF, CASE WHEN,
NULL_OR_EMPTY).
+ // These expressions may produce non-null values from null-padded rows,
// so pre-aggregation before the join loses those contributions.
- if (!(context.hasDecomposedAggIf || context.hasCaseWhen)) {
+ if (!(context.hasDecomposedAggIf || context.containsNullToNonNull)) {
return Pair.of(toLeft, toRight);
}
JoinType joinType = join.getJoinType();
@@ -274,8 +274,8 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
return Pair.of(toLeft, toRight);
}
- // Do not push agg(literal) or agg(preserved_side_col) to the nullable
side of outer joins.
- // Aggregates like count(*), sum(2), min(1) etc. aggregate over all
physical rows,
+ // Do not push count(*)/agg(literal)(e.g. sum(2), min(1)) to the nullable
side of outer joins.
+ // Aggregates without input from the nullable side aggregate over all
physical rows,
// including null-extended rows from the outer join.
// After pushdown to the nullable side, unmatched rows produce NULL for
the pre-aggregated value,
// losing the contribution of those rows (e.g. sum(2) should add 2 per
unmatched row,
@@ -383,21 +383,23 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
aggFunctions.add(newAggFunc);
}
// After pushing expressions past the project, the agg functions may
now
- // contain If/CaseWhen that were hidden behind slot references before.
- // e.g. count(#slot) where #slot = if(cond, a, b) in the project.
- // We must re-check and update hasCaseWhen accordingly.
- boolean newHasCaseWhen = context.hasCaseWhen;
- if (!newHasCaseWhen) {
+ // contain NullToNonNull expressions that were hidden behind slot
references before.
+ // e.g. count(#slot) where #slot = coalesce(a, 0) in the project.
+ // We must re-check and update containsNullToNonNull accordingly.
+ boolean newContainsNullToNonNull = context.containsNullToNonNull;
+ if (!newContainsNullToNonNull) {
for (AggregateFunction aggFunc : aggFunctions) {
- if (aggFunc.anyMatch(e -> e instanceof CaseWhen || e
instanceof If)) {
- newHasCaseWhen = true;
+ if (aggFunc.children().stream().anyMatch(
+ arg -> arg.anyMatch(e ->
+
NullToNonNullFunction.canConvertNullToNonNull((Expression) e)))) {
+ newContainsNullToNonNull = true;
break;
}
}
}
PushDownAggContext newContext = new PushDownAggContext(aggFunctions,
groupKeys, aliasMap,
context.getCascadesContext(), context.isPassThroughBigJoin(),
- context.hasDecomposedAggIf, newHasCaseWhen,
+ context.hasDecomposedAggIf, newContainsNullToNonNull,
context.getBilateralState(), context.needOutputCount());
return newContext;
}
@@ -528,7 +530,7 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
.collect(Collectors.toList());
PushDownAggContext contextForChild = new
PushDownAggContext(aggFunctionsForChild, groupKeysForChild,
aliasMapForChild, context.getCascadesContext(),
- context.isPassThroughBigJoin(),
context.hasDecomposedAggIf, context.hasCaseWhen,
+ context.isPassThroughBigJoin(),
context.hasDecomposedAggIf, context.containsNullToNonNull,
context.getBilateralState(), context.needOutputCount());
inheritHintActionsToUnionChild(context, contextForChild,
aggFunctionsForChild);
Plan newChild = child.accept(this, contextForChild);
@@ -791,21 +793,23 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
aggOutputExpressions.add(outputCountAlias.get());
}
LogicalAggregate genAgg = new
LogicalAggregate(context.getGroupKeys(), aggOutputExpressions, child);
- NormalizeAggregate normalizeAggregate = new NormalizeAggregate();
- Plan normalized = normalizeAggregate.normalizeAgg(genAgg,
Optional.empty(),
- context.getCascadesContext());
for (AggregateFunction func : context.getAggFunctions()) {
Alias a = context.getAliasMap().get(func);
- Slot pushedSlot = normalized.getOutput().stream()
+ Slot pushedSlot = genAgg.getOutput().stream()
.filter(slot -> slot.getExprId().equals(a.getExprId()))
.findFirst()
.orElse(a.toSlot());
context.getBilateralState().registerPushedAggFuncSlot(a.getExprId(),
pushedSlot);
}
outputCountAlias.ifPresent(
- alias ->
context.getBilateralState().registerCountSlot(normalized, alias.toSlot()));
- return normalized;
+ alias ->
context.getBilateralState().registerCountSlot(genAgg, alias.toSlot()));
+ // Return non-normalized aggregate directly. The optimizer
fix-point loop
+ // will invoke NormalizeAggregate rule in the next iteration if
needed.
+ // Explicit normalization here is redundant (the original
aggregate was
+ // already normalized) and could alter expression shapes that the
push-down
+ // logic relies on for correct NullToNonNullFunction detection.
+ return genAgg;
} else {
return child;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggContext.java
index 0d2832617e6..cd315da374a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggContext.java
@@ -44,10 +44,15 @@ public class PushDownAggContext {
// count(if(...)): if(...) push down as a whole
// sum/min/max(if(truePart, elsePart)): if(...) can be split to
sum(truePart) and sum(elsePart)
public final boolean hasDecomposedAggIf;
- // When aggFunc(if(...)) is present, pushing down the null-supplemented
side of the outer join is avoided.
- // This is because null values are highly error-prone,
- // so the push-down operation is not performed during hashCaseWhen.
- public final boolean hasCaseWhen;
+ // When aggFunc contains expressions that can convert NULL to non-NULL
+ // (e.g. COALESCE, NVL, IF, CASE WHEN), pushing down to the nullable side
of
+ // an outer join is blocked — null-extended rows would be wrongly counted.
+ //
+ // TODO: This is conservative — a per-function check (rather than a global
flag)
+ // would allow pushing non-NullToNonNull aggregate functions (e.g. sum(a))
to
+ // the nullable side even when another agg function contains a
NullToNonNull
+ // expression. Currently, one problematic agg function blocks all
push-down.
+ public final boolean containsNullToNonNull;
private final List<AggregateFunction> aggFunctions;
private final List<SlotReference> groupKeys;
private final HashMap<AggregateFunction, Alias> aliasMap;
@@ -66,10 +71,10 @@ public class PushDownAggContext {
public PushDownAggContext(List<AggregateFunction> aggFunctions,
List<SlotReference> groupKeys, Map<AggregateFunction, Alias>
aliasMap, CascadesContext cascadesContext,
- boolean passThroughBigJoin, boolean hasDecomposedAggIf, boolean
hasCaseWhen,
+ boolean passThroughBigJoin, boolean hasDecomposedAggIf, boolean
containsNullToNonNull,
BilateralState bilateralState) {
- this(aggFunctions, groupKeys, aliasMap, cascadesContext,
passThroughBigJoin, hasDecomposedAggIf, hasCaseWhen,
- bilateralState, false);
+ this(aggFunctions, groupKeys, aliasMap, cascadesContext,
passThroughBigJoin,
+ hasDecomposedAggIf, containsNullToNonNull, bilateralState,
false);
}
/**
@@ -77,7 +82,7 @@ public class PushDownAggContext {
*/
public PushDownAggContext(List<AggregateFunction> aggFunctions,
List<SlotReference> groupKeys, Map<AggregateFunction, Alias>
aliasMap, CascadesContext cascadesContext,
- boolean passThroughBigJoin, boolean hasDecomposedAggIf, boolean
hasCaseWhen,
+ boolean passThroughBigJoin, boolean hasDecomposedAggIf, boolean
containsNullToNonNull,
BilateralState bilateralState, boolean needOutputCount) {
this.groupKeys =
groupKeys.stream().distinct().collect(Collectors.toList());
this.aggFunctions = ImmutableList.copyOf(aggFunctions);
@@ -105,7 +110,7 @@ public class PushDownAggContext {
.collect(ImmutableSet.toImmutableSet());
this.passThroughBigJoin = passThroughBigJoin;
this.hasDecomposedAggIf = hasDecomposedAggIf;
- this.hasCaseWhen = hasCaseWhen;
+ this.containsNullToNonNull = containsNullToNonNull;
this.needOutputCount = needOutputCount;
this.bilateralState = Objects.requireNonNull(bilateralState,
"bilateralState cannot be null");
for (Map.Entry<AggregateFunction, Alias> entry :
this.aliasMap.entrySet()) {
@@ -143,7 +148,7 @@ public class PushDownAggContext {
public PushDownAggContext withGroupKeys(List<SlotReference> groupKeys) {
return new PushDownAggContext(aggFunctions, groupKeys, aliasMap,
- cascadesContext, passThroughBigJoin, hasDecomposedAggIf,
hasCaseWhen,
+ cascadesContext, passThroughBigJoin, hasDecomposedAggIf,
containsNullToNonNull,
bilateralState, needOutputCount);
}
@@ -154,7 +159,7 @@ public class PushDownAggContext {
Map<AggregateFunction, Alias> branchAliasMap, List<SlotReference>
groupKeys,
boolean passThroughBigJoin, boolean needOutputCount) {
return new PushDownAggContext(branchAggFunctions, groupKeys,
branchAliasMap,
- cascadesContext, passThroughBigJoin, hasDecomposedAggIf,
hasCaseWhen,
+ cascadesContext, passThroughBigJoin, hasDecomposedAggIf,
containsNullToNonNull,
bilateralState, needOutputCount);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
index ef75f37a18f..0cb410c1e3c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
@@ -18,13 +18,12 @@
package org.apache.doris.nereids.rules.rewrite.eageraggregation;
import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.rules.analysis.NormalizeAggregate;
import org.apache.doris.nereids.rules.rewrite.AdjustNullable;
import org.apache.doris.nereids.trees.expressions.Alias;
-import org.apache.doris.nereids.trees.expressions.CaseWhen;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
@@ -57,7 +56,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -129,7 +127,7 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
Set<AggregateFunction> aggFunctions = Sets.newHashSet();
boolean hasDecomposedAggIf = false;
- boolean hasCaseWhen = false;
+ boolean containsNullToNonNull = false;
Map<NamedExpression, List<AggregateFunction>>
aggFunctionsForOutputExpressions = Maps.newHashMap();
for (NamedExpression aggOutput : agg.getOutputExpressions()) {
List<AggregateFunction> funcs = Lists.newArrayList();
@@ -143,14 +141,18 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
if (aggFunction.containsVolatileExpression()) {
return agg;
}
- // CaseWhen and If (which CASE WHEN is normalized into)
must both be checked.
- // When an agg function contains an If/CaseWhen whose
condition tests IS NULL
- // (e.g. count(if(col IS NULL, value, NULL))), pushing it
to the nullable side
- // of an outer join produces wrong results: null-extended
rows make "col IS NULL"
- // TRUE at the top level, but the pre-aggregated count
slot becomes NULL after
- // null-extension, and ifnull(sum(NULL), 0) = 0 instead of
the correct 1.
- if (!hasCaseWhen && aggFunction.anyMatch(e -> e instanceof
CaseWhen || e instanceof If)) {
- hasCaseWhen = true;
+ // NullToNonNullFunction / AlwaysNotNullable: expressions
that can convert NULL
+ // input to non-NULL output (e.g. COALESCE, NVL, IF, CASE
WHEN, Array).
+ // When an agg function contains such an expression
wrapping a column from the
+ // nullable side of an outer join, null-extended rows
would produce non-NULL values
+ // that get counted by the aggregation. But the
pre-aggregation on the base table
+ // cannot see null-extended rows (they are produced by the
join), so the push-down
+ // would lose those contributions — producing wrong
results.
+ if (!containsNullToNonNull
+ && aggFunction.children().stream().anyMatch(
+ arg -> arg.anyMatch(e ->
+
NullToNonNullFunction.canConvertNullToNonNull((Expression) e)))) {
+ containsNullToNonNull = true;
}
if (aggFunction.arity() > 0 && aggFunction.child(0)
instanceof If
&& !(aggFunction instanceof Count)) {
@@ -189,13 +191,13 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
}
}
- groupKeys = groupKeys.stream().distinct().collect(Collectors.toList());
if (!checkSubTreePattern(agg.child())) {
return agg;
}
+ groupKeys = groupKeys.stream().distinct().collect(Collectors.toList());
PushDownAggContext pushDownContext = new PushDownAggContext(new
ArrayList<>(aggFunctions),
- groupKeys, null, context.getCascadesContext(), false,
hasDecomposedAggIf, hasCaseWhen,
+ groupKeys, null, context.getCascadesContext(), false,
hasDecomposedAggIf, containsNullToNonNull,
new BilateralState());
if (groupKeys.isEmpty()) {
return agg;
@@ -265,9 +267,7 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
}
LogicalAggregate<Plan> eagerAgg =
agg.withAggOutputChild(newOutputExpressions, child);
- NormalizeAggregate normalizeAggregate = new
NormalizeAggregate();
- return normalizeAggregate.normalizeAgg(eagerAgg,
Optional.empty(),
- context.getCascadesContext());
+ return eagerAgg;
}
} catch (RuntimeException e) {
String msg = "PushDownAggregation failed: " + e.getMessage() +
"\n" + agg.treeString();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java
index 4c81393e6b0..a2cd9cb4730 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java
@@ -43,7 +43,7 @@ import java.util.function.Supplier;
*
* Children layout: [value?, WhenClause+, defaultValue?]
*/
-public class CaseWhen extends Expression implements NeedSessionVarGuard {
+public class CaseWhen extends Expression implements NeedSessionVarGuard,
NullToNonNullFunction {
private final Optional<Expression> value;
private final List<WhenClause> whenClauses;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java
index fdd17c296fc..09f2a77039f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java
@@ -33,7 +33,8 @@ import java.util.Objects;
/**
* expr is false predicate.
*/
-public class IsFalse extends Expression implements UnaryExpression,
AlwaysNotNullable {
+public class IsFalse extends Expression
+ implements UnaryExpression, AlwaysNotNullable, NullToNonNullFunction {
public IsFalse(Expression e) {
super(ImmutableList.of(e));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java
index a587ac6f8e2..56371c4c77c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java
@@ -33,7 +33,7 @@ import java.util.Objects;
/**
* expr is null predicate.
*/
-public class IsNull extends Expression implements UnaryExpression,
AlwaysNotNullable {
+public class IsNull extends Expression implements UnaryExpression,
AlwaysNotNullable, NullToNonNullFunction {
public IsNull(Expression e) {
super(ImmutableList.of(e));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java
index 366ab0056dc..1876a6dec54 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java
@@ -33,7 +33,8 @@ import java.util.Objects;
/**
* expr is true predicate.
*/
-public class IsTrue extends Expression implements UnaryExpression,
AlwaysNotNullable {
+public class IsTrue extends Expression
+ implements UnaryExpression, AlwaysNotNullable, NullToNonNullFunction {
public IsTrue(Expression e) {
super(ImmutableList.of(e));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
index 6eda6c738b7..5bd0769e259 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
@@ -29,7 +29,7 @@ import java.util.List;
* Null safe equal expression: a <=> b.
* Unlike normal equal to expression, null <=> null is true.
*/
-public class NullSafeEqual extends EqualPredicate implements AlwaysNotNullable
{
+public class NullSafeEqual extends EqualPredicate implements
AlwaysNotNullable, NullToNonNullFunction {
public NullSafeEqual(Expression left, Expression right) {
this(left, right, false);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullToNonNullFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullToNonNullFunction.java
new file mode 100644
index 00000000000..e281b4f03e4
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullToNonNullFunction.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
+
+/**
+ * Marker interface for expressions that can convert NULL input into a
non-NULL output.
+ *
+ * For example: Coalesce(NULL, 2) → 2, Nvl(NULL, 0) → 0, NullOrEmpty(NULL) →
true.
+ *
+ * This is significant for outer-join push-down safety: when an aggregate
function contains
+ * a NullToNonNull expression wrapping a column from the nullable side of an
outer join,
+ * the aggregation must NOT be pushed down. Null-extended rows (produced by
the join for
+ * unmatched rows) have NULL for all nullable-side columns. The NullToNonNull
expression
+ * would convert those NULLs to non-NULL values, and the pre-aggregation would
miss those
+ * contributions because null-extended rows do not exist in the base table.
+ *
+ * <p>Note: {@link AlwaysNotNullable} expressions with input slots (e.g.
Array, JsonArray,
+ * JsonObject, CreateStruct, CreateMap) are also blocked from being pushed to
the nullable
+ * side of outer joins via a separate check in {@link
#canConvertNullToNonNull(Expression)}.
+ */
+public interface NullToNonNullFunction {
+
+ /**
+ * Check whether an expression can convert NULL input to non-NULL output.
+ * This covers both {@link NullToNonNullFunction} (e.g. Coalesce, Nvl, If,
CaseWhen,
+ * NullOrEmpty, IsNull, IsTrue, IsFalse, NonNullable)
+ * and {@link AlwaysNotNullable} expressions with input slots (e.g. Array,
JsonArray,
+ * CreateStruct, CreateMap), which always produce non-NULL output
regardless of NULL inputs.
+ *
+ * <p>In outer-join push-down safety checks, any expression matching this
predicate
+ * must NOT be pushed to the nullable side, because null-extended rows
(produced by the
+ * join for unmatched rows) would produce non-NULL values that get
aggregated, but the
+ * pre-aggregation on the base table cannot see those rows — resulting in
wrong results.
+ */
+ static boolean canConvertNullToNonNull(Expression e) {
+ return e instanceof NullToNonNullFunction
+ || (e instanceof AlwaysNotNullable
+ && !e.getInputSlots().isEmpty());
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Coalesce.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Coalesce.java
index 2bc8a67ff08..2c63b17ef2e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Coalesce.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Coalesce.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NeedSessionVarGuard;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
@@ -39,7 +40,7 @@ import java.util.stream.Collectors;
/**
* ScalarFunction 'coalesce'. This class is generated by GenerateFunction.
*/
-public class Coalesce extends ScalarFunction implements CustomSignature,
NeedSessionVarGuard {
+public class Coalesce extends ScalarFunction implements CustomSignature,
NeedSessionVarGuard, NullToNonNullFunction {
/**
* constructor with 1 or more arguments.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/If.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/If.java
index 118ed8622ec..abfb322a717 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/If.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/If.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.analyzer.Unbound;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NeedSessionVarGuard;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression;
@@ -38,7 +39,7 @@ import java.util.Optional;
* ScalarFunction 'if'. This class is generated by GenerateFunction.
*/
public class If extends ScalarFunction
- implements TernaryExpression, CustomSignature, NeedSessionVarGuard {
+ implements TernaryExpression, CustomSignature, NeedSessionVarGuard,
NullToNonNullFunction {
/**
* constructor with 3 arguments.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv4StringToNumOrDefault.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv4StringToNumOrDefault.java
index 35eedb5c621..fd837d22157 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv4StringToNumOrDefault.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv4StringToNumOrDefault.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -36,7 +37,7 @@ import java.util.List;
* scalar function ipv4_string_to_num_or_default
*/
public class Ipv4StringToNumOrDefault extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, NullToNonNullFunction {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv6StringToNumOrDefault.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv6StringToNumOrDefault.java
index b431fb0741f..beea54c451e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv6StringToNumOrDefault.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv6StringToNumOrDefault.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -35,7 +36,7 @@ import java.util.List;
* scalar function ipv6_string_to_num_or_default
*/
public class Ipv6StringToNumOrDefault extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, NullToNonNullFunction {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java
index a4bc35b414d..c5fa29c5020 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -32,7 +33,8 @@ import java.util.List;
/**
* change nullable input col to non_nullable col
*/
-public class NonNullable extends ScalarFunction implements UnaryExpression,
CustomSignature, AlwaysNotNullable {
+public class NonNullable extends ScalarFunction
+ implements UnaryExpression, CustomSignature, AlwaysNotNullable,
NullToNonNullFunction {
public NonNullable(Expression expr) {
super("non_nullable", expr);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NotNullOrEmpty.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NotNullOrEmpty.java
index fa3741b9491..a7e8e8b2f5f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NotNullOrEmpty.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NotNullOrEmpty.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'not_null_or_empty'. This class is generated by
GenerateFunction.
*/
public class NotNullOrEmpty extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, NullToNonNullFunction {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NullOrEmpty.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NullOrEmpty.java
index 8b511e3f68d..68b5978c179 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NullOrEmpty.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NullOrEmpty.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'null_or_empty'. This class is generated by GenerateFunction.
*/
public class NullOrEmpty extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements NullToNonNullFunction, UnaryExpression,
ExplicitlyCastableSignature, AlwaysNotNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java
index 8cdb95a7ca1..4c522b6b333 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NeedSessionVarGuard;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -36,7 +37,7 @@ import java.util.Optional;
* ScalarFunction 'nvl'. This class is generated by GenerateFunction.
*/
public class Nvl extends ScalarFunction
- implements BinaryExpression, CustomSignature, NeedSessionVarGuard {
+ implements BinaryExpression, CustomSignature, NeedSessionVarGuard,
NullToNonNullFunction {
/**
* constructor with 2 arguments.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv4OrDefault.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv4OrDefault.java
index d9f61e8a7fb..17a06327e28 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv4OrDefault.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv4OrDefault.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -36,7 +37,7 @@ import java.util.List;
* scalar function to_ipv4_or_default
*/
public class ToIpv4OrDefault extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, NullToNonNullFunction {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(IPv4Type.INSTANCE).args(VarcharType.SYSTEM_DEFAULT),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv6OrDefault.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv6OrDefault.java
index 142e35b4ca2..8b7db83d227 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv6OrDefault.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToIpv6OrDefault.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -36,7 +37,7 @@ import java.util.List;
* scalar function to_ipv6_or_default
*/
public class ToIpv6OrDefault extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, NullToNonNullFunction {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(IPv6Type.INSTANCE).args(VarcharType.SYSTEM_DEFAULT),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
index 5f2d912a8dd..660ecef4b53 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
@@ -435,7 +435,7 @@ public class LogicalUnion extends LogicalSetOperation
implements Union, OutputPr
return changed ? castedRow.build() : row;
}
- public LogicalSetOperation withChildrenAndOutputs(List<Plan> children,
List<NamedExpression> newOuptuts,
+ public LogicalUnion withChildrenAndOutputs(List<Plan> children,
List<NamedExpression> newOuptuts,
List<List<SlotReference>> childrenOutputs) {
Preconditions.checkArgument(children.size() == childrenOutputs.size(),
"children size %s is not equals with children outputs size %s",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
index 758e23a4ba0..5e58f6a8ef5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
@@ -272,10 +272,10 @@ class EagerAggRewriterTest extends TestWithFeService
implements MemoPatternMatch
void testNotPushCountCaseWhenWithElseToNullableSideViaProject() {
// When CASE WHEN has an ELSE clause (e.g. CASE WHEN cond THEN -121
ELSE 2 END),
// NormalizeAggregate extracts if(cond, -121, 2) into a Project as a
slot,
- // so the aggregate becomes count(#slot). At the agg level,
hasCaseWhen=false
+ // so the aggregate becomes count(#slot). At the agg level,
containsNullToNonNull=false
// because count(#slot) contains no If/CaseWhen.
//
- // EagerAggRewriter must recheck hasCaseWhen after substituting
through the
+ // EagerAggRewriter must recheck containsNullToNonNull after
substituting through the
// Project (in createContextFromProject), otherwise count(if(cond,
-121, 2))
// is incorrectly pushed to the nullable side of an outer join.
//
@@ -551,6 +551,241 @@ class EagerAggRewriterTest extends TestWithFeService
implements MemoPatternMatch
}
}
+ //
=========================================================================
+ // NullToNonNullFunction safety guard tests:
+ // Agg functions wrapping expressions that convert NULL to non-NULL values
+ // (IsNull, NullSafeEqual, COALESCE, NVL, NULL_OR_EMPTY, NOT_NULL_OR_EMPTY)
+ // must NOT be pushed to the nullable side of outer joins. Null-extended
rows
+ // would produce non-NULL values at the top level, but the pre-aggregation
+ // cannot see those rows, producing wrong results.
+ //
=========================================================================
+
+ @Test
+ void testNotPushBareIsNullToNullableSideOfOuterJoin() {
+ // count(col IS NULL): IsNull converts NULL input to TRUE,
+ // so null-extended rows contribute 1. Pushed pre-agg on the base table
+ // would miss those rows.
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ // RIGHT JOIN: t1 is the nullable side (left side of RIGHT JOIN)
+ // count(t1.name IS NULL) must NOT be pushed to t1
+ String sql = "select count(t1.name IS NULL), t2.id2"
+ + " from t1 right join t2 on t1.id1 = t2.id2 group by
t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(logicalAggregate(), any()))
+ .printlnTree();
+
+ // LEFT JOIN: t2 is the nullable side (right side of LEFT JOIN)
+ // count(t2.name IS NULL) must NOT be pushed to t2
+ sql = "select count(t2.name IS NULL), t1.id1"
+ + " from t1 left join t2 on t1.id1 = t2.id2 group by
t1.id1";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(any(), logicalAggregate()))
+ .printlnTree();
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
+ @Test
+ void testNotPushNullSafeEqualToNullableSideOfOuterJoin() {
+ // count(col <=> value): NullSafeEqual always returns TRUE/FALSE
(never NULL),
+ // so null-extended rows contribute to the count. Pushed pre-agg would
miss them.
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ // RIGHT JOIN: t1 is the nullable side
+ // count(t1.name <=> NULL) must NOT be pushed to t1
+ String sql = "select count(t1.name <=> NULL), t2.id2"
+ + " from t1 right join t2 on t1.id1 = t2.id2 group by
t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(logicalAggregate(), any()))
+ .printlnTree();
+
+ // LEFT JOIN: t2 is the nullable side
+ // count(t2.name <=> 'x') must NOT be pushed to t2
+ sql = "select count(t2.name <=> 'x'), t1.id1"
+ + " from t1 left join t2 on t1.id1 = t2.id2 group by
t1.id1";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(any(), logicalAggregate()))
+ .printlnTree();
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
+ @Test
+ void testNotPushCoalesceToNullableSideOfOuterJoin() {
+ // count(coalesce(col, default)): COALESCE converts NULL input to
default,
+ // so null-extended rows contribute 1. Pushed pre-agg would miss them.
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ // RIGHT JOIN: t1 is the nullable side
+ String sql = "select count(coalesce(t1.name, 'default')), t2.id2"
+ + " from t1 right join t2 on t1.id1 = t2.id2 group by
t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(logicalAggregate(), any()))
+ .printlnTree();
+
+ // LEFT JOIN: t2 is the nullable side
+ sql = "select count(coalesce(t2.name, 'x')), t1.id1"
+ + " from t1 left join t2 on t1.id1 = t2.id2 group by
t1.id1";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(any(), logicalAggregate()))
+ .printlnTree();
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
+ @Test
+ void testNotPushNvlToNullableSideOfOuterJoin() {
+ // count(nvl(col, default)): NVL converts NULL input to default.
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ // RIGHT JOIN: t1 is the nullable side
+ String sql = "select count(nvl(t1.name, 'default')), t2.id2"
+ + " from t1 right join t2 on t1.id1 = t2.id2 group by
t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(logicalAggregate(), any()))
+ .printlnTree();
+
+ // LEFT JOIN: t2 is the nullable side
+ sql = "select count(nvl(t2.name, 0)), t1.id1"
+ + " from t1 left join t2 on t1.id1 = t2.id2 group by
t1.id1";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(any(), logicalAggregate()))
+ .printlnTree();
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
+ @Test
+ void testNotPushNullOrEmptyToNullableSideOfOuterJoin() {
+ // count(null_or_empty(col)): NULL_OR_EMPTY converts NULL to TRUE.
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ // RIGHT JOIN: t1 is the nullable side
+ String sql = "select count(null_or_empty(t1.name)), t2.id2"
+ + " from t1 right join t2 on t1.id1 = t2.id2 group by
t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(logicalAggregate(), any()))
+ .printlnTree();
+
+ // LEFT JOIN: t2 is the nullable side
+ sql = "select count(null_or_empty(t2.name)), t1.id1"
+ + " from t1 left join t2 on t1.id1 = t2.id2 group by
t1.id1";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(any(), logicalAggregate()))
+ .printlnTree();
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
+ @Test
+ void testNotPushNotNullOrEmptyToNullableSideOfOuterJoin() {
+ // count(not_null_or_empty(col)): NOT_NULL_OR_EMPTY converts NULL to
FALSE
+ // (non-NULL).
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ // RIGHT JOIN: t1 is the nullable side
+ String sql = "select count(not_null_or_empty(t1.name)), t2.id2"
+ + " from t1 right join t2 on t1.id1 = t2.id2 group by
t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(logicalAggregate(), any()))
+ .printlnTree();
+
+ // LEFT JOIN: t2 is the nullable side
+ sql = "select count(not_null_or_empty(t2.name)), t1.id1"
+ + " from t1 left join t2 on t1.id1 = t2.id2 group by
t1.id1";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(any(), logicalAggregate()))
+ .printlnTree();
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
+ @Test
+ void testNotPushAlwaysNotNullableToNullableSideOfOuterJoin() {
+ // count(array(col)): Array is AlwaysNotNullable (not
NullToNonNullFunction),
+ // but it still converts NULL input to non-NULL output (array(NULL) →
[NULL]),
+ // so it must be caught by the AlwaysNotNullable fallback branch in
+ // NullToNonNullFunction.canConvertNullToNonNull().
+ // Pushing pre-agg under the nullable side would miss null-extended
row contributions.
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ // RIGHT JOIN: t1 is the nullable side, array(t1.name) on the
nullable side
+ String sql = "select count(array(t1.name)), t2.id2"
+ + " from t1 right join t2 on t1.id1 = t2.id2 group by
t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(logicalAggregate(), any()))
+ .printlnTree();
+
+ // LEFT JOIN: t2 is the nullable side, array(t2.name) on the
nullable side
+ sql = "select count(array(t2.name)), t1.id1"
+ + " from t1 left join t2 on t1.id1 = t2.id2 group by
t1.id1";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .nonMatch(logicalJoin(any(), logicalAggregate()))
+ .printlnTree();
+
+ // Verify that array on the non-nullable side CAN still be pushed.
+ // INNER JOIN: array on either side is safe because no
null-extended rows
+ sql = "select count(array(t1.name)), t2.id2"
+ + " from t1 join t2 on t1.id1 = t2.id2 group by t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+
.matches(logicalAggregate(logicalProject(logicalJoin(logicalAggregate(),
any()))))
+ .printlnTree();
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
@Test
void testInvalidFilterContextFallsBackToCurrentFilter() {
connectContext.getSessionVariable().setEagerAggregationMode(1);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
