This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 432420eac78 branch-4.2: [fix](rbo) Preserve column mappings when
merging nested set operations #69131 (#69428)
432420eac78 is described below
commit 432420eac78f7c536dff74efce21040408b82bac
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 22 18:42:55 2026 +0800
branch-4.2: [fix](rbo) Preserve column mappings when merging nested set
operations #69131 (#69428)
Cherry-picked from #69131
Co-authored-by: feiniaofeiafei <[email protected]>
---
.../nereids/rules/rewrite/MergeSetOperations.java | 41 +++++++++-
.../rules/rewrite/MergeSetOperationsExcept.java | 5 +-
.../rewrite/SetOperationOutputMappingTest.java | 53 +++++++++++++
.../merge_union_output_mapping.out | 12 +++
.../merge_union_output_mapping.groovy | 88 ++++++++++++++++++++++
5 files changed, 195 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
index 57fedeb350e..53817b658e4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -27,9 +28,12 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Merge nodes of the same type and same qualifier.
@@ -83,8 +87,13 @@ public class MergeSetOperations implements
RewriteRuleFactory {
if (canMerge(parentUnion, child) && child.getOutput().size() ==
parentUnion.getOutput().size()) {
LogicalUnion childUnion = (LogicalUnion) child;
newChildren.addAll(childUnion.children());
-
newChildrenOutputs.addAll(childUnion.getRegularChildrenOutputs());
- constantList.addAll(childUnion.getConstantExprsList());
+ List<Integer> outputOrder = getChildOutputOrder(parentUnion,
i, childUnion);
+ for (List<SlotReference> outputs :
childUnion.getRegularChildrenOutputs()) {
+ newChildrenOutputs.add(reorderOutputs(outputs,
outputOrder));
+ }
+ for (List<NamedExpression> constants :
childUnion.getConstantExprsList()) {
+ constantList.add(reorderOutputs(constants, outputOrder));
+ }
needMerge = true;
} else {
newChildren.add(child);
@@ -108,7 +117,10 @@ public class MergeSetOperations implements
RewriteRuleFactory {
if (canMerge(parentSetIntersect, child)) {
LogicalIntersect childIntersect = (LogicalIntersect) child;
newChildren.addAll(childIntersect.children());
-
newChildrenOutputs.addAll(childIntersect.getRegularChildrenOutputs());
+ List<Integer> outputOrder =
getChildOutputOrder(parentSetIntersect, i, childIntersect);
+ for (List<SlotReference> outputs :
childIntersect.getRegularChildrenOutputs()) {
+ newChildrenOutputs.add(reorderOutputs(outputs,
outputOrder));
+ }
} else {
newChildren.add(child);
newChildrenOutputs.add(parentSetIntersect.getRegularChildOutput(i));
@@ -118,6 +130,29 @@ public class MergeSetOperations implements
RewriteRuleFactory {
newChildren.build(), newChildrenOutputs.build());
}
+ static List<Integer> getChildOutputOrder(
+ LogicalSetOperation parent, int childIndex, LogicalSetOperation
child) {
+ Map<ExprId, Integer> outputIndexes = new HashMap<>();
+ for (int i = 0; i < child.getOutput().size(); i++) {
+ outputIndexes.put(child.getOutput().get(i).getExprId(), i);
+ }
+ ImmutableList.Builder<Integer> order = ImmutableList.builder();
+ for (SlotReference slot : parent.getRegularChildOutput(childIndex)) {
+ Integer index = outputIndexes.get(slot.getExprId());
+ Preconditions.checkState(index != null, "Set operation input must
belong to child output");
+ order.add(index);
+ }
+ return order.build();
+ }
+
+ static <T> List<T> reorderOutputs(List<T> outputs, List<Integer> order) {
+ ImmutableList.Builder<T> reordered =
ImmutableList.builderWithExpectedSize(order.size());
+ for (int index : order) {
+ reordered.add(outputs.get(index));
+ }
+ return reordered.build();
+ }
+
/** canMerge */
public static boolean canMerge(LogicalSetOperation parent) {
Plan left = parent.child(0);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
index 2eb284e69c5..79d62285bca 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java
@@ -54,7 +54,10 @@ public class MergeSetOperationsExcept extends
OneRewriteRuleFactory {
newChildren.addAll(childExcept.children());
newChildren.addAll(parentExcept.children().subList(1,
parentExcept.children().size()));
-
newChildrenOutputs.addAll(childExcept.getRegularChildrenOutputs());
+ List<Integer> outputOrder =
MergeSetOperations.getChildOutputOrder(parentExcept, 0, childExcept);
+ for (List<SlotReference> outputs :
childExcept.getRegularChildrenOutputs()) {
+
newChildrenOutputs.add(MergeSetOperations.reorderOutputs(outputs, outputOrder));
+ }
newChildrenOutputs.addAll(
parentExcept.getRegularChildrenOutputs().subList(1,
parentExcept.children().size()));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
index a42c974dabf..e1d4489e18c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SetOperationOutputMappingTest.java
@@ -26,6 +26,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.types.IntegerType;
@@ -63,6 +64,58 @@ class SetOperationOutputMappingTest extends
TestWithFeService implements MemoPat
StatementScopeIdGenerator.clear();
}
+ @Test
+ void testMergeSetOperationsComposesChildOutputMapping() {
+ Alias first = new Alias(new ExprId(101), new IntegerLiteral(10),
"campaign");
+ Alias second = new Alias(new ExprId(102), new IntegerLiteral(20),
"country");
+ LogicalOneRowRelation row = new LogicalOneRowRelation(
+ new RelationId(101), ImmutableList.of(first, second));
+ SlotReference campaign = new SlotReference(new ExprId(103), "campaign",
+ IntegerType.INSTANCE, false, ImmutableList.of());
+ SlotReference country = new SlotReference(new ExprId(104), "country",
+ IntegerType.INSTANCE, false, ImmutableList.of());
+ LogicalUnion child = new LogicalUnion(Qualifier.ALL,
ImmutableList.of(campaign, country),
+ ImmutableList.of(ImmutableList.of((SlotReference)
first.toSlot(), (SlotReference) second.toSlot())),
+ ImmutableList.of(ImmutableList.of(first, second)), false,
ImmutableList.of(row));
+ LogicalUnion parent = new LogicalUnion(Qualifier.ALL,
ImmutableList.of(country, campaign),
+ ImmutableList.of(ImmutableList.of(country, campaign)),
ImmutableList.of(),
+ false, ImmutableList.of(child));
+
+ LogicalUnion rewritten = (LogicalUnion)
PlanChecker.from(MemoTestUtils.createConnectContext(), parent)
+ .applyTopDown(new MergeSetOperations()).getPlan();
+ Assertions.assertEquals(ImmutableList.of(row), rewritten.children());
+ Assertions.assertEquals(ImmutableList.of(second.toSlot(),
first.toSlot()),
+ rewritten.getRegularChildOutput(0));
+ Assertions.assertEquals(ImmutableList.of(second, first),
rewritten.getConstantExprsList().get(0));
+
+ Alias rightFirst = new Alias(new ExprId(105), new IntegerLiteral(30),
"right_first");
+ Alias rightSecond = new Alias(new ExprId(106), new IntegerLiteral(40),
"right_second");
+ LogicalOneRowRelation right = new LogicalOneRowRelation(
+ new RelationId(102), ImmutableList.of(rightFirst,
rightSecond));
+ List<SlotReference> rightOutputs = ImmutableList.of(
+ (SlotReference) rightFirst.toSlot(), (SlotReference)
rightSecond.toSlot());
+ LogicalExcept childExcept = new LogicalExcept(Qualifier.DISTINCT,
ImmutableList.of(campaign, country),
+ ImmutableList.of(child.getRegularChildOutput(0),
rightOutputs), ImmutableList.of(row, right));
+ Alias outerFirst = new Alias(new ExprId(107), new IntegerLiteral(50),
"outer_first");
+ Alias outerSecond = new Alias(new ExprId(108), new IntegerLiteral(60),
"outer_second");
+ LogicalOneRowRelation outerRight = new LogicalOneRowRelation(
+ new RelationId(103), ImmutableList.of(outerFirst,
outerSecond));
+ List<SlotReference> outerRightOutputs = ImmutableList.of(
+ (SlotReference) outerFirst.toSlot(), (SlotReference)
outerSecond.toSlot());
+ LogicalExcept parentExcept = new LogicalExcept(Qualifier.DISTINCT,
ImmutableList.of(country, campaign),
+ ImmutableList.of(ImmutableList.of(country, campaign),
outerRightOutputs),
+ ImmutableList.of(childExcept, outerRight));
+ LogicalExcept rewrittenExcept = (LogicalExcept) PlanChecker.from(
+ MemoTestUtils.createConnectContext(), parentExcept)
+ .applyTopDown(new MergeSetOperationsExcept()).getPlan();
+ Assertions.assertEquals(ImmutableList.of(row, right, outerRight),
rewrittenExcept.children());
+ Assertions.assertEquals(ImmutableList.of(second.toSlot(),
first.toSlot()),
+ rewrittenExcept.getRegularChildOutput(0));
+ Assertions.assertEquals(ImmutableList.of(rightSecond.toSlot(),
rightFirst.toSlot()),
+ rewrittenExcept.getRegularChildOutput(1));
+ Assertions.assertEquals(outerRightOutputs,
rewrittenExcept.getRegularChildOutput(2));
+ }
+
@Test
void testMergeOneRowRelationUsesRegularChildOutput() {
Alias firstProject = new Alias(new ExprId(1), new IntegerLiteral(10),
"first_col");
diff --git
a/regression-test/data/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.out
b/regression-test/data/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.out
new file mode 100644
index 00000000000..41dab4ef646
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !country --
+FR
+
+-- !country_date --
+FR 2026-01-01
+
+-- !except_reordered --
+FR AD_A
+
+-- !except_removed --
+
diff --git
a/regression-test/suites/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.groovy
b/regression-test/suites/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.groovy
new file mode 100644
index 00000000000..6a95bceba9a
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/merge_one_row_relation/merge_union_output_mapping.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("merge_union_output_mapping", "p0") {
+ sql "DROP TABLE IF EXISTS merge_union_output_mapping"
+ sql """CREATE TABLE merge_union_output_mapping (
+ id INT, campaign VARCHAR(40), country VARCHAR(40), dt VARCHAR(40)
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')"""
+ sql "INSERT INTO merge_union_output_mapping VALUES (1, 'AD_A', 'FR',
'2026-01-01')"
+ sql "SET enable_sql_cache=false"
+ order_qt_country """WITH sa AS (
+ SELECT CAST(dt AS DATE) AS dt, campaign, country, campaign AS tag FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT CAST(dt AS DATE), campaign, country, campaign
+ FROM (
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ ) u
+), c AS (
+ SELECT dt, country, tag, 'SA' AS ad_type FROM sa
+), e AS (
+ SELECT 'FR' AS country
+)
+SELECT a.country
+FROM c a
+LEFT JOIN e ON CASE
+ WHEN a.country = 'PL' AND a.ad_type = 'DSP' THEN 'PL_DSP'
+ ELSE a.country
+END = e.country
+LEFT JOIN c b ON a.tag = b.tag
+GROUP BY a.country"""
+ order_qt_country_date """WITH sa AS (
+ SELECT CAST(dt AS DATE) AS dt, campaign, country, campaign AS tag FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT CAST(dt AS DATE), campaign, country, campaign
+ FROM (
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ UNION ALL
+ SELECT campaign, country, dt, id AS x0, id AS x1 FROM
merge_union_output_mapping
+ ) u
+), c AS (
+ SELECT dt, country, tag, 'SA' AS ad_type FROM sa
+), e AS (
+ SELECT 'FR' AS country
+)
+SELECT a.country, a.dt
+FROM c a
+LEFT JOIN e ON CASE
+ WHEN a.country = 'PL' AND a.ad_type = 'DSP' THEN 'PL_DSP'
+ ELSE a.country
+END = e.country
+LEFT JOIN c b ON a.tag = b.tag
+GROUP BY a.country, a.dt"""
+ order_qt_except_reordered """
+ SELECT country, campaign FROM (
+ SELECT campaign, country FROM merge_union_output_mapping
+ EXCEPT
+ SELECT campaign, country FROM merge_union_output_mapping WHERE id
< 0
+ ) u
+ EXCEPT
+ SELECT country, campaign FROM merge_union_output_mapping WHERE id < 0
+ """
+ order_qt_except_removed """
+ SELECT country, campaign FROM (
+ SELECT campaign, country FROM merge_union_output_mapping
+ EXCEPT
+ SELECT campaign, country FROM merge_union_output_mapping WHERE id
< 0
+ ) u
+ EXCEPT
+ SELECT country, campaign FROM merge_union_output_mapping
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
