This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6accacc3929af1f254d0a2f90c250ccd7e0d6bf5 Author: minghong <[email protected]> AuthorDate: Thu Sep 10 09:21:10 2026 +0800 branch-4.1 [fix](fd) Propagate constants out of one row relation to enable partition pruning (#67400) Cherry-pick of #66905 to branch-4.1 [fix](fd) Propagate constants out of one row relation to enable partition pruning --- .../trees/plans/logical/LogicalOneRowRelation.java | 13 ++- .../nereids/rules/rewrite/ConstantCteTest.java | 121 +++++++++++++++++++++ .../rules/rewrite/PruneNestedColumnTest.java | 9 +- .../infer_predicate/pull_up_predicate_literal.out | 4 +- .../cte/test_constant_cte_partition_prune.groovy | 80 ++++++++++++++ 5 files changed, 221 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 5f8446fb015..e2584b04fb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -169,7 +169,18 @@ public class LogicalOneRowRelation extends LogicalRelation implements OneRowRela @Override public void computeUniform(DataTrait.Builder builder) { - getOutput().forEach(builder::addUniformSlot); + for (NamedExpression project : getProjects()) { + if (project instanceof Alias && project.child(0).isConstant()) { + // A one row relation's constant projects are known literals/constant expressions, + // register the value so that constant propagation and predicate inference outside + // the relation can fold predicates over these slots (e.g. a constant CTE whose + // consumers reference `date_sub(params.begin_time, ...)`), which enables + // partition pruning on the referenced tables. + builder.addUniformSlotAndLiteral(project.toSlot(), project.child(0)); + } else { + builder.addUniformSlot(project.toSlot()); + } + } } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantCteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantCteTest.java new file mode 100644 index 00000000000..cc6242a0fc6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantCteTest.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.FeConstants; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Set; + +/** + * Test that a CTE defining a constant single row (e.g. `params`) propagates the constants + * out, so that downstream predicates over the CTE columns can be constant-folded and the + * partition pruning on the referenced tables works. + * + * <p>After the constant CTE is inlined, the first consumer predicate + * `dt BETWEEN params.begin_time AND params.end_time` is folded and the scan prunes to the + * day partition; the second consumer predicate + * `dt BETWEEN DATE_SUB(params.begin_time, INTERVAL params.period_days DAY) + * AND DATE_SUB(params.begin_time, INTERVAL 1 DAY)` + * was not folded, so the scan read all partitions. + */ +class ConstantCteTest extends TestWithFeService implements MemoPatternMatchSupported { + + // the constant CTE `params` participates in two joins; the second join uses + // DATE_SUB() over the CTE columns, which needs constant folding to prune partitions. + private static final String SQL = "WITH params AS (\n" + + " SELECT\n" + + " CAST('2026-07-28 00:00:00' AS DATETIME) AS begin_time,\n" + + " CAST('2026-07-28 23:59:59' AS DATETIME) AS end_time,\n" + + " DATEDIFF(CAST('2026-07-28 23:59:59' AS DATETIME), " + + "CAST('2026-07-28 00:00:00' AS DATETIME)) + 1 AS period_days\n" + + "),\n" + + "current_data AS (\n" + + " SELECT SUM(v) AS total_value\n" + + " FROM cte_prune_t\n" + + " JOIN params ON 1=1\n" + + " WHERE dt BETWEEN params.begin_time AND params.end_time\n" + + "),\n" + + "last_period_data AS (\n" + + " SELECT SUM(v) AS total_value\n" + + " FROM cte_prune_t\n" + + " JOIN params ON 1=1\n" + + " WHERE dt BETWEEN DATE_SUB(params.begin_time, INTERVAL params.period_days DAY)\n" + + " AND DATE_SUB(params.begin_time, INTERVAL 1 DAY)\n" + + ")\n" + + "SELECT * FROM current_data, last_period_data"; + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + useDatabase("test"); + createTable("CREATE TABLE `test`.`cte_prune_t` (\n" + + " `dt` DATE NULL COMMENT \"\",\n" + + " `sn` VARCHAR(50) NULL COMMENT \"\",\n" + + " `v` DOUBLE NULL COMMENT \"\"\n" + + ") DUPLICATE KEY(`dt`, `sn`)\n" + + "PARTITION BY RANGE(`dt`)\n" + + "(PARTITION p20260101 VALUES [(\"2026-01-01\"), (\"2026-01-02\")),\n" + + " PARTITION p20260726 VALUES [(\"2026-07-26\"), (\"2026-07-27\")),\n" + + " PARTITION p20260727 VALUES [(\"2026-07-27\"), (\"2026-07-28\")),\n" + + " PARTITION p20260728 VALUES [(\"2026-07-28\"), (\"2026-07-29\")),\n" + + " PARTITION p20260729 VALUES [(\"2026-07-29\"), (\"2026-07-30\")),\n" + + " PARTITION p20260901 VALUES [(\"2026-09-01\"), (\"2026-09-02\")))\n" + + "DISTRIBUTED BY HASH(`sn`) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); + FeConstants.runningUnitTest = true; + } + + @Test + void testConstantCteFoldJoinPredicateAndPrunePartition() { + // params has 2 consumers; force inline to match the reported scenario + connectContext.getSessionVariable().inlineCTEReferencedThreshold = 2; + + PlanChecker planChecker = PlanChecker.from(connectContext) + .analyze(SQL) + .rewrite(); + Plan plan = planChecker.getCascadesContext().getRewritePlan(); + String planString = plan.treeString(); + + List<LogicalOlapScan> scans = plan.collectToList(LogicalOlapScan.class::isInstance); + Assertions.assertEquals(2, scans.size(), + "both current_data and last_period_data should scan cte_prune_t, plan: " + planString); + Set<String> selectedPartitions = Sets.newHashSet(); + for (LogicalOlapScan scan : scans) { + // current_data only needs p20260728, last_period_data only needs p20260727; + // both must prune to exactly one partition + Assertions.assertEquals(1, scan.getSelectedPartitionIds().size(), + "scan on cte_prune_t should prune to exactly one partition, plan: " + planString); + selectedPartitions.add(((OlapTable) scan.getTable()) + .getPartition(scan.getSelectedPartitionIds().get(0)).getName()); + } + Assertions.assertEquals(Sets.newHashSet("p20260727", "p20260728"), selectedPartitions, + "current_data should prune to p20260728 and last_period_data to p20260727, plan: " + + planString); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 725c0c4cd08..4d7a8a4488e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -778,9 +778,12 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt logicalOlapScan() ) ).when(p -> { - Assertions.assertEquals(2, p.getProjects().size()); - Assertions.assertTrue(p.getProjects().stream() - .anyMatch(o -> o instanceof Alias && o.child(0) instanceof ElementAt)); + // the one-row relation's constant `id` is propagated into the + // left side (`id = 100` pushed into the filter below), so the + // project above the filter only keeps the pushed-down access + Assertions.assertEquals(1, p.getProjects().size()); + Assertions.assertTrue(p.getProjects().get(0) instanceof Alias + && p.getProjects().get(0).child(0) instanceof ElementAt); return true; }), logicalOneRowRelation() diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out index adc6bb8087c..e1baa520726 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out @@ -2,7 +2,7 @@ -- !test_pull_up_literal1 -- PhysicalResultSink --PhysicalProject -----hashJoin[INNER_JOIN] hashCondition=((col1 = ds.col1) and (col2 = ds.col2)) otherCondition=() +----NestedLoopJoin[INNER_JOIN] ------PhysicalOneRowRelation ------filter((ds.col1 = 'abc') and (ds.col2 = 'def')) --------PhysicalOlapScan[test_pull_up_predicate_literal] @@ -10,7 +10,7 @@ PhysicalResultSink -- !test_pull_up_literal2 -- PhysicalResultSink --PhysicalProject -----hashJoin[INNER_JOIN] hashCondition=((col1 = ds.col1) and (col2 = ds.col2)) otherCondition=() +----NestedLoopJoin[INNER_JOIN] ------PhysicalOneRowRelation ------filter((ds.col1 = 'abc') and (ds.col2 = 'def')) --------PhysicalOlapScan[test_pull_up_predicate_literal] diff --git a/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy b/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy new file mode 100644 index 00000000000..be08e756747 --- /dev/null +++ b/regression-test/suites/nereids_p0/cte/test_constant_cte_partition_prune.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_constant_cte_partition_prune") { + sql "DROP TABLE IF EXISTS constant_cte_prune_t" + sql """ + CREATE TABLE constant_cte_prune_t ( + dt DATE, + sn VARCHAR(50), + v DOUBLE + ) ENGINE=OLAP + DUPLICATE KEY(dt, sn) + PARTITION BY RANGE(dt) + (PARTITION p20260101 VALUES [("2026-01-01"), ("2026-01-02")), + PARTITION p20260726 VALUES [("2026-07-26"), ("2026-07-27")), + PARTITION p20260727 VALUES [("2026-07-27"), ("2026-07-28")), + PARTITION p20260728 VALUES [("2026-07-28"), ("2026-07-29")), + PARTITION p20260729 VALUES [("2026-07-29"), ("2026-07-30")), + PARTITION p20260901 VALUES [("2026-09-01"), ("2026-09-02"))) + DISTRIBUTED BY HASH(sn) BUCKETS 3 + PROPERTIES ("replication_num" = "1") + """ + + // force the constant CTE `params` to be inlined (it is referenced twice), matching the + // reported scenario; after inlining the constants must propagate out so that predicates + // over the CTE columns (including DATE_SUB() over them) are folded and the scans prune + // to a single day partition + sql "SET inline_cte_referenced_threshold=2" + + // seed every partition so that PRUNE_EMPTY_PARTITION does not eliminate the scans + // (an all-empty table would rewrite the scan to an empty relation and the + // partition-pruning info below would not show up in the explain output) + sql """INSERT INTO constant_cte_prune_t VALUES + ('2026-01-01', 'sn1', 1.0), + ('2026-07-26', 'sn2', 2.0), + ('2026-07-27', 'sn3', 3.0), + ('2026-07-28', 'sn4', 4.0), + ('2026-07-29', 'sn5', 5.0), + ('2026-09-01', 'sn6', 6.0)""" + + explain { + sql """ + WITH params AS ( + SELECT CAST('2026-07-28 00:00:00' AS DATETIME) AS begin_time, + CAST('2026-07-28 23:59:59' AS DATETIME) AS end_time, + DATEDIFF(CAST('2026-07-28 23:59:59' AS DATETIME), + CAST('2026-07-28 00:00:00' AS DATETIME)) + 1 AS period_days + ), + current_data AS ( + SELECT SUM(v) AS total_value + FROM constant_cte_prune_t JOIN params ON 1=1 + WHERE dt BETWEEN params.begin_time AND params.end_time + ), + last_period_data AS ( + SELECT SUM(v) AS total_value + FROM constant_cte_prune_t JOIN params ON 1=1 + WHERE dt BETWEEN DATE_SUB(params.begin_time, INTERVAL params.period_days DAY) + AND DATE_SUB(params.begin_time, INTERVAL 1 DAY) + ) + SELECT * FROM current_data, last_period_data + """ + // current_data only needs 2026-07-28, last_period_data only needs 2026-07-27 + contains("partitions=1/6 (p20260728)") + contains("partitions=1/6 (p20260727)") + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
