This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5ce7604934b [fix](mtmv) Fix rewrite fail by materialized view when
filter or join condition has alias (#44779)
5ce7604934b is described below
commit 5ce7604934b364216ea8ff41c984feee903a502b
Author: seawinde <[email protected]>
AuthorDate: Thu Dec 5 23:13:19 2024 +0800
[fix](mtmv) Fix rewrite fail by materialized view when filter or join
condition has alias (#44779)
### What problem does this PR solve?
Related PR: #27922
Problem Summary:
query and mv def are as following,` partsupp.public_col as public_col `
is alias, this would cause rewritting fail by materialized view with
msg, the graph logic between query and view is different.
select
o_custkey,
o_orderdate,
o_shippriority,
o_comment,
o_orderkey,
orders.public_col as col1,
l_orderkey,
l_partkey,
l_suppkey,
lineitem.public_col as col2,
ps_partkey,
ps_suppkey,
partsupp.public_col as col3,
partsupp.public_col * 2 as col4,
o_orderkey + l_orderkey + ps_partkey * 2,
sum(
o_orderkey + l_orderkey + ps_partkey * 2
),
count() as count_all
from
(
select
o_custkey,
o_orderdate,
o_shippriority,
o_comment,
o_orderkey,
orders.public_col as public_col
from
orders
) orders
left join (
select
l_orderkey,
l_partkey,
l_suppkey,
lineitem.public_col as public_col
from
lineitem
where
lineitem.public_col is null
or lineitem.public_col <> 1
) lineitem on l_orderkey = o_orderkey
inner join (
select
ps_partkey,
ps_suppkey,
partsupp.public_col as public_col
from
partsupp
) partsupp on ps_partkey = o_orderkey
where
lineitem.public_col is null
or lineitem.public_col <> 1
and o_orderkey = 2
group by
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14;
### Release note
Fix rewrite fail by materialized view when filter or join condition has
alias
---
.../jobs/joinorder/hypergraph/HyperElement.java | 27 ++
.../jobs/joinorder/hypergraph/edge/Edge.java | 4 +-
.../joinorder/hypergraph/node/AbstractNode.java | 8 +-
.../rules/exploration/mv/HyperGraphComparator.java | 103 +++--
.../mv/LogicalCompatibilityContext.java | 74 +++-
.../nereids/rules/exploration/mv/StructInfo.java | 107 +++--
.../aggregate_without_roll_up.out | 24 ++
.../aggregate_without_roll_up.groovy | 463 ++++++++++++++++++++-
.../range_datetime_part_up_rewrite.groovy | 20 +-
9 files changed, 738 insertions(+), 92 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java
new file mode 100644
index 00000000000..6d8d7c6326c
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.jobs.joinorder.hypergraph;
+
+/**
+ * This is the common base class for all
+ * */
+public interface HyperElement {
+
+ // Get the references nodes
+ long getReferenceNodes();
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
index 7698d881c66..f75ed832501 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.jobs.joinorder.hypergraph.edge;
import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -32,7 +33,7 @@ import java.util.Set;
/**
* Edge in HyperGraph
*/
-public abstract class Edge {
+public abstract class Edge implements HyperElement {
private final int index;
private final double selectivity;
@@ -192,6 +193,7 @@ public abstract class Edge {
return LongBitmap.isSubset(getReferenceNodes(), otherBitmap);
}
+ @Override
public long getReferenceNodes() {
return LongBitmap.newBitmapUnion(leftExtendedNodes,
rightExtendedNodes);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
index a4a64e0449d..686576de771 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.jobs.joinorder.hypergraph.node;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
@@ -33,7 +34,7 @@ import java.util.Set;
/**
* HyperGraph Node.
*/
-public class AbstractNode {
+public class AbstractNode implements HyperElement {
protected final int index;
protected final List<JoinEdge> joinEdges;
protected final List<FilterEdge> filterEdges;
@@ -65,6 +66,11 @@ public class AbstractNode {
.build();
}
+ @Override
+ public long getReferenceNodes() {
+ return getNodeMap();
+ }
+
public int getIndex() {
return index;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index d4594583c31..22282a23516 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -19,12 +19,14 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.ConflictRulesMaker;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
+import
org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -51,6 +53,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
@@ -79,9 +82,9 @@ public class HyperGraphComparator {
private final Map<Edge, List<? extends Expression>> pullUpViewExprWithEdge
= new HashMap<>();
private final LogicalCompatibilityContext logicalCompatibilityContext;
// this records the slots which needs to reject null
- // the key is the target join which should reject null, the value is a
pair, the first value of the pair is the
- // join type, the second value is also a pair which left represents the
slots in the left of join that should
- // reject null, right represents the slots in the right of join that
should reject null.
+ // the key is the view join edge which should reject null, the value is a
pair, the first value of the pair is the
+ // query join type, the second value is also a pair which left represents
the slots in the left of view join that
+ // should reject null, right represents the slots in the right of view
join that should reject null.
private final Map<JoinEdge, Pair<JoinType, Pair<Set<Slot>, Set<Slot>>>>
inferredViewEdgeWithCond = new HashMap<>();
private List<JoinEdge> viewJoinEdgesAfterInferring;
private List<FilterEdge> viewFilterEdgesAfterInferring;
@@ -249,9 +252,17 @@ public class HyperGraphComparator {
}
int size = queryExprSetList.size();
for (int i = 0; i < size; i++) {
- Set<Expression> mappingQueryExprSet =
queryExprSetList.get(i).stream()
- .map(logicalCompatibilityContext::getViewNodeExprFromQuery)
- .collect(Collectors.toSet());
+ Set<Expression> queryExpressions = queryExprSetList.get(i);
+ Set<Expression> mappingQueryExprSet = new HashSet<>();
+ for (Expression queryExpression : queryExpressions) {
+ Optional<Expression> mappingViewExprByQueryExpr =
getMappingViewExprByQueryExpr(queryExpression, query,
+ this.logicalCompatibilityContext,
+ ExpressionPosition.NODE);
+ if (!mappingViewExprByQueryExpr.isPresent()) {
+ return false;
+ }
+ mappingQueryExprSet.add(mappingViewExprByQueryExpr.get());
+ }
if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) {
return false;
}
@@ -407,7 +418,10 @@ public class HyperGraphComparator {
if (edgeMap.containsKey(entry.getValue())) {
continue;
}
- Expression viewExpr =
logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey());
+ Expression viewExpr = getMappingViewExprByQueryExpr(entry.getKey(),
+ entry.getValue(),
+ logicalCompatibilityContext,
+ ExpressionPosition.JOIN_EDGE).orElse(null);
if (viewExprToEdge.containsKey(viewExpr)) {
edgeMap.put(entry.getValue(),
Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
}
@@ -441,15 +455,19 @@ public class HyperGraphComparator {
HashMap<Edge, Edge> queryToViewEdgeMap = new HashMap<>();
for (Entry<Expression, Collection<Edge>> entry :
queryExprToEdge.asMap().entrySet()) {
- Expression queryExprViewBased =
logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey());
- if (queryExprViewBased == null) {
- continue;
- }
- Collection<Edge> viewEdges =
viewExprToEdge.get(queryExprViewBased);
- if (viewEdges.isEmpty()) {
- continue;
- }
+ Expression queryExprViewBased = null;
for (Edge queryEdge : entry.getValue()) {
+ queryExprViewBased =
getMappingViewExprByQueryExpr(entry.getKey(),
+ queryEdge,
+ logicalCompatibilityContext,
+ ExpressionPosition.FILTER_EDGE).orElse(null);
+ if (queryExprViewBased == null) {
+ continue;
+ }
+ Collection<Edge> viewEdges =
viewExprToEdge.get(queryExprViewBased);
+ if (viewEdges.isEmpty()) {
+ continue;
+ }
for (Edge viewEdge : viewEdges) {
if (!isSubTreeNodesEquals(queryEdge, viewEdge,
logicalCompatibilityContext)) {
// Such as query filter edge is <{1} --FILTER-- {}>
but view filter edge is
@@ -514,17 +532,17 @@ public class HyperGraphComparator {
}
private boolean compareFilterEdgeWithNode(FilterEdge query, FilterEdge
view) {
- return rewriteQueryNodeMap(query.getReferenceNodes()) ==
view.getReferenceNodes();
+ return getViewNodesByQuery(query.getReferenceNodes()) ==
view.getReferenceNodes();
}
private boolean compareJoinEdgeWithNode(JoinEdge query, JoinEdge view) {
boolean res = false;
if (query.getJoinType().swap() == view.getJoinType()) {
- res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) ==
view.getRightExtendedNodes()
- && rewriteQueryNodeMap(query.getRightExtendedNodes()) ==
view.getLeftExtendedNodes();
+ res |= getViewNodesByQuery(query.getLeftExtendedNodes()) ==
view.getRightExtendedNodes()
+ && getViewNodesByQuery(query.getRightExtendedNodes()) ==
view.getLeftExtendedNodes();
}
- res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) ==
view.getLeftExtendedNodes()
- && rewriteQueryNodeMap(query.getRightExtendedNodes()) ==
view.getRightExtendedNodes();
+ res |= getViewNodesByQuery(query.getLeftExtendedNodes()) ==
view.getLeftExtendedNodes()
+ && getViewNodesByQuery(query.getRightExtendedNodes()) ==
view.getRightExtendedNodes();
return res;
}
@@ -547,8 +565,8 @@ public class HyperGraphComparator {
}
private boolean tryInferEdge(JoinEdge query, JoinEdge view) {
- if (rewriteQueryNodeMap(query.getLeftRequiredNodes()) !=
view.getLeftRequiredNodes()
- || rewriteQueryNodeMap(query.getRightRequiredNodes()) !=
view.getRightRequiredNodes()) {
+ if (getViewNodesByQuery(query.getLeftRequiredNodes()) !=
view.getLeftRequiredNodes()
+ || getViewNodesByQuery(query.getRightRequiredNodes()) !=
view.getRightRequiredNodes()) {
return false;
}
if (!query.getJoinType().equals(view.getJoinType())) {
@@ -569,7 +587,7 @@ public class HyperGraphComparator {
return true;
}
- private long rewriteQueryNodeMap(long bitmap) {
+ private long getViewNodesByQuery(long bitmap) {
long newBitmap = LongBitmap.newBitmap();
for (int i : LongBitmap.getIterator(bitmap)) {
int newIdx = getQueryToViewNodeIdMap().getOrDefault(i, 0);
@@ -578,6 +596,35 @@ public class HyperGraphComparator {
return newBitmap;
}
+ private Optional<Expression> getMappingViewExprByQueryExpr(Expression
queryExpression,
+ HyperElement queryExpressionBelongedHyperElement,
+ LogicalCompatibilityContext context,
+ ExpressionPosition expressionPosition) {
+ Expression queryShuttledExpr;
+ Collection<Pair<Expression, HyperElement>> viewExpressions;
+ if (ExpressionPosition.JOIN_EDGE.equals(expressionPosition)) {
+ queryShuttledExpr =
context.getQueryJoinShuttledExpr(queryExpression);
+ viewExpressions =
context.getViewJoinExprFromQuery(queryShuttledExpr);
+ } else if (ExpressionPosition.FILTER_EDGE.equals(expressionPosition)) {
+ queryShuttledExpr =
context.getQueryFilterShuttledExpr(queryExpression);
+ viewExpressions =
context.getViewFilterExprFromQuery(queryShuttledExpr);
+ } else {
+ queryShuttledExpr =
context.getQueryNodeShuttledExpr(queryExpression);
+ viewExpressions =
context.getViewNodeExprFromQuery(queryShuttledExpr);
+ }
+ if (viewExpressions.size() == 1) {
+ return Optional.of(viewExpressions.iterator().next().key());
+ }
+ long queryReferenceNodes =
queryExpressionBelongedHyperElement.getReferenceNodes();
+ long viewReferenceNodes = getViewNodesByQuery(queryReferenceNodes);
+ for (Pair<Expression, HyperElement> viewExpressionPair :
viewExpressions) {
+ if (viewExpressionPair.value().getReferenceNodes() ==
viewReferenceNodes) {
+ return Optional.of(viewExpressionPair.key());
+ }
+ }
+ return Optional.empty();
+ }
+
private void compareJoinEdgeWithExpr(Edge query, Edge view) {
Set<? extends Expression> queryExprSet = query.getExpressionSet();
Set<? extends Expression> viewExprSet = view.getExpressionSet();
@@ -585,7 +632,10 @@ public class HyperGraphComparator {
Set<Expression> exprMappedOfView = new HashSet<>();
List<Expression> residualQueryExpr = new ArrayList<>();
for (Expression queryExpr : queryExprSet) {
- Expression viewExpr =
logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr);
+ Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr,
+ query,
+ logicalCompatibilityContext,
+ ExpressionPosition.JOIN_EDGE).orElse(null);
if (viewExprSet.contains(viewExpr)) {
exprMappedOfView.add(viewExpr);
} else {
@@ -604,7 +654,10 @@ public class HyperGraphComparator {
Set<Expression> exprMappedOfView = new HashSet<>();
List<Expression> residualQueryExpr = new ArrayList<>();
for (Expression queryExpr : queryExprSet) {
- Expression viewExpr =
logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr);
+ Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr,
+ query,
+ logicalCompatibilityContext,
+ ExpressionPosition.FILTER_EDGE).orElse(null);
if (viewExprSet.contains(viewExpr)) {
exprMappedOfView.add(viewExpr);
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
index ca13c9701da..77ab37873d0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.rules.exploration.mv;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.GroupExpression;
import
org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition;
@@ -36,8 +38,10 @@ import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Suppliers;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
-import java.util.HashMap;
+import java.util.Collection;
import java.util.Map;
import java.util.function.Supplier;
@@ -48,11 +52,15 @@ public class LogicalCompatibilityContext {
private final BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping;
private final BiMap<Integer, Integer> queryToViewNodeIDMapping;
private final ObjectId planNodeId;
- private final Supplier<BiMap<Expression, Expression>>
queryToViewJoinEdgeExpressionMappingSupplier;
- private final Supplier<BiMap<Expression, Expression>>
queryToViewNodeExpressionMappingSupplier;
- private final Supplier<BiMap<Expression, Expression>>
queryToViewFilterEdgeExpressionMappingSupplier;
- @Deprecated
- private BiMap<Expression, Expression> queryToViewAllExpressionMapping;
+ private final Supplier<Multimap<Expression, Pair<Expression,
HyperElement>>>
+ queryToViewJoinEdgeExpressionMappingSupplier;
+ private final Supplier<Map<Expression, Expression>>
queryToQueryShuttledJoinExpressionMappingSupplier;
+ private final Supplier<Multimap<Expression, Pair<Expression,
HyperElement>>>
+ queryToViewNodeExpressionMappingSupplier;
+ private final Supplier<Map<Expression, Expression>>
queryToQueryShuttledNodeExpressionMappingSupplier;
+ private final Supplier<Multimap<Expression, Pair<Expression,
HyperElement>>>
+ queryToViewFilterEdgeExpressionMappingSupplier;
+ private final Supplier<Map<Expression, Expression>>
queryToQueryShuttledFilterExpressionMappingSupplier;
/**
* LogicalCompatibilityContext
@@ -66,16 +74,25 @@ public class LogicalCompatibilityContext {
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE)));
+ this.queryToQueryShuttledJoinExpressionMappingSupplier =
Suppliers.memoize(
+ () ->
queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.JOIN_EDGE));
+
this.queryToViewNodeExpressionMappingSupplier =
Suppliers.memoize(() ->
generateExpressionMapping(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE)));
+ this.queryToQueryShuttledNodeExpressionMappingSupplier =
Suppliers.memoize(
+ () ->
queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.NODE));
+
this.queryToViewFilterEdgeExpressionMappingSupplier =
Suppliers.memoize(() ->
generateExpressionMapping(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE)));
+ this.queryToQueryShuttledFilterExpressionMappingSupplier =
Suppliers.memoize(
+ () ->
queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.FILTER_EDGE));
+
this.queryToViewNodeMapping = queryToViewNodeMapping;
this.queryToViewNodeIDMapping = HashBiMap.create();
queryToViewNodeMapping.forEach((k, v) ->
queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex()));
@@ -92,18 +109,30 @@ public class LogicalCompatibilityContext {
return queryToViewNodeIDMapping;
}
- public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) {
+ public Collection<Pair<Expression, HyperElement>>
getViewJoinExprFromQuery(Expression queryJoinExpr) {
return
queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
}
- public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) {
+ public Expression getQueryJoinShuttledExpr(Expression queryJoinExpr) {
+ return
queryToQueryShuttledJoinExpressionMappingSupplier.get().get(queryJoinExpr);
+ }
+
+ public Collection<Pair<Expression, HyperElement>>
getViewFilterExprFromQuery(Expression queryJoinExpr) {
return
queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
}
- public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) {
+ public Expression getQueryFilterShuttledExpr(Expression queryFilterExpr) {
+ return
queryToQueryShuttledFilterExpressionMappingSupplier.get().get(queryFilterExpr);
+ }
+
+ public Collection<Pair<Expression, HyperElement>>
getViewNodeExprFromQuery(Expression queryJoinExpr) {
return
queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr);
}
+ public Expression getQueryNodeShuttledExpr(Expression queryNodeExpr) {
+ return
queryToQueryShuttledNodeExpressionMappingSupplier.get().get(queryNodeExpr);
+ }
+
/**
* Generate logical compatibility context,
* this make expression mapping between query and view by relation and the
slot in relation mapping
@@ -134,24 +163,31 @@ public class LogicalCompatibilityContext {
viewStructInfo);
}
- private static BiMap<Expression, Expression> generateExpressionMapping(
+ /**
+ * The result is multimap
+ * the key is shuttled query expr
+ * the value is original view expr collection
+ * */
+ private static Multimap<Expression, Pair<Expression, HyperElement>>
generateExpressionMapping(
Map<SlotReference, SlotReference> viewToQuerySlotMapping,
- Map<Expression, Expression> queryShuttledExprToExprMap,
- Map<Expression, Expression> viewShuttledExprToExprMap) {
- final Map<Expression, Expression> viewEdgeToConjunctsMapQueryBased =
new HashMap<>();
- BiMap<Expression, Expression> queryToViewEdgeMapping =
HashBiMap.create();
+ Multimap<Expression, Pair<Expression, HyperElement>>
queryShuttledExprToExprMap,
+ Multimap<Expression, Pair<Expression, HyperElement>>
viewShuttledExprToExprMap) {
+ Multimap<Expression, Pair<Expression, HyperElement>>
queryToViewEdgeMapping = HashMultimap.create();
if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap ==
null
|| queryShuttledExprToExprMap.isEmpty() ||
viewShuttledExprToExprMap.isEmpty()) {
return queryToViewEdgeMapping;
}
+ final Multimap<Expression, Pair<Expression, HyperElement>>
viewShuttledExprToExprMapQueryBased =
+ HashMultimap.create();
viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> {
- viewEdgeToConjunctsMapQueryBased.put(
+ viewShuttledExprToExprMapQueryBased.put(
orderSlotAsc(ExpressionUtils.replace(shuttledExpr,
viewToQuerySlotMapping)), expr);
});
- queryShuttledExprToExprMap.forEach((exprSet, edge) -> {
- Expression viewExpr =
viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet));
- if (viewExpr != null) {
- queryToViewEdgeMapping.put(edge, viewExpr);
+ queryShuttledExprToExprMap.forEach((shuttledExpr, expr) -> {
+ Collection<Pair<Expression, HyperElement>> viewExpressions =
viewShuttledExprToExprMapQueryBased.get(
+ orderSlotAsc(shuttledExpr));
+ if (viewExpressions != null) {
+ queryToViewEdgeMapping.putAll(shuttledExpr, viewExpressions);
}
});
return queryToViewEdgeMapping;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 3de48dc7ff6..365360e06b0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -23,10 +23,10 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.executor.Rewriter;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
-import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import
org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.TableQueryOperatorChecker;
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
@@ -65,12 +65,15 @@ import
org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
import org.apache.doris.nereids.util.ExpressionUtils;
+import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -113,9 +116,23 @@ public class StructInfo {
// split predicates is shuttled
private SplitPredicate splitPredicate;
private EquivalenceClass equivalenceClass;
- // Key is the expression shuttled and the value is the origin expression
+ // For value of Map, the key is the position of expression
+ // the value is the expressions and the hyper element of expression pair
+ // Key of pair is the expression shuttled and the value is the origin
expression and the hyper element it belonged
+ // Sometimes origin expressions are different and shuttled expression is
same
+ // Such as origin expressions are l_partkey#0 > 1 and l_partkey#10 > 1 and
shuttled expression is l_partkey#10 > 1
// this is for building LogicalCompatibilityContext later.
- private final Map<ExpressionPosition, Map<Expression, Expression>>
shuttledExpressionsToExpressionsMap;
+ private final Map<ExpressionPosition, Multimap<Expression,
Pair<Expression, HyperElement>>>
+ shuttledExpressionsToExpressionsMap;
+ // For value of Map, the key is the position of expression
+ // the value is the original expression and shuttled expression map
+ // Such as origin expressions are l_partkey#0 > 1 and shuttled expression
is l_partkey#10 > 1
+ // the map would be {ExpressionPosition.FILTER, {
+ // l_partkey#0 > 1 : l_partkey#10 > 1
+ // }}
+ // this is for building LogicalCompatibilityContext later.
+ private final Map<ExpressionPosition, Map<Expression, Expression>>
expressionToShuttledExpressionToMap;
+
// Record the exprId and the corresponding expr map, this is used by
expression shuttled
private final Map<ExprId, Expression> namedExprIdAndExprMapping;
private final List<? extends Expression> planOutputShuttledExpressions;
@@ -127,7 +144,9 @@ public class StructInfo {
Plan bottomPlan, List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
@Nullable Predicates predicates,
- Map<ExpressionPosition, Map<Expression, Expression>>
shuttledExpressionsToExpressionsMap,
+ Map<ExpressionPosition, Multimap<Expression, Pair<Expression,
HyperElement>>>
+ shuttledExpressionsToExpressionsMap,
+ Map<ExpressionPosition, Map<Expression, Expression>>
expressionToShuttledExpressionToMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
BitSet tableIdSet,
SplitPredicate splitPredicate,
@@ -146,6 +165,7 @@ public class StructInfo {
this.splitPredicate = splitPredicate;
this.equivalenceClass = equivalenceClass;
this.shuttledExpressionsToExpressionsMap =
shuttledExpressionsToExpressionsMap;
+ this.expressionToShuttledExpressionToMap =
expressionToShuttledExpressionToMap;
this.namedExprIdAndExprMapping = namedExprIdAndExprMapping;
this.planOutputShuttledExpressions = planOutputShuttledExpressions;
}
@@ -156,7 +176,8 @@ public class StructInfo {
public StructInfo withPredicates(Predicates predicates) {
return new StructInfo(this.originalPlan, this.originalPlanId,
this.hyperGraph, this.valid, this.topPlan,
this.bottomPlan, this.relations,
this.relationIdStructInfoNodeMap, predicates,
- this.shuttledExpressionsToExpressionsMap,
this.namedExprIdAndExprMapping, this.tableBitSet,
+ this.shuttledExpressionsToExpressionsMap,
this.expressionToShuttledExpressionToMap,
+ this.namedExprIdAndExprMapping, this.tableBitSet,
null, null, this.planOutputShuttledExpressions);
}
@@ -166,13 +187,16 @@ public class StructInfo {
public StructInfo withTableBitSet(BitSet tableBitSet) {
return new StructInfo(this.originalPlan, this.originalPlanId,
this.hyperGraph, this.valid, this.topPlan,
this.bottomPlan, this.relations,
this.relationIdStructInfoNodeMap, this.predicates,
- this.shuttledExpressionsToExpressionsMap,
this.namedExprIdAndExprMapping, tableBitSet,
+ this.shuttledExpressionsToExpressionsMap,
this.expressionToShuttledExpressionToMap,
+ this.namedExprIdAndExprMapping, tableBitSet,
this.splitPredicate, this.equivalenceClass,
this.planOutputShuttledExpressions);
}
private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
Plan topPlan,
- Map<ExpressionPosition, Map<Expression, Expression>>
shuttledExpressionsToExpressionsMap,
+ Map<ExpressionPosition, Multimap<Expression, Pair<Expression,
HyperElement>>>
+ shuttledExpressionsToExpressionsMap,
+ Map<ExpressionPosition, Map<Expression, Expression>>
expressionToShuttledExpressionToMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
@@ -200,8 +224,9 @@ public class StructInfo {
structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE,
replaceContext);
// Replace expressions by expression map
List<Expression> replacedExpressions =
replaceContext.getReplacedExpressions();
-
putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
- ExpressionPosition.NODE,
replacedExpressions.get(0), expression);
+
putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap,
+ expressionToShuttledExpressionToMap,
+ ExpressionPosition.NODE,
replacedExpressions.get(0), expression, node);
// Record this, will be used in top level expression
shuttle later, see the method
// ExpressionLineageReplacer#visitGroupPlan
namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap());
@@ -227,8 +252,10 @@ public class StructInfo {
// Replace expressions by expression map
List<Expression> replacedExpressions =
replaceContext.getReplacedExpressions();
for (int i = 0; i < replacedExpressions.size(); i++) {
-
putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
- ExpressionPosition.JOIN_EDGE,
replacedExpressions.get(i), joinConjunctExpressions.get(i));
+
putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap,
+ expressionToShuttledExpressionToMap,
+ ExpressionPosition.JOIN_EDGE,
replacedExpressions.get(i), joinConjunctExpressions.get(i),
+ edge);
}
// Record this, will be used in top level expression shuttle
later, see the method
// ExpressionLineageReplacer#visitGroupPlan
@@ -240,10 +267,11 @@ public class StructInfo {
filterExpressions.forEach(predicate -> {
// this is used for LogicalCompatibilityContext
ExpressionUtils.extractConjunction(predicate).forEach(expr ->
-
putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
+
putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap,
+ expressionToShuttledExpressionToMap,
ExpressionPosition.FILTER_EDGE,
ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan, new BitSet()),
- predicate));
+ predicate, filterEdge));
});
});
return true;
@@ -315,11 +343,13 @@ public class StructInfo {
// collect struct info fromGraph
List<CatalogRelation> relationList = new ArrayList<>();
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap = new
LinkedHashMap<>();
- Map<ExpressionPosition, Map<Expression, Expression>>
shuttledHashConjunctsToConjunctsMap =
- new LinkedHashMap<>();
+ Map<ExpressionPosition, Multimap<Expression, Pair<Expression,
HyperElement>>>
+ shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>();
Map<ExprId, Expression> namedExprIdAndExprMapping = new
LinkedHashMap<>();
BitSet tableBitSet = new BitSet();
+ Map<ExpressionPosition, Map<Expression, Expression>>
expressionToShuttledExpressionToMap = new HashMap<>();
boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan,
shuttledHashConjunctsToConjunctsMap,
+ expressionToShuttledExpressionToMap,
namedExprIdAndExprMapping,
relationList,
relationIdStructInfoNodeMap,
@@ -341,19 +371,11 @@ public class StructInfo {
ExpressionUtils.shuttleExpressionWithLineage(originalPlan.getOutput(),
originalPlan, new BitSet());
return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid,
topPlan, bottomPlan,
relationList, relationIdStructInfoNodeMap, predicates,
shuttledHashConjunctsToConjunctsMap,
+ expressionToShuttledExpressionToMap,
namedExprIdAndExprMapping, tableBitSet, null, null,
planOutputShuttledExpressions);
}
- /**
- * Build Struct info from group.
- * Maybe return multi structInfo when original plan already be rewritten
by mv
- */
- public static StructInfo of(Group group) {
- // TODO build graph from original plan and get relations and
predicates from graph
- return null;
- }
-
public List<CatalogRelation> getRelations() {
return relations;
}
@@ -410,21 +432,36 @@ public class StructInfo {
return relationIdStructInfoNodeMap;
}
- public Map<ExpressionPosition, Map<Expression, Expression>>
getShuttledExpressionsToExpressionsMap() {
+ public Map<ExpressionPosition, Multimap<Expression, Pair<Expression,
HyperElement>>>
+ getShuttledExpressionsToExpressionsMap() {
return shuttledExpressionsToExpressionsMap;
}
- private static void putShuttledExpressionsToExpressionsMap(
- Map<ExpressionPosition, Map<Expression, Expression>>
shuttledExpressionsToExpressionsMap,
+ public Map<ExpressionPosition, Map<Expression, Expression>>
getExpressionToShuttledExpressionToMap() {
+ return expressionToShuttledExpressionToMap;
+ }
+
+ private static void putShuttledExpressionToExpressionsMap(
+ Map<ExpressionPosition, Multimap<Expression, Pair<Expression,
HyperElement>>>
+ shuttledExpressionsToExpressionsMap,
+ Map<ExpressionPosition, Map<Expression, Expression>>
expressionPositionToExpressionToMap,
ExpressionPosition expressionPosition,
- Expression key, Expression value) {
- Map<Expression, Expression> expressionExpressionMap =
shuttledExpressionsToExpressionsMap.get(
- expressionPosition);
- if (expressionExpressionMap == null) {
- expressionExpressionMap = new LinkedHashMap<>();
- shuttledExpressionsToExpressionsMap.put(expressionPosition,
expressionExpressionMap);
- }
- expressionExpressionMap.put(key, value);
+ Expression shuttledExpression, Expression originalExpression,
HyperElement valueBelongedElement) {
+ Multimap<Expression, Pair<Expression, HyperElement>>
shuttledExpressionToExpressionMap =
+ shuttledExpressionsToExpressionsMap.get(expressionPosition);
+ if (shuttledExpressionToExpressionMap == null) {
+ shuttledExpressionToExpressionMap = HashMultimap.create();
+ shuttledExpressionsToExpressionsMap.put(expressionPosition,
shuttledExpressionToExpressionMap);
+ }
+ shuttledExpressionToExpressionMap.put(shuttledExpression,
Pair.of(originalExpression, valueBelongedElement));
+
+ Map<Expression, Expression> originalExprToShuttledExprMap =
+ expressionPositionToExpressionToMap.get(expressionPosition);
+ if (originalExprToShuttledExprMap == null) {
+ originalExprToShuttledExprMap = new HashMap<>();
+ expressionPositionToExpressionToMap.put(expressionPosition,
originalExprToShuttledExprMap);
+ }
+ originalExprToShuttledExprMap.put(originalExpression,
shuttledExpression);
}
public List<? extends Expression> getExpressions() {
diff --git
a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
index 5c9df6b7f92..c400e078daf 100644
---
a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
+++
b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
@@ -323,3 +323,27 @@ c 3 6 c,c,c 5.333333333333333 mi
3 2
1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 \N \N 8 8 1
1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 1 2 8 8 1
+-- !query29_0_before --
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 \N \N 8 8 1
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 1 2 8 8 1
+
+-- !query29_0_after --
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 \N \N 8 8 1
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 1 2 8 8 1
+
+-- !query30_0_before --
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 \N \N 8 8 1
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 1 2 8 8 1
+
+-- !query30_0_after --
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 \N \N 8 8 1
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 1 2 8 8 1
+
+-- !query31_0_before --
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 \N \N 8 8 1
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 1 2 8 8 1
+
+-- !query31_0_after --
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 \N \N 8 8 1
+1 2023-12-09 1 yy 2 2 2 4 3
\N 2 3 1 2 8 8 1
+
diff --git
a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 3af41bda690..356b96267a8 100644
---
a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -45,11 +45,9 @@ suite("aggregate_without_roll_up") {
"replication_num" = "1"
);
"""
-
sql """
drop table if exists lineitem
"""
-
sql"""
CREATE TABLE IF NOT EXISTS lineitem (
l_orderkey INTEGER NOT NULL,
@@ -76,11 +74,9 @@ suite("aggregate_without_roll_up") {
"replication_num" = "1"
)
"""
-
sql """
drop table if exists partsupp
"""
-
sql """
CREATE TABLE IF NOT EXISTS partsupp (
ps_partkey INTEGER NOT NULL,
@@ -1517,4 +1513,463 @@ suite("aggregate_without_roll_up") {
order_qt_query28_0_after "${query28_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv28_0"""
+
+
+ // query and mv has the same filter but position is different, should
rewrite successfully
+ def mv29_0 = """
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as col1,
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as col2,
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as col3,
+ partsupp.public_col * 2 as col4,
+ o_orderkey + l_orderkey + ps_partkey * 2,
+ sum(
+ o_orderkey + l_orderkey + ps_partkey * 2
+ ),
+ count() as count_all
+ from
+ (
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as public_col
+ from
+ orders
+ ) orders
+ left join (
+ select
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as public_col
+ from
+ lineitem
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ ) lineitem on l_orderkey = o_orderkey
+ inner join (
+ select
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as public_col
+ from
+ partsupp
+ ) partsupp on ps_partkey = o_orderkey
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ group by
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14;
+ """
+ def query29_0 = """
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as col1,
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as col2,
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as col3,
+ partsupp.public_col * 2 as col4,
+ o_orderkey + l_orderkey + ps_partkey * 2,
+ sum(
+ o_orderkey + l_orderkey + ps_partkey * 2
+ ),
+ count() as count_all
+ from
+ (
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as public_col
+ from
+ orders
+ ) orders
+ left join (
+ select
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as public_col
+ from
+ lineitem
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ ) lineitem on l_orderkey = o_orderkey
+ inner join (
+ select
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as public_col
+ from
+ partsupp
+ ) partsupp on ps_partkey = o_orderkey
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ group by
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14;
+ """
+ order_qt_query29_0_before "${query29_0}"
+ async_mv_rewrite_success(db, mv29_0, query29_0, "mv29_0")
+ order_qt_query29_0_after "${query29_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0"""
+
+
+ // query and mv has the same filter but position is different, should
rewrite successfully
+ // mv join condition has alias
+ def mv30_0 = """
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey_alias,
+ orders.public_col as col1,
+ l_orderkey_alias,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as col2,
+ ps_partkey_alias,
+ ps_suppkey,
+ partsupp.public_col as col3,
+ partsupp.public_col * 2 as col4,
+ o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2,
+ sum(
+ o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2
+ ),
+ count() as count_all
+ from
+ (
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey as o_orderkey_alias,
+ orders.public_col as public_col
+ from
+ orders
+ ) orders
+ left join (
+ select
+ l_orderkey as l_orderkey_alias,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as public_col
+ from
+ lineitem
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias
+ inner join (
+ select
+ ps_partkey as ps_partkey_alias,
+ ps_suppkey,
+ partsupp.public_col as public_col
+ from
+ partsupp
+ ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ and o_orderkey_alias = 2
+ group by
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14;
+ """
+ def query30_0 = """
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as col1,
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as col2,
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as col3,
+ partsupp.public_col * 2 as col4,
+ o_orderkey + l_orderkey + ps_partkey * 2,
+ sum(
+ o_orderkey + l_orderkey + ps_partkey * 2
+ ),
+ count() as count_all
+ from
+ (
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as public_col
+ from
+ orders
+ ) orders
+ left join (
+ select
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as public_col
+ from
+ lineitem
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ ) lineitem on l_orderkey = o_orderkey
+ inner join (
+ select
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as public_col
+ from
+ partsupp
+ ) partsupp on ps_partkey = o_orderkey
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ and o_orderkey = 2
+ group by
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14;
+ """
+ order_qt_query30_0_before "${query30_0}"
+ async_mv_rewrite_success(db, mv30_0, query30_0, "mv30_0")
+ order_qt_query30_0_after "${query30_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv30_0"""
+
+
+ // query and mv has the same filter but position is different, should
rewrite successfully
+ // query join condition has alias
+ def mv31_0 = """
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as col1,
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as col2,
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as col3,
+ partsupp.public_col * 2 as col4,
+ o_orderkey + l_orderkey + ps_partkey * 2,
+ sum(
+ o_orderkey + l_orderkey + ps_partkey * 2
+ ),
+ count() as count_all
+ from
+ (
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey,
+ orders.public_col as public_col
+ from
+ orders
+ ) orders
+ left join (
+ select
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as public_col
+ from
+ lineitem
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ ) lineitem on l_orderkey = o_orderkey
+ inner join (
+ select
+ ps_partkey,
+ ps_suppkey,
+ partsupp.public_col as public_col
+ from
+ partsupp
+ ) partsupp on ps_partkey = o_orderkey
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ and o_orderkey = 2
+ group by
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14;
+ """
+ def query31_0 = """
+select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey_alias,
+ orders.public_col as col1,
+ l_orderkey_alias,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as col2,
+ ps_partkey_alias,
+ ps_suppkey,
+ partsupp.public_col as col3,
+ partsupp.public_col * 2 as col4,
+ o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2,
+ sum(
+ o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2
+ ),
+ count() as count_all
+ from
+ (
+ select
+ o_custkey,
+ o_orderdate,
+ o_shippriority,
+ o_comment,
+ o_orderkey as o_orderkey_alias,
+ orders.public_col as public_col
+ from
+ orders
+ ) orders
+ left join (
+ select
+ l_orderkey as l_orderkey_alias,
+ l_partkey,
+ l_suppkey,
+ lineitem.public_col as public_col
+ from
+ lineitem
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias
+ inner join (
+ select
+ ps_partkey as ps_partkey_alias,
+ ps_suppkey,
+ partsupp.public_col as public_col
+ from
+ partsupp
+ ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias
+ where
+ lineitem.public_col is null
+ or lineitem.public_col <> 1
+ and o_orderkey_alias = 2
+ group by
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14;
+ """
+ order_qt_query31_0_before "${query31_0}"
+ async_mv_rewrite_success(db, mv31_0, query31_0, "mv31_0")
+ order_qt_query31_0_after "${query31_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv31_0"""
}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
index f8e601e64f5..140a91edd7c 100644
---
a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_datetime_part_up_rewrite.groovy
@@ -168,7 +168,7 @@ suite("mtmv_range_datetime_part_up_rewrite") {
for (int i = 0; i < mv_name_list.size(); i++) {
def job_name = getJobName(db, mv_name_list[i])
waitingMTMVTaskFinished(job_name)
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
@@ -178,13 +178,15 @@ suite("mtmv_range_datetime_part_up_rewrite") {
(1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18',
'2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-29 03:00:00')"""
sql """alter table lineitem_range_datetime_union modify column l_comment
set stats ('row_count'='8');"""
for (int i = 0; i < mv_name_list.size(); i++) {
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ // both mv should rewrite success
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
for (int i = 0; i < mv_name_list.size(); i++) {
sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ // both mv should rewrite success
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
@@ -192,25 +194,29 @@ suite("mtmv_range_datetime_part_up_rewrite") {
(3, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18',
'2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-29 03:00:00');"""
sql """alter table lineitem_range_datetime_union modify column l_comment
set stats ('row_count'='9');"""
for (int i = 0; i < mv_name_list.size(); i++) {
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ // both mv should rewrite success
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
for (int i = 0; i < mv_name_list.size(); i++) {
sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ // both mv should rewrite success
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
sql """ALTER TABLE lineitem_range_datetime_union DROP PARTITION IF EXISTS
p4 FORCE"""
for (int i = 0; i < mv_name_list.size(); i++) {
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ // both mv should rewrite success
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
for (int i = 0; i < mv_name_list.size(); i++) {
sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ // both mv should rewrite success
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
