This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9af493f3f9d [fix](mtmv) Fix table id overturn and optimize get table
qualifier method (#34768) (#35381)
9af493f3f9d is described below
commit 9af493f3f9d7be22ab3b16e882ff7517a8219f7c
Author: seawinde <[email protected]>
AuthorDate: Fri May 24 21:19:15 2024 +0800
[fix](mtmv) Fix table id overturn and optimize get table qualifier method
(#34768) (#35381)
commitid: 806e241
pr: #34768
Table id may be the same but actually they are different tables. so we
optimize the
org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping#getTableQualifier
with following code:
Objects.hash(table.getDatabase().getCatalog().getId(),
table.getDatabase().getId(), table.getId())
table id is long, we identify the table used in mv rewrite is bitSet. the
bitSet can only use int, so we mapping the long id to init id in every query
when mv rewrite
---
.../doris/catalog/constraint/TableIdentifier.java | 10 ++--
.../org/apache/doris/nereids/NereidsPlanner.java | 13 +++--
.../org/apache/doris/nereids/StatementContext.java | 24 +++++++++
.../apache/doris/nereids/memo/StructInfoMap.java | 30 ++++++-----
.../exploration/mv/MaterializationContext.java | 25 +++++----
.../exploration/mv/MaterializedViewUtils.java | 6 +--
.../nereids/rules/exploration/mv/StructInfo.java | 29 +++++++----
.../exploration/mv/mapping/RelationMapping.java | 21 ++++----
.../expressions/StatementScopeIdGenerator.java | 8 +++
.../apache/doris/nereids/trees/plans/TableId.java | 59 ++++++++++++++++++++++
.../joinorder/hypergraph/CompareOuterJoinTest.java | 35 ++++---------
.../jobs/joinorder/hypergraph/InferJoinTest.java | 24 ++-------
.../joinorder/hypergraph/InferPredicateTest.java | 19 +------
.../joinorder/hypergraph/PullupExpressionTest.java | 25 ++-------
.../doris/nereids/memo/StructInfoMapTest.java | 21 ++++----
.../rules/exploration/mv/EliminateJoinTest.java | 29 +++--------
.../exploration/mv/HyperGraphComparatorTest.java | 22 ++------
.../apache/doris/nereids/sqltest/SqlTestBase.java | 19 +++++++
18 files changed, 235 insertions(+), 184 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java
index 2688fd5784d..8e510ec7a93 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java
@@ -40,8 +40,9 @@ public class TableIdentifier {
Preconditions.checkArgument(tableIf != null,
"Table can not be null in constraint");
tableId = tableIf.getId();
- databaseId = tableIf.getDatabase().getId();
- catalogId = tableIf.getDatabase().getCatalog().getId();
+ databaseId = tableIf.getDatabase() == null ? 0L :
tableIf.getDatabase().getId();
+ catalogId = tableIf.getDatabase() == null ||
tableIf.getDatabase().getCatalog() == null
+ ? 0L : tableIf.getDatabase().getCatalog().getId();
}
public TableIf toTableIf() {
@@ -69,13 +70,14 @@ public class TableIdentifier {
return false;
}
TableIdentifier that = (TableIdentifier) o;
- return databaseId == that.databaseId
+ return catalogId == that.catalogId
+ && databaseId == that.databaseId
&& tableId == that.tableId;
}
@Override
public int hashCode() {
- return Objects.hash(databaseId, tableId);
+ return Objects.hash(catalogId, databaseId, tableId);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 2a354eb9c4b..21c950c40c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -494,10 +494,17 @@ public class NereidsPlanner extends Planner {
plan = super.getExplainString(explainOptions)
+
MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(),
this.getPhysicalPlan());
+ if (statementContext != null) {
+ if (statementContext.isHasUnknownColStats()) {
+ plan += "\n\nStatistics\n planed with unknown column
statistics\n";
+ }
+ }
}
- if (statementContext != null &&
!statementContext.getHints().isEmpty()) {
- String hint = getHintExplainString(statementContext.getHints());
- return plan + hint;
+ if (statementContext != null) {
+ if (!statementContext.getHints().isEmpty()) {
+ String hint =
getHintExplainString(statementContext.getHints());
+ return plan + hint;
+ }
}
return plan;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 3274233c16b..6e4b3c23fc0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.constraint.TableIdentifier;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.hint.Hint;
@@ -29,8 +30,10 @@ import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.TableId;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -55,6 +58,7 @@ import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -99,6 +103,7 @@ public class StatementContext implements Closeable {
private final IdGenerator<ObjectId> objectIdGenerator =
ObjectId.createGenerator();
private final IdGenerator<RelationId> relationIdGenerator =
RelationId.createGenerator();
private final IdGenerator<CTEId> cteIdGenerator = CTEId.createGenerator();
+ private final IdGenerator<TableId> talbeIdGenerator =
TableId.createGenerator();
private final Map<CTEId, Set<LogicalCTEConsumer>> cteIdToConsumers = new
HashMap<>();
private final Map<CTEId, Set<Slot>> cteIdToOutputIds = new HashMap<>();
@@ -138,6 +143,9 @@ public class StatementContext implements Closeable {
// and value is the new string used for replacement.
private final TreeMap<Pair<Integer, Integer>, String> indexInSqlToString
= new TreeMap<>(new Pair.PairComparator<>());
+ // Record table id mapping, the key is the hash code of union catalogId,
databaseId, tableId
+ // the value is the auto-increment id in the cascades context
+ private final Map<TableIdentifier, TableId> tableIdMapping = new
LinkedHashMap<>();
public StatementContext() {
this(ConnectContext.get(), null, 0);
@@ -290,6 +298,10 @@ public class StatementContext implements Closeable {
return relationIdGenerator.getNextId();
}
+ public TableId getNextTableId() {
+ return talbeIdGenerator.getNextId();
+ }
+
public void setParsedStatement(StatementBase parsedStatement) {
this.parsedStatement = parsedStatement;
}
@@ -485,4 +497,16 @@ public class StatementContext implements Closeable {
+ ",\n sql:\n" + sql + "\n}";
}
}
+
+ /** Get table id with lazy */
+ public TableId getTableId(TableIf tableIf) {
+ TableIdentifier tableIdentifier = new TableIdentifier(tableIf);
+ TableId tableId = this.tableIdMapping.get(tableIdentifier);
+ if (tableId != null) {
+ return tableId;
+ }
+ tableId = StatementScopeIdGenerator.newTableId();
+ this.tableIdMapping.put(tableIdentifier, tableId);
+ return tableId;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
index df41eb3eb48..0c11b5fbb22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.memo;
import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
@@ -51,20 +52,21 @@ public class StructInfoMap {
* @param group the group that the mv matched
* @return struct info or null if not found
*/
- public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap,
Group group, Plan originPlan) {
+ public @Nullable StructInfo getStructInfo(CascadesContext cascadesContext,
BitSet tableMap, Group group,
+ Plan originPlan) {
StructInfo structInfo = infoMap.get(tableMap);
if (structInfo != null) {
return structInfo;
}
if (groupExpressionMap.isEmpty() ||
!groupExpressionMap.containsKey(tableMap)) {
- refresh(group, memo.getRefreshVersion());
-
group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion());
+ refresh(group, cascadesContext);
+
group.getstructInfoMap().setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
}
if (groupExpressionMap.containsKey(tableMap)) {
Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair =
getGroupExpressionWithChildren(
tableMap);
- structInfo = constructStructInfo(groupExpressionBitSetPair.first,
- groupExpressionBitSetPair.second, tableMap, originPlan);
+ structInfo = constructStructInfo(groupExpressionBitSetPair.first,
groupExpressionBitSetPair.second,
+ tableMap, originPlan, cascadesContext);
infoMap.put(tableMap, structInfo);
}
return structInfo;
@@ -87,10 +89,11 @@ public class StructInfoMap {
}
private StructInfo constructStructInfo(GroupExpression groupExpression,
List<BitSet> children,
- BitSet tableMap, Plan originPlan) {
+ BitSet tableMap, Plan originPlan, CascadesContext cascadesContext)
{
// this plan is not origin plan, should record origin plan in struct
info
Plan plan = constructPlan(groupExpression, children, tableMap);
- return originPlan == null ? StructInfo.of(plan) : StructInfo.of(plan,
originPlan);
+ return originPlan == null ? StructInfo.of(plan, cascadesContext)
+ : StructInfo.of(plan, originPlan, cascadesContext);
}
private Plan constructPlan(GroupExpression groupExpression, List<BitSet>
children, BitSet tableMap) {
@@ -112,8 +115,9 @@ public class StructInfoMap {
* @param group the root group
*
*/
- public void refresh(Group group, long memoVersion) {
+ public void refresh(Group group, CascadesContext cascadesContext) {
StructInfoMap structInfoMap = group.getstructInfoMap();
+ long memoVersion = cascadesContext.getMemo().getRefreshVersion();
if (!structInfoMap.getTableMaps().isEmpty() && memoVersion ==
structInfoMap.refreshVersion) {
return;
}
@@ -121,14 +125,14 @@ public class StructInfoMap {
for (GroupExpression groupExpression : group.getLogicalExpressions()) {
List<Set<BitSet>> childrenTableMap = new LinkedList<>();
if (groupExpression.children().isEmpty()) {
- BitSet leaf = constructLeaf(groupExpression);
+ BitSet leaf = constructLeaf(groupExpression, cascadesContext);
groupExpressionMap.put(leaf, Pair.of(groupExpression, new
LinkedList<>()));
continue;
}
for (Group child : groupExpression.children()) {
StructInfoMap childStructInfoMap = child.getstructInfoMap();
if (!refreshedGroup.contains(child.getGroupId().asInt())) {
- childStructInfoMap.refresh(child, memoVersion);
+ childStructInfoMap.refresh(child, cascadesContext);
childStructInfoMap.setRefreshVersion(memoVersion);
}
refreshedGroup.add(child.getGroupId().asInt());
@@ -156,12 +160,12 @@ public class StructInfoMap {
}
}
- private BitSet constructLeaf(GroupExpression groupExpression) {
+ private BitSet constructLeaf(GroupExpression groupExpression,
CascadesContext cascadesContext) {
Plan plan = groupExpression.getPlan();
BitSet tableMap = new BitSet();
if (plan instanceof LogicalCatalogRelation) {
- // TODO: Bitset is not compatible with long, use tree map instead
- tableMap.set((int) ((LogicalCatalogRelation)
plan).getTable().getId());
+ tableMap.set(cascadesContext.getStatementContext()
+ .getTableId(((LogicalCatalogRelation)
plan).getTable()).asInt());
}
// one row relation / CTE consumer
return tableMap;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index c2c2448b24c..ebb231c3c68 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -107,13 +107,21 @@ public abstract class MaterializationContext {
// mv output expression shuttle, this will be used to expression
rewrite
this.mvExprToMvScanExprMapping =
ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
this.mvScanPlan.getOutput());
- // copy the plan from cache, which the plan in cache may change
- List<StructInfo> viewStructInfos =
MaterializedViewUtils.extractStructInfo(
- mvPlan, cascadesContext, new BitSet());
- if (viewStructInfos.size() > 1) {
- // view struct info should only have one, log error and use the
first struct info
- LOG.warn(String.format("view strut info is more than one,
materialization name is %s, mv plan is %s",
- getMaterializationQualifier(), getMvPlan().treeString()));
+ // Construct mv struct info, catch exception which may cause planner
roll back
+ List<StructInfo> viewStructInfos;
+ try {
+ viewStructInfos = MaterializedViewUtils.extractStructInfo(mvPlan,
cascadesContext, new BitSet());
+ if (viewStructInfos.size() > 1) {
+ // view struct info should only have one, log error and use
the first struct info
+ LOG.warn(String.format("view strut info is more than one,
materialization name is %s, mv plan is %s",
+ getMaterializationQualifier(),
getMvPlan().treeString()));
+ }
+ } catch (Exception exception) {
+ LOG.warn(String.format("construct mv struct info fail,
materialization name is %s, mv plan is %s",
+ getMaterializationQualifier(), getMvPlan().treeString()),
exception);
+ this.available = false;
+ this.structInfo = null;
+ return;
}
this.structInfo = viewStructInfos.get(0);
}
@@ -276,9 +284,8 @@ public abstract class MaterializationContext {
// rewrite success and chosen
builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
if (!chosenMaterializationQualifiers.isEmpty()) {
- builder.append(" Names: ");
chosenMaterializationQualifiers.forEach(materializationQualifier ->
-
builder.append(generateQualifierName(materializationQualifier)).append(", "));
+
builder.append(generateQualifierName(materializationQualifier)).append(", \n"));
}
// rewrite success but not chosen
builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index f1a90fb940c..e4dc120c20e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -149,7 +149,7 @@ public class MaterializedViewUtils {
Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup();
StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
// Refresh struct info in current level plan from top to bottom
- structInfoMap.refresh(ownerGroup,
cascadesContext.getMemo().getRefreshVersion());
+ structInfoMap.refresh(ownerGroup, cascadesContext);
structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
@@ -161,7 +161,7 @@ public class MaterializedViewUtils {
&&
!materializedViewTableSet.equals(queryTableSet)) {
continue;
}
- StructInfo structInfo =
structInfoMap.getStructInfo(cascadesContext.getMemo(),
+ StructInfo structInfo =
structInfoMap.getStructInfo(cascadesContext,
queryTableSet, ownerGroup, plan);
if (structInfo != null) {
structInfosBuilder.add(structInfo);
@@ -171,7 +171,7 @@ public class MaterializedViewUtils {
}
}
// if plan doesn't belong to any group, construct it directly
- return ImmutableList.of(StructInfo.of(plan));
+ return ImmutableList.of(StructInfo.of(plan, cascadesContext));
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 55649c658b5..fec66265880 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -96,7 +96,7 @@ public class StructInfo {
// bottom plan which top plan only contain join or scan. this is needed by
hyper graph
private final Plan bottomPlan;
private final List<CatalogRelation> relations;
- private final BitSet tableBitSet = new BitSet();
+ private final BitSet tableBitSet;
// this is for LogicalCompatibilityContext later
private final Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap;
// this recorde the predicates which can pull up, not shuttled
@@ -113,12 +113,13 @@ public class StructInfo {
/**
* The construct method for StructInfo
*/
- public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph
hyperGraph, boolean valid, Plan topPlan,
+ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph
hyperGraph, boolean valid, Plan topPlan,
Plan bottomPlan, List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
@Nullable Predicates predicates,
Map<ExpressionPosition, Map<Expression, Expression>>
shuttledExpressionsToExpressionsMap,
- Map<ExprId, Expression> namedExprIdAndExprMapping) {
+ Map<ExprId, Expression> namedExprIdAndExprMapping,
+ BitSet talbeIdSet) {
this.originalPlan = originalPlan;
this.originalPlanId = originalPlanId;
this.hyperGraph = hyperGraph;
@@ -127,7 +128,7 @@ public class StructInfo {
this.topPlan = topPlan;
this.bottomPlan = bottomPlan;
this.relations = relations;
- relations.forEach(relation -> this.tableBitSet.set((int)
(relation.getTable().getId())));
+ this.tableBitSet = talbeIdSet;
this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap;
this.predicates = predicates;
if (predicates == null) {
@@ -150,7 +151,7 @@ public class StructInfo {
public StructInfo withPredicates(Predicates predicates) {
return new StructInfo(this.originalPlan, this.originalPlanId,
this.hyperGraph, this.valid, this.topPlan,
this.bottomPlan, this.relations,
this.relationIdStructInfoNodeMap, predicates,
- this.shuttledExpressionsToExpressionsMap,
this.namedExprIdAndExprMapping);
+ this.shuttledExpressionsToExpressionsMap,
this.namedExprIdAndExprMapping, this.tableBitSet);
}
private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
@@ -265,15 +266,15 @@ public class StructInfo {
* Build Struct info from plan.
* Maybe return multi structInfo when original plan already be rewritten
by mv
*/
- public static StructInfo of(Plan originalPlan) {
- return of(originalPlan, originalPlan);
+ public static StructInfo of(Plan originalPlan, CascadesContext
cascadesContext) {
+ return of(originalPlan, originalPlan, cascadesContext);
}
/**
* Build Struct info from plan.
* Maybe return multi structInfo when original plan already be rewritten
by mv
*/
- public static StructInfo of(Plan derivedPlan, Plan originalPlan) {
+ public static StructInfo of(Plan derivedPlan, Plan originalPlan,
CascadesContext cascadesContext) {
// Split plan by the boundary which contains multi child
LinkedHashSet<Class<? extends Plan>> set = Sets.newLinkedHashSet();
set.add(LogicalJoin.class);
@@ -281,14 +282,15 @@ public class StructInfo {
// if single table without join, the bottom is
derivedPlan.accept(PLAN_SPLITTER, planSplitContext);
return StructInfo.of(originalPlan, planSplitContext.getTopPlan(),
planSplitContext.getBottomPlan(),
-
HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build());
+
HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build(),
cascadesContext);
}
/**
* The construct method for init StructInfo
*/
public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan,
@Nullable Plan bottomPlan,
- HyperGraph hyperGraph) {
+ HyperGraph hyperGraph,
+ CascadesContext cascadesContext) {
ObjectId originalPlanId = originalPlan.getGroupExpression()
.map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1));
// if any of topPlan or bottomPlan is null, split the top plan to two
parts by join node
@@ -310,9 +312,14 @@ public class StructInfo {
namedExprIdAndExprMapping,
relationList,
relationIdStructInfoNodeMap);
+ // Get mapped table id in relation and set
+ BitSet tableBitSet = new BitSet();
+ for (CatalogRelation relation : relationList) {
+
tableBitSet.set(cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt());
+ }
return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid,
topPlan, bottomPlan,
relationList, relationIdStructInfoNodeMap, null,
shuttledHashConjunctsToConjunctsMap,
- namedExprIdAndExprMapping);
+ namedExprIdAndExprMapping, tableBitSet);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
index eb53923da53..a6f68d047ba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.constraint.TableIdentifier;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
@@ -62,22 +63,22 @@ public class RelationMapping extends Mapping {
*/
public static List<RelationMapping> generate(List<CatalogRelation>
sources, List<CatalogRelation> targets) {
// Construct tmp map, key is the table qualifier, value is the
corresponding catalog relations
- HashMultimap<Long, MappedRelation> sourceTableRelationIdMap =
HashMultimap.create();
+ HashMultimap<TableIdentifier, MappedRelation> sourceTableRelationIdMap
= HashMultimap.create();
for (CatalogRelation relation : sources) {
-
sourceTableRelationIdMap.put(getTableQualifier(relation.getTable()),
+
sourceTableRelationIdMap.put(getTableIdentifier(relation.getTable()),
MappedRelation.of(relation.getRelationId(), relation));
}
- HashMultimap<Long, MappedRelation> targetTableRelationIdMap =
HashMultimap.create();
+ HashMultimap<TableIdentifier, MappedRelation> targetTableRelationIdMap
= HashMultimap.create();
for (CatalogRelation relation : targets) {
-
targetTableRelationIdMap.put(getTableQualifier(relation.getTable()),
+
targetTableRelationIdMap.put(getTableIdentifier(relation.getTable()),
MappedRelation.of(relation.getRelationId(), relation));
}
- Set<Long> sourceTableKeySet = sourceTableRelationIdMap.keySet();
+ Set<TableIdentifier> sourceTableKeySet =
sourceTableRelationIdMap.keySet();
List<List<BiMap<MappedRelation, MappedRelation>>> mappedRelations =
new ArrayList<>();
- for (Long sourceTableId : sourceTableKeySet) {
- Set<MappedRelation> sourceMappedRelations =
sourceTableRelationIdMap.get(sourceTableId);
- Set<MappedRelation> targetMappedRelations =
targetTableRelationIdMap.get(sourceTableId);
+ for (TableIdentifier tableIdentifier : sourceTableKeySet) {
+ Set<MappedRelation> sourceMappedRelations =
sourceTableRelationIdMap.get(tableIdentifier);
+ Set<MappedRelation> targetMappedRelations =
targetTableRelationIdMap.get(tableIdentifier);
if (targetMappedRelations.isEmpty()) {
continue;
}
@@ -141,8 +142,8 @@ public class RelationMapping extends Mapping {
return RelationMapping.of(mappingBuilder.build());
}
- private static Long getTableQualifier(TableIf tableIf) {
- return tableIf.getId();
+ private static TableIdentifier getTableIdentifier(TableIf tableIf) {
+ return new TableIdentifier(tableIf);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java
index 170c7cd5c70..df7ef2ab69a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.TableId;
import org.apache.doris.qe.ConnectContext;
import com.google.common.annotations.VisibleForTesting;
@@ -65,6 +66,13 @@ public class StatementScopeIdGenerator {
return ConnectContext.get().getStatementContext().getNextCTEId();
}
+ public static TableId newTableId() {
+ if (ConnectContext.get() == null ||
ConnectContext.get().getStatementContext() == null) {
+ return statementContext.getNextTableId();
+ }
+ return ConnectContext.get().getStatementContext().getNextTableId();
+ }
+
/**
* Reset Id Generator
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java
new file mode 100644
index 00000000000..03e0ec450a3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans;
+
+import org.apache.doris.common.Id;
+import org.apache.doris.common.IdGenerator;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+
+/**
+ * Table id
+ */
+public class TableId extends Id<TableId> {
+
+ public TableId(int id) {
+ super(id);
+ }
+
+ /**
+ * Should be only called by {@link StatementScopeIdGenerator}.
+ */
+ public static IdGenerator<TableId> createGenerator() {
+ return new IdGenerator<TableId>() {
+ @Override
+ public TableId getNextId() {
+ return new TableId(nextId++);
+ }
+ };
+ }
+
+ @Override
+ public String toString() {
+ return "TableId#" + id;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
index ca543a0fd52..490090a43fb 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
@@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
-import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
-import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
-import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.JoinType;
@@ -37,7 +32,6 @@ import com.google.common.collect.Sets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import java.util.BitSet;
import java.util.List;
class CompareOuterJoinTest extends SqlTestBase {
@@ -68,7 +62,7 @@ class CompareOuterJoinTest extends SqlTestBase {
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
Assertions.assertFalse(
- HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2)).isInvalid());
+ HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1)).isInvalid());
}
@Test
@@ -76,9 +70,10 @@ class CompareOuterJoinTest extends SqlTestBase {
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
Plan p1 = new HyperGraphBuilder(Sets.newHashSet(JoinType.INNER_JOIN))
.randomBuildPlanWith(3, 3);
- p1 = PlanChecker.from(connectContext, p1)
+ PlanChecker planChecker = PlanChecker.from(connectContext, p1)
.analyze()
- .rewrite()
+ .rewrite();
+ p1 = planChecker
.getPlan();
Plan p2 = PlanChecker.from(connectContext, p1)
.analyze()
@@ -88,7 +83,8 @@ class CompareOuterJoinTest extends SqlTestBase {
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
Assertions.assertFalse(
- HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2)).isInvalid());
+ HyperGraphComparator.isLogicCompatible(h1, h2,
+ constructContext(p1, p2,
planChecker.getCascadesContext())).isInvalid());
}
@Test
@@ -113,7 +109,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(id = 0)",
res.getQueryExpressions().get(0).toSql());
}
@@ -140,7 +136,8 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- List<Expression> exprList = HyperGraphComparator.isLogicCompatible(h1,
h2, constructContext(p1, p2)).getQueryExpressions();
+ List<Expression> exprList = HyperGraphComparator.isLogicCompatible(h1,
h2,
+ constructContext(p1, p2, c1)).getQueryExpressions();
Assertions.assertEquals(0, exprList.size());
}
@@ -166,7 +163,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(id = 0)",
res.getQueryExpressions().get(0).toSql());
}
@@ -193,17 +190,7 @@ class CompareOuterJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(res.isInvalid());
}
-
- LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
- StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null, new BitSet()).get(0);
- StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
- SlotMapping sm = SlotMapping.generate(rm);
- return LogicalCompatibilityContext.from(rm, sm, st1, st2);
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
index c7d93a363e3..026c354d787 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
@@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
-import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
-import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
-import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -35,7 +30,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import java.util.BitSet;
import java.util.stream.Collectors;
class InferJoinTest extends SqlTestBase {
@@ -61,7 +55,7 @@ class InferJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
Assertions.assertEquals("[id, score]",
@@ -90,7 +84,7 @@ class InferJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
Assertions.assertEquals("[id, score]",
@@ -127,7 +121,7 @@ class InferJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
Assertions.assertEquals("[id, score]",
@@ -157,17 +151,7 @@ class InferJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(res.isInvalid());
}
-
- LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
- StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null, new BitSet()).get(0);
- StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
- SlotMapping sm = SlotMapping.generate(rm);
- return LogicalCompatibilityContext.from(rm, sm, st1, st2);
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
index b6f3fbb45b6..acc4c90b67e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
@@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
-import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
-import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
-import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.PlanChecker;
@@ -33,8 +28,6 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import java.util.BitSet;
-
class InferPredicateTest extends SqlTestBase {
@Test
void testPullUpQueryFilter() {
@@ -58,18 +51,8 @@ class InferPredicateTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(0).toSql());
}
-
- LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
- StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null, new BitSet()).get(0);
- StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
- SlotMapping sm = SlotMapping.generate(rm);
- return LogicalCompatibilityContext.from(rm, sm, st1, st2);
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
index def3281932a..26daa649ec0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
@@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
-import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
-import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
-import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.PlanChecker;
@@ -33,8 +28,6 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import java.util.BitSet;
-
class PullupExpressionTest extends SqlTestBase {
@Test
void testPullUpQueryFilter() {
@@ -58,7 +51,7 @@ class PullupExpressionTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(0).toSql());
}
@@ -85,7 +78,7 @@ class PullupExpressionTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(score = score)",
res.getQueryExpressions().get(0).toSql());
}
@@ -112,7 +105,7 @@ class PullupExpressionTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertEquals(2, res.getViewExpressions().size());
Assertions.assertEquals("(id = 1)",
res.getViewExpressions().get(0).toSql());
Assertions.assertEquals("(id = 1)",
res.getViewExpressions().get(1).toSql());
@@ -140,18 +133,8 @@ class PullupExpressionTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertEquals(1, res.getViewExpressions().size());
Assertions.assertEquals("(score = score)",
res.getViewExpressions().get(0).toSql());
}
-
- LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
- StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null, new BitSet()).get(0);
- StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
- SlotMapping sm = SlotMapping.generate(rm);
- return LogicalCompatibilityContext.from(rm, sm, st1, st2);
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 80670dc366d..0645cf8515b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -22,6 +22,7 @@ import org.apache.doris.mtmv.MTMVRelationManager;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
@@ -59,7 +60,7 @@ class StructInfoMapTest extends SqlTestBase {
Group root = c1.getMemo().getRoot();
Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertTrue(tableMaps.isEmpty());
- root.getstructInfoMap().refresh(root, 1);
+ root.getstructInfoMap().refresh(root, c1);
Assertions.assertEquals(1, tableMaps.size());
new MockUp<MTMVRelationManager>() {
@Mock
@@ -87,7 +88,7 @@ class StructInfoMapTest extends SqlTestBase {
.optimize()
.printlnBestPlanTree();
root = c1.getMemo().getRoot();
- root.getstructInfoMap().refresh(root, 1);
+ root.getstructInfoMap().refresh(root, c1);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
dropMvByNereids("drop materialized view mv1");
@@ -116,8 +117,8 @@ class StructInfoMapTest extends SqlTestBase {
Group root = c1.getMemo().getRoot();
Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertTrue(tableMaps.isEmpty());
- root.getstructInfoMap().refresh(root, 1);
- root.getstructInfoMap().refresh(root, 1);
+ root.getstructInfoMap().refresh(root, c1);
+ root.getstructInfoMap().refresh(root, c1);
Assertions.assertEquals(1, tableMaps.size());
new MockUp<MTMVRelationManager>() {
@Mock
@@ -144,8 +145,8 @@ class StructInfoMapTest extends SqlTestBase {
.optimize()
.printlnBestPlanTree();
root = c1.getMemo().getRoot();
- root.getstructInfoMap().refresh(root, 1);
- root.getstructInfoMap().refresh(root, 1);
+ root.getstructInfoMap().refresh(root, c1);
+ root.getstructInfoMap().refresh(root, c1);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
dropMvByNereids("drop materialized view mv1");
@@ -191,16 +192,18 @@ class StructInfoMapTest extends SqlTestBase {
.rewrite()
.optimize();
Group root = c1.getMemo().getRoot();
- root.getstructInfoMap().refresh(root, 1);
+ root.getstructInfoMap().refresh(root, c1);
StructInfoMap structInfoMap = root.getstructInfoMap();
Assertions.assertEquals(2, structInfoMap.getTableMaps().size());
BitSet mvMap = structInfoMap.getTableMaps().stream()
.filter(b -> b.cardinality() == 2)
.collect(Collectors.toList()).get(0);
- StructInfo structInfo = structInfoMap.getStructInfo(c1.getMemo(),
mvMap, root, null);
+ StructInfo structInfo = structInfoMap.getStructInfo(c1, mvMap, root,
null);
System.out.println(structInfo.getOriginalPlan().treeString());
BitSet bitSet = new BitSet();
- structInfo.getRelations().forEach(r -> bitSet.set((int)
r.getTable().getId()));
+ for (CatalogRelation relation : structInfo.getRelations()) {
+
bitSet.set(c1.getStatementContext().getTableId(relation.getTable()).asInt());
+ }
Assertions.assertEquals(bitSet, mvMap);
dropMvByNereids("drop materialized view mv1");
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
index 1756107b287..2bec767d46f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
@@ -20,8 +20,6 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.rules.RuleSet;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.PlanChecker;
@@ -30,8 +28,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import java.util.BitSet;
-
class EliminateJoinTest extends SqlTestBase {
@Test
void testLOJWithGroupBy() {
@@ -67,9 +63,10 @@ class EliminateJoinTest extends SqlTestBase {
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
HyperGraph h3 = HyperGraph.builderForMv(p3).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
- Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3,
constructContext(p1, p2)).isInvalid());
+ Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3,
+ constructContext(p1, p2, c1)).isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
}
@@ -97,7 +94,7 @@ class EliminateJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
dropConstraint("alter table T2 drop constraint uk");
@@ -139,10 +136,10 @@ class EliminateJoinTest extends SqlTestBase {
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
HyperGraph h3 = HyperGraph.builderForMv(p3).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
- Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3,
constructContext(p1, p2)).isInvalid());
+ Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3,
constructContext(p1, p2, c1)).isInvalid());
dropConstraint("alter table T2 drop constraint pk");
}
@@ -173,7 +170,7 @@ class EliminateJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
dropConstraint("alter table T2 drop constraint pk");
@@ -206,20 +203,10 @@ class EliminateJoinTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
dropConstraint("alter table T2 drop constraint pk");
dropConstraint("alter table T3 drop constraint uk");
}
-
- LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
- StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null, new BitSet()).get(0);
- StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
- SlotMapping sm = SlotMapping.generate(rm);
- return LogicalCompatibilityContext.from(rm, sm, st1, st2);
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
index 360f1310e51..6d752cd6cac 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
@@ -20,8 +20,6 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.rules.RuleSet;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
-import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.PlanChecker;
@@ -30,8 +28,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import java.util.BitSet;
-
class HyperGraphComparatorTest extends SqlTestBase {
@Test
void testInnerJoinAndLOJ() {
@@ -59,7 +55,7 @@ class HyperGraphComparatorTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
}
@@ -90,7 +86,7 @@ class HyperGraphComparatorTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
}
@@ -122,7 +118,7 @@ class HyperGraphComparatorTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
}
@@ -157,18 +153,8 @@ class HyperGraphComparatorTest extends SqlTestBase {
.getAllPlan().get(0).child(0);
HyperGraph h1 = HyperGraph.builderForMv(p1).build();
HyperGraph h2 = HyperGraph.builderForMv(p2).build();
- ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
+ ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2, c1));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
}
-
- LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
- StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null, new BitSet()).get(0);
- StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
- SlotMapping sm = SlotMapping.generate(rm);
- return LogicalCompatibilityContext.from(rm, sm, st1, st2);
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
index 5c7aaed36e3..e879f461012 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
@@ -17,10 +17,19 @@
package org.apache.doris.nereids.sqltest;
+import org.apache.doris.nereids.CascadesContext;
+import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.utframe.TestWithFeService;
+import java.util.BitSet;
+
public abstract class SqlTestBase extends TestWithFeService implements
MemoPatternMatchSupported {
@Override
protected void runBeforeAll() throws Exception {
@@ -82,4 +91,14 @@ public abstract class SqlTestBase extends TestWithFeService
implements MemoPatte
protected void runBeforeEach() throws Exception {
StatementScopeIdGenerator.clear();
}
+
+ protected LogicalCompatibilityContext constructContext(Plan p1, Plan p2,
CascadesContext context) {
+ StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
+ context, new BitSet()).get(0);
+ StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
+ context, new BitSet()).get(0);
+ RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
+ SlotMapping sm = SlotMapping.generate(rm);
+ return LogicalCompatibilityContext.from(rm, sm, st1, st2);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
