This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new de56be4d97d branch-3.1: [fix](mtmv) fix when compatible fail, will
throw NPE #49875 (#52276)
de56be4d97d is described below
commit de56be4d97df0373c01470868376d69ba1ed2c43
Author: zhangdong <[email protected]>
AuthorDate: Thu Jun 26 10:52:35 2025 +0800
branch-3.1: [fix](mtmv) fix when compatible fail, will throw NPE #49875
(#52276)
Cherry pick from #49875
---
.../main/java/org/apache/doris/alter/Alter.java | 5 ++
.../main/java/org/apache/doris/catalog/Env.java | 6 ++-
.../main/java/org/apache/doris/catalog/MTMV.java | 16 +++++++
.../java/org/apache/doris/mtmv/BaseTableInfo.java | 16 ++++++-
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 2 +-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 56 ++++++++++------------
.../org/apache/doris/mtmv/MTMVRefreshSnapshot.java | 2 +-
.../java/org/apache/doris/mtmv/MTMVRelation.java | 4 +-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 43 ++++++++---------
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 5 +-
.../java/org/apache/doris/mtmv/MTMVStatus.java | 6 +++
.../org/apache/doris/nereids/StatementContext.java | 6 +++
.../nereids/rules/analysis/CollectRelation.java | 8 +++-
.../InitConsistentMaterializationContextHook.java | 8 +---
.../mv/InitMaterializationContextHook.java | 7 +--
.../java/org/apache/doris/mtmv/AlterMTMVTest.java | 2 +-
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 12 +++--
.../doris/nereids/memo/StructInfoMapTest.java | 18 +++++++
.../doris/nereids/mv/IdStatisticsMapTest.java | 6 +++
.../doris/nereids/mv/MvTableIdIsLongTest.java | 6 +++
.../org/apache/doris/nereids/util/PlanChecker.java | 11 +++++
.../suites/mtmv_p0/test_paimon_mtmv.groovy | 4 +-
.../mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy | 4 +-
.../suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy | 4 +-
.../mv/dml/insert/dml_insert_and_overwrite.groovy | 9 ++--
.../mv/dml/outfile/dml_into_outfile.groovy | 5 +-
26 files changed, 179 insertions(+), 92 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 8f2677cbd35..e1f60949be1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -1040,6 +1040,11 @@ public class Alter {
case ADD_TASK:
mtmv.addTaskResult(alterMTMV.getTask(),
alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(),
isReplay);
+ // If it is not a replay thread, it means that the current
service is already a new version
+ // and does not require compatibility
+ if (isReplay) {
+ mtmv.compatible(Env.getCurrentEnv().getCatalogMgr());
+ }
break;
default:
throw new RuntimeException("Unknown type value: " +
alterMTMV.getOpType());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 2585087d25b..474c5199df6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4193,7 +4193,11 @@ public class Env {
public void replayCreateTable(CreateTableInfo info) throws
MetaNotFoundException {
if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName()
.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
- getInternalCatalog().replayCreateTable(info.getDbName(),
info.getTable());
+ Table table = info.getTable();
+ getInternalCatalog().replayCreateTable(info.getDbName(), table);
+ if (table instanceof MTMV) {
+ ((MTMV) table).compatible(Env.getCurrentEnv().getCatalogMgr());
+ }
} else {
ExternalCatalog externalCatalog = (ExternalCatalog)
catalogMgr.getCatalog(info.getCtlName());
if (externalCatalog != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 337ad38bbf9..5685dcb06f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -480,6 +480,10 @@ public class MTMV extends OlapTable {
this.refreshSnapshot = refreshSnapshot;
}
+ public boolean canBeCandidate() {
+ return getStatus().canBeCandidate();
+ }
+
public void readMvLock() {
this.mvRwLock.readLock().lock();
}
@@ -557,6 +561,18 @@ public class MTMV extends OlapTable {
* The logic here is to be compatible with older versions by converting ID
to name
*/
public void compatible(CatalogMgr catalogMgr) {
+ try {
+ compatibleInternal(catalogMgr);
+ Env.getCurrentEnv().getMtmvService().unregisterMTMV(this);
+ Env.getCurrentEnv().getMtmvService().registerMTMV(this,
this.getDatabase().getId());
+ } catch (Throwable e) {
+ LOG.warn("MTMV compatible failed, dbName: {}, mvName: {}, errMsg:
{}", getDBName(), name, e.getMessage());
+ status.setState(MTMVState.SCHEMA_CHANGE);
+ status.setSchemaChangeDetail("compatible failed, please refresh or
recreate it, reason: " + e.getMessage());
+ }
+ }
+
+ private void compatibleInternal(CatalogMgr catalogMgr) throws Exception {
if (mvPartitionInfo != null) {
mvPartitionInfo.compatible(catalogMgr);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
index 625fe7d9187..b2b20ec0bab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
@@ -159,10 +159,18 @@ public class BaseTableInfo {
+ '}';
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
if (!StringUtils.isEmpty(ctlName)) {
return;
}
+ // should not get meta from external catalog when replay, because the
timeout period may be very long
+ if (ctlId != InternalCatalog.INTERNAL_CATALOG_ID) {
+ String msg = String.format(
+ "Can not compatibility external table, ctlId: %s, dbId:
%s, tableId: %s",
+ ctlId, dbId, tableId);
+ LOG.warn(msg);
+ throw new Exception(msg);
+ }
try {
CatalogIf catalog =
catalogMgr.getCatalogOrAnalysisException(ctlId);
DatabaseIf db = catalog.getDbOrAnalysisException(dbId);
@@ -171,7 +179,11 @@ public class BaseTableInfo {
this.dbName = db.getFullName();
this.tableName = table.getName();
} catch (AnalysisException e) {
- LOG.warn("MTMV compatible failed, ctlId: {}, dbId: {}, tableId:
{}", ctlId, dbId, tableId, e);
+ String msg = String.format(
+ "Failed to get name based on id during compatibility
process, ctlId: %s, dbId: %s, tableId: %s",
+ ctlId, dbId, tableId);
+ LOG.warn(msg, e);
+ throw new Exception(msg);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index 5a14867c7e1..d34580f6608 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -154,7 +154,7 @@ public class MTMVPartitionInfo {
}
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
if (relatedTable == null) {
return;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
index d7d9b65f4a5..490fc8ca2fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
@@ -20,8 +20,9 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
-import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -77,29 +78,23 @@ public class MTMVRefreshPartitionSnapshot {
+ '}';
}
- public void compatible(MTMV mtmv) {
- try {
- // snapshot add partitionId resolve problem of insert overwrite
- compatiblePartitions(mtmv);
- } catch (Throwable e) {
- LOG.warn("MTMV compatiblePartitions failed, mtmv: {}",
mtmv.getName(), e);
- }
- try {
- // change table id to BaseTableInfo
- compatibleTables(mtmv);
- } catch (Throwable e) {
- LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
- }
-
- try {
- // snapshot add tableId resolve problem of recreate table
- compatibleTablesSnapshot();
- } catch (Throwable e) {
- LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
- }
+ public void compatible(MTMV mtmv) throws Exception {
+ // snapshot add partitionId resolve problem of insert overwrite
+ compatiblePartitions(mtmv);
+ // change table id to BaseTableInfo
+ compatibleTables(mtmv);
+ // snapshot add tableId resolve problem of recreate table
+ compatibleTablesSnapshot();
}
private void compatiblePartitions(MTMV mtmv) throws AnalysisException {
+ if
(mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE))
{
+ return;
+ }
+ // Only olapTable has historical data issues that require compatibility
+ if (mtmv.getMvPartitionInfo().getRelatedTableInfo().getCtlId() !=
InternalCatalog.INTERNAL_CATALOG_ID) {
+ return;
+ }
MTMVRelatedTableIf relatedTableIf =
mtmv.getMvPartitionInfo().getRelatedTable();
// Only olapTable has historical data issues that require compatibility
if (!(relatedTableIf instanceof OlapTable)) {
@@ -113,6 +108,8 @@ public class MTMVRefreshPartitionSnapshot {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot)
entry.getValue();
if (versionSnapshot.getId() == 0) {
Partition partition =
relatedTable.getPartition(entry.getKey());
+ // if not find partition, may be partition has been dropped,
+ // the impact is that MTMV will consider this partition to be
async
if (partition != null) {
(versionSnapshot).setId(partition.getId());
}
@@ -136,12 +133,7 @@ public class MTMVRefreshPartitionSnapshot {
for (Entry<BaseTableInfo, MTMVSnapshotIf> entry :
tablesInfo.entrySet()) {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot)
entry.getValue();
if (versionSnapshot.getId() == 0) {
- try {
- TableIf table = MTMVUtil.getTable(entry.getKey());
- versionSnapshot.setId(table.getId());
- } catch (AnalysisException e) {
- LOG.warn("MTMV compatibleTablesSnapshot failed, can not
get table by: {}", entry.getKey());
- }
+ versionSnapshot.setId(entry.getKey().getTableId());
}
}
}
@@ -155,7 +147,7 @@ public class MTMVRefreshPartitionSnapshot {
return false;
}
- private void compatibleTables(MTMV mtmv) {
+ private void compatibleTables(MTMV mtmv) throws Exception {
if (tables.size() == tablesInfo.size()) {
return;
}
@@ -169,8 +161,12 @@ public class MTMVRefreshPartitionSnapshot {
if (tableInfo.isPresent()) {
tablesInfo.put(tableInfo.get(), entry.getValue());
} else {
- LOG.warn("MTMV compatibleTables failed, tableId: {},
relationTables: {}", entry.getKey(),
- relation.getBaseTablesOneLevel());
+ String msg = String.format(
+ "Failed to get table info based on id during
compatibility process, "
+ + "tableId: %s, relationTables: %s",
+ entry.getKey(), relation.getBaseTablesOneLevel());
+ LOG.warn(msg);
+ throw new Exception(msg);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
index 74fc3cc1c5c..0d9665cb446 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
@@ -91,7 +91,7 @@ public class MTMVRefreshSnapshot {
+ '}';
}
- public void compatible(MTMV mtmv) {
+ public void compatible(MTMV mtmv) throws Exception {
if (MapUtils.isEmpty(partitionSnapshots)) {
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
index 87a0199f128..148d2d00884 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
@@ -65,13 +65,13 @@ public class MTMVRelation {
+ '}';
}
- public void compatible(CatalogMgr catalogMgr) {
+ public void compatible(CatalogMgr catalogMgr) throws Exception {
compatible(catalogMgr, baseTables);
compatible(catalogMgr, baseViews);
compatible(catalogMgr, baseTablesOneLevel);
}
- private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos) {
+ private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos)
throws Exception {
if (CollectionUtils.isEmpty(infos)) {
return;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index bd9244af61e..29cba417f1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -77,48 +77,43 @@ public class MTMVRelationManager implements MTMVHookService
{
/**
* if At least one partition is available, return this mtmv
*
- * @param tableInfos
+ * @param candidateMTMVs
* @param ctx
* @return
*/
- public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos,
ConnectContext ctx,
+ public Set<MTMV> getAvailableMTMVs(Set<MTMV> candidateMTMVs,
ConnectContext ctx,
boolean forceConsistent, BiPredicate<ConnectContext, MTMV>
predicate) {
Set<MTMV> res = Sets.newLinkedHashSet();
- Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
Map<List<String>, Set<String>> queryUsedPartitions =
PartitionCompensator.getQueryUsedPartitions(
ctx.getStatementContext());
-
- for (BaseTableInfo tableInfo : mvInfos) {
- try {
- MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
- if (predicate.test(ctx, mtmv)) {
- continue;
- }
- if (!mtmv.isUseForRewrite()) {
- continue;
- }
- BaseTableInfo relatedTableInfo =
mtmv.getMvPartitionInfo().getRelatedTableInfo();
- if (isMVPartitionValid(mtmv, ctx, forceConsistent,
- relatedTableInfo == null ? null :
queryUsedPartitions.get(relatedTableInfo.toList()))) {
- res.add(mtmv);
- }
- } catch (Exception e) {
- // not throw exception to client, just ignore it
- LOG.warn("getTable failed: {}", tableInfo.toString(), e);
+ for (MTMV mtmv : candidateMTMVs) {
+ if (predicate.test(ctx, mtmv)) {
+ continue;
+ }
+ if (!mtmv.isUseForRewrite()) {
+ continue;
+ }
+ BaseTableInfo relatedTableInfo =
mtmv.getMvPartitionInfo().getRelatedTableInfo();
+ if (isMVPartitionValid(mtmv, ctx, forceConsistent,
+ relatedTableInfo == null ? null :
queryUsedPartitions.get(relatedTableInfo.toList()))) {
+ res.add(mtmv);
}
}
return res;
}
/**
- * get all mtmv related to tableInfos.
+ * get candidate mtmv related to tableInfos.
*/
- public Set<MTMV> getAllMTMVs(List<BaseTableInfo> tableInfos) {
+ public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo> tableInfos) {
Set<MTMV> mtmvs = Sets.newLinkedHashSet();
Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
for (BaseTableInfo tableInfo : mvInfos) {
try {
- mtmvs.add((MTMV) MTMVUtil.getTable(tableInfo));
+ MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
+ if (mtmv.canBeCandidate()) {
+ mtmvs.add(mtmv);
+ }
} catch (Exception e) {
// not throw exception to client, just ignore it
LOG.warn("getTable failed: {}", tableInfo.toString(), e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index afaad55a34b..58b2a37d504 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -20,8 +20,6 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState;
-import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
@@ -56,8 +54,7 @@ public class MTMVRewriteUtil {
return res;
}
// check mv is normal
- MTMVStatus mtmvStatus = mtmv.getStatus();
- if (mtmvStatus.getState() != MTMVState.NORMAL ||
mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) {
+ if (!mtmv.canBeCandidate()) {
return res;
}
// if relatedPartitions is empty but not null, which means query no
partitions
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
index b1761b9e973..aa058e628c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
@@ -82,6 +82,12 @@ public class MTMVStatus {
return this;
}
+ public boolean canBeCandidate() {
+ // MTMVRefreshState.FAIL also can be candidate, because may have some
sync partitions
+ return getState() == MTMVState.NORMAL
+ && getRefreshState() != MTMVRefreshState.INIT;
+ }
+
@Override
public String toString() {
return "MTMVStatus{"
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 66b2f124d10..305436f8062 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
@@ -187,6 +188,7 @@ public class StatementContext implements Closeable {
// if query is: select * from t2 join t5
// mtmvRelatedTables is mv1, mv2, mv3, t1, t2, t3, t4, t5
private final Map<List<String>, TableIf> mtmvRelatedTables =
Maps.newHashMap();
+ private final Set<MTMV> candidateMTMVs = Sets.newHashSet();
// insert into target tables
private final Map<List<String>, TableIf> insertTargetTables =
Maps.newHashMap();
// save view's def and sql mode to avoid them change before lock
@@ -310,6 +312,10 @@ public class StatementContext implements Closeable {
return mtmvRelatedTables;
}
+ public Set<MTMV> getCandidateMTMVs() {
+ return candidateMTMVs;
+ }
+
public Map<List<String>, TableIf> getTables() {
return tables;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
index efcefb2951f..3f5edb81e91 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -208,9 +208,12 @@ public class CollectRelation implements
AnalysisRuleFactory {
}
if (shouldCollect) {
Set<MTMV> mtmvSet =
Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
- LOG.info("table {} related mv set is {}", new
BaseTableInfo(table), mtmvSet);
+ .getCandidateMTMVs(Lists.newArrayList(new
BaseTableInfo(table)));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("table {} related mv set is {}", new
BaseTableInfo(table), mtmvSet);
+ }
for (MTMV mtmv : mtmvSet) {
+
cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv);
cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(),
mtmv);
mtmv.readMvLock();
try {
@@ -222,6 +225,7 @@ public class CollectRelation implements AnalysisRuleFactory
{
LOG.debug("mtmv {} related base table include {}",
new BaseTableInfo(mtmv), baseTableInfo);
}
try {
+ // Collect all base tables and lock them before
querying
cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(),
TableFrom.MTMV);
} catch (AnalysisException exception) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
index fbcf4726a10..e86cca263d3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
@@ -20,16 +20,13 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.PlannerHook;
import com.google.common.annotations.VisibleForTesting;
-import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* If enable query rewrite with mv in dml, should init consistent
materialization context after analyze
@@ -49,10 +46,9 @@ public class InitConsistentMaterializationContextHook
extends InitMaterializatio
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables,
CascadesContext cascadesContext) {
- List<BaseTableInfo> usedBaseTables =
-
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAvailableMTMVs(usedBaseTables,
cascadesContext.getConnectContext(),
+
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
+ cascadesContext.getConnectContext(),
true, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) &&
(!connectContext.getSessionVariable()
.isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 5baeff7c585..1ee22743749 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -26,7 +26,6 @@ import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.mtmv.MTMVPlanUtil;
import org.apache.doris.mtmv.MTMVUtil;
@@ -50,7 +49,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* If enable query rewrite with mv, should init materialization context after
analyze
@@ -107,10 +105,9 @@ public class InitMaterializationContextHook implements
PlannerHook {
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables,
CascadesContext cascadesContext) {
- List<BaseTableInfo> usedBaseTables =
-
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
- .getAvailableMTMVs(usedBaseTables,
cascadesContext.getConnectContext(),
+
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
+ cascadesContext.getConnectContext(),
false, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) &&
(!connectContext.getSessionVariable()
.isEnableMaterializedViewRewriteWhenBaseTableUnawareness());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
index 17ec145f583..342f9fd60c8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
@@ -51,7 +51,7 @@ public class AlterMTMVTest extends TestWithFeService {
MTMVRelationManager relationManager =
Env.getCurrentEnv().getMtmvService().getRelationManager();
Table table =
Env.getCurrentInternalCatalog().getDb("test").get().getTableOrMetaException("stu");
- Set<MTMV> allMTMVs =
relationManager.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
+ Set<MTMV> allMTMVs =
relationManager.getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
boolean hasMvA = false;
boolean hasMvB = false;
for (MTMV mtmv : allMTMVs) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
index 82c7eaac631..e4788c18409 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
@@ -122,6 +122,10 @@ public class MTMVRewriteUtilTest {
MTMVUtil.mtmvContainsExternalTable((MTMV) any);
minTimes = 0;
result = false;
+
+ mtmv.canBeCandidate();
+ minTimes = 0;
+ result = true;
}
};
}
@@ -279,9 +283,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsStateAbnormal() {
new Expectations() {
{
- status.getState();
+ mtmv.canBeCandidate();
minTimes = 0;
- result = MTMVState.SCHEMA_CHANGE;
+ result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
@@ -309,9 +313,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsRefreshStateInit() {
new Expectations() {
{
- status.getRefreshState();
+ mtmv.canBeCandidate();
minTimes = 0;
- result = MTMVRefreshState.INIT;
+ result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 19d1efdbbd8..0d3181d15e8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -70,6 +70,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
@@ -129,6 +135,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
@@ -177,6 +189,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
index 1403a9fee5e..3e93919ed22 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
@@ -59,6 +59,12 @@ public class IdStatisticsMapTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv100 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
index 4fa0a68e77c..5c9fb3c0392 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
@@ -55,6 +55,12 @@ public class MvTableIdIsLongTest extends SqlTestBase {
return true;
}
};
+ new MockUp<MTMV>() {
+ @Mock
+ public boolean canBeCandidate() {
+ return true;
+ }
+ };
connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
index e900937d2ba..48803d1d252 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.util;
import org.apache.doris.analysis.ExplainOptions;
+import org.apache.doris.nereids.CTEContext;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlanProcess;
@@ -49,6 +50,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.RuleType;
+import
org.apache.doris.nereids.rules.exploration.mv.InitConsistentMaterializationContextHook;
import
org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.GroupPlan;
@@ -119,6 +121,9 @@ public class PlanChecker {
}
public PlanChecker analyze() {
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;
@@ -126,6 +131,9 @@ public class PlanChecker {
public PlanChecker analyze(Plan plan) {
this.cascadesContext =
MemoTestUtils.createCascadesContext(connectContext, plan);
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
Set<String> originDisableRules =
connectContext.getSessionVariable().getDisableNereidsRuleNames();
Set<String> disableRuleWithAuth = Sets.newHashSet(originDisableRules);
disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name());
@@ -139,6 +147,9 @@ public class PlanChecker {
public PlanChecker analyze(String sql) {
this.cascadesContext =
MemoTestUtils.createCascadesContext(connectContext, sql);
+
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
+ this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;
diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
index 98477ab92d5..abd0f64450e 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
@@ -54,7 +54,9 @@ suite("test_paimon_mtmv",
"p0,external,mtmv,external_docker,external_docker_dori
"s3.access_key" = "admin",
"s3.secret_key" = "password",
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
- "s3.region" = "us-east-1"
+ "s3.region" = "us-east-1",
+ "fs.oss.connection.timeout" = "1000",
+ "fs.oss.connection.establish.timeout" = "1000"
);"""
order_qt_base_table """ select * from
${catalogName}.test_paimon_spark.test_tb_mix_format ; """
diff --git
a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
index 225ecc22cfb..068597aede6 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
@@ -56,7 +56,9 @@ suite("test_paimon_olap_rewrite_mtmv",
"p0,external,mtmv,external_docker,externa
"s3.access_key" = "admin",
"s3.secret_key" = "password",
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
- "s3.region" = "us-east-1"
+ "s3.region" = "us-east-1",
+ "fs.oss.connection.timeout" = "1000",
+ "fs.oss.connection.establish.timeout" = "1000"
);"""
sql """analyze table ${catalogName}.`test_paimon_spark`.test_tb_mix_format
with sync"""
diff --git a/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
index 024d48da8a7..a1bba2391ed 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_rewrite_mtmv.groovy
@@ -39,7 +39,9 @@ suite("test_paimon_rewrite_mtmv",
"p0,external,mtmv,external_docker,external_doc
"s3.access_key" = "admin",
"s3.secret_key" = "password",
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
- "s3.region" = "us-east-1"
+ "s3.region" = "us-east-1",
+ "fs.oss.connection.timeout" = "1000",
+ "fs.oss.connection.establish.timeout" = "1000"
);"""
sql """analyze table ${catalogName}.`test_paimon_spark`.test_tb_mix_format
with sync"""
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
index 54e9b57f7e5..753217f922d 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy
@@ -106,7 +106,8 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ // todo: Temporarily turn off, otherwise usable materialized views will
not be collected and will need to be changed back in the future
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -155,7 +156,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -203,7 +204,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -249,7 +250,7 @@ suite("dml_insert_and_overwrite") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
diff --git
a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
index 350e49057d0..dc23130b990 100644
---
a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy
@@ -122,7 +122,8 @@ suite("dml_into_outfile", "p0") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ // todo: Temporarily turn off, otherwise usable materialized views will
not be collected and will need to be changed back in the future
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
@@ -185,7 +186,7 @@ suite("dml_into_outfile", "p0") {
ps_comment;""")
// disable query rewrite by mv
- sql "set enable_materialized_view_rewrite=false";
+ sql "set enable_materialized_view_rewrite=true";
// enable dml rewrite by mv
sql "set enable_dml_materialized_view_rewrite=true";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
