This is an automated email from the ASF dual-hosted git repository.
seawinde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c0abfa4083f [fix](mtmv) Refresh MTMV after excluded trigger tables
change (#64041)
c0abfa4083f is described below
commit c0abfa4083fea8e64df158e74ab58b7190e8db55
Author: seawinde <[email protected]>
AuthorDate: Fri Jun 12 07:39:55 2026 +0800
[fix](mtmv) Refresh MTMV after excluded trigger tables change (#64041)
Changing the effective `excluded_trigger_tables` set changes which base
tables participate in MTMV freshness decisions. Reusing the old
`refreshSnapshot` can make a later AUTO refresh skip work under the new
exclude rules, leaving MV data based on the old snapshot.
Root cause: In `MTMVTask.calculateNeedRefreshPartitions()`, AUTO refresh
calls `MTMVPartitionUtil.isMTMVSync()` before verifying whether the
refresh baseline is still complete. After `ALTER MATERIALIZED VIEW ...
SET ("excluded_trigger_tables"=...)`, the new exclude rules can skip
changed base tables and incorrectly mark the MV as fresh.
| File | Change Description |
|------|-------------------|
| `MTMV.java` | Detect effective `excluded_trigger_tables` set changes,
bump `schemaChangeVersion`, and clear `refreshSnapshot` without setting
`MTMVStatus.state` to `SCHEMA_CHANGE`. |
| `MTMVTask.java` | Treat incomplete refresh snapshots as requiring a
complete AUTO refresh before calling `isMTMVSync()`. Manual
specified-partition refresh keeps existing behavior. |
| `MTMVTest.java` | Cover qualified-name changes, same-set order
changes, reduced/cleared excluded table sets, and unrelated property
changes. |
| `MTMVTaskTest.java` | Cover incomplete-baseline AUTO refresh and
manual specified-partition behavior. |
| `test_excluded_trigger_table_mtmv.groovy` | Align the case comments
with the expected refresh behavior. |
Design rationale:
This uses the existing `refreshSnapshot` completeness as the
invalid-baseline signal instead of adding a new persisted flag. That
keeps the change scoped to the ALTER property path and AUTO refresh
decision path. Query rewrite behavior is unchanged.
---
.../main/java/org/apache/doris/catalog/MTMV.java | 47 +++++++--
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 5 +
.../java/org/apache/doris/mtmv/MTMVTaskTest.java | 30 ++++++
.../test/java/org/apache/doris/mtmv/MTMVTest.java | 105 +++++++++++++++++++++
.../data/mtmv_p0/test_create_mtmv_with_view.out | 3 +-
.../mtmv_p0/test_excluded_trigger_table_mtmv.out | 6 ++
.../test_excluded_trigger_table_mtmv.groovy | 6 +-
7 files changed, 189 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index b67894b2be9..c760d63563e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -280,7 +280,21 @@ public class MTMV extends OlapTable {
public Map<String, String> alterMvProperties(Map<String, String>
mvProperties) {
writeMvLock();
try {
+ boolean containsExcludedTriggerTables = mvProperties.containsKey(
+ PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES);
+ Set<TableNameInfo> oldExcludedTriggerTables =
containsExcludedTriggerTables
+ ? parseExcludedTriggerTables()
+ : Sets.newHashSet();
this.mvProperties.putAll(mvProperties);
+ if (containsExcludedTriggerTables) {
+ Set<TableNameInfo> newExcludedTriggerTables =
parseExcludedTriggerTables();
+ if
(!oldExcludedTriggerTables.equals(newExcludedTriggerTables)) {
+ // excluded_trigger_tables changes the refresh baseline
semantics. Invalidate the old
+ // snapshots so the next AUTO refresh rebuilds a complete
baseline with the new rules.
+ this.schemaChangeVersion++;
+ this.refreshSnapshot = new MTMVRefreshSnapshot();
+ }
+ }
return this.mvProperties;
} finally {
writeMvUnlock();
@@ -341,22 +355,26 @@ public class MTMV extends OlapTable {
}
public Set<TableNameInfo> getExcludedTriggerTables() {
- Set<TableNameInfo> res = Sets.newHashSet();
readMvLock();
try {
- if
(StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES)))
{
- return res;
- }
- String[] split =
mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES).split(",");
- for (String alias : split) {
- res.add(new TableNameInfo(alias));
- }
- return res;
+ return parseExcludedTriggerTables();
} finally {
readMvUnlock();
}
}
+ private Set<TableNameInfo> parseExcludedTriggerTables() {
+ Set<TableNameInfo> res = Sets.newHashSet();
+ if
(StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES)))
{
+ return res;
+ }
+ String[] split =
mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES).split(",");
+ for (String alias : split) {
+ res.add(new TableNameInfo(alias));
+ }
+ return res;
+ }
+
public Set<TableNameInfo> getQueryRewriteConsistencyRelaxedTables() {
Set<TableNameInfo> res = Sets.newHashSet();
readMvLock();
@@ -448,6 +466,17 @@ public class MTMV extends OlapTable {
return refreshSnapshot;
}
+ public boolean hasCompleteRefreshSnapshot() {
+ Set<String> partitionNames = getPartitionNames();
+ readMvLock();
+ try {
+ // A refresh baseline is complete only when every current MV
partition has a snapshot.
+ return
refreshSnapshot.getPartitionSnapshots().keySet().containsAll(partitionNames);
+ } finally {
+ readMvUnlock();
+ }
+ }
+
public long getSchemaChangeVersion() {
readMvLock();
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index 7c9e65906ac..9766a175a56 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -645,6 +645,11 @@ public class MTMVTask extends AbstractTask {
if (mtmv.getRefreshInfo().getRefreshMethod() ==
RefreshMethod.COMPLETE) {
return Lists.newArrayList(mtmv.getPartitionNames());
}
+ // An incomplete baseline cannot be checked by isMTMVSync, because the
current exclude rules may
+ // skip the changed base tables and incorrectly mark the MV as fresh.
Rebuild it with a full refresh.
+ if (!mtmv.hasCompleteRefreshSnapshot()) {
+ return Lists.newArrayList(mtmv.getPartitionNames());
+ }
// check if data is fresh
// We need to use a newly generated relationship and cannot retrieve
it using mtmv.getRelation()
// to avoid rebuilding the baseTable and causing a change in the
tableId
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
index 505f210fc2f..f0bf165b2db 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
@@ -85,6 +85,8 @@ public class MTMVTaskTest {
Mockito.when(mtmv.getRefreshInfo()).thenReturn(mtmvRefreshInfo);
Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.COMPLETE);
+
+ Mockito.when(mtmv.hasCompleteRefreshSnapshot()).thenReturn(true);
}
@After
@@ -127,6 +129,34 @@ public class MTMVTaskTest {
Assert.assertEquals(allPartitionNames, result);
}
+ @Test
+ public void
testCalculateNeedRefreshPartitionsSystemIncompleteRefreshSnapshot() throws
AnalysisException {
+
Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.AUTO);
+ Mockito.when(mtmv.hasCompleteRefreshSnapshot()).thenReturn(false);
+
+ MTMVTaskContext context = new
MTMVTaskContext(MTMVTaskTriggerMode.SYSTEM);
+ MTMVTask task = new MTMVTask(mtmv, relation, context);
+ List<String> result = task.calculateNeedRefreshPartitions(null);
+
+ Assert.assertEquals(allPartitionNames, result);
+ mtmvPartitionUtilStatic.verify(() -> MTMVPartitionUtil.isMTMVSync(
+ Mockito.nullable(MTMVRefreshContext.class),
Mockito.nullable(Set.class), Mockito.nullable(Set.class)),
+ Mockito.never());
+ }
+
+ @Test
+ public void
testCalculateNeedRefreshPartitionsManualPartitionsIncompleteRefreshSnapshot()
+ throws AnalysisException {
+ Mockito.when(mtmv.hasCompleteRefreshSnapshot()).thenReturn(false);
+
+ MTMVTaskContext context = new
MTMVTaskContext(MTMVTaskTriggerMode.MANUAL, Lists.newArrayList(poneName),
+ false, null);
+ MTMVTask task = new MTMVTask(mtmv, relation, context);
+ List<String> result = task.calculateNeedRefreshPartitions(null);
+
+ Assert.assertEquals(Lists.newArrayList(poneName), result);
+ }
+
@Test
public void testCalculateNeedRefreshPartitionsSystemNotSyncComplete()
throws AnalysisException {
mtmvPartitionUtilStatic.when(() ->
MTMVPartitionUtil.isMTMVSync(Mockito.nullable(MTMVRefreshContext.class),
Mockito.nullable(Set.class), Mockito.nullable(Set.class))).thenReturn(false);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
index 59d669a3c02..25c62451e43 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
@@ -177,6 +177,111 @@ public class MTMVTest {
Assert.assertTrue(excludedTriggerTables.contains(new
TableNameInfo(null, null, "t3")));
}
+ @Test
+ public void testAlterMvPropertiesWithExcludedTriggerTablesChange() {
+ Map<String, String> mvProperties = Maps.newHashMap();
+ mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"t1");
+ MTMV mtmv = new MTMV();
+ mtmv.setMvProperties(mvProperties);
+ MTMVStatus status = new MTMVStatus(MTMVState.NORMAL, null);
+ mtmv.setStatus(status);
+ MTMVRefreshSnapshot refreshSnapshot = new MTMVRefreshSnapshot();
+ refreshSnapshot.getPartitionSnapshots().put("p1", new
MTMVRefreshPartitionSnapshot());
+ mtmv.setRefreshSnapshot(refreshSnapshot);
+
+ long oldSchemaChangeVersion = mtmv.getSchemaChangeVersion();
+ Map<String, String> newProperties = Maps.newHashMap();
+ newProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"db1.t1");
+
+ mtmv.alterMvProperties(newProperties);
+
+ Assert.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState());
+ Assert.assertEquals(oldSchemaChangeVersion + 1,
mtmv.getSchemaChangeVersion());
+
Assert.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty());
+
+ mtmv.getRefreshSnapshot().getPartitionSnapshots().put("p1", new
MTMVRefreshPartitionSnapshot());
+ oldSchemaChangeVersion = mtmv.getSchemaChangeVersion();
+ newProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"internal.db1.t1");
+
+ mtmv.alterMvProperties(newProperties);
+
+ Assert.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState());
+ Assert.assertEquals(oldSchemaChangeVersion + 1,
mtmv.getSchemaChangeVersion());
+
Assert.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty());
+ }
+
+ @Test
+ public void testAlterMvPropertiesWithSameExcludedTriggerTables() {
+ Map<String, String> mvProperties = Maps.newHashMap();
+ mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"t1,t2");
+ MTMV mtmv = new MTMV();
+ mtmv.setMvProperties(mvProperties);
+ MTMVRefreshSnapshot refreshSnapshot = new MTMVRefreshSnapshot();
+ refreshSnapshot.getPartitionSnapshots().put("p1", new
MTMVRefreshPartitionSnapshot());
+ mtmv.setRefreshSnapshot(refreshSnapshot);
+
+ long oldSchemaChangeVersion = mtmv.getSchemaChangeVersion();
+ Map<String, String> newProperties = Maps.newHashMap();
+ newProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"t2,t1");
+
+ mtmv.alterMvProperties(newProperties);
+
+ Assert.assertEquals(oldSchemaChangeVersion,
mtmv.getSchemaChangeVersion());
+
Assert.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty());
+ }
+
+ @Test
+ public void testAlterMvPropertiesWithReducedExcludedTriggerTables() {
+ Map<String, String> mvProperties = Maps.newHashMap();
+ mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"t1,t2");
+ MTMV mtmv = new MTMV();
+ mtmv.setMvProperties(mvProperties);
+ mtmv.setStatus(new MTMVStatus(MTMVState.NORMAL, null));
+ MTMVRefreshSnapshot refreshSnapshot = new MTMVRefreshSnapshot();
+ refreshSnapshot.getPartitionSnapshots().put("p1", new
MTMVRefreshPartitionSnapshot());
+ mtmv.setRefreshSnapshot(refreshSnapshot);
+
+ long oldSchemaChangeVersion = mtmv.getSchemaChangeVersion();
+ Map<String, String> newProperties = Maps.newHashMap();
+ newProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"t1");
+
+ mtmv.alterMvProperties(newProperties);
+
+ Assert.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState());
+ Assert.assertEquals(oldSchemaChangeVersion + 1,
mtmv.getSchemaChangeVersion());
+
Assert.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty());
+
+ mtmv.getRefreshSnapshot().getPartitionSnapshots().put("p1", new
MTMVRefreshPartitionSnapshot());
+ oldSchemaChangeVersion = mtmv.getSchemaChangeVersion();
+ newProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"");
+
+ mtmv.alterMvProperties(newProperties);
+
+ Assert.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState());
+ Assert.assertEquals(oldSchemaChangeVersion + 1,
mtmv.getSchemaChangeVersion());
+
Assert.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty());
+ }
+
+ @Test
+ public void testAlterMvPropertiesWithOtherProperty() {
+ Map<String, String> mvProperties = Maps.newHashMap();
+ mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
"t1");
+ MTMV mtmv = new MTMV();
+ mtmv.setMvProperties(mvProperties);
+ MTMVRefreshSnapshot refreshSnapshot = new MTMVRefreshSnapshot();
+ refreshSnapshot.getPartitionSnapshots().put("p1", new
MTMVRefreshPartitionSnapshot());
+ mtmv.setRefreshSnapshot(refreshSnapshot);
+
+ long oldSchemaChangeVersion = mtmv.getSchemaChangeVersion();
+ Map<String, String> newProperties = Maps.newHashMap();
+ newProperties.put(PropertyAnalyzer.PROPERTIES_GRACE_PERIOD, "10");
+
+ mtmv.alterMvProperties(newProperties);
+
+ Assert.assertEquals(oldSchemaChangeVersion,
mtmv.getSchemaChangeVersion());
+
Assert.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty());
+ }
+
@Test
public void testAlterStatus() {
MTMV mtmv = new MTMV();
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
b/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
index 90c7a97c096..f083632719a 100644
--- a/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
@@ -22,9 +22,10 @@ COMPLETE
2 2
-- !trigger_table_not_need_refresh --
-NOT_REFRESH
+COMPLETE
-- !after_trigger_table --
1 1
2 2
+3 3
diff --git a/regression-test/data/mtmv_p0/test_excluded_trigger_table_mtmv.out
b/regression-test/data/mtmv_p0/test_excluded_trigger_table_mtmv.out
index 4a2ede2fe66..4c0bc1f0076 100644
--- a/regression-test/data/mtmv_p0/test_excluded_trigger_table_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_excluded_trigger_table_mtmv.out
@@ -4,12 +4,18 @@
-- !true_table --
1 1
+2 2
-- !true_db_table --
1 1
+2 2
+3 3
-- !true_ctl_db_table --
1 1
+2 2
+3 3
+4 4
-- !false_ctl_db_table --
1 1
diff --git
a/regression-test/suites/mtmv_p0/test_excluded_trigger_table_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_excluded_trigger_table_mtmv.groovy
index 6a2264be699..4794470cd3c 100644
--- a/regression-test/suites/mtmv_p0/test_excluded_trigger_table_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_excluded_trigger_table_mtmv.groovy
@@ -65,7 +65,7 @@ suite("test_excluded_trigger_table_mtmv","mtmv") {
REFRESH MATERIALIZED VIEW ${mvName} AUTO
"""
waitingMTMVTaskFinishedByMvName(mvName)
- // should not refresh
+ // should refresh because excluded_trigger_tables changed and refresh
baseline should be rebuilt
order_qt_true_table "SELECT * FROM ${mvName}"
sql """
@@ -78,7 +78,7 @@ suite("test_excluded_trigger_table_mtmv","mtmv") {
REFRESH MATERIALIZED VIEW ${mvName} AUTO
"""
waitingMTMVTaskFinishedByMvName(mvName)
- // should not refresh
+ // should refresh because excluded_trigger_tables changed and refresh
baseline should be rebuilt
order_qt_true_db_table "SELECT * FROM ${mvName}"
sql """
@@ -91,7 +91,7 @@ suite("test_excluded_trigger_table_mtmv","mtmv") {
REFRESH MATERIALIZED VIEW ${mvName} AUTO
"""
waitingMTMVTaskFinishedByMvName(mvName)
- // should not refresh
+ // should refresh because excluded_trigger_tables changed and refresh
baseline should be rebuilt
order_qt_true_ctl_db_table "SELECT * FROM ${mvName}"
sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
