This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 204d45d9cea [fix](decommission) fix cann't decommission mtmv (#33823)
204d45d9cea is described below
commit 204d45d9cea106fd5fa612deae4f313e20fc16eb
Author: yujun <[email protected]>
AuthorDate: Wed Apr 24 22:22:03 2024 +0800
[fix](decommission) fix cann't decommission mtmv (#33823)
---
.../java/org/apache/doris/alter/SystemHandler.java | 2 +-
.../main/java/org/apache/doris/catalog/Table.java | 25 -----
.../clone/ColocateTableCheckerAndBalancer.java | 25 ++++-
.../java/org/apache/doris/clone/TabletChecker.java | 30 ++++--
.../doris/cloud/catalog/CloudTabletRebalancer.java | 3 +-
.../doris/cluster/DecommissionBackendTest.java | 106 ++++++++++++++++++++-
.../apache/doris/utframe/TestWithFeService.java | 6 +-
.../data/clone_p0/test_decommission_mtmv.out | 13 +++
.../org/apache/doris/regression/Config.groovy | 3 +
.../apache/doris/regression/ConfigOptions.groovy | 8 ++
.../org/apache/doris/regression/suite/Suite.groovy | 13 ++-
.../doris/regression/suite/SuiteCluster.groovy | 9 +-
.../suites/clone_p0/test_decommission_mtmv.groovy | 93 ++++++++++++++++++
run-regression-test.sh | 1 +
14 files changed, 292 insertions(+), 45 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java
index 394c827e163..26842806483 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java
@@ -316,7 +316,7 @@ public class SystemHandler extends AlterHandler {
for (Table table : db.getTables()) {
table.readLock();
try {
- if (!table.needSchedule()) {
+ if (!table.isManagedTable()) {
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index f1734e9b84f..136a0e04f2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -598,31 +598,6 @@ public abstract class Table extends MetaObject implements
Writable, TableIf {
return table;
}
- /*
- * 1. Only schedule OLAP table.
- * 2. If table is colocate with other table, not schedule it.
- * 3. (deprecated). if table's state is ROLLUP or SCHEMA_CHANGE, but alter
job's state is FINISHING, we should also
- * schedule the tablet to repair it(only for VERSION_INCOMPLETE case,
this will be checked in
- * TabletScheduler).
- * 4. Even if table's state is ROLLUP or SCHEMA_CHANGE, check it. Because
we can repair the tablet of base index.
- */
- public boolean needSchedule() {
- if (type != TableType.OLAP) {
- return false;
- }
-
- OlapTable olapTable = (OlapTable) this;
-
- if (Env.getCurrentColocateIndex().isColocateTable(olapTable.getId())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("table {} is a colocate table, skip tablet
checker.", name);
- }
- return false;
- }
-
- return true;
- }
-
public boolean isHasCompoundKey() {
return hasCompoundKey;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
index 945183cc488..740acd331c3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
@@ -852,6 +852,8 @@ public class ColocateTableCheckerAndBalancer extends
MasterDaemon {
int targetSeqIndex = -1;
long minDataSizeDiff = Long.MAX_VALUE;
+ boolean destBeContainsAllBuckets = true;
+ boolean theSameHostContainsAllBuckets = true;
for (int seqIndex : seqIndexes) {
// the bucket index.
// eg: 0 / 3 = 0, so that the bucket index of the 4th
backend id in flatBackendsPerBucketSeq is 0.
@@ -859,9 +861,15 @@ public class ColocateTableCheckerAndBalancer extends
MasterDaemon {
List<Long> backendsSet =
backendsPerBucketSeq.get(bucketIndex);
List<String> hostsSet = hostsPerBucketSeq.get(bucketIndex);
// the replicas of a tablet can not locate in same Backend
or same host
- if (backendsSet.contains(destBeId) ||
hostsSet.contains(destBe.getHost())) {
+ if (backendsSet.contains(destBeId)) {
continue;
}
+ destBeContainsAllBuckets = false;
+
+ if (!Config.allow_replica_on_same_host &&
hostsSet.contains(destBe.getHost())) {
+ continue;
+ }
+ theSameHostContainsAllBuckets = false;
Preconditions.checkState(backendsSet.contains(srcBeId),
srcBeId);
long bucketDataSize =
@@ -890,8 +898,19 @@ public class ColocateTableCheckerAndBalancer extends
MasterDaemon {
if (targetSeqIndex < 0) {
// we use next node as dst node
- LOG.info("unable to replace backend {} with backend {} in
colocate group {}",
- srcBeId, destBeId, groupId);
+ String failedReason;
+ if (destBeContainsAllBuckets) {
+ failedReason = "dest be contains all the same buckets";
+ } else if (theSameHostContainsAllBuckets) {
+ failedReason = "dest be's host contains all the same
buckets "
+ + "and
Config.allow_replica_on_same_host=false";
+ } else {
+ failedReason = "dest be has no fit path, maybe disk
usage is exceeds "
+ +
"Config.storage_high_watermark_usage_percent";
+ }
+ LOG.info("unable to replace backend {} with dest backend
{} in colocate group {}, "
+ + "failed reason: {}",
+ srcBeId, destBeId, groupId, failedReason);
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
index afd62ec8819..0a23894ab3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
@@ -19,6 +19,7 @@ package org.apache.doris.clone;
import org.apache.doris.analysis.AdminCancelRepairTableStmt;
import org.apache.doris.analysis.AdminRepairTableStmt;
+import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
@@ -241,6 +242,8 @@ public class TabletChecker extends MasterDaemon {
copiedPrios = HashBasedTable.create(prios);
}
+ ColocateTableIndex colocateTableIndex = Env.getCurrentColocateIndex();
+
OUT:
for (long dbId : copiedPrios.rowKeySet()) {
Database db = env.getInternalCatalog().getDbNullable(dbId);
@@ -250,17 +253,21 @@ public class TabletChecker extends MasterDaemon {
List<Long> aliveBeIds = infoService.getAllBackendIds(true);
Map<Long, Set<PrioPart>> tblPartMap = copiedPrios.row(dbId);
for (long tblId : tblPartMap.keySet()) {
- OlapTable tbl = (OlapTable) db.getTableNullable(tblId);
- if (tbl == null) {
+ Table tbl = db.getTableNullable(tblId);
+ if (tbl == null || !tbl.isManagedTable()) {
continue;
}
- tbl.readLock();
+ OlapTable olapTable = (OlapTable) tbl;
+ olapTable.readLock();
try {
- if (!tbl.needSchedule()) {
+ if (colocateTableIndex.isColocateTable(olapTable.getId()))
{
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("table {} is a colocate table, skip
tablet checker.", olapTable.getName());
+ }
continue;
}
- for (Partition partition : tbl.getAllPartitions()) {
- LoopControlStatus st = handlePartitionTablet(db, tbl,
partition, true, aliveBeIds, start,
+ for (Partition partition : olapTable.getAllPartitions()) {
+ LoopControlStatus st = handlePartitionTablet(db,
olapTable, partition, true, aliveBeIds, start,
counter);
if (st == LoopControlStatus.BREAK_OUT) {
break OUT;
@@ -269,7 +276,7 @@ public class TabletChecker extends MasterDaemon {
}
}
} finally {
- tbl.readUnlock();
+ olapTable.readUnlock();
}
}
}
@@ -291,9 +298,16 @@ public class TabletChecker extends MasterDaemon {
List<Long> aliveBeIds = infoService.getAllBackendIds(true);
for (Table table : tableList) {
+ if (!table.isManagedTable()) {
+ continue;
+ }
+
table.readLock();
try {
- if (!table.needSchedule()) {
+ if (colocateTableIndex.isColocateTable(table.getId())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("table {} is a colocate table, skip
tablet checker.", table.getName());
+ }
continue;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
index 262fdaad2f0..749914920f3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
@@ -25,7 +25,6 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.cloud.persist.UpdateCloudReplicaInfo;
import org.apache.doris.cloud.proto.Cloud;
@@ -521,7 +520,7 @@ public class CloudTabletRebalancer extends MasterDaemon {
}
List<Table> tableList = db.getTables();
for (Table table : tableList) {
- if (table.getType() != TableType.OLAP) {
+ if (!table.isManagedTable()) {
continue;
}
OlapTable olapTable = (OlapTable) table;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/cluster/DecommissionBackendTest.java
b/fe/fe-core/src/test/java/org/apache/doris/cluster/DecommissionBackendTest.java
index f1efadcbc83..e50a7ba6323 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/cluster/DecommissionBackendTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/cluster/DecommissionBackendTest.java
@@ -22,6 +22,10 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Replica;
+import org.apache.doris.catalog.Tablet;
+import org.apache.doris.clone.RebalancerTestUtil;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
@@ -56,10 +60,14 @@ public class DecommissionBackendTest extends
TestWithFeService {
@Override
protected void beforeCreatingConnectContext() throws Exception {
FeConstants.default_scheduler_interval_millisecond = 1000;
- Config.tablet_checker_interval_ms = 1000;
+ Config.tablet_checker_interval_ms = 100;
+ Config.tablet_schedule_interval_ms = 100;
Config.tablet_repair_delay_factor_second = 1;
Config.allow_replica_on_same_host = true;
Config.disable_balance = true;
+ Config.schedule_batch_size = 1000;
+ Config.schedule_slot_num_per_hdd_path = 1000;
+ Config.heartbeat_interval_second = 5;
}
@Test
@@ -76,6 +84,7 @@ public class DecommissionBackendTest extends
TestWithFeService {
// 3. create table tbl1
createTable("create table db1.tbl1(k1 int) distributed by hash(k1)
buckets 3 properties('replication_num' = '1');");
+ RebalancerTestUtil.updateReplicaPathHash();
// 4. query tablet num
int tabletNum =
Env.getCurrentInvertedIndex().getTabletMetaMap().size();
@@ -125,6 +134,7 @@ public class DecommissionBackendTest extends
TestWithFeService {
// 3. create table tbl1
createTable("create table db2.tbl1(k1 int) distributed by hash(k1)
buckets 3 properties('replication_num' = '1');");
+ RebalancerTestUtil.updateReplicaPathHash();
// 4. query tablet num
int tabletNum =
Env.getCurrentInvertedIndex().getTabletMetaMap().size();
@@ -182,6 +192,7 @@ public class DecommissionBackendTest extends
TestWithFeService {
createTable("create table db3.tbl1(k1 int) distributed by hash(k1)
buckets 3 properties('replication_num' = '"
+ availableBeNum + "');");
createTable("create table db3.tbl2(k1 int) distributed by hash(k1)
buckets 3 properties('replication_num' = '1');");
+ RebalancerTestUtil.updateReplicaPathHash();
// 4. query tablet num
int tabletNum =
Env.getCurrentInvertedIndex().getTabletMetaMap().size();
@@ -229,11 +240,104 @@ public class DecommissionBackendTest extends
TestWithFeService {
// recover tbl1, because tbl1 has more than one replica, so it still
can be recovered
Assertions.assertDoesNotThrow(() -> recoverTable("db3.tbl1"));
Assertions.assertDoesNotThrow(() -> showCreateTable(sql));
+ dropTable("db3.tbl1", false);
addNewBackend();
Assertions.assertEquals(backendNum(),
Env.getCurrentSystemInfo().getIdToBackend().size());
}
+ @Test
+ public void testDecommissionBackendWithMTMV() throws Exception {
+ // 1. create connect context
+ connectContext = createDefaultCtx();
+
+ ImmutableMap<Long, Backend> idToBackendRef =
Env.getCurrentSystemInfo().getIdToBackend();
+ Assertions.assertEquals(backendNum(), idToBackendRef.size());
+
+ // 2. create database db1
+ createDatabase("db4");
+ System.out.println(Env.getCurrentInternalCatalog().getDbNames());
+
+ // 3. create table
+ createTable("CREATE TABLE db4.table1 (\n"
+ + " `c1` varchar(20) NULL,\n"
+ + " `c2` bigint(20) NULL,\n"
+ + " `c3` int(20) not NULL,\n"
+ + " `k4` bitmap BITMAP_UNION NULL,\n"
+ + " `k5` bitmap BITMAP_UNION NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n"
+ + "COMMENT 'OLAP'\n"
+ + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n"
+ + ";");
+
+ createTable("CREATE TABLE db4.table2 (\n"
+ + " `c1` bigint(20) NULL,\n"
+ + " `c2` bigint(20) NULL,\n"
+ + " `c3` bigint(20) not NULL,\n"
+ + " `k4` bitmap BITMAP_UNION NULL,\n"
+ + " `k5` bitmap BITMAP_UNION NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n"
+ + "COMMENT 'OLAP'\n"
+ + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n"
+ + ";");
+
+ createMvByNereids("create materialized view db4.mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
+ + "DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + "as "
+ + "select t1.c1, t2.c2, t2.k4 "
+ + "from db4.table1 t1 "
+ + "inner join db4.table2 t2 on t1.c1= t2.c2;");
+
+ createMvByNereids("create materialized view db4.mv2 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
+ + "DISTRIBUTED BY HASH(c1) BUCKETS 20 \n"
+ + "PROPERTIES ( 'colocate_with' = 'foo', 'replication_num' =
'3' ) "
+ + "as "
+ + "select t1.c1 as c1, t2.c3, t2.k5 "
+ + "from db4.table1 t1 "
+ + "inner join db4.table2 t2 on t1.c1= t2.c3;");
+
+ RebalancerTestUtil.updateReplicaPathHash();
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrMetaException("db4");
+ OlapTable tbl = (OlapTable) db.getTableOrMetaException("mv1");
+ Assertions.assertNotNull(tbl);
+
+ Partition partition = tbl.getPartitions().iterator().next();
+ Tablet tablet =
partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)
+ .iterator().next().getTablets().iterator().next();
+ Assertions.assertNotNull(tablet);
+ Backend srcBackend =
Env.getCurrentSystemInfo().getBackend(tablet.getReplicas().get(0).getBackendId());
+ Assertions.assertNotNull(srcBackend);
+
+ // 4. query tablet num
+ int tabletNum =
Env.getCurrentInvertedIndex().getTabletMetaMap().size();
+
+ String decommissionStmtStr = "alter system decommission backend
\"127.0.0.1:"
+ + srcBackend.getHeartbeatPort() + "\"";
+ AlterSystemStmt decommissionStmt = (AlterSystemStmt)
parseAndAnalyzeStmt(decommissionStmtStr);
+
Env.getCurrentEnv().getAlterInstance().processAlterCluster(decommissionStmt);
+
+ Assertions.assertTrue(srcBackend.isDecommissioned());
+ long startTimestamp = System.currentTimeMillis();
+ while (System.currentTimeMillis() - startTimestamp < 90000
+ &&
Env.getCurrentSystemInfo().getIdToBackend().containsKey(srcBackend.getId())) {
+ Thread.sleep(1000);
+ }
+ Assertions.assertEquals(backendNum() - 1,
Env.getCurrentSystemInfo().getIdToBackend().size());
+ // For now, we have pre-built internal table: analysis_job and
column_statistics
+ Assertions.assertEquals(tabletNum,
+ Env.getCurrentInvertedIndex().getTabletMetaMap().size());
+
+ for (Replica replica : tablet.getReplicas()) {
+ Assertions.assertTrue(replica.getBackendId() !=
srcBackend.getId());
+ }
+
+ // 6. add backend
+ addNewBackend();
+ Assertions.assertEquals(backendNum(),
Env.getCurrentSystemInfo().getIdToBackend().size());
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 37bc5f431f8..b590234a3e8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -431,7 +431,7 @@ public abstract class TestWithFeService {
}
private void checkBEHeartbeat(List<Backend> bes) throws
InterruptedException {
- int maxTry = 10;
+ int maxTry = Config.heartbeat_interval_second + 2;
boolean allAlive = false;
while (maxTry-- > 0 && !allAlive) {
Thread.sleep(1000);
@@ -463,7 +463,9 @@ public abstract class TestWithFeService {
}
protected Backend addNewBackend() throws IOException, InterruptedException
{
- return createBackend("127.0.0.1", lastFeRpcPort);
+ Backend be = createBackend("127.0.0.1", lastFeRpcPort);
+ checkBEHeartbeat(Lists.newArrayList(be));
+ return be;
}
protected Backend createBackend(String beHost, int feRpcPort) throws
IOException, InterruptedException {
diff --git a/regression-test/data/clone_p0/test_decommission_mtmv.out
b/regression-test/data/clone_p0/test_decommission_mtmv.out
new file mode 100644
index 00000000000..054a55377be
--- /dev/null
+++ b/regression-test/data/clone_p0/test_decommission_mtmv.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select1 --
+1 100 200
+
+-- !select2 --
+2 1000 2000
+
+-- !select3 --
+1 100 200
+
+-- !select4 --
+2 1000 2000
+
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 2b7cd480019..85bae191f76 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -76,6 +76,7 @@ class Config {
public String image
public String dockerCoverageOutputDir
public Boolean dockerEndDeleteFiles
+ public Boolean dockerEndNoKill
public Boolean excludeDockerTest
public String testGroups
@@ -277,6 +278,7 @@ class Config {
config.sslCertificatePath =
FileUtils.getCanonicalPath(cmd.getOptionValue(sslCertificateOpt,
config.sslCertificatePath))
config.dorisComposePath =
FileUtils.getCanonicalPath(config.dorisComposePath)
config.image = cmd.getOptionValue(imageOpt, config.image)
+ config.dockerEndNoKill = cmd.hasOption(noKillDockerOpt)
config.suiteWildcard = cmd.getOptionValue(suiteOpt, config.testSuites)
.split(",")
.collect({s -> s.trim()})
@@ -532,6 +534,7 @@ class Config {
config.image = configToString(obj.image)
config.dockerCoverageOutputDir =
configToString(obj.dockerCoverageOutputDir)
config.dockerEndDeleteFiles = configToBoolean(obj.dockerEndDeleteFiles)
+ config.dockerEndNoKill = configToBoolean(obj.dockerEndNoKill)
config.excludeDockerTest = configToBoolean(obj.excludeDockerTest)
def declareFileNames = config.getClass()
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
index 77fb592cab7..c76560ba745 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
@@ -56,6 +56,7 @@ class ConfigOptions {
static Option pluginOpt
static Option sslCertificateOpt
static Option imageOpt
+ static Option noKillDockerOpt
static Option suiteOpt
static Option excludeSuiteOpt
static Option groupsOpt
@@ -218,6 +219,12 @@ class ConfigOptions {
.desc("the docker image")
.build()
+ noKillDockerOpt = Option.builder("noKillDocker")
+ .required(false)
+ .hasArg(false)
+ .desc("don't kill docker containers")
+ .build()
+
suiteOpt = Option.builder("s")
.argName("suiteName")
.required(false)
@@ -583,6 +590,7 @@ class ConfigOptions {
.addOption(pluginOpt)
.addOption(sslCertificateOpt)
.addOption(imageOpt)
+ .addOption(noKillDockerOpt)
.addOption(confOpt)
.addOption(suiteOpt)
.addOption(excludeSuiteOpt)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index c426a395ed1..8b440ac60df 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -286,7 +286,16 @@ class Suite implements GroovyInterceptable {
}
Thread.sleep(1000)
}
+
assertNotNull(fe)
+ if (!dockerIsCloud) {
+ for (def be : cluster.getAllBackends()) {
+ be_report_disk(be.host, be.httpPort)
+ }
+ }
+
+ // wait be report
+ Thread.sleep(5000)
def url = String.format(
"jdbc:mysql://%s:%s/?useLocalSessionState=false&allowLoadLocalInfile=false",
fe.host, fe.queryPort)
@@ -299,7 +308,9 @@ class Suite implements GroovyInterceptable {
logger.info("connect to docker cluster: suite={}, url={}", name,
url)
connect(user, password, url, actionSupplier)
} finally {
- cluster.destroy(context.config.dockerEndDeleteFiles)
+ if (!context.config.dockerEndNoKill) {
+ cluster.destroy(context.config.dockerEndDeleteFiles)
+ }
}
}
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index e556c1896bb..a915851b938 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -34,8 +34,13 @@ class ClusterOptions {
int feNum = 1
int beNum = 3
- List<String> feConfigs = ['heartbeat_interval_second=5']
- List<String> beConfigs = []
+ List<String> feConfigs = [
+ 'heartbeat_interval_second=5',
+ ]
+
+ List<String> beConfigs = [
+ 'report_random_wait=false',
+ ]
boolean connectToFollower = false
// 1. cloudMode = true, only create cloud cluster.
diff --git a/regression-test/suites/clone_p0/test_decommission_mtmv.groovy
b/regression-test/suites/clone_p0/test_decommission_mtmv.groovy
new file mode 100644
index 00000000000..24853aa718c
--- /dev/null
+++ b/regression-test/suites/clone_p0/test_decommission_mtmv.groovy
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+
+suite('test_decommission_mtmv') {
+ def options = new ClusterOptions()
+ options.feConfigs += [
+ 'disable_balance=true',
+ 'tablet_checker_interval_ms=100',
+ 'tablet_schedule_interval_ms=100',
+ 'schedule_batch_size=1000',
+ 'schedule_slot_num_per_hdd_path=1000',
+ 'storage_high_watermark_usage_percent=99',
+ 'storage_flood_stage_usage_percent=99',
+ ]
+
+ docker(options) {
+ sql '''CREATE TABLE t1 (k1 INT, k2 INT, v INT)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
+ '''
+
+ sql '''CREATE TABLE t2 (k1 INT, k2 INT, v INT)
+ DISTRIBUTED BY HASH(k2) BUCKETS 1
+ '''
+
+ sql '''CREATE MATERIALIZED VIEW mv1 BUILD DEFERRED REFRESH AUTO ON
MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ AS
+ SELECT t1.k1, t1.v as v1, t2.v as v2
+ FROM t1 INNER JOIN t2
+ ON t1.k1 = t2.k1
+ '''
+
+ sql '''CREATE MATERIALIZED VIEW mv2 BUILD DEFERRED REFRESH AUTO ON
MANUAL
+ DISTRIBUTED BY HASH(k2) BUCKETS 6
+ PROPERTIES ( 'colocate_with' = 'foo' )
+ AS
+ SELECT t1.k2 as k2, t1.v as v1, t2.v as v2
+ FROM t1 INNER JOIN t2
+ ON t1.k2 = t2.k2
+ '''
+
+ sql 'INSERT INTO t1 VALUES (1, 0, 100), (0, 2, 1000)'
+ sql 'INSERT INTO t2 VALUES (1, 1, 200), (2, 2, 2000)'
+
+ sql 'REFRESH MATERIALIZED VIEW mv1 AUTO'
+ sql 'REFRESH MATERIALIZED VIEW mv2 AUTO'
+
+ def dbName = context.config.getDbNameByFile(context.file)
+ waitingMTMVTaskFinished(getJobName(dbName, 'mv1'))
+ waitingMTMVTaskFinished(getJobName(dbName, 'mv2'))
+
+ order_qt_select1 'SELECT * FROM mv1'
+ order_qt_select2 'SELECT * FROM mv2'
+
+ def newBeIndex = cluster.addBackend(1)[0]
+ sleep(10 * 1000)
+
+ def decommissionBeIdx = 1
+ def decommissionBe = cluster.getBeByIndex(decommissionBeIdx)
+ cluster.decommissionBackends(decommissionBeIdx)
+
+ def backends = sql_return_maparray 'show backends'
+ assertEquals(3, backends.size())
+ for (def be : backends) {
+ assertNotEquals(be.BackendId as long, decommissionBe.backendId)
+ }
+
+ cluster.stopBackends(2, 3)
+ sleep(10 * 1000)
+ cluster.checkBeIsAlive(2, false)
+ cluster.checkBeIsAlive(3, false)
+ cluster.checkBeIsAlive(4, true)
+
+ order_qt_select3 'SELECT * FROM mv1'
+ order_qt_select4 'SELECT * FROM mv2'
+ }
+}
diff --git a/run-regression-test.sh b/run-regression-test.sh
index dc90f4b648d..e04d1d7cefb 100755
--- a/run-regression-test.sh
+++ b/run-regression-test.sh
@@ -44,6 +44,7 @@ Usage: $0 <shell_options> <framework_options>
-forceGenOut delete and generate .out file
-parallel run tests using specified threads
-randomOrder run tests in a random order
+ -noKillDocker don't kill container when finish docker
suites
-times rum tests {times} times
Eg.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
