This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f217e052d3 [fix](dynamic_partition) fix dynamic partition not work
when drop and recover olap table (#19031)
f217e052d3 is described below
commit f217e052d364304c611de80106ae28306c59042e
Author: caiconghui <[email protected]>
AuthorDate: Mon May 29 13:02:10 2023 +0800
[fix](dynamic_partition) fix dynamic partition not work when drop and
recover olap table (#19031)
when olap table is dynamic partition enable, if drop and recover olap
table, the table should be added to DynamicPartitionScheduler again
---------
Co-authored-by: caiconghui1 <[email protected]>
---
.../apache/doris/catalog/CatalogRecycleBin.java | 2 +
.../doris/clone/DynamicPartitionScheduler.java | 11 ++++-
.../java/org/apache/doris/catalog/RecoverTest.java | 47 ++++++++++++++++++++++
3 files changed, 58 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index fa86a9241e..d05e61ff01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.common.util.RangeUtils;
import org.apache.doris.common.util.TimeUtils;
@@ -696,6 +697,7 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
RecoverInfo recoverInfo = new RecoverInfo(db.getId(),
table.getId(), -1L, "", newTableName, "");
Env.getCurrentEnv().getEditLog().logRecoverTable(recoverInfo);
}
+
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(),
(OlapTable) table, isReplay);
} finally {
table.writeUnlock();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 7c20455742..eac10bb2d0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -109,6 +109,11 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
dynamicPartitionTableInfo.add(Pair.of(dbId, tableId));
}
+ // only for test
+ public boolean containsDynamicPartitionTable(Long dbId, Long tableId) {
+ return dynamicPartitionTableInfo.contains(Pair.of(dbId, tableId));
+ }
+
public void removeDynamicPartitionTable(Long dbId, Long tableId) {
dynamicPartitionTableInfo.remove(Pair.of(dbId, tableId));
}
@@ -273,8 +278,10 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
} catch (Exception e) {
isPartitionExists = true;
if (addPartitionKeyRange.equals(partitionItem.getItems()))
{
- LOG.info("partition range {} exist in table {}, clear
fail msg",
- addPartitionKeyRange, olapTable.getName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("partition range {} exist in table {},
clear fail msg",
+ addPartitionKeyRange, olapTable.getName());
+ }
clearCreatePartitionFailedMsg(olapTable.getId());
} else {
recordCreatePartitionFailedMsg(db.getFullName(),
olapTable.getName(),
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java
index cf096da8b2..df4ad65747 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java
@@ -135,6 +135,10 @@ public class RecoverTest {
.flatMap(db -> db.getTable(tblName)).isPresent();
}
+ private static boolean checkTableInDynamicScheduler(Long dbId, Long
tableId) {
+ return
Env.getCurrentEnv().getDynamicPartitionScheduler().containsDynamicPartitionTable(dbId,
tableId);
+ }
+
private static boolean checkPartitionExist(String dbName, String tblName,
String partName) {
return Env.getCurrentInternalCatalog()
.getDb(ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, dbName))
@@ -389,4 +393,47 @@ public class RecoverTest {
recoverPartition("test2", "table2", "p1", -1);
Assert.assertTrue(checkPartitionExist("test2", "table2", "p1"));
}
+
+
+ @Test
+ public void testDynamicTableRecover() throws Exception {
+ createDb("test3");
+ createTable("CREATE TABLE test3.`table3` (\n"
+ + " `event_date` datetime(3) NOT NULL COMMENT \"\",\n"
+ + " `app_name` varchar(64) NOT NULL COMMENT \"\",\n"
+ + " `package_name` varchar(64) NOT NULL COMMENT \"\",\n"
+ + " `age` varchar(32) NOT NULL COMMENT \"\",\n"
+ + " `gender` varchar(32) NOT NULL COMMENT \"\",\n"
+ + " `level` varchar(64) NOT NULL COMMENT \"\",\n"
+ + " `city` varchar(64) NOT NULL COMMENT \"\",\n"
+ + " `model` varchar(64) NOT NULL COMMENT \"\",\n"
+ + " `brand` varchar(64) NOT NULL COMMENT \"\",\n"
+ + " `hours` varchar(16) NOT NULL COMMENT \"\",\n"
+ + " `use_num` int(11) SUM NOT NULL COMMENT \"\",\n"
+ + " `use_time` double SUM NOT NULL COMMENT \"\",\n"
+ + " `start_times` bigint(20) SUM NOT NULL COMMENT \"\"\n"
+ + ") ENGINE=OLAP\n"
+ + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`,
`age`, `gender`, `level`, `city`, \n"
+ + " `model`, `brand`, `hours`) COMMENT \"OLAP\"\n"
+ + "PARTITION BY RANGE(`event_date`)\n"
+ + "(PARTITION p1 VALUES [('2020-02-27 00:00:00'), ('2020-03-02
00:00:00')),\n"
+ + "PARTITION p2 VALUES [('2020-03-02 00:00:00'), ('2020-03-07
00:00:00')))\n"
+ + "DISTRIBUTED BY HASH(`event_date`, `app_name`,
`package_name`, `age`, `gender`, `level`, `city`, \n"
+ + " `model`, `brand`, `hours`) BUCKETS 1 PROPERTIES (\n"
+ + "\"replication_num\" = \"1\",\n"
+ + "\"dynamic_partition.enable\" = \"true\",\n"
+ + "\"dynamic_partition.time_unit\" = \"DAY\",\n"
+ + "\"dynamic_partition.end\" = \"3\",\n"
+ + "\"dynamic_partition.prefix\" = \"p\",\n"
+ + "\"dynamic_partition.buckets\" = \"1\",\n"
+ + "\"dynamic_partition.replication_num\" = \"1\",\n"
+ + "\"dynamic_partition.create_history_partition\"=\"true\",\n"
+ + "\"dynamic_partition.start\" = \"-3\"\n"
+ + ");\n");
+ Long dbId = getDbId("test3");
+ Long tableId = getTableId("test3", "table3");
+ dropTable("test3", "table3");
+ recoverTable("test3", "table3", -1);
+ Assert.assertTrue(checkTableInDynamicScheduler(dbId, tableId));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
